Compare commits
69 Commits
df58595d1e
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| e433c86203 | |||
| a2a144dffa | |||
| 8ef6e99d69 | |||
| 2545faef6c | |||
| 80be8be4d6 | |||
| 801adb385d | |||
| feba7b9d74 | |||
| b89224bbde | |||
| 131ffd9887 | |||
| af492c9e97 | |||
| f39fc94610 | |||
| 4e4eff6ba7 | |||
| 8ff1b4fa66 | |||
| 702f622e18 | |||
| 9da2c1e144 | |||
| 32653f54f9 | |||
| 72a200968a | |||
| b39b68add7 | |||
| d9fa1d9328 | |||
| 8375ad83f3 | |||
| 4158394dcf | |||
| eac7e155a5 | |||
| 0cf2cbfeb3 | |||
| 361dbb4ca8 | |||
| d6deccf3e8 | |||
| ee747243fe | |||
| a1ceb457e9 | |||
| 9900211fa4 | |||
| 60cebf0e4b | |||
| 7bd575187e | |||
| ab5a7e8e3d | |||
| 99b2e1cd81 | |||
| 363313d99c | |||
| 18ddf00d3d | |||
| 59f3fe3d1d | |||
| 1dccf5f140 | |||
| 0b40cf8026 | |||
| a7ec195587 | |||
| 13de820931 | |||
| 14ef59aaed | |||
| f387222fce | |||
| 9cb9008dfc | |||
| 083decc5b4 | |||
| 57cac5d3f7 | |||
| 0a772e03b4 | |||
| 0920062a38 | |||
| 39afe644eb | |||
| e3ee3de10a | |||
| 9c72db56e9 | |||
| bb2d606dbb | |||
| 9850767a8a | |||
| 74e2d21de5 | |||
| 7cb18a1a40 | |||
| b556fc2f4f | |||
| b99bd38eb4 | |||
| 701b6726d7 | |||
| 665039f4dc | |||
| ef8dae776e | |||
| d01775b68a | |||
| 0d6f2dd0ce | |||
| df40cbec6e | |||
| 0341e0c7c0 | |||
| 39af7d4f3c | |||
| bba582b4ca | |||
| 1f16a85330 | |||
| f9482639d4 | |||
| dce721cdbd | |||
| 98734644d6 | |||
| 951383226c |
@@ -2,8 +2,33 @@ when:
|
|||||||
- event: tag
|
- event: tag
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: build-release-assets
|
validate:
|
||||||
image: golang:1.25
|
image: golang:1.25
|
||||||
|
commands:
|
||||||
|
- go test ./...
|
||||||
|
- go test -race ./...
|
||||||
|
- go vet ./...
|
||||||
|
- go build ./...
|
||||||
|
- go test ./internal/doccheck
|
||||||
|
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||||
|
|
||||||
|
cross-build:
|
||||||
|
image: golang:1.25
|
||||||
|
depends_on: validate
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
set -eu
|
||||||
|
output_dir="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$output_dir"' EXIT
|
||||||
|
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
|
||||||
|
goos="${target%/*}"
|
||||||
|
goarch="${target#*/}"
|
||||||
|
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
|
||||||
|
done
|
||||||
|
|
||||||
|
build-release-assets:
|
||||||
|
image: golang:1.25
|
||||||
|
depends_on: [validate, cross-build]
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
set -eu
|
set -eu
|
||||||
@@ -33,7 +58,7 @@ steps:
|
|||||||
build_binary windows amd64 ".exe"
|
build_binary windows amd64 ".exe"
|
||||||
build_binary windows arm64 ".exe"
|
build_binary windows arm64 ".exe"
|
||||||
|
|
||||||
- name: publish-release
|
publish-release:
|
||||||
image: woodpeckerci/plugin-release
|
image: woodpeckerci/plugin-release
|
||||||
depends_on:
|
depends_on:
|
||||||
- build-release-assets
|
- build-release-assets
|
||||||
|
|||||||
8
.woodpecker/shuffle.yml
Normal file
8
.woodpecker/shuffle.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
when:
|
||||||
|
- event: cron
|
||||||
|
|
||||||
|
steps:
|
||||||
|
shuffled-race-tests:
|
||||||
|
image: golang:1.25
|
||||||
|
commands:
|
||||||
|
- go test -race -shuffle=on -count=3 ./...
|
||||||
47
.woodpecker/verify.yml
Normal file
47
.woodpecker/verify.yml
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
when:
|
||||||
|
- event: [push, pull_request]
|
||||||
|
|
||||||
|
steps:
|
||||||
|
tests:
|
||||||
|
image: golang:1.25
|
||||||
|
commands:
|
||||||
|
- go test ./...
|
||||||
|
|
||||||
|
race-tests:
|
||||||
|
image: golang:1.25
|
||||||
|
depends_on: tests
|
||||||
|
commands:
|
||||||
|
- go test -race ./...
|
||||||
|
|
||||||
|
static-analysis:
|
||||||
|
image: golang:1.25
|
||||||
|
depends_on: tests
|
||||||
|
commands:
|
||||||
|
- go vet ./...
|
||||||
|
|
||||||
|
build:
|
||||||
|
image: golang:1.25
|
||||||
|
depends_on: tests
|
||||||
|
commands:
|
||||||
|
- go build ./...
|
||||||
|
|
||||||
|
documentation-and-examples:
|
||||||
|
image: golang:1.25
|
||||||
|
depends_on: tests
|
||||||
|
commands:
|
||||||
|
- go test ./internal/doccheck
|
||||||
|
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||||
|
|
||||||
|
cross-build:
|
||||||
|
image: golang:1.25
|
||||||
|
depends_on: [race-tests, static-analysis, build, documentation-and-examples]
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
set -eu
|
||||||
|
output_dir="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$output_dir"' EXIT
|
||||||
|
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
|
||||||
|
goos="${target%/*}"
|
||||||
|
goarch="${target#*/}"
|
||||||
|
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
|
||||||
|
done
|
||||||
@@ -1,7 +1,8 @@
|
|||||||
# narratio
|
# narratio
|
||||||
|
|
||||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
||||||
polished transcripts and generated artifacts.
|
polished transcripts, validated Notarius extraction lanes, and generated
|
||||||
|
artifacts.
|
||||||
|
|
||||||
It runs a deterministic workflow with manifest-driven continuation, remote
|
It runs a deterministic workflow with manifest-driven continuation, remote
|
||||||
publish, and restore support.
|
publish, and restore support.
|
||||||
|
|||||||
25
docs/cli.md
25
docs/cli.md
@@ -47,7 +47,11 @@ Rules:
|
|||||||
- `--campaign` and `--campaign-file` are mutually exclusive.
|
- `--campaign` and `--campaign-file` are mutually exclusive.
|
||||||
- `--session` is not used by `session init`.
|
- `--session` is not used by `session init`.
|
||||||
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
||||||
|
- `--previous-session-id` is a strict expectation: the selected session file
|
||||||
|
must contain the same `previous_session_id`.
|
||||||
- `clean --all` cannot be combined with campaign/session selectors.
|
- `clean --all` cannot be combined with campaign/session selectors.
|
||||||
|
- notification delivery is currently limited to the configured `noop` mode; see
|
||||||
|
the [configuration reference](./config.md#notifications).
|
||||||
|
|
||||||
## Session ID Input Rules
|
## Session ID Input Rules
|
||||||
|
|
||||||
@@ -75,7 +79,10 @@ narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common co
|
|||||||
Behavior:
|
Behavior:
|
||||||
|
|
||||||
- evaluates full stage order;
|
- evaluates full stage order;
|
||||||
- skips already-succeeded stages unless `--force` is set;
|
- runs `extract` between `trim` and `render`; an omitted or disabled Notarius
|
||||||
|
configuration records an explicit `notarius_disabled` self-skip;
|
||||||
|
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
||||||
|
resume check finds its durable result obsolete;
|
||||||
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||||
- writes session and run manifests.
|
- writes session and run manifests.
|
||||||
|
|
||||||
@@ -93,6 +100,7 @@ Valid stage names:
|
|||||||
- `polish`
|
- `polish`
|
||||||
- `normalize`
|
- `normalize`
|
||||||
- `trim`
|
- `trim`
|
||||||
|
- `extract`
|
||||||
- `render`
|
- `render`
|
||||||
- `analyze`
|
- `analyze`
|
||||||
- `publish`
|
- `publish`
|
||||||
@@ -164,7 +172,8 @@ Read-only preflight checks for config validity, required inputs, audio mode, pre
|
|||||||
narratio session status <session_id> [...common config flags]
|
narratio session status <session_id> [...common config flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Prints local manifest state and, when storage is available, remote current-state and published-output status.
|
Prints local manifest state and, when storage is available, status for the
|
||||||
|
pointer-selected remote commit and its declared published outputs.
|
||||||
|
|
||||||
### `session init`
|
### `session init`
|
||||||
|
|
||||||
@@ -207,7 +216,8 @@ Behavior:
|
|||||||
- discovers committed remote current state;
|
- discovers committed remote current state;
|
||||||
- plans local restores;
|
- plans local restores;
|
||||||
- writes an execution report;
|
- writes an execution report;
|
||||||
- blocks conflicting overwrites unless `--force` is set.
|
- blocks unresolved conflicts. `--force` permits replacement only of eligible
|
||||||
|
regular files.
|
||||||
|
|
||||||
See [Operations: Restore Workflow](./operations.md#restore-workflow) for the
|
See [Operations: Restore Workflow](./operations.md#restore-workflow) for the
|
||||||
default restore scope, report location, and conflict-handling workflow.
|
default restore scope, report location, and conflict-handling workflow.
|
||||||
@@ -218,7 +228,10 @@ default restore scope, report location, and conflict-handling workflow.
|
|||||||
narratio session artifacts <session_id> [--remote] [...common config flags]
|
narratio session artifacts <session_id> [--remote] [...common config flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Lists effective built-in and configured artifact sources, publish rules, lock state, and optional remote published-state availability.
|
Lists effective built-in, configured Scriptorium, and configured extraction
|
||||||
|
sources; reports planned, available, unavailable, and published state without
|
||||||
|
reading payload bodies; and includes publish rules, lock state, and optional
|
||||||
|
remote published-state availability.
|
||||||
|
|
||||||
### `session locks`
|
### `session locks`
|
||||||
|
|
||||||
@@ -248,7 +261,9 @@ Effects:
|
|||||||
|
|
||||||
- filters analyze execution to selected configured artifacts;
|
- filters analyze execution to selected configured artifacts;
|
||||||
- filters publish rules that source `narratio.artifact.<name>`;
|
- filters publish rules that source `narratio.artifact.<name>`;
|
||||||
- does not filter built-in transcript/bounds publish sources.
|
- does not filter built-in transcript/bounds or explicitly configured
|
||||||
|
`narratio.extraction.<name>` publish sources; and
|
||||||
|
- does not select or filter Notarius lanes.
|
||||||
|
|
||||||
## Common Workflows
|
## Common Workflows
|
||||||
|
|
||||||
|
|||||||
113
docs/config.md
113
docs/config.md
@@ -38,9 +38,25 @@ If local session discovery fails and a `session_id` is known, Narratio attempts
|
|||||||
|
|
||||||
using configured object storage.
|
using configured object storage.
|
||||||
|
|
||||||
|
The downloaded remote session file is command-scoped: Narratio removes it after
|
||||||
|
the command finishes and records only the remote object provenance alongside
|
||||||
|
the durable copied session input.
|
||||||
|
|
||||||
|
### Identity segments
|
||||||
|
|
||||||
|
Campaign IDs (`campaign_id` and `default_campaign_id`), session IDs, previous
|
||||||
|
session IDs, and Narratio run IDs are opaque portable segments. They must use
|
||||||
|
only ASCII letters, digits, `.`, `_`, and `-`; empty values, `.`/`..`, path
|
||||||
|
separators, drive forms, whitespace, control characters, and non-ASCII text are
|
||||||
|
rejected. Narratio does not trim or rewrite these values. Existing manifests or
|
||||||
|
remote state with an unsafe legacy identity must be migrated before use.
|
||||||
|
|
||||||
## Validation and Merge Rules
|
## Validation and Merge Rules
|
||||||
|
|
||||||
- YAML decode is strict (`KnownFields(true)`): unknown fields fail load.
|
- YAML decode is strict (`KnownFields(true)`) and accepts exactly one document:
|
||||||
|
unknown fields or trailing documents fail load.
|
||||||
|
- Configured timeout and retry-delay durations must be positive. An omitted
|
||||||
|
artifact timeout continues to inherit its configured Scriptorium timeout.
|
||||||
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
||||||
- Pipeline defaults are applied before validation.
|
- Pipeline defaults are applied before validation.
|
||||||
- Campaign and session identities must agree.
|
- Campaign and session identities must agree.
|
||||||
@@ -85,7 +101,13 @@ inputs:
|
|||||||
|
|
||||||
- Do not place raw secrets in YAML.
|
- Do not place raw secrets in YAML.
|
||||||
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
|
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
|
||||||
- Optionally load env files from `pipeline.secrets.env_dir`.
|
- Optionally load credential files from `pipeline.secrets.env_dir`. Each valid
|
||||||
|
environment-variable filename supplies one value; trailing CR/LF is removed.
|
||||||
|
- An existing process environment value takes precedence over a credential file.
|
||||||
|
- Credential directories and files must not be symlinks and must be regular,
|
||||||
|
bounded files (at most 8 KiB per value). On POSIX, provision the directory
|
||||||
|
with no group/other access (normally `0700`) and files with no group/other
|
||||||
|
access (normally `0600`).
|
||||||
- Commands that need storage/auth load filesystem secrets before constructing adapters.
|
- Commands that need storage/auth load filesystem secrets before constructing adapters.
|
||||||
|
|
||||||
## Publish Configuration Summary
|
## Publish Configuration Summary
|
||||||
@@ -118,6 +140,9 @@ Rules:
|
|||||||
|
|
||||||
- `outputs[].source` is required.
|
- `outputs[].source` is required.
|
||||||
- `outputs[].dest` may be omitted when derivable from source.
|
- `outputs[].dest` may be omitted when derivable from source.
|
||||||
|
- extraction sources require an explicit `outputs[].dest` and publish only when
|
||||||
|
a rule names that source; the Notarius index and complete bundle are not
|
||||||
|
publish sources.
|
||||||
- `outputs[].required` defaults to `true`.
|
- `outputs[].required` defaults to `true`.
|
||||||
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
||||||
|
|
||||||
@@ -132,8 +157,8 @@ Rules:
|
|||||||
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||||
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||||
| `pipeline.secrets.env_dir` | string | No | empty |
|
| `pipeline.secrets.env_dir` | string | No | empty |
|
||||||
| `pipeline.storage.backend` | string | No | empty |
|
| `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
|
||||||
| `pipeline.storage.s3.bucket` | string | Conditional | required for S3 session-audio and for publish upload when backend is `s3` |
|
| `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
|
||||||
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
||||||
| `pipeline.storage.s3.region` | string | No | empty |
|
| `pipeline.storage.s3.region` | string | No | empty |
|
||||||
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
||||||
@@ -153,7 +178,7 @@ Rules:
|
|||||||
| `pipeline.publish.locks[]` | list | No | empty |
|
| `pipeline.publish.locks[]` | list | No | empty |
|
||||||
| `pipeline.publish.locks[].source` | string | Yes (per lock) | must reference supported publish source |
|
| `pipeline.publish.locks[].source` | string | Yes (per lock) | must reference supported publish source |
|
||||||
| `pipeline.publish.locks[].reason` | string | No | empty |
|
| `pipeline.publish.locks[].reason` | string | No | empty |
|
||||||
| `pipeline.whisperx.transcribe_url` | string | Yes | valid URL |
|
| `pipeline.whisperx.transcribe_url` | string | Yes | absolute `http` or `https` URL |
|
||||||
| `pipeline.whisperx.language` | string | No | `en` |
|
| `pipeline.whisperx.language` | string | No | `en` |
|
||||||
| `pipeline.whisperx.timeout` | duration | No | `30m` |
|
| `pipeline.whisperx.timeout` | duration | No | `30m` |
|
||||||
| `pipeline.whisperx.retries` | int | No | `3` |
|
| `pipeline.whisperx.retries` | int | No | `3` |
|
||||||
@@ -196,6 +221,13 @@ Rules:
|
|||||||
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
||||||
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
||||||
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||||
|
| `pipeline.notarius.enabled` | bool | No | `false` |
|
||||||
|
| `pipeline.notarius.binary` | string | No | `notarius` |
|
||||||
|
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
||||||
|
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
||||||
|
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
||||||
|
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
||||||
|
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
||||||
| `pipeline.render.enabled` | bool | No | `true` |
|
| `pipeline.render.enabled` | bool | No | `true` |
|
||||||
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
|
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
|
||||||
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
|
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
|
||||||
@@ -207,9 +239,27 @@ Rules:
|
|||||||
| `pipeline.scriptorium.timeout` | duration | No | `10m` |
|
| `pipeline.scriptorium.timeout` | duration | No | `10m` |
|
||||||
| `pipeline.scriptorium.render_debug` | bool | No | `false` |
|
| `pipeline.scriptorium.render_debug` | bool | No | `false` |
|
||||||
| `pipeline.scriptorium.artifacts` | map | No | empty |
|
| `pipeline.scriptorium.artifacts` | map | No | empty |
|
||||||
| `pipeline.notification.backend` | string | No | empty |
|
| `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
|
||||||
| `pipeline.notification.recipient` | string | No | empty |
|
|
||||||
| `pipeline.notification.timeout` | duration | No | empty |
|
### Notarius Output Entries
|
||||||
|
|
||||||
|
For each `pipeline.notarius.outputs.<name>`:
|
||||||
|
|
||||||
|
| Field | Type | Required | Rule |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `lane_id` | string | Yes | unique Notarius lane ID |
|
||||||
|
| `media_type` | string | Yes | exact accepted descriptor media type |
|
||||||
|
| `schema_id` | string | Yes | exact accepted descriptor schema ID |
|
||||||
|
| `schema_version` | string | Yes | exact accepted descriptor schema version |
|
||||||
|
| `module_key` | string | No | exact accepted module key when set |
|
||||||
|
|
||||||
|
Output names must match `^[a-z][a-z0-9_]*$` and become selectable sources named
|
||||||
|
`narratio.extraction.<name>`. Lane IDs must be unique. Every declared output is
|
||||||
|
required from a successful Notarius result; a missing, rejected, duplicate, or
|
||||||
|
contract-incompatible lane fails extraction. See the
|
||||||
|
[complete maintained example](../examples/pipeline.full.annotated.yml) for the
|
||||||
|
current ten-lane D&D mapping and the [Notarius contract](./integrations/notarius.md)
|
||||||
|
for compatibility ownership.
|
||||||
|
|
||||||
### Scriptorium Artifact Entries
|
### Scriptorium Artifact Entries
|
||||||
|
|
||||||
@@ -229,20 +279,39 @@ For each `pipeline.scriptorium.artifacts.<name>`:
|
|||||||
|
|
||||||
Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium request for sticky upstream LLM routing. If an artifact config sets `vars.session_id`, Narratio replaces that value before invoking Scriptorium. Use a different variable name if a prompt needs the raw Narratio session ID as content.
|
Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium request for sticky upstream LLM routing. If an artifact config sets `vars.session_id`, Narratio replaces that value before invoking Scriptorium. Use a different variable name if a prompt needs the raw Narratio session ID as content.
|
||||||
|
|
||||||
|
Without `--artifacts`, analyze executes enabled configured artifacts. With an
|
||||||
|
explicit `--artifacts` list, the exact named configured artifacts are the
|
||||||
|
one-invocation execution set even if their `enabled` values are false; the list
|
||||||
|
does not automatically include dependencies. Named artifacts must therefore be
|
||||||
|
configured with valid executable fields, and their configured dependencies must
|
||||||
|
already be available to analyze. This override affects analyze planning only;
|
||||||
|
publish uses the list only to filter configured
|
||||||
|
`narratio.artifact.<name>` output rules.
|
||||||
|
|
||||||
For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_name>`:
|
For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_name>`:
|
||||||
|
|
||||||
| Field | Type | Required | Rule |
|
| Field | Type | Required | Rule |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.extraction.<name>`, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
||||||
| `artifact` | string | No | optional passthrough adapter field |
|
|
||||||
| `path` | string | No | optional passthrough adapter field |
|
|
||||||
| `required` | bool | No | optional input requirement |
|
| `required` | bool | No | optional input requirement |
|
||||||
|
|
||||||
|
`artifact` and `path` are obsolete and rejected by strict configuration
|
||||||
|
loading. Use the canonical `source` identifier to select the input; Narratio
|
||||||
|
does not provide adapter-specific input passthrough fields.
|
||||||
|
|
||||||
|
### Notifications
|
||||||
|
|
||||||
|
Narratio currently supports only `notification.mode: noop`, which is also the
|
||||||
|
default when the section is omitted. The notify stage performs no delivery in
|
||||||
|
this mode. Backend, recipient, timeout, and other provider settings are
|
||||||
|
rejected by strict configuration loading until Narratio has a provider
|
||||||
|
integration.
|
||||||
|
|
||||||
### Campaign
|
### Campaign
|
||||||
|
|
||||||
| Field | Type | Required | Notes |
|
| Field | Type | Required | Notes |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `campaign_id` | string | Yes | canonical campaign identity |
|
| `campaign_id` | string | Yes | canonical opaque campaign identity |
|
||||||
| `session_template_file` | string | No | used by `session init` when set |
|
| `session_template_file` | string | No | used by `session init` when set |
|
||||||
| `inputs.speakers_file` | string | Yes | stable input default |
|
| `inputs.speakers_file` | string | Yes | stable input default |
|
||||||
| `inputs.autocorrect_file` | string | Yes | stable input default |
|
| `inputs.autocorrect_file` | string | Yes | stable input default |
|
||||||
@@ -254,9 +323,9 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
|
|||||||
|
|
||||||
| Field | Type | Required in session file | Notes |
|
| Field | Type | Required in session file | Notes |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `session_id` | string | Yes | must match CLI session target when provided |
|
| `session_id` | string | Yes | opaque identity; must match CLI session target when provided |
|
||||||
| `previous_session_id` | string | No | must not equal `session_id` |
|
| `previous_session_id` | string | No | opaque identity; must not equal `session_id` |
|
||||||
| `campaign` | string | No | filled from `campaign_id` during resolve if omitted |
|
| `campaign` | string | No | opaque identity; filled from `campaign_id` during resolve if omitted |
|
||||||
| `date` | string | No | metadata |
|
| `date` | string | No | metadata |
|
||||||
| `title` | string | No | metadata |
|
| `title` | string | No | metadata |
|
||||||
| `inputs.speakers_file` | string | No | overrides campaign stable input |
|
| `inputs.speakers_file` | string | No | overrides campaign stable input |
|
||||||
@@ -271,6 +340,20 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
|
|||||||
Audio rules:
|
Audio rules:
|
||||||
|
|
||||||
- configure local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
- configure local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
||||||
|
- `audio_s3` requires `pipeline.storage.backend: s3` and a configured S3 bucket.
|
||||||
|
|
||||||
|
### Storage backend selection
|
||||||
|
|
||||||
|
`local` is the default and disables remote object-store operations. Configure
|
||||||
|
`s3` explicitly before supplying `storage.s3`; a populated S3 block does not
|
||||||
|
select a backend on its own. Unknown backend names and an S3 block paired with
|
||||||
|
`local` are rejected during configuration validation.
|
||||||
|
|
||||||
|
### Previous-session expectation
|
||||||
|
|
||||||
|
`previous_session_id` is optional in a session file. When a command supplies
|
||||||
|
`--previous-session-id`, however, the session file must contain the same value;
|
||||||
|
an omitted or different value is rejected before the command performs work.
|
||||||
|
|
||||||
## Maintained Examples
|
## Maintained Examples
|
||||||
|
|
||||||
|
|||||||
@@ -25,19 +25,34 @@ polished transcripts and generated artifacts. Start with the
|
|||||||
| Adapters or external tool contracts | [Adapter Internals](internal/adapters.md) and [Integration Contracts](integrations/README.md) | The internal guide owns adapter composition and mechanics; integration documents own external formats and protocols. |
|
| Adapters or external tool contracts | [Adapter Internals](internal/adapters.md) and [Integration Contracts](integrations/README.md) | The internal guide owns adapter composition and mechanics; integration documents own external formats and protocols. |
|
||||||
| Manifests, artifacts, workspace paths, or publish behavior | [Manifest Internals](internal/manifest.md), [Artifact Internals](internal/artifacts.md), [Workspace Internals](internal/workspace.md), [Publish Internals](internal/stage-publish.md), and [Operations](operations.md) | These separate implementation state and resolution from operator-visible layout and lifecycle. |
|
| Manifests, artifacts, workspace paths, or publish behavior | [Manifest Internals](internal/manifest.md), [Artifact Internals](internal/artifacts.md), [Workspace Internals](internal/workspace.md), [Publish Internals](internal/stage-publish.md), and [Operations](operations.md) | These separate implementation state and resolution from operator-visible layout and lifecycle. |
|
||||||
| Maintained configuration or input examples | [Configuration](config.md) and [Examples](../examples/README.md) | The reference owns field meanings; the examples directory owns complete copyable files. |
|
| Maintained configuration or input examples | [Configuration](config.md) and [Examples](../examples/README.md) | The reference owns field meanings; the examples directory owns complete copyable files. |
|
||||||
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
|
| Proposed or unimplemented behavior | `docs/roadmap/` | Future work belongs only in roadmap documentation until implemented. |
|
||||||
|
|
||||||
For an existing subsystem, also inspect its focused tests and package-level
|
For an existing subsystem, also inspect its focused tests and package-level
|
||||||
contracts before changing behavior.
|
contracts before changing behavior.
|
||||||
|
|
||||||
## Validation
|
## Validation
|
||||||
|
|
||||||
Use focused package tests while iterating. Run the repository-wide checks when a
|
Use focused package tests while iterating. Every pull request and push runs the
|
||||||
change affects shared contracts, application behavior, or maintained
|
following repository-wide checks before it can be accepted:
|
||||||
documentation examples:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go test ./...
|
go test ./...
|
||||||
|
go test -race ./...
|
||||||
go vet ./...
|
go vet ./...
|
||||||
go build ./cmd/narratio
|
go build ./...
|
||||||
|
go test ./internal/doccheck
|
||||||
|
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The documentation check verifies local Markdown links and the dependency graph
|
||||||
|
of the Woodpecker workflows. The configuration check loads every maintained
|
||||||
|
pipeline and session example. Release automation repeats these checks and
|
||||||
|
cross-compiles the CLI before it builds release assets; publishing depends on
|
||||||
|
that validation path, so a failure cannot publish a release.
|
||||||
|
|
||||||
|
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
|
||||||
|
job to expose ordering and repeatability defects. Current runners cross-compile
|
||||||
|
for macOS and Windows, but do not provide native macOS or Windows execution.
|
||||||
|
Those cross-builds establish compilation only, not platform-equivalent runtime
|
||||||
|
evidence. Add native checks only when official runner labels and successful
|
||||||
|
native-run evidence are available.
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ focused stage documents.
|
|||||||
## Integration Contracts
|
## Integration Contracts
|
||||||
|
|
||||||
- [Audita](./audita.md): transcript polishing (`audita process`).
|
- [Audita](./audita.md): transcript polishing (`audita process`).
|
||||||
|
- [Notarius](./notarius.md): complete pipeline execution and safe JSON bundle
|
||||||
|
discovery (`notarius run`).
|
||||||
- [Seriatim](./seriatim.md): merge, normalize, trim, and render operations.
|
- [Seriatim](./seriatim.md): merge, normalize, trim, and render operations.
|
||||||
- [Scriptorium](./scriptorium.md): artifact generation and debug rendering
|
- [Scriptorium](./scriptorium.md): artifact generation and debug rendering
|
||||||
(`scriptorium run|render`).
|
(`scriptorium run|render`).
|
||||||
|
|||||||
@@ -15,7 +15,12 @@ runner composition is documented in
|
|||||||
- required transcript/glossary/output/work-dir paths;
|
- required transcript/glossary/output/work-dir paths;
|
||||||
- optional report path (required when report mode is enabled);
|
- optional report path (required when report mode is enabled);
|
||||||
- generated config and stdout/stderr log paths;
|
- generated config and stdout/stderr log paths;
|
||||||
- optional module/model/base-url/config/output-schema/concurrency settings.
|
- optional per-invocation module override.
|
||||||
|
|
||||||
|
The constructed runner owns static Audita settings: binary, timeout,
|
||||||
|
credentials, default modules, model and endpoint settings, validation and output
|
||||||
|
settings, report mode, and concurrency. The `polish` stage supplies only
|
||||||
|
invocation-specific paths and may override modules for that invocation.
|
||||||
|
|
||||||
## Result Contract
|
## Result Contract
|
||||||
`PolishResult` returns:
|
`PolishResult` returns:
|
||||||
@@ -41,6 +46,9 @@ Run fails for:
|
|||||||
- invalid processed transcript JSON (`segments` array required);
|
- invalid processed transcript JSON (`segments` array required);
|
||||||
- invalid report JSON when reporting is enabled.
|
- invalid report JSON when reporting is enabled.
|
||||||
|
|
||||||
|
Processed transcript JSON is limited to 64 MiB and optional report JSON to 16
|
||||||
|
MiB. Both must be regular files without symlinked path components.
|
||||||
|
|
||||||
Failure results still include output/log/config/exit metadata for diagnostics.
|
Failure results still include output/log/config/exit metadata for diagnostics.
|
||||||
|
|
||||||
## Deterministic Behavior
|
## Deterministic Behavior
|
||||||
|
|||||||
83
docs/integrations/notarius.md
Normal file
83
docs/integrations/notarius.md
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
# Notarius Integration Contract
|
||||||
|
|
||||||
|
## Boundary
|
||||||
|
|
||||||
|
Narratio uses Notarius as a subprocess to extract configured structured JSON
|
||||||
|
lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
|
||||||
|
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
||||||
|
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
||||||
|
|
||||||
|
Canonical Notarius references:
|
||||||
|
|
||||||
|
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/subprocess.md)
|
||||||
|
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/dnd-pipeline.md)
|
||||||
|
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/run-result.md)
|
||||||
|
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/json-output.md)
|
||||||
|
|
||||||
|
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
||||||
|
records the exact current constraints for all ten D&D lanes. Treat the linked
|
||||||
|
Notarius documents as canonical when changing those values; Narratio does not
|
||||||
|
duplicate the complete schemas.
|
||||||
|
|
||||||
|
## Invocation
|
||||||
|
|
||||||
|
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
||||||
|
configuration path, input path, output directory, and working directory to
|
||||||
|
absolute paths and invokes:
|
||||||
|
|
||||||
|
```text
|
||||||
|
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> --json
|
||||||
|
```
|
||||||
|
|
||||||
|
Standard output is reserved for the JSON receipt. Standard error is captured
|
||||||
|
separately as diagnostic output. Narratio applies the configured timeout and
|
||||||
|
does not interpret stdout as a receipt unless the subprocess exits successfully.
|
||||||
|
It does not pass a Narratio session ID or run `notarius config validate`
|
||||||
|
automatically; the configured working directory and Narratio's minimal child
|
||||||
|
environment apply to the subprocess.
|
||||||
|
|
||||||
|
## Accepted Result
|
||||||
|
|
||||||
|
Narratio currently accepts receipt schema `notarius.run-result.v1`. The receipt
|
||||||
|
must identify the configured pipeline, and its `index_file` must be exactly
|
||||||
|
`index.json` beneath the reported bundle root. The production index must name
|
||||||
|
the management files exactly as `manifest.json`, `rejected.json`, and
|
||||||
|
`warnings.json`. All receipt, index, and lane paths must stay inside that
|
||||||
|
bundle; symlinks and non-regular lane payloads are rejected.
|
||||||
|
|
||||||
|
Supported receipt and index shapes tolerate unknown fields for forward
|
||||||
|
compatibility, while required identity, validation, count, manifest,
|
||||||
|
rejection, warning, and lane-list fields remain mandatory. Narratio applies
|
||||||
|
bounded reads to the receipt, index, rejection, and warning documents. Optional
|
||||||
|
chunk-map and evidence-context descriptors must carry their complete generic
|
||||||
|
contract metadata when present.
|
||||||
|
|
||||||
|
For every entry in `pipeline.notarius.outputs`, Narratio requires exactly one
|
||||||
|
index descriptor with the configured lane ID, media type, schema ID, schema
|
||||||
|
version, and, when configured, module key. Missing, duplicate, rejected, or
|
||||||
|
incompatible required lanes fail extraction even if Notarius exited zero.
|
||||||
|
Unconfigured lanes may remain in the preserved bundle but do not become
|
||||||
|
selectable Narratio sources.
|
||||||
|
|
||||||
|
Each accepted configured lane is registered as
|
||||||
|
`narratio.extraction.<output_key>`. The bundle index is retained for audit and
|
||||||
|
resume validation but is not selectable. Scriptorium and publish rules consume
|
||||||
|
only explicitly named lane sources; `--artifacts` never selects Notarius lanes.
|
||||||
|
|
||||||
|
## Failure And Compatibility Behavior
|
||||||
|
|
||||||
|
- Startup and nonzero-exit errors fail extraction and retain captured diagnostics.
|
||||||
|
- Invalid receipt JSON or an unsupported receipt schema fails before bundle use.
|
||||||
|
- Unsafe or incompatible index data and required-lane rejection fail before the
|
||||||
|
staged bundle is promoted to durable storage.
|
||||||
|
- Contract and external provenance metadata are preserved on lane artifact
|
||||||
|
records and through explicit publication.
|
||||||
|
|
||||||
|
Rejection and warning summaries retain structured stage, scope, lane, and
|
||||||
|
reason-code fields for diagnostics without exposing free-form external messages
|
||||||
|
or reading lane payload bodies.
|
||||||
|
|
||||||
|
Configuration fields and defaults are in [Configuration](../config.md).
|
||||||
|
Operator paths, rerun procedures, and bundle retention are in
|
||||||
|
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
||||||
|
for failure recovery.
|
||||||
@@ -46,6 +46,9 @@ Run behavior:
|
|||||||
- `run` exit code `2` is mapped to `ValidationFailed=true`;
|
- `run` exit code `2` is mapped to `ValidationFailed=true`;
|
||||||
- successful subprocess still fails if output file is missing or empty.
|
- successful subprocess still fails if output file is missing or empty.
|
||||||
|
|
||||||
|
Each artifact result is limited to 64 MiB and must be a regular file without
|
||||||
|
symlinked path components.
|
||||||
|
|
||||||
Render behavior:
|
Render behavior:
|
||||||
- subprocess errors propagate;
|
- subprocess errors propagate;
|
||||||
- output file must exist and be non-empty.
|
- output file must exist and be non-empty.
|
||||||
|
|||||||
@@ -41,6 +41,8 @@ Invocation fails on:
|
|||||||
- empty render output files.
|
- empty render output files.
|
||||||
|
|
||||||
When report paths are provided/enabled, report files must parse as JSON.
|
When report paths are provided/enabled, report files must parse as JSON.
|
||||||
|
Each Seriatim JSON or rendered-text result is limited to 64 MiB and must be a
|
||||||
|
regular file without symlinked path components.
|
||||||
|
|
||||||
## Deterministic Behavior
|
## Deterministic Behavior
|
||||||
- argument ordering is deterministic per command construction.
|
- argument ordering is deterministic per command construction.
|
||||||
|
|||||||
@@ -17,6 +17,11 @@ Narratio sends an HTTP `POST` to the configured transcription URL using
|
|||||||
The server must return a `2xx` response whose body is valid JSON. Narratio does
|
The server must return a `2xx` response whose body is valid JSON. Narratio does
|
||||||
not currently require a more specific response schema at this boundary.
|
not currently require a more specific response schema at this boundary.
|
||||||
|
|
||||||
|
The transcription URL must be an absolute `http` or `https` URL. The audio body
|
||||||
|
is streamed through a fresh multipart writer for every attempt, so its memory
|
||||||
|
use is bounded by the transport buffer rather than by the complete audio file.
|
||||||
|
WhisperX response acquisition is capped at 10 MiB.
|
||||||
|
|
||||||
## Request And Result Contract
|
## Request And Result Contract
|
||||||
|
|
||||||
Each adapter request identifies a speaker, a readable audio file, and the
|
Each adapter request identifies a speaker, a readable audio file, and the
|
||||||
@@ -40,7 +45,7 @@ transcript output.
|
|||||||
|
|
||||||
## Validation And Failure Semantics
|
## Validation And Failure Semantics
|
||||||
|
|
||||||
Client construction rejects a missing or invalid absolute transcription URL,
|
Client construction rejects a missing or non-HTTP(S) absolute transcription URL,
|
||||||
a missing language, a non-positive timeout, negative retries, or a negative
|
a missing language, a non-positive timeout, negative retries, or a negative
|
||||||
retry delay. A request fails before transmission when its audio or output path
|
retry delay. A request fails before transmission when its audio or output path
|
||||||
is missing.
|
is missing.
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ Primary adapters:
|
|||||||
- `seriatim.Runner`
|
- `seriatim.Runner`
|
||||||
- `audita.Runner`
|
- `audita.Runner`
|
||||||
- `scriptorium.Runner`
|
- `scriptorium.Runner`
|
||||||
|
- `notarius.Runner`
|
||||||
- `storage.ObjectStore`
|
- `storage.ObjectStore`
|
||||||
- `notify.Sender`
|
- `notify.Sender`
|
||||||
|
|
||||||
@@ -40,9 +41,13 @@ Adapters do not own:
|
|||||||
- Seriatim subprocess runner.
|
- Seriatim subprocess runner.
|
||||||
- Audita subprocess runner.
|
- Audita subprocess runner.
|
||||||
- Scriptorium subprocess runner.
|
- Scriptorium subprocess runner.
|
||||||
|
- Notarius subprocess runner when extraction is enabled.
|
||||||
- Noop notifier (`notify.NoopSender`).
|
- Noop notifier (`notify.NoopSender`).
|
||||||
- Object store only when required by selected stages/config.
|
- Object store only when required by selected stages/config.
|
||||||
|
|
||||||
|
Notarius is composed only when extraction is enabled; the extract stage owns
|
||||||
|
receipt, bundle, and configured-lane policy rather than the adapter.
|
||||||
|
|
||||||
Object-store construction goes through `newCommandObjectStore`, which loads
|
Object-store construction goes through `newCommandObjectStore`, which loads
|
||||||
configured filesystem secrets before adapter initialization.
|
configured filesystem secrets before adapter initialization.
|
||||||
|
|
||||||
@@ -51,21 +56,35 @@ configured filesystem secrets before adapter initialization.
|
|||||||
- Constructor errors fail stage execution setup early.
|
- Constructor errors fail stage execution setup early.
|
||||||
- Runtime adapter errors propagate to stage code and then manifest failure handling.
|
- Runtime adapter errors propagate to stage code and then manifest failure handling.
|
||||||
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
||||||
|
- Shared subprocess execution starts an owned process group on Linux/macOS or a
|
||||||
|
kill-on-close job object on Windows. Every terminal path disposes of that
|
||||||
|
owned tree before returning. After a natural leader exit, Unix checks for
|
||||||
|
remaining group members and uses bounded graceful then forceful termination;
|
||||||
|
Windows closes the job so kill-on-close applies. Cancellation, deadlines, and
|
||||||
|
diagnostic limits use the same terminal disposal path without losing their
|
||||||
|
original result classification. Child environments contain only the execution
|
||||||
|
baseline and adapter-specified values; configured credentials are explicit
|
||||||
|
sensitive values. Stdout and stderr are redacted while streaming into separate
|
||||||
|
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
|
||||||
|
departed leader's descendant. Unsupported platforms reject owned command
|
||||||
|
execution.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
|
|
||||||
- Composition: `internal/app/runner.go`, `internal/app/object_store.go`
|
- Composition: `internal/app/runner.go`, `internal/app/object_store.go`
|
||||||
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
||||||
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,storage,notify}`
|
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
|
||||||
- `internal/adapters/whisperx/http_test.go`
|
- `internal/adapters/whisperx/http_test.go`
|
||||||
- `internal/adapters/seriatim/subprocess_test.go`
|
- `internal/adapters/seriatim/subprocess_test.go`
|
||||||
- `internal/adapters/audita/subprocess_test.go`
|
- `internal/adapters/audita/subprocess_test.go`
|
||||||
- `internal/adapters/scriptorium/subprocess_test.go`
|
- `internal/adapters/scriptorium/subprocess_test.go`
|
||||||
|
- `internal/adapters/notarius/subprocess_test.go`
|
||||||
- `internal/adapters/storage/*_test.go`
|
- `internal/adapters/storage/*_test.go`
|
||||||
- `internal/app/runner_test.go`
|
- `internal/app/runner_test.go`
|
||||||
|
|
||||||
See the [WhisperX](../integrations/whisperx.md),
|
See the [WhisperX](../integrations/whisperx.md),
|
||||||
[Seriatim](../integrations/seriatim.md), [Audita](../integrations/audita.md),
|
[Seriatim](../integrations/seriatim.md), [Audita](../integrations/audita.md),
|
||||||
and [Scriptorium](../integrations/scriptorium.md) contracts before changing an
|
[Scriptorium](../integrations/scriptorium.md), and
|
||||||
|
[Notarius](../integrations/notarius.md) contracts before changing an
|
||||||
externally visible boundary. Operator-selected values belong in
|
externally visible boundary. Operator-selected values belong in
|
||||||
[Configuration](../config.md).
|
[Configuration](../config.md).
|
||||||
|
|||||||
@@ -24,22 +24,36 @@ Registry entries bind each ID to its producer, output kind, canonical fallback,
|
|||||||
and content validator. The focused stage documents own their input/output flow;
|
and content validator. The focused stage documents own their input/output flow;
|
||||||
[Configuration](../config.md) owns where operators may select these IDs.
|
[Configuration](../config.md) owns where operators may select these IDs.
|
||||||
|
|
||||||
## Configured and Previous-Session Sources
|
## Configured, Extraction, And Previous-Session Sources
|
||||||
|
|
||||||
- configured source ID format: `narratio.artifact.<artifact_key>`
|
- configured source ID format: `narratio.artifact.<artifact_key>`
|
||||||
|
- extraction source ID format: `narratio.extraction.<output_key>`
|
||||||
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
||||||
|
|
||||||
Both formats are validated by strict source-policy rules.
|
All formats are validated by strict source-policy rules. Configured artifact and
|
||||||
|
extraction keys use `^[a-z][a-z0-9_]*$`; source parsers never normalize an
|
||||||
|
unrecognized token into a valid source. Extraction sources are registered only
|
||||||
|
from `pipeline.notarius.outputs`; the Notarius index has no selectable source
|
||||||
|
ID.
|
||||||
|
|
||||||
## Runtime Catalog
|
## Runtime Catalog
|
||||||
|
|
||||||
`ArtifactCatalog` tracks:
|
`ArtifactCatalog` tracks:
|
||||||
|
|
||||||
- `planned`: source registered for run context;
|
- `planned`: source registered for run context;
|
||||||
- `executable`: selected and enabled for analyze execution;
|
- `executable`: included in the effective analyze artifact set;
|
||||||
- `available`: local file exists and validates;
|
- `available`: local file exists and validates;
|
||||||
- `provenance`: availability source.
|
- `provenance`: availability source.
|
||||||
|
|
||||||
|
Configured definitions are always registered. Without an explicit selection,
|
||||||
|
the effective analyze set contains enabled definitions. With `--artifacts`, the
|
||||||
|
exact named configured definitions become the effective set for that invocation,
|
||||||
|
regardless of their `enabled` value; dependencies are not added implicitly.
|
||||||
|
Availability is separate from executability: a non-executable configured output
|
||||||
|
may be reused from a canonical non-empty file, while an executable definition
|
||||||
|
is generated by analyze. Extraction entries are registered from configuration
|
||||||
|
and become available only after compatible extraction evidence is hydrated.
|
||||||
|
|
||||||
Current provenance values:
|
Current provenance values:
|
||||||
|
|
||||||
- `generated.current_analyze_run`
|
- `generated.current_analyze_run`
|
||||||
@@ -58,12 +72,28 @@ Configured sources (`narratio.artifact.*`):
|
|||||||
|
|
||||||
- resolve only through runtime catalog availability.
|
- resolve only through runtime catalog availability.
|
||||||
|
|
||||||
|
Extraction sources (`narratio.extraction.*`):
|
||||||
|
|
||||||
|
- use the shared typed bundle evidence inspection in `extraction_evidence.go`;
|
||||||
|
- require a current successful extract record with the exact configured source,
|
||||||
|
compatible contract and Notarius provenance, a confined regular durable
|
||||||
|
payload, matching checksum, and the current resolved trimmed-transcript
|
||||||
|
identity;
|
||||||
|
- remain unavailable unless catalog hydration receives valid evidence. Resume
|
||||||
|
treats absent or obsolete evidence as a rerun decision and unsafe evidence as
|
||||||
|
an error; and
|
||||||
|
- are never inferred by scanning the Notarius bundle directory.
|
||||||
|
|
||||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||||
|
|
||||||
- resolve only from local `previous/` cache state;
|
- resolve only from local `previous/` cache state;
|
||||||
- prefer manifest-backed previous-input paths;
|
- prefer manifest-backed previous-input paths;
|
||||||
- fallback to existing previous-cache filesystem paths.
|
- fallback to existing previous-cache filesystem paths.
|
||||||
|
|
||||||
|
Source absence is evaluated by the consuming artifact input. An optional input
|
||||||
|
is omitted from that invocation; a required input fails resolution. This is
|
||||||
|
separate from a stage's lifecycle outcome.
|
||||||
|
|
||||||
Validation by content type:
|
Validation by content type:
|
||||||
|
|
||||||
- transcript JSON built-ins: JSON with top-level `segments` array;
|
- transcript JSON built-ins: JSON with top-level `segments` array;
|
||||||
@@ -75,7 +105,7 @@ Validation by content type:
|
|||||||
|
|
||||||
`CollectPreviousArtifactRequirements`:
|
`CollectPreviousArtifactRequirements`:
|
||||||
|
|
||||||
- scans enabled configured artifacts only;
|
- scans the effective configured artifact set;
|
||||||
- extracts only canonical previous-session sources;
|
- extracts only canonical previous-session sources;
|
||||||
- deduplicates by artifact key;
|
- deduplicates by artifact key;
|
||||||
- merges required and optional references (required wins);
|
- merges required and optional references (required wins);
|
||||||
@@ -86,12 +116,31 @@ Validation by content type:
|
|||||||
Artifacts package owns shared remote current-state loading mechanics used by
|
Artifacts package owns shared remote current-state loading mechanics used by
|
||||||
restore, status and validation checks, and previous-cache planning.
|
restore, status and validation checks, and previous-cache planning.
|
||||||
|
|
||||||
|
For a new-protocol current state, the pointer-selected immutable commit is the
|
||||||
|
complete restore authority. Callers receive its declared object identities and
|
||||||
|
must not supplement them by listing mutable session prefixes. The legacy reader
|
||||||
|
is intentionally separate and remains migration-only support.
|
||||||
|
|
||||||
|
The reader opens each small control object directly and enforces owner-specific
|
||||||
|
limits before decoding: 64 KiB for the mutable commit pointer, 4 MiB for the
|
||||||
|
immutable commit manifest, and 8 MiB for the selected session manifest. Legacy
|
||||||
|
compatibility applies a 4 KiB limit to `current/run_id.txt` and the same 8 MiB
|
||||||
|
manifest limit to `current/manifest.json`. These are exposed as
|
||||||
|
`MaxCurrentCommitPointerBytes`, `MaxRemoteCommitManifestBytes`,
|
||||||
|
`MaxRemoteSessionManifestBytes`, `MaxLegacyCurrentRunPointerBytes`, and
|
||||||
|
`MaxLegacyCurrentManifestBytes`.
|
||||||
|
|
||||||
|
Each read uses the generation and size metadata returned with its opened body.
|
||||||
|
Actual bytes remain subject to a limit-plus-one read even if size metadata is
|
||||||
|
absent or inaccurate. Immutable selections then retain their declared-size,
|
||||||
|
checksum, generation, and identity checks. No current-state control object is
|
||||||
|
downloaded through a temporary file.
|
||||||
|
|
||||||
Core helpers:
|
Core helpers:
|
||||||
|
|
||||||
- `LoadCurrentRunPointer`
|
|
||||||
- `LoadCurrentManifest`
|
|
||||||
- `LoadCurrentState`
|
- `LoadCurrentState`
|
||||||
- `ValidateCurrentStateIdentity`
|
- `ValidateCurrentStateIdentity`
|
||||||
|
- `RemoteCommitManifest` and `CurrentCommitPointer`
|
||||||
|
|
||||||
Typed missing-state errors:
|
Typed missing-state errors:
|
||||||
|
|
||||||
@@ -119,6 +168,18 @@ Caller policy is intentionally outside artifacts helpers:
|
|||||||
- spool/cache paths;
|
- spool/cache paths;
|
||||||
- S3 session/run/current-state key layout.
|
- S3 session/run/current-state key layout.
|
||||||
|
|
||||||
|
New publication creates run-scoped immutable objects, including
|
||||||
|
`runs/{run_id}/commit.json` and `runs/{run_id}/session-manifest.json`. The sole
|
||||||
|
mutable selector is `current/commit-pointer.json`; readers verify its selected
|
||||||
|
commit and declared object generations/checksums. Legacy current-pair loading
|
||||||
|
is confined to `current_state_legacy.go` for migration only.
|
||||||
|
|
||||||
|
Campaign, session, and Narratio run IDs are validated as portable opaque
|
||||||
|
segments at configuration and artifact boundaries before they can be used in a
|
||||||
|
workspace or S3 namespace. Previous-artifact destinations remain typed,
|
||||||
|
multi-segment relative paths and are confined beneath `previous/artifacts`; they
|
||||||
|
are not treated as opaque identifiers.
|
||||||
|
|
||||||
See [Workspace Internals](workspace.md) for how callers consume local helpers
|
See [Workspace Internals](workspace.md) for how callers consume local helpers
|
||||||
and [Operations](../operations.md#local-state-layout) for the authoritative
|
and [Operations](../operations.md#local-state-layout) for the authoritative
|
||||||
physical layout.
|
physical layout.
|
||||||
@@ -127,19 +188,27 @@ physical layout.
|
|||||||
|
|
||||||
- source ID formats are stable contracts;
|
- source ID formats are stable contracts;
|
||||||
- artifact resolution is deterministic and manifest-aware;
|
- artifact resolution is deterministic and manifest-aware;
|
||||||
|
- extraction sources are available only from a compatible successful manifest
|
||||||
|
record;
|
||||||
- previous-session source resolution in `analyze` is local-only;
|
- previous-session source resolution in `analyze` is local-only;
|
||||||
- remote current-state key construction remains centralized in artifacts helpers.
|
- remote current-state key construction remains centralized in artifacts helpers.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
|
|
||||||
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
||||||
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`
|
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`,
|
||||||
- Current state: `internal/artifacts/current_state.go`
|
`internal/artifacts/extraction_catalog.go`,
|
||||||
|
`internal/artifacts/extraction_evidence.go`,
|
||||||
|
`internal/artifacts/extraction_input.go`
|
||||||
|
- Current state: `internal/artifacts/current_state.go`,
|
||||||
|
`internal/artifacts/current_state_commit.go`,
|
||||||
|
`internal/artifacts/current_state_legacy.go`
|
||||||
- Paths and keys: `internal/artifacts/paths.go`,
|
- Paths and keys: `internal/artifacts/paths.go`,
|
||||||
`internal/artifacts/s3_keys.go`
|
`internal/artifacts/s3_keys.go`
|
||||||
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
||||||
- Tests: `internal/artifacts/artifact_resolver_test.go`,
|
- Tests: `internal/artifacts/artifact_resolver_test.go`,
|
||||||
`internal/artifacts/catalog_test.go`,
|
`internal/artifacts/catalog_test.go`,
|
||||||
|
`internal/artifacts/extraction_catalog_test.go`,
|
||||||
`internal/artifacts/current_state_test.go`,
|
`internal/artifacts/current_state_test.go`,
|
||||||
`internal/artifacts/paths_model_test.go`,
|
`internal/artifacts/paths_model_test.go`,
|
||||||
`internal/artifacts/previous_requirements_test.go`
|
`internal/artifacts/previous_requirements_test.go`
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ reporting flow in `internal/app`. User invocation belongs in
|
|||||||
physical restore scope belong in
|
physical restore scope belong in
|
||||||
[Operations](../operations.md#restore-workflow).
|
[Operations](../operations.md#restore-workflow).
|
||||||
|
|
||||||
Restore is split into explicit phases so remote authority, local conflict
|
Restore separates remote authority, local conflict policy, and filesystem
|
||||||
policy, and filesystem mutation can be tested independently.
|
mutation so each remains testable independently.
|
||||||
|
|
||||||
## Discovery Contract
|
## Discovery Contract
|
||||||
|
|
||||||
@@ -18,6 +18,7 @@ Discovery delegates current-state pointer and manifest loading to
|
|||||||
|
|
||||||
- campaign must match;
|
- campaign must match;
|
||||||
- session ID must match.
|
- session ID must match.
|
||||||
|
- run ID must match the pointer-selected committed run.
|
||||||
|
|
||||||
Restore treats any missing or invalid remote current state as a command error.
|
Restore treats any missing or invalid remote current state as a command error.
|
||||||
|
|
||||||
@@ -31,13 +32,20 @@ Restore planner action kinds:
|
|||||||
|
|
||||||
Planner behavior:
|
Planner behavior:
|
||||||
|
|
||||||
- remote list scope is the resolved session prefix;
|
- a new-protocol restore uses only the selected commit's declared artifact set;
|
||||||
|
each action carries that artifact's immutable key, checksum, size, and
|
||||||
|
generation. Coherent legacy state remains on the isolated compatibility path;
|
||||||
- remote-to-local mapping is traversal-safe;
|
- remote-to-local mapping is traversal-safe;
|
||||||
- actions are sorted by local relative path and then remote key;
|
- actions are sorted by local relative path and then remote key;
|
||||||
- force converts differing local targets from conflicts to downloads.
|
- force converts differing eligible regular files from conflicts to downloads;
|
||||||
|
directories and other non-regular targets remain conflicts.
|
||||||
|
|
||||||
Previous-cache files are planned separately through `previouscache.BuildPlan`
|
For a non-dry-run restore, planning/classification happens only after acquiring
|
||||||
when configured previous-session requirements exist.
|
the session lock. Runner manifest/reuse checks acquire that same lock first.
|
||||||
|
|
||||||
|
Previous-cache readiness is resolved through `previouscache.Resolve` for restore,
|
||||||
|
prepare, status, and validation. A committed source is selected only by its
|
||||||
|
exact source identity; legacy fallback remains isolated and rejects ambiguity.
|
||||||
|
|
||||||
## Execution Contract
|
## Execution Contract
|
||||||
|
|
||||||
@@ -47,17 +55,32 @@ Execution order and safety:
|
|||||||
- `manifest.json` installs last;
|
- `manifest.json` installs last;
|
||||||
- downloads use sibling temp files plus atomic rename;
|
- downloads use sibling temp files plus atomic rename;
|
||||||
- manifest replacement is validated before rename;
|
- manifest replacement is validated before rename;
|
||||||
|
- each committed object is verified against its declared checksum, size, and
|
||||||
|
generation before installation;
|
||||||
|
- a committed manifest already verified during discovery is retained for the
|
||||||
|
matching restore action and revalidated before installation, avoiding a
|
||||||
|
second body transfer;
|
||||||
- failed installs do not roll back files already written in the same execution.
|
- failed installs do not roll back files already written in the same execution.
|
||||||
|
- a durable `.restore-incomplete.json` marker is written before installation.
|
||||||
|
It blocks runners until a restore retry completes all verified installs and
|
||||||
|
the local manifest replacement, at which point it is removed.
|
||||||
|
- restored manifest local references are rebased beneath the selected local
|
||||||
|
session root. Unsafe relative references and producer-machine absolute paths
|
||||||
|
outside the manifest's producer session root are rejected; producer-local
|
||||||
|
spool/cache and cleanup locations are not restored as authority.
|
||||||
|
|
||||||
Audio restore path:
|
Audio restore path:
|
||||||
|
|
||||||
- uses `audio.MaterializeS3Audio`;
|
- uses `audio.MaterializeS3Audio`;
|
||||||
- integrates spool and S3 audio cache paths;
|
- integrates spool and S3 audio cache paths;
|
||||||
- supports cache-hit reuse without object redownload.
|
- reuses cached audio only when its no-follow regular file, content digest, and
|
||||||
|
identity sidecar all match the selected remote object version; otherwise it
|
||||||
|
refreshes through the durable download path.
|
||||||
|
|
||||||
## Reporting Contract
|
## Reporting Contract
|
||||||
|
|
||||||
- dry-run mode prints a summary and performs no local writes;
|
- dry-run mode prints a summary, performs no durable session writes, and may
|
||||||
|
read remote current-state or object-identity data to produce that summary;
|
||||||
- execution mode persists the canonical restore report described in
|
- execution mode persists the canonical restore report described in
|
||||||
[Operations](../operations.md#restore-workflow);
|
[Operations](../operations.md#restore-workflow);
|
||||||
- report includes plan counts, per-action status, and execution failures.
|
- report includes plan counts, per-action status, and execution failures.
|
||||||
@@ -65,7 +88,13 @@ Audio restore path:
|
|||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- restore uses committed remote current state as authority;
|
- restore uses committed remote current state as authority;
|
||||||
- `current/run_id.txt` is the remote publish commit marker;
|
- one restore or status inspection observes the single pointer-selected commit
|
||||||
|
loaded at discovery; later pointer changes cannot add objects or substitute a
|
||||||
|
different run into its plan;
|
||||||
|
- a verified `current/commit-pointer.json` and its selected immutable commit
|
||||||
|
establish new-protocol remote commitment; coherent legacy
|
||||||
|
`current/run_id.txt` plus `current/manifest.json` remains read-only migration
|
||||||
|
support;
|
||||||
- restore does not execute pipeline stages.
|
- restore does not execute pipeline stages.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
|
|||||||
71
docs/internal/fileops.md
Normal file
71
docs/internal/fileops.md
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
# Internal: File Operations
|
||||||
|
|
||||||
|
`internal/fileops` owns the narrow mechanics for durable replacement of one
|
||||||
|
byte file. Callers keep ownership of serialization, validation, cancellation,
|
||||||
|
and destination-directory policy.
|
||||||
|
|
||||||
|
## Destination Confinement
|
||||||
|
|
||||||
|
Before it creates, replaces, or installs a destination file, `fileops` opens
|
||||||
|
each ancestor from the filesystem root and rejects symbolic links or components
|
||||||
|
that change during traversal. The resulting parent-directory handle is retained
|
||||||
|
for sibling temporary-file creation and rename, so a later pathname swap cannot
|
||||||
|
redirect the replacement. Existing destination symlinks are replaced as leaf
|
||||||
|
entries; their targets are never followed.
|
||||||
|
|
||||||
|
Remote object acquisition uses a writer supplied by the storage owner. The
|
||||||
|
writer receives a `fileops`-owned, already-open sibling temporary file rather
|
||||||
|
than a mutable destination path. Callers still own remote object selection,
|
||||||
|
validation, conflict handling, and final mode.
|
||||||
|
|
||||||
|
Directory promotion keeps the verified destination parent open while it creates
|
||||||
|
the temporary tree, copies regular source entries, and performs the platform
|
||||||
|
no-replace rename. Platforms without a verified handle-relative atomic
|
||||||
|
no-replace primitive reject promotion before writing a temporary tree.
|
||||||
|
|
||||||
|
## Cleanup Contract
|
||||||
|
|
||||||
|
`RemoveAllUnderRoot` accepts an explicit root and a proper descendant. It opens
|
||||||
|
the root and each target ancestor without following symlinks, then removes the
|
||||||
|
tree through those directory handles. It rejects root deletion and any symlink
|
||||||
|
encountered in the target path or tree; repeated removal of a missing target is
|
||||||
|
successful. Command and post-publish policy remains owned by `internal/app`.
|
||||||
|
|
||||||
|
## Confined Reads
|
||||||
|
|
||||||
|
`ReadRegularFileUnderRoot` is the no-follow, bounded read primitive for a
|
||||||
|
caller-selected root and relative file path; `ReadRegularFile` is its
|
||||||
|
path-based convenience wrapper. They verify every ancestor through directory
|
||||||
|
handles and admit only a stable regular-file handle. Callers enforce their own
|
||||||
|
byte limits and access policy. Credential mode policy and environment
|
||||||
|
precedence remain owned by `internal/app`.
|
||||||
|
|
||||||
|
## Replacement Contract
|
||||||
|
|
||||||
|
`ReplaceFileAtomic` requires an existing destination directory. It creates a
|
||||||
|
sibling temporary file, writes the complete byte sequence, applies the
|
||||||
|
caller-supplied mode, syncs and closes the file, runs an optional pre-rename
|
||||||
|
check, replaces the destination with a rename, then syncs the containing
|
||||||
|
directory.
|
||||||
|
|
||||||
|
The pre-rename check is the last point at which a caller can cancel without
|
||||||
|
installing a new destination. A failure before the rename leaves the old
|
||||||
|
destination unchanged and removes the temporary file; any cleanup failure is
|
||||||
|
returned alongside the primary failure. A failure after the rename may leave
|
||||||
|
the new file visible, but it is not reported as crash-durable.
|
||||||
|
|
||||||
|
Replacement follows the operating system's same-filesystem rename semantics.
|
||||||
|
If a platform cannot replace an existing destination, the operation returns an
|
||||||
|
error and never removes the old file as an emulation step.
|
||||||
|
|
||||||
|
## Directory-Sync Support
|
||||||
|
|
||||||
|
Linux and macOS attempt to sync the destination directory. Windows opens the
|
||||||
|
directory with backup semantics and flushes its buffers. If either operation
|
||||||
|
is unavailable for the platform, directory handle, or filesystem,
|
||||||
|
`ErrDirectorySyncUnsupported` is returned. Narratio does not treat that result
|
||||||
|
as successful crash-durable replacement.
|
||||||
|
|
||||||
|
`WriteFileAtomic`, copy helpers, and downloaded temporary-file installation
|
||||||
|
retain their compatibility behavior of creating the destination parent with
|
||||||
|
the repository's workspace permissions before using this contract.
|
||||||
@@ -16,6 +16,14 @@ Explain the session-progress and invocation-audit models implemented by
|
|||||||
- `inputs` records
|
- `inputs` records
|
||||||
- durable `artifacts` records
|
- durable `artifacts` records
|
||||||
- per-stage `stages` map
|
- per-stage `stages` map
|
||||||
|
- an optional `post_publish_cleanup` obligation, which binds a committed run,
|
||||||
|
remote commit identity, and each exact root-confined local target to its
|
||||||
|
completion evidence
|
||||||
|
|
||||||
|
Session, campaign, and run identities in local and downloaded manifests must be
|
||||||
|
portable opaque segments. Unsafe legacy identities are rejected with migration
|
||||||
|
guidance rather than being normalized into a different workspace or remote
|
||||||
|
namespace.
|
||||||
|
|
||||||
The model admits these stage states:
|
The model admits these stage states:
|
||||||
|
|
||||||
@@ -37,40 +45,112 @@ The model admits these stage states:
|
|||||||
- per-stage status
|
- per-stage status
|
||||||
- overall run status (`running`, `succeeded`, `failed`)
|
- overall run status (`running`, `succeeded`, `failed`)
|
||||||
|
|
||||||
|
## Remote Commit Manifest
|
||||||
|
|
||||||
|
`artifacts.RemoteCommitManifest` is a separate, versioned remote snapshot
|
||||||
|
contract. It is not a serialized session manifest and contains no local
|
||||||
|
post-publication assertion such as `current_pointer_written`. A remote commit
|
||||||
|
identifies one campaign, session, and run and declares its immutable artifact
|
||||||
|
set. Each artifact has a typed source, immutable destination key, SHA-256
|
||||||
|
checksum, size, and storage generation.
|
||||||
|
|
||||||
|
`current/commit-pointer.json` is the sole mutable selector for the new
|
||||||
|
contract. It identifies exactly one run-scoped `runs/{run_id}/commit.json` and
|
||||||
|
binds that object by checksum, size, and generation. Readers strictly reject
|
||||||
|
unknown fields, version mismatches, pointer/commit identity mismatches, and
|
||||||
|
objects that do not match their declaration.
|
||||||
|
|
||||||
|
The reader retains a temporary, clearly isolated compatibility path for a
|
||||||
|
coherent legacy `current/manifest.json` plus `current/run_id.txt` pair. That
|
||||||
|
path is removable after migration and is never used to write new state.
|
||||||
|
|
||||||
## Persistence Semantics
|
## Persistence Semantics
|
||||||
|
|
||||||
`manifest.LocalStore`:
|
`manifest.LocalStore`:
|
||||||
|
|
||||||
- validates loaded documents;
|
- validates loaded documents;
|
||||||
- normalizes missing maps/stage records;
|
- normalizes missing maps/stage records;
|
||||||
- writes atomically via temp file + rename;
|
- writes through a sibling temporary file, syncing the completed file and
|
||||||
|
destination directory after atomic replacement;
|
||||||
- updates `updated_at` on save.
|
- updates `updated_at` on save.
|
||||||
|
|
||||||
|
If the operating system or filesystem cannot sync a directory, save returns an
|
||||||
|
explicit error instead of claiming crash-durable replacement. A returned error
|
||||||
|
after the rename can therefore leave the new manifest visible but not confirmed
|
||||||
|
durable; callers must reload it before retrying.
|
||||||
|
|
||||||
## Execution Semantics
|
## Execution Semantics
|
||||||
|
|
||||||
The application runner marks an executing stage running and then succeeded or
|
The application runner marks an executing stage running and then succeeded or
|
||||||
failed in both manifests, persisting each transition. On success it records
|
failed in both manifests, persisting each transition. On success it records
|
||||||
outputs, logs, generated configuration references, and metadata. A successful
|
outputs, logs, generated configuration references, and metadata. Artifact
|
||||||
forced rerun marks only succeeded downstream session-stage records stale.
|
records may include optional contract and external provenance objects; old
|
||||||
|
manifests remain compatible when those fields are absent. A successful forced
|
||||||
|
rerun marks only succeeded downstream session-stage records stale.
|
||||||
|
|
||||||
|
Starting an execution clears the current session-stage record's prior outputs,
|
||||||
|
logs, generated configuration references, and metadata. Failed and skipped
|
||||||
|
transitions enforce the same clearing rule directly, while success repopulates
|
||||||
|
only fields returned by the new result. Marking a record stale does not clear
|
||||||
|
those details because resume validation and diagnosis may still require them
|
||||||
|
before execution begins. Invocation run manifests remain immutable audit
|
||||||
|
records of their own outcomes.
|
||||||
|
|
||||||
|
A stage may explicitly return a skipped disposition and stable reason. The
|
||||||
|
runner persists that outcome in both manifests, clears older outputs for the
|
||||||
|
session-stage record along with older logs, generated configuration references,
|
||||||
|
and metadata, then applies any bounded details from the current skip and
|
||||||
|
continues. This self-skip is distinct from deciding not to execute an
|
||||||
|
already-succeeded stage and is reconsidered on later runs. Skipped results
|
||||||
|
cannot contain outputs.
|
||||||
|
|
||||||
When an already-succeeded stage is skipped, the invocation run manifest records
|
When an already-succeeded stage is skipped, the invocation run manifest records
|
||||||
the `skip` action and reason. The session manifest deliberately retains its
|
the `skip` action and reason. The session manifest deliberately retains its
|
||||||
existing succeeded record because it remains the cross-invocation progress
|
existing succeeded record because it remains the cross-invocation progress
|
||||||
authority.
|
authority. Stages with a resume validator, currently extraction, may reject an
|
||||||
|
otherwise eligible skip when the recorded durable result is obsolete; the
|
||||||
|
runner marks it stale and executes it.
|
||||||
|
|
||||||
Session manifest is the authoritative stage-progress ledger across invocations.
|
Session manifest is the authoritative stage-progress ledger across invocations.
|
||||||
Run manifest is invocation-scoped audit state.
|
Run manifest is invocation-scoped audit state.
|
||||||
|
|
||||||
|
After a publish commits remotely, any configured local cleanup is first recorded
|
||||||
|
as a session-manifest obligation before deletion begins. Each target becomes
|
||||||
|
complete only after its confined deletion (or safe absence check) and a
|
||||||
|
successful manifest save. An incomplete obligation is retried on later
|
||||||
|
invocations independently of their selected stages and retains the committed
|
||||||
|
run and remote identity that authorized it.
|
||||||
|
|
||||||
|
Each invocation derives campaign, session, run, local-path, and remote-prefix
|
||||||
|
metadata from the validated resolved configuration as one projection. A persisted
|
||||||
|
session manifest must agree on campaign and session identity before execution;
|
||||||
|
the current projection is refreshed for every invocation while stage progress,
|
||||||
|
inputs, and durable artifacts remain session history.
|
||||||
|
|
||||||
|
For handled failures after an invocation record is created, the runner records
|
||||||
|
the failure on the session ledger and persists it before persisting the failed
|
||||||
|
run audit record. This preserves the resume authority while making a partial
|
||||||
|
persistence disagreement visible. Abrupt process death remains an accepted case
|
||||||
|
where a durable running record can require operator interpretation.
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- stage resume/skip decisions are session-manifest driven.
|
- stage resume/skip decisions are session-manifest driven.
|
||||||
|
- running, failed, and self-skipped stages do not retain result payloads from
|
||||||
|
an earlier success.
|
||||||
|
- stale stages retain prior details until replacement execution starts.
|
||||||
- force reruns stale downstream succeeded stages.
|
- force reruns stale downstream succeeded stages.
|
||||||
- run manifest does not replace session manifest as progress authority.
|
- run manifest does not replace session manifest as progress authority.
|
||||||
|
- remote commitment is established by a verified current pointer and remote
|
||||||
|
commit relationship, never by a mutable session-manifest boolean.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
|
|
||||||
- Models and transitions: `internal/manifest/manifest.go`,
|
- Models and transitions: `internal/manifest/manifest.go`,
|
||||||
`internal/manifest/run_manifest.go`
|
`internal/manifest/run_manifest.go`
|
||||||
|
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
|
||||||
|
`internal/artifacts/current_state_commit.go`,
|
||||||
|
`internal/artifacts/current_state_legacy.go`
|
||||||
- Persistence and validation: `internal/manifest/store.go`
|
- Persistence and validation: `internal/manifest/store.go`
|
||||||
- Package tests: `internal/manifest/*_test.go`
|
- Package tests: `internal/manifest/*_test.go`
|
||||||
- Assembled execution behavior: `internal/app/runner_test.go`,
|
- Assembled execution behavior: `internal/app/runner_test.go`,
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ progress and artifact services resolve durable inputs and outputs.
|
|||||||
| Artifacts and paths | `internal/artifacts`, `internal/pathsafe` | Artifact identities and resolution, local and remote path/key models, current-state discovery, and confined relative destinations. |
|
| Artifacts and paths | `internal/artifacts`, `internal/pathsafe` | Artifact identities and resolution, local and remote path/key models, current-state discovery, and confined relative destinations. |
|
||||||
| Previous-session cache | `internal/previouscache` | Deterministic planning and materialization requirements for configured previous-session inputs. |
|
| Previous-session cache | `internal/previouscache` | Deterministic planning and materialization requirements for configured previous-session inputs. |
|
||||||
| Artifact policy | `internal/artifactpolicy` | Source and destination policy, configured artifact identity validation, and publish destination safety. |
|
| Artifact policy | `internal/artifactpolicy` | Source and destination policy, configured artifact identity validation, and publish destination safety. |
|
||||||
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, `internal/fileops` | Transcript and artifact data contracts plus narrow atomic filesystem helpers. |
|
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, [`internal/fileops`](fileops.md) | Transcript and artifact data contracts plus durable single-file replacement helpers; unsupported directory syncing is reported explicitly. |
|
||||||
| Logging | `internal/logging` | Application logger construction and shared structured logging behavior. |
|
| Logging | `internal/logging` | Application logger construction and shared structured logging behavior. |
|
||||||
|
|
||||||
The application boundary composes concrete implementations. Stages depend on
|
The application boundary composes concrete implementations. Stages depend on
|
||||||
@@ -53,14 +53,15 @@ The implemented canonical order is:
|
|||||||
4. [`polish`](stage-polish.md)
|
4. [`polish`](stage-polish.md)
|
||||||
5. [`normalize`](stage-normalize.md)
|
5. [`normalize`](stage-normalize.md)
|
||||||
6. [`trim`](stage-trim.md)
|
6. [`trim`](stage-trim.md)
|
||||||
7. [`render`](stage-render.md)
|
7. [`extract`](stage-extract.md)
|
||||||
8. [`analyze`](stage-analyze.md)
|
8. [`render`](stage-render.md)
|
||||||
9. [`publish`](stage-publish.md)
|
9. [`analyze`](stage-analyze.md)
|
||||||
10. `notify` (placeholder)
|
10. [`publish`](stage-publish.md)
|
||||||
|
11. `notify` (no-op)
|
||||||
|
|
||||||
`notify` currently has optional notifier call behavior and no persisted pipeline
|
`notify` currently has no persisted pipeline outputs and uses the explicit
|
||||||
outputs; its default collaborator is a no-op sender. The focused stage
|
`noop` notification mode. The focused stage documents own implementation
|
||||||
documents own implementation mechanics. The
|
mechanics. The
|
||||||
[CLI](../cli.md) and [Operations](../operations.md) own user-visible invocation
|
[CLI](../cli.md) and [Operations](../operations.md) own user-visible invocation
|
||||||
and execution semantics.
|
and execution semantics.
|
||||||
|
|
||||||
@@ -83,6 +84,7 @@ and execution semantics.
|
|||||||
- [`polish`](stage-polish.md)
|
- [`polish`](stage-polish.md)
|
||||||
- [`normalize`](stage-normalize.md)
|
- [`normalize`](stage-normalize.md)
|
||||||
- [`trim`](stage-trim.md)
|
- [`trim`](stage-trim.md)
|
||||||
|
- [`extract`](stage-extract.md)
|
||||||
- [`render`](stage-render.md)
|
- [`render`](stage-render.md)
|
||||||
- [`analyze`](stage-analyze.md)
|
- [`analyze`](stage-analyze.md)
|
||||||
- [`publish`](stage-publish.md)
|
- [`publish`](stage-publish.md)
|
||||||
|
|||||||
@@ -8,13 +8,15 @@ Execute selected configured Scriptorium artifacts in dependency order and materi
|
|||||||
|
|
||||||
- configured artifacts from `pipeline.scriptorium.artifacts`
|
- configured artifacts from `pipeline.scriptorium.artifacts`
|
||||||
- optional selected artifact keys supplied through the stage environment
|
- optional selected artifact keys supplied through the stage environment
|
||||||
- built-in/configured/previous-session source references in artifact inputs
|
- built-in, configured, extraction, and previous-session source references in
|
||||||
|
artifact inputs
|
||||||
|
|
||||||
Supported source families:
|
Supported source families:
|
||||||
- built-ins: `narratio.transcript.*`, `narratio.bounds.session`
|
- built-ins: `narratio.transcript.*`, `narratio.bounds.session`
|
||||||
- prepared stable inputs: `narratio.input.players`, `narratio.input.party`,
|
- prepared stable inputs: `narratio.input.players`, `narratio.input.party`,
|
||||||
`narratio.input.glossary`
|
`narratio.input.glossary`
|
||||||
- configured artifacts: `narratio.artifact.<key>`
|
- configured artifacts: `narratio.artifact.<key>`
|
||||||
|
- extraction lanes: `narratio.extraction.<key>`
|
||||||
- previous-session cache: `narratio.previous_session.artifact.<key>`
|
- previous-session cache: `narratio.previous_session.artifact.<key>`
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
@@ -24,11 +26,22 @@ Supported source families:
|
|||||||
|
|
||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- skips with metadata when Scriptorium config is missing or no executable artifacts remain.
|
- when Scriptorium is absent or no configured artifact is executable, completes
|
||||||
- builds runtime artifact catalog (built-ins + configured artifacts).
|
successfully with no outputs and records explanatory metadata. This is not an
|
||||||
|
explicit self-skip: both manifests record success, satisfy publish's
|
||||||
|
prerequisite, and an ordinary later run reuses the result until forced.
|
||||||
|
- builds a runtime artifact catalog containing built-ins, configured artifacts,
|
||||||
|
and configured extraction lanes. Extraction availability is hydrated only
|
||||||
|
from compatible successful extraction evidence.
|
||||||
|
- uses enabled configured artifacts by default. An explicit `--artifacts`
|
||||||
|
selection is a one-invocation override: it makes exactly the named configured
|
||||||
|
artifacts executable even when disabled, and does not automatically include
|
||||||
|
dependencies. A selected artifact's dependencies must instead already be
|
||||||
|
available to the catalog.
|
||||||
- marks non-executable configured artifacts as reusable when output files already exist.
|
- marks non-executable configured artifacts as reusable when output files already exist.
|
||||||
- validates selected artifact dependency order (cycle-safe topo ordering).
|
- validates selected artifact dependency order (cycle-safe topo ordering).
|
||||||
- resolves required/optional inputs per artifact source definition.
|
- resolves required/optional inputs per artifact source definition.
|
||||||
|
- omits an unavailable optional input; an unavailable required input fails.
|
||||||
- resolves prepared stable input sources from `inputs/*.yml` materialized by `prepare`.
|
- resolves prepared stable input sources from `inputs/*.yml` materialized by `prepare`.
|
||||||
- resolves previous-session sources from local `previous/` cache only.
|
- resolves previous-session sources from local `previous/` cache only.
|
||||||
- runs optional render-debug, then artifact execution.
|
- runs optional render-debug, then artifact execution.
|
||||||
|
|||||||
90
docs/internal/stage-extract.md
Normal file
90
docs/internal/stage-extract.md
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
# Internal: Extract Stage
|
||||||
|
|
||||||
|
## Responsibility
|
||||||
|
|
||||||
|
`extract` runs after `trim` and before `render`. It converts the canonical
|
||||||
|
`narratio.transcript.final_trimmed` JSON into configured Notarius lane artifacts.
|
||||||
|
An omitted or disabled Notarius section makes the stage explicitly self-skip
|
||||||
|
with reason `notarius_disabled`, no outputs, and no Notarius runner.
|
||||||
|
|
||||||
|
The external protocol is documented in the
|
||||||
|
[Notarius integration contract](../integrations/notarius.md). Configuration
|
||||||
|
fields belong in [Configuration](../config.md), and physical paths and force
|
||||||
|
procedures belong in [Operations](../operations.md).
|
||||||
|
|
||||||
|
## Lifecycle
|
||||||
|
|
||||||
|
`internal/stage/extract.go`:
|
||||||
|
|
||||||
|
1. resolves the final trimmed transcript from the shared artifact catalog;
|
||||||
|
2. resolves and fingerprints the Notarius invocation contract;
|
||||||
|
3. creates a run-local staging directory and invokes the injected
|
||||||
|
`notarius.Runner`;
|
||||||
|
4. validates the successful receipt, confined index, configured required lane
|
||||||
|
descriptors, and regular payload files;
|
||||||
|
5. atomically promotes the complete bundle to its immutable durable location;
|
||||||
|
6. records one non-selectable `notarius_index` output and one selectable
|
||||||
|
`notarius_lane` output per configured lane; and
|
||||||
|
7. registers each lane as `narratio.extraction.<output_key>` for downstream
|
||||||
|
Scriptorium and publish resolution.
|
||||||
|
|
||||||
|
Lane records retain checksum, contract, producer run ID, and Notarius system,
|
||||||
|
run, pipeline, and lane provenance. Stage metadata retains the durable bundle
|
||||||
|
root, receipt, diagnostic paths, rejection/warning summaries, producing
|
||||||
|
Narratio run ID, the resolved trimmed-input identity, and invocation
|
||||||
|
fingerprint. The input identity binds the exact transcript bytes, canonical
|
||||||
|
source ID, producer stage/output/run identity, and resolution provenance.
|
||||||
|
Validation completes before
|
||||||
|
promotion, so a rejected result cannot expose a partial durable bundle.
|
||||||
|
|
||||||
|
Any executed extraction outcome that replaces a different effective outcome
|
||||||
|
marks succeeded downstream stages stale. Repeating the same disabled self-skip
|
||||||
|
with no outputs is stable and does not repeatedly invalidate downstream stages.
|
||||||
|
|
||||||
|
## Resume Validation
|
||||||
|
|
||||||
|
`internal/stage/extract_resume.go` permits a skip only when the existing stage
|
||||||
|
record succeeded and still matches the current invocation fingerprint. The
|
||||||
|
fingerprint covers the resolved executable and config paths, pipeline ID,
|
||||||
|
timeout, working directory, sorted configured output contracts, and the current
|
||||||
|
direct trimmed-transcript identity. The same identity is resolved again for
|
||||||
|
artifact evidence, so changing the current transcript bytes or producer
|
||||||
|
identity makes the prior extraction obsolete.
|
||||||
|
|
||||||
|
The validator then checks the producing run identity, canonical immutable
|
||||||
|
bundle root, path confinement and absence of symlink components, receipt
|
||||||
|
identity, exactly one canonical index, the exact configured source set,
|
||||||
|
contracts and provenance, regular-file status, and stored checksums. Missing or
|
||||||
|
obsolete results are non-resumable and run again; unsafe filesystem conditions
|
||||||
|
return an error rather than silently accepting or replacing data.
|
||||||
|
|
||||||
|
The fingerprint cannot observe files imported by Notarius configuration,
|
||||||
|
profile contents, prompt/module definitions, or other transitive inputs.
|
||||||
|
Operators must force extraction after changing any such input.
|
||||||
|
|
||||||
|
## Failure Behavior
|
||||||
|
|
||||||
|
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
|
||||||
|
index compatibility, required-lane rejection, payload inspection, checksum, or
|
||||||
|
promotion errors fail the stage through ordinary manifest transition handling.
|
||||||
|
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
|
||||||
|
not given selectable extraction sources unless the complete configured result
|
||||||
|
has passed validation and promotion.
|
||||||
|
|
||||||
|
When a replacement attempt begins, the current session-stage record no longer
|
||||||
|
advertises payload from the previous success. A failed replacement therefore
|
||||||
|
has no current outputs, logs, generated configuration references, or metadata,
|
||||||
|
while the earlier invocation manifest and immutable promoted bundle remain
|
||||||
|
available for audit and recovery.
|
||||||
|
|
||||||
|
## Implementation And Focused Tests
|
||||||
|
|
||||||
|
- Stage execution, selection, and resume validation: `internal/stage/extract.go`,
|
||||||
|
`internal/stage/extract_resume.go`,
|
||||||
|
`internal/stage/extract_test.go`
|
||||||
|
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
||||||
|
`internal/adapters/notarius/subprocess_test.go`
|
||||||
|
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
||||||
|
`internal/artifacts/extraction_catalog_test.go`
|
||||||
|
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
||||||
|
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
||||||
@@ -17,7 +17,8 @@ Run Audita polishing on base transcript and produce polished transcript.
|
|||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- resolves base transcript from merge outputs/canonical fallback.
|
- resolves base transcript from merge outputs/canonical fallback.
|
||||||
- invokes Audita with configured model/module/runtime options.
|
- invokes an Audita runner configured with static model/runtime options; the
|
||||||
|
invocation supplies paths and modules.
|
||||||
- validates processed transcript structure (`segments` array required).
|
- validates processed transcript structure (`segments` array required).
|
||||||
- validates optional report JSON.
|
- validates optional report JSON.
|
||||||
- materializes canonical outputs; records logs/generated config and adapter metadata.
|
- materializes canonical outputs; records logs/generated config and adapter metadata.
|
||||||
|
|||||||
@@ -30,21 +30,26 @@ Materialize canonical current-session inputs before processing stages.
|
|||||||
|
|
||||||
- validates required config/store state.
|
- validates required config/store state.
|
||||||
- enforces local audio vs S3 audio mutual exclusivity.
|
- enforces local audio vs S3 audio mutual exclusivity.
|
||||||
|
- rejects duplicate explicit local audio sources after resolution.
|
||||||
|
- gives distinct local source paths with the same basename deterministic unique
|
||||||
|
prepared filenames so neither source is overwritten.
|
||||||
- materializes S3 audio through spool/cache-aware logic.
|
- materializes S3 audio through spool/cache-aware logic.
|
||||||
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
|
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
|
||||||
- when previous requirements exist:
|
- clears managed `previous/` state on every invocation, then, when requirements exist:
|
||||||
- clears managed `previous/` state;
|
- resolves the pointer-selected previous source through the shared resolver;
|
||||||
- builds previous-cache remote plan;
|
|
||||||
- downloads previous manifest/artifacts;
|
- downloads previous manifest/artifacts;
|
||||||
- records previous inputs in `manifest.inputs`.
|
- records previous inputs in `manifest.inputs`.
|
||||||
|
|
||||||
Required previous-session inputs fail when unavailable; optional missing inputs are skipped.
|
Required previous-session inputs fail when unavailable; optional missing inputs
|
||||||
|
are typed skipped results. Committed sources use their exact source-to-destination
|
||||||
|
mapping, while the isolated legacy reader rejects ambiguous fallback matches.
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- only `prepare` hydrates canonical `previous/` cache state.
|
- only `prepare` hydrates canonical `previous/` cache state.
|
||||||
- managed previous artifacts are stored under `previous/artifacts/**` without
|
- managed previous artifacts are stored under `previous/artifacts/**` without
|
||||||
duplicate `artifacts/artifacts/` nesting.
|
duplicate `artifacts/artifacts/` nesting.
|
||||||
|
- managed `previous/` state represents only the current requirement set.
|
||||||
- `manifest.inputs` ordering is deterministic (`kind`, `path`).
|
- `manifest.inputs` ordering is deterministic (`kind`, `path`).
|
||||||
|
|
||||||
## Related Contracts And Tests
|
## Related Contracts And Tests
|
||||||
|
|||||||
@@ -9,29 +9,52 @@ Upload run/session outputs to object storage and atomically advance remote curre
|
|||||||
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
|
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
|
||||||
- invocation-scoped run files
|
- invocation-scoped run files
|
||||||
- resolved publish output rules
|
- resolved publish output rules
|
||||||
- effective publish locks (static + remote merged lock set)
|
- effective publish locks (static + remote merged lock set), revalidated at the
|
||||||
|
remote commit boundary
|
||||||
- durable previous-session cache files when present
|
- durable previous-session cache files when present
|
||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
- uploaded invocation record and selected publish outputs;
|
- uploaded invocation record and selected publish outputs;
|
||||||
- uploaded durable previous-session cache files when present;
|
- uploaded durable previous-session cache files when present;
|
||||||
- updated remote current manifest; and
|
- immutable run-scoped commit manifest; and
|
||||||
- remote current-run commit marker, written last.
|
- current commit pointer, written last.
|
||||||
|
|
||||||
Exact remote placement and the operator workflow belong in
|
Exact remote placement and the operator workflow belong in
|
||||||
[Operations](../operations.md#publish-workflow).
|
[Operations](../operations.md#publish-workflow).
|
||||||
|
|
||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- stage can self-skip when publish disabled or run upload disabled.
|
- when publishing or run upload is disabled, completes successfully with no
|
||||||
|
outputs and records explanatory metadata. This is not an explicit self-skip:
|
||||||
|
both manifests record success, and an ordinary later run reuses that result
|
||||||
|
until publish is forced.
|
||||||
- validates prerequisite stage success and object-store availability.
|
- validates prerequisite stage success and object-store availability.
|
||||||
- collects deterministic run file list plus run `manifest.json`.
|
- derives a deterministic run-archive allowlist from the validated run
|
||||||
|
`manifest.json`: declared run-local outputs, logs, generated configs, and the
|
||||||
|
manifest itself. Unlisted workspace files are not archive candidates.
|
||||||
|
- opens each archive candidate beneath its archive root without following
|
||||||
|
symlinked ancestors or leaf entries, verifies that it is a regular file and
|
||||||
|
checks a declared checksum when present, then streams the opened descriptor.
|
||||||
|
- derives the durable previous-cache archive from its validated manifest using
|
||||||
|
the same confinement and regular-file checks.
|
||||||
- resolves publish output sources through runtime artifact catalog and manifest-aware resolution.
|
- resolves publish output sources through runtime artifact catalog and manifest-aware resolution.
|
||||||
|
- publishes extraction lanes only through explicit configured output rules;
|
||||||
|
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
|
||||||
- selected artifact filter applies to configured artifact sources only.
|
- selected artifact filter applies to configured artifact sources only.
|
||||||
- locked outputs are skipped intentionally (including required ones).
|
- locked outputs are skipped intentionally (including required ones).
|
||||||
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
||||||
- writes remote current manifest before current run pointer.
|
- creates one complete immutable source-to-destination mapping before upload;
|
||||||
|
- uploads and verifies every declared immutable object and the commit manifest;
|
||||||
|
- updates `current/commit-pointer.json` exactly once, last; and
|
||||||
|
- does not write the legacy `current/manifest.json` or `current/run_id.txt` pair.
|
||||||
|
- rechecks remote lock state immediately before the pointer update. A newly
|
||||||
|
committed lock aborts selection, leaving any uploaded immutable attempt
|
||||||
|
unselected.
|
||||||
|
- reads the mutable remote lock document through a direct limit-plus-one read
|
||||||
|
capped by `MaxRemoteLockStoreBytes` (1 MiB), retaining the generation returned
|
||||||
|
with the opened body for conditional updates. Oversized lock documents fail
|
||||||
|
before YAML decoding; published artifact payloads do not use this limit.
|
||||||
|
|
||||||
## Metadata Signals
|
## Metadata Signals
|
||||||
|
|
||||||
@@ -42,14 +65,23 @@ Includes counts/lists for:
|
|||||||
- skipped optional outputs
|
- skipped optional outputs
|
||||||
- skipped unselected outputs
|
- skipped unselected outputs
|
||||||
- locked outputs
|
- locked outputs
|
||||||
- current-state key paths
|
- remote commit and current-pointer key paths
|
||||||
- `current_pointer_written`
|
- the run identifier selected by the commit
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- `current/run_id.txt` is the remote commit marker and is written last.
|
- `current/commit-pointer.json` is the remote commit marker and is written last.
|
||||||
- run upload excludes `audio/**`.
|
- run files, selected outputs, previous-cache files, and the committed session
|
||||||
- publish locks are not overridden by `--force`.
|
manifest are all declared by an immutable commit under the run prefix.
|
||||||
|
- run and previous uploads contain only manifest-declared regular files opened
|
||||||
|
from verified descriptors; symlinks, special files, replacement races, and
|
||||||
|
undeclared entries are rejected or ignored before uploads begin.
|
||||||
|
- run-local diagnostics, including Notarius receipt and stderr files, are
|
||||||
|
archived only when recorded by the run manifest.
|
||||||
|
- publish locks are not overridden by `--force`; remote locks are revalidated
|
||||||
|
immediately before current-state selection.
|
||||||
|
- post-commit local cleanup is authorized by the committed publish metadata and
|
||||||
|
is durably recorded by the application lifecycle before any local deletion.
|
||||||
|
|
||||||
The commit boundary and cleanup gate are normative architecture invariants; see
|
The commit boundary and cleanup gate are normative architecture invariants; see
|
||||||
[Architecture](../policy/architecture.md#publish-commit-boundary).
|
[Architecture](../policy/architecture.md#publish-commit-boundary).
|
||||||
|
|||||||
@@ -20,7 +20,9 @@ Render Markdown transcript artifacts from normalized JSON transcripts via Seriat
|
|||||||
- resolves inputs manifest-first, then canonical fallback.
|
- resolves inputs manifest-first, then canonical fallback.
|
||||||
- writes run-local outputs first, then materializes canonical session outputs.
|
- writes run-local outputs first, then materializes canonical session outputs.
|
||||||
- records input provenance, output paths, adapter metadata, logs, and generated config refs.
|
- records input provenance, output paths, adapter metadata, logs, and generated config refs.
|
||||||
- skips with stage metadata when `pipeline.render.enabled=false`.
|
- when `pipeline.render.enabled=false`, completes successfully with no outputs
|
||||||
|
and records explanatory metadata. This is not an explicit self-skip: both
|
||||||
|
manifests record success, and enabling render later requires a forced run.
|
||||||
|
|
||||||
## Failure Semantics
|
## Failure Semantics
|
||||||
|
|
||||||
|
|||||||
@@ -15,16 +15,21 @@ Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
|||||||
## Key Behavior
|
## Key Behavior
|
||||||
|
|
||||||
- discovers prepared audio from manifest inputs or canonical audio directory.
|
- discovers prepared audio from manifest inputs or canonical audio directory.
|
||||||
- derives speaker ID from `.flac` basename.
|
- derives the transcript identity from the prepared `.flac` filename.
|
||||||
- dispatches WhisperX requests through a bounded worker pool.
|
- dispatches WhisperX requests through a bounded worker pool.
|
||||||
- validates each output as JSON.
|
- validates each output as JSON.
|
||||||
- writes run-local outputs then materializes canonical transcript outputs.
|
- writes run-local outputs then materializes canonical transcript outputs only
|
||||||
|
after every planned request succeeds.
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- speaker basenames must be unique.
|
- prepared audio identities must be unique; prepare disambiguates distinct
|
||||||
|
source paths that share a basename.
|
||||||
- output path returned by adapter must match requested output path.
|
- output path returned by adapter must match requested output path.
|
||||||
- each successful output is validated before stage success.
|
- an empty adapter result path means the requested path; adapters cannot select
|
||||||
|
an alternate destination.
|
||||||
|
- each successful output is validated before stage success, and cancellation or
|
||||||
|
incomplete dispatch cannot be reported as a successful result.
|
||||||
|
|
||||||
## Related Contracts And Tests
|
## Related Contracts And Tests
|
||||||
|
|
||||||
|
|||||||
@@ -12,14 +12,23 @@ operator-selected storage fields and credential mechanisms belong in
|
|||||||
`storage.ObjectStore` interface:
|
`storage.ObjectStore` interface:
|
||||||
|
|
||||||
- `List(ctx, prefix)`
|
- `List(ctx, prefix)`
|
||||||
|
- `Read(ctx, key)` returns an object body and the generation observed with it
|
||||||
- `Download(ctx, key, localPath)`
|
- `Download(ctx, key, localPath)`
|
||||||
- `Upload(ctx, localPath, key, opts)`
|
- `Upload(ctx, localPath, key, opts)`
|
||||||
|
- `UploadConditional(ctx, source, key, opts, condition)`
|
||||||
- `Exists(ctx, key)`
|
- `Exists(ctx, key)`
|
||||||
|
|
||||||
Key invariant:
|
Key invariant:
|
||||||
- callers pass full bucket-relative keys;
|
- callers pass full bucket-relative keys;
|
||||||
- storage implementations do not infer campaign/session/run prefixes.
|
- storage implementations do not infer campaign/session/run prefixes.
|
||||||
|
|
||||||
|
`ReadObjectBounded` is the shared mechanism for small control objects. It opens
|
||||||
|
one object version, returns the metadata observed with that body, rejects an
|
||||||
|
oversized known size before transfer, and still performs a context-aware
|
||||||
|
limit-plus-one read. It closes the body on every exit. Callers own the policy
|
||||||
|
limit and add the control-object category to errors; this helper is not used for
|
||||||
|
large artifact payloads.
|
||||||
|
|
||||||
## Composition
|
## Composition
|
||||||
|
|
||||||
`NewObjectStoreFromConfig` constructs the S3-backed implementation from
|
`NewObjectStoreFromConfig` constructs the S3-backed implementation from
|
||||||
@@ -31,13 +40,20 @@ not own discovery, defaults, or configuration validation.
|
|||||||
|
|
||||||
- normalizes object keys.
|
- normalizes object keys.
|
||||||
- `List` paginates and returns normalized `ObjectInfo`.
|
- `List` paginates and returns normalized `ObjectInfo`.
|
||||||
|
- A truncated S3 listing must supply a new, non-empty continuation token;
|
||||||
|
otherwise listing fails with bucket and prefix context instead of looping.
|
||||||
- `Download` writes local files with parent directory creation.
|
- `Download` writes local files with parent directory creation.
|
||||||
- `Upload` streams local file and returns remote metadata.
|
- `Upload` streams local file and returns remote metadata.
|
||||||
|
- `Read` binds a returned body to its S3 ETag. `UploadConditional` maps an ETag
|
||||||
|
match or absence precondition directly to the provider request and reports a
|
||||||
|
failed precondition without performing a local check-then-write replacement.
|
||||||
- `Exists` maps not-found responses to `false`.
|
- `Exists` maps not-found responses to `false`.
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- storage layer is stateless regarding manifest/stage progression.
|
- storage layer is stateless regarding manifest/stage progression.
|
||||||
|
- bounded reads never retain more than the caller's limit plus one byte and do
|
||||||
|
not replace owner-specific size policy.
|
||||||
- publish ordering semantics are owned by stage/app code, not storage adapters.
|
- publish ordering semantics are owned by stage/app code, not storage adapters.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
|
|||||||
@@ -13,8 +13,16 @@ previous-cache path construction. `SessionPathsFor` provides the session-scoped
|
|||||||
path model, and layout creation goes through `EnsureLayoutFor`. Callers should
|
path model, and layout creation goes through `EnsureLayoutFor`. Callers should
|
||||||
consume those helpers instead of rebuilding relative paths.
|
consume those helpers instead of rebuilding relative paths.
|
||||||
|
|
||||||
`internal/pathsafe` and application cleanup helpers enforce confinement for
|
`internal/pathsafe` validates relative destinations. `internal/fileops` opens
|
||||||
relative destinations and deletion targets.
|
cleanup roots and their descendants through no-follow directory handles before
|
||||||
|
removing them.
|
||||||
|
|
||||||
|
`internal/fileops` owns the ordinary workspace mode contract. On POSIX,
|
||||||
|
`WorkspaceDirectoryMode` is setgid `02775` and `WorkspaceFileMode` is `0664`.
|
||||||
|
`EnsureWorkspaceDirectory` reapplies the directory mode after creation so a
|
||||||
|
restrictive umask cannot remove group access, while retaining existing ownership
|
||||||
|
and group. Credential paths are outside this contract; the platform-specific
|
||||||
|
operational requirements are in [Operations](../operations.md#workspace-permissions).
|
||||||
|
|
||||||
## Run-Local Stage Layout
|
## Run-Local Stage Layout
|
||||||
|
|
||||||
@@ -24,23 +32,42 @@ materialized into canonical session paths before stage success. Managed
|
|||||||
previous-session cache paths remain session-durable and are never redirected
|
previous-session cache paths remain session-durable and are never redirected
|
||||||
into run-local output space.
|
into run-local output space.
|
||||||
|
|
||||||
|
Extraction uses run-local receipt, stderr, and output-root helpers, then
|
||||||
|
promotes the validated external bundle to the unique immutable Notarius bundle
|
||||||
|
path supplied by `internal/artifacts`. `internal/fileops.PromoteDirectory`
|
||||||
|
copies only regular files and directories to a same-filesystem temporary
|
||||||
|
sibling. Source traversal uses confined directory handles and identity checks
|
||||||
|
so replacing an inspected root, directory, or file is rejected rather than
|
||||||
|
followed. The completed tree is atomically renamed without replacing an
|
||||||
|
existing destination. Exact physical paths belong in
|
||||||
|
[Operations](../operations.md#extraction-workflow).
|
||||||
|
|
||||||
## Locking
|
## Locking
|
||||||
|
|
||||||
`artifacts.LocalStore` enforces the single-writer session lock via `.lock`
|
`artifacts.LocalStore` enforces the single-writer session lock via an
|
||||||
(`ErrLockConflict` on contention).
|
operating-system lock held on `.lock` (`ErrLockConflict` on contention). The
|
||||||
|
file retains owner metadata after release or process death; its existence is
|
||||||
|
not evidence that a lock is active. Command and restore flows wait for this
|
||||||
|
lock only while their context remains active, and report a release failure.
|
||||||
|
|
||||||
## Cleanup Semantics
|
## Cleanup Semantics
|
||||||
|
|
||||||
Automatic post-publish cleanup:
|
Automatic post-publish cleanup:
|
||||||
|
|
||||||
- only runs when publish actually executed and succeeded;
|
- is created only after a successful publish commit with complete publish
|
||||||
- requires `uploaded=true` and `current_pointer_written=true` metadata;
|
metadata, then is persisted before any deletion;
|
||||||
|
- requires `uploaded=true`, a remote commit key, and a current commit-pointer
|
||||||
|
key in publish metadata;
|
||||||
- consumes the resolved cleanup policy described in
|
- consumes the resolved cleanup policy described in
|
||||||
[Configuration](../config.md);
|
[Configuration](../config.md);
|
||||||
- refuses unsafe deletes (root delete, out-of-root delete, symlink paths).
|
- refuses unsafe deletes (root delete, out-of-root delete, and symlinked
|
||||||
|
ancestors or entries);
|
||||||
|
- retries any recorded incomplete target on later invocations even when no
|
||||||
|
publish work is selected. Missing targets are a successful, idempotent
|
||||||
|
cleanup result only after the completion evidence is saved.
|
||||||
|
|
||||||
Manual cleanup uses the same scoped-target checks. Invocation syntax and exact
|
Manual cleanup uses the same root-confined deletion mechanism. Invocation
|
||||||
deletion scope belong in [CLI](../cli.md#clean) and
|
syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
||||||
[Operations](../operations.md#cleanup).
|
[Operations](../operations.md#cleanup).
|
||||||
|
|
||||||
## Invariants
|
## Invariants
|
||||||
@@ -48,15 +75,20 @@ deletion scope belong in [CLI](../cli.md#clean) and
|
|||||||
- campaign-aware session root is mandatory.
|
- campaign-aware session root is mandatory.
|
||||||
- manifest-driven stage state is durable across runs.
|
- manifest-driven stage state is durable across runs.
|
||||||
- cleanup guardrails prevent destructive root/out-of-scope deletion.
|
- cleanup guardrails prevent destructive root/out-of-scope deletion.
|
||||||
|
- ordinary workspace paths retain group-writable directory and file modes across
|
||||||
|
nested creation, replacement, and Notarius promotion.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
|
|
||||||
- Path model and local store: `internal/artifacts/paths.go`,
|
- Path model and local store: `internal/artifacts/paths.go`,
|
||||||
`internal/artifacts/local.go`
|
`internal/artifacts/local.go`
|
||||||
- Run-local materialization: `internal/stage/run_local.go`
|
- Run-local materialization: `internal/stage/run_local.go`
|
||||||
- Cleanup confinement: `internal/app/cleanup_targets.go`,
|
- Immutable bundle promotion: `internal/fileops/directory.go`
|
||||||
`internal/app/post_publish_cleanup.go`
|
- Workspace modes: `internal/fileops/modes.go`
|
||||||
|
- Cleanup confinement: `internal/fileops/cleanup.go`,
|
||||||
|
`internal/app/cleanup_targets.go`, `internal/app/post_publish_cleanup.go`
|
||||||
- Tests: `internal/artifacts/paths_model_test.go`,
|
- Tests: `internal/artifacts/paths_model_test.go`,
|
||||||
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
||||||
`internal/app/cleanup_targets_test.go`,
|
`internal/fileops/directory_test.go`, `internal/fileops/modes_posix_test.go`,
|
||||||
|
`internal/fileops/cleanup_test.go`, `internal/app/cleanup_targets_test.go`,
|
||||||
`internal/app/post_publish_cleanup_test.go`
|
`internal/app/post_publish_cleanup_test.go`
|
||||||
|
|||||||
@@ -75,16 +75,30 @@ Canonical stage order:
|
|||||||
4. `polish`
|
4. `polish`
|
||||||
5. `normalize`
|
5. `normalize`
|
||||||
6. `trim`
|
6. `trim`
|
||||||
7. `render`
|
7. `extract`
|
||||||
8. `analyze`
|
8. `render`
|
||||||
9. `publish`
|
9. `analyze`
|
||||||
10. `notify`
|
10. `publish`
|
||||||
|
11. `notify`
|
||||||
|
|
||||||
Execution rules:
|
Execution rules:
|
||||||
|
|
||||||
- succeeded stages are skipped unless `--force` is set;
|
- succeeded stages are skipped unless `--force` is set;
|
||||||
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||||
- force rerunning a succeeded upstream stage marks succeeded downstream stages as `stale`.
|
- forcing an upstream stage marks succeeded downstream stages as `stale` before
|
||||||
|
the replacement runs; and
|
||||||
|
- an executed failure, changed self-skip, or success that replaces a different
|
||||||
|
effective upstream outcome also marks succeeded downstream stages stale. A
|
||||||
|
repeated self-skip with the same reason and no outputs is stable and does not
|
||||||
|
perpetually rerun downstream work.
|
||||||
|
|
||||||
|
An explicit self-skip is a durable `skipped` stage outcome that later runs
|
||||||
|
reconsider. It differs from successful no-output execution: disabled `render`
|
||||||
|
and `publish`, and absent or no-executable `analyze`, record `succeeded` with
|
||||||
|
metadata and no outputs. Ordinary later runs reuse those successful results;
|
||||||
|
force the affected stage after enabling or configuring it. Optional artifact
|
||||||
|
inputs are omitted only from the consuming artifact invocation and do not make
|
||||||
|
the stage self-skip.
|
||||||
|
|
||||||
Single-stage execution:
|
Single-stage execution:
|
||||||
|
|
||||||
@@ -101,7 +115,80 @@ Selection behavior:
|
|||||||
- validates names against `pipeline.scriptorium.artifacts`;
|
- validates names against `pipeline.scriptorium.artifacts`;
|
||||||
- filters analyze execution to selected configured artifacts;
|
- filters analyze execution to selected configured artifacts;
|
||||||
- filters publish rules for `narratio.artifact.<name>` sources only;
|
- filters publish rules for `narratio.artifact.<name>` sources only;
|
||||||
- does not suppress built-in transcript or bounds publish sources.
|
- does not suppress built-in transcript, bounds, or explicitly configured
|
||||||
|
`narratio.extraction.<name>` publish sources; and
|
||||||
|
- never partially selects Notarius lanes.
|
||||||
|
|
||||||
|
## Extraction Workflow
|
||||||
|
|
||||||
|
When Notarius is omitted or disabled, `extract` records an explicit skipped
|
||||||
|
outcome with reason `notarius_disabled` and no outputs. A later invocation
|
||||||
|
reconsiders the skipped stage, so enabling Notarius does not require force.
|
||||||
|
|
||||||
|
When Notarius extraction is enabled, the stage consumes the final trimmed JSON
|
||||||
|
and preserves the complete validated Notarius bundle at:
|
||||||
|
|
||||||
|
- `artifacts/notarius/{narratio_run_id}/`
|
||||||
|
|
||||||
|
The directory is immutable once promoted. Configured lanes become
|
||||||
|
`narratio.extraction.<name>` sources for Scriptorium and explicit publish rules;
|
||||||
|
the bundle and `index.json` are retained for audit and resume validation but
|
||||||
|
are not selectable or published implicitly.
|
||||||
|
|
||||||
|
Starting a replacement clears the previous extraction payload from the current
|
||||||
|
session-stage record. If that replacement fails or self-skips, the current
|
||||||
|
record does not fall back to the earlier outputs. The earlier run manifest and
|
||||||
|
immutable bundle remain available for inspection, but downstream resolution
|
||||||
|
requires a new current successful extraction record.
|
||||||
|
|
||||||
|
Atomic Notarius bundle promotion is supported on Linux and macOS. On Windows
|
||||||
|
and other operating systems, extraction fails before copying the bundle into a
|
||||||
|
temporary promotion tree because Narratio has no verified atomic no-replace
|
||||||
|
directory primitive there. This is an extraction limitation, not a broader
|
||||||
|
platform-support guarantee for every Narratio workflow.
|
||||||
|
|
||||||
|
## External Command Lifecycle
|
||||||
|
|
||||||
|
When an external command is cancelled or times out, Narratio terminates its
|
||||||
|
owned descendants as well as the command itself. Cancellation first requests
|
||||||
|
termination where the platform supports it, then force terminates after a
|
||||||
|
bounded wait. A command is not considered finished until its leader has been
|
||||||
|
reaped, and descendants that keep standard output or error open cannot keep
|
||||||
|
the invocation blocked. Other operating systems fail closed rather than launch
|
||||||
|
a command without tree ownership.
|
||||||
|
|
||||||
|
Subprocess stdout and stderr diagnostics are separately redacted and capped at
|
||||||
|
8 MiB per invocation. Narratio does not retain configured credential values in
|
||||||
|
these logs or their error tails; reaching a capture limit terminates the command
|
||||||
|
tree and reports which stream exceeded the limit.
|
||||||
|
|
||||||
|
Run-local diagnostics are:
|
||||||
|
|
||||||
|
- `runs/{run_id}/extract/notarius.receipt.json`
|
||||||
|
- `runs/{run_id}/extract/notarius.stderr.log`
|
||||||
|
- `runs/{run_id}/extract/notarius-output/` before durable promotion
|
||||||
|
|
||||||
|
The run-record upload is an allowlist derived from the validated run manifest,
|
||||||
|
not a workspace scan. Each declared source is opened without following
|
||||||
|
symlinked ancestors or the leaf, verified as a regular file, and streamed from
|
||||||
|
that verified descriptor. Unlisted files and unsafe entries are never uploaded.
|
||||||
|
The durable bundle is never scanned for implicit publication; only lanes named
|
||||||
|
by explicit `pipeline.publish.outputs` rules are uploaded.
|
||||||
|
|
||||||
|
To intentionally replace the current extraction result, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage extract 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Narratio automatically reruns extraction when its recorded invocation contract
|
||||||
|
or durable output validation changes. It cannot fingerprint configuration
|
||||||
|
files, profiles, prompts, modules, or references loaded transitively by
|
||||||
|
Notarius. Force extraction after changing any of those inputs, even when the
|
||||||
|
top-level Narratio and Notarius config paths remain the same. A forced extract
|
||||||
|
marks successful downstream stages stale. Ordinary extraction failures or
|
||||||
|
outcome changes also stale affected downstream stages, while an identical
|
||||||
|
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
||||||
|
|
||||||
## Publish Workflow
|
## Publish Workflow
|
||||||
|
|
||||||
@@ -119,13 +206,31 @@ narratio run-stage publish 2026-04-04 --force
|
|||||||
|
|
||||||
Publish commit model:
|
Publish commit model:
|
||||||
|
|
||||||
- uploads run files under `{session_prefix}/runs/{run_id}/`;
|
- uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
|
||||||
- uploads configured published outputs;
|
audio and the run-local Notarius staging bundle;
|
||||||
- uploads `previous/**` cache files when present;
|
- uploads configured published outputs and `previous/**` cache files into the
|
||||||
- writes `current/manifest.json`;
|
same immutable run scope, including only explicitly configured extraction
|
||||||
- writes `current/run_id.txt` last.
|
lanes;
|
||||||
|
- writes `{session_prefix}/runs/{run_id}/commit.json` after all declared
|
||||||
|
immutable objects are uploaded and verified; and
|
||||||
|
- writes `{session_prefix}/current/commit-pointer.json` once, last.
|
||||||
|
|
||||||
`current/run_id.txt` is the remote current-state commit marker.
|
`current/commit-pointer.json` is the remote current-state commit marker. It
|
||||||
|
selects exactly one immutable commit, which declares the complete object set.
|
||||||
|
|
||||||
|
## Remote Commit Migration
|
||||||
|
|
||||||
|
The immutable remote commit contract uses
|
||||||
|
`runs/{run_id}/commit.json` to declare a run's complete object set and a small
|
||||||
|
`current/commit-pointer.json` to select it. The pointer binds the selected
|
||||||
|
commit by version, checksum, size, and storage generation; committed artifacts
|
||||||
|
are also checksum- and generation-bound. Readers accept this contract now and
|
||||||
|
strictly reject mismatched or unknown data.
|
||||||
|
|
||||||
|
Legacy reads are limited to a coherent `current/manifest.json` and
|
||||||
|
`current/run_id.txt` pair; a torn pair is rejected. New publication does not
|
||||||
|
write that pair and remote commit state does not carry local
|
||||||
|
`current_pointer_written` metadata.
|
||||||
|
|
||||||
## Publish Locks
|
## Publish Locks
|
||||||
|
|
||||||
@@ -139,7 +244,14 @@ Effective lock rules:
|
|||||||
- static and remote locks are merged;
|
- static and remote locks are merged;
|
||||||
- static locks win on source collisions;
|
- static locks win on source collisions;
|
||||||
- locked outputs are intentional skips;
|
- locked outputs are intentional skips;
|
||||||
- lock add/remove commands mutate only remote lock state.
|
- lock add/remove commands mutate only remote lock state through generation-bound
|
||||||
|
conditional writes. A command retries a bounded number of concurrent
|
||||||
|
conflicts while its invocation context remains active, so it never replaces a
|
||||||
|
different lock-document generation; and
|
||||||
|
- a publish re-reads remote locks immediately before it writes the current
|
||||||
|
commit pointer. A lock committed before that recheck prevents selecting the
|
||||||
|
new snapshot, even though its already-uploaded immutable objects may remain
|
||||||
|
available for a later retry.
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
|
|
||||||
@@ -165,20 +277,30 @@ Apply:
|
|||||||
narratio session restore 2026-04-04
|
narratio session restore 2026-04-04
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`--dry-run` does not write durable session files. It still reads the selected
|
||||||
|
remote current state and may read object identity/content needed to classify the
|
||||||
|
plan, so it is not a network-free operation.
|
||||||
|
|
||||||
Default restore scope:
|
Default restore scope:
|
||||||
|
|
||||||
- `manifest.json`
|
- the committed session manifest and the committed transcript/artifact objects
|
||||||
- `transcripts/**`
|
declared by the selected remote commit
|
||||||
- `artifacts/**`
|
|
||||||
- `previous/**` when needed by configured previous-session artifact inputs
|
- `previous/**` when needed by configured previous-session artifact inputs
|
||||||
|
|
||||||
Optional:
|
Optional:
|
||||||
|
|
||||||
- `--include-audio` to include `audio/**`
|
- `--include-audio` to include `audio/**`
|
||||||
- `--force` to overwrite local conflicts
|
- `--force` to overwrite eligible conflicting regular files; it never replaces
|
||||||
|
directories or other non-regular local targets
|
||||||
|
|
||||||
Restore writes an execution report at `reports/restore-latest.json`.
|
Restore writes an execution report at `reports/restore-latest.json`.
|
||||||
|
|
||||||
|
If restore fails after beginning installation, it leaves a durable
|
||||||
|
`.restore-incomplete.json` marker in the session root. Pipeline runs will stop
|
||||||
|
until you rerun the same restore command and it completes. Restore intentionally
|
||||||
|
does not try to roll back files already installed; retrying the selected remote
|
||||||
|
snapshot is the recovery procedure.
|
||||||
|
|
||||||
## Local State Layout
|
## Local State Layout
|
||||||
|
|
||||||
Session root:
|
Session root:
|
||||||
@@ -198,6 +320,10 @@ Durable session paths:
|
|||||||
- `config/**`
|
- `config/**`
|
||||||
- `runs/**`
|
- `runs/**`
|
||||||
|
|
||||||
|
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
||||||
|
stderr, and pre-promotion output remain in the producing run's `extract`
|
||||||
|
directory as described in [Extraction Workflow](#extraction-workflow).
|
||||||
|
|
||||||
Run-local layout:
|
Run-local layout:
|
||||||
|
|
||||||
- `runs/{run_id}/{stage}/outputs`
|
- `runs/{run_id}/{stage}/outputs`
|
||||||
@@ -215,6 +341,38 @@ Cache layout (durable S3 audio cache):
|
|||||||
|
|
||||||
- `{cache.root}/s3/{bucket}/...`
|
- `{cache.root}/s3/{bucket}/...`
|
||||||
|
|
||||||
|
Each cached audio file has an adjacent managed identity record. It binds the
|
||||||
|
file to its remote object version and verified digest; deleting or altering the
|
||||||
|
record simply causes Narratio to download and verify the object again.
|
||||||
|
|
||||||
|
### Workspace Permissions
|
||||||
|
|
||||||
|
Ordinary Narratio workspace content is intentionally shareable with the
|
||||||
|
workspace group. On POSIX systems, Narratio-created workspace, spool, and cache
|
||||||
|
directories converge on setgid `02775`; ordinary files, including manifests,
|
||||||
|
transcripts, generated configuration, logs, reports, and Notarius artifacts,
|
||||||
|
converge on `0664`. Narratio explicitly applies these modes so a restrictive
|
||||||
|
caller umask does not remove group write or setgid. It does not change file or
|
||||||
|
directory ownership: the configured workspace's existing group is inherited.
|
||||||
|
|
||||||
|
Windows does not implement POSIX mode bits or setgid semantics. Configure the
|
||||||
|
workspace, spool, and cache locations with an ACL that grants the collaborating
|
||||||
|
group read/write access, and configure credential locations with an ACL limited
|
||||||
|
to the intended credential owner. Do not use POSIX mode displays as evidence of
|
||||||
|
Windows access control.
|
||||||
|
|
||||||
|
API keys are credentials, not ordinary workspace data. Store them outside the
|
||||||
|
shared workspace or in a separately restricted credential location; ordinary
|
||||||
|
workspace group access must never be treated as authorization to read keys.
|
||||||
|
On POSIX, provision a credential directory as `0700` and credential files as
|
||||||
|
`0600`; Narratio rejects group- or other-readable configured credential paths.
|
||||||
|
On Windows, restrict the directory and files with ACLs to the credential owner.
|
||||||
|
|
||||||
|
External adapter results are individually bounded before Narratio validates or
|
||||||
|
materializes them. These per-file limits do not reserve disk space: prevent hard
|
||||||
|
disk exhaustion with filesystem, service, container, or volume quotas sized for
|
||||||
|
the session workload.
|
||||||
|
|
||||||
## Cleanup
|
## Cleanup
|
||||||
|
|
||||||
Session-scoped cleanup:
|
Session-scoped cleanup:
|
||||||
@@ -240,9 +398,14 @@ Rules:
|
|||||||
|
|
||||||
- `clean` deletes work/spool session state;
|
- `clean` deletes work/spool session state;
|
||||||
- cache is preserved unless `--clear-cache` is set;
|
- cache is preserved unless `--clear-cache` is set;
|
||||||
|
- each deletion is confined beneath its configured workspace, spool, or cache
|
||||||
|
root and refuses symlinked paths;
|
||||||
- automatic post-publish cleanup is gated by successful publish commit plus:
|
- automatic post-publish cleanup is gated by successful publish commit plus:
|
||||||
- `pipeline.spool.delete_audio_after_publish=true`
|
- `pipeline.spool.delete_audio_after_publish=true`
|
||||||
- `pipeline.workspace.cleanup_after_publish=true`
|
- `pipeline.workspace.cleanup_after_publish=true`
|
||||||
|
- Narratio first records the exact run-scoped cleanup obligation. If cleanup
|
||||||
|
reports incomplete, the remote committed snapshot remains current; rerun
|
||||||
|
Narratio to retry only the outstanding confined local cleanup.
|
||||||
|
|
||||||
## Operational Caveats
|
## Operational Caveats
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ their domains:
|
|||||||
|
|
||||||
- WhisperX performs transcription;
|
- WhisperX performs transcription;
|
||||||
- Seriatim performs deterministic transcript processing and rendering;
|
- Seriatim performs deterministic transcript processing and rendering;
|
||||||
- Audita performs transcript correction and polishing; and
|
- Audita performs transcript correction and polishing;
|
||||||
|
- Notarius extracts validated structured artifact bundles; and
|
||||||
- Scriptorium executes prompts and produces configured artifacts.
|
- Scriptorium executes prompts and produces configured artifacts.
|
||||||
|
|
||||||
Narratio owns orchestration, configuration resolution, session and run state,
|
Narratio owns orchestration, configuration resolution, session and run state,
|
||||||
@@ -51,9 +52,9 @@ HTTP, subprocess, notification, and object-storage mechanics, including command
|
|||||||
construction, transport behavior, provider response handling, and external
|
construction, transport behavior, provider response handling, and external
|
||||||
error adaptation. External dependency types must remain inside the adapter that
|
error adaptation. External dependency types must remain inside the adapter that
|
||||||
owns them unless that dependency is the adapter's explicit public contract.
|
owns them unless that dependency is the adapter's explicit public contract.
|
||||||
WhisperX HTTP behavior, Seriatim, Audita, and Scriptorium command construction,
|
WhisperX HTTP behavior, Seriatim, Audita, Notarius, and Scriptorium command
|
||||||
notification transport, and object-storage SDK details remain behind these
|
construction, notification transport, and object-storage SDK details remain
|
||||||
boundaries.
|
behind these boundaries.
|
||||||
|
|
||||||
State and path services must not infer stage policy. Storage implementations
|
State and path services must not infer stage policy. Storage implementations
|
||||||
receive explicit bucket-relative keys and do not infer campaign, session, run,
|
receive explicit bucket-relative keys and do not infer campaign, session, run,
|
||||||
@@ -89,6 +90,13 @@ should preserve enough local state and diagnostics for inspection, recovery,
|
|||||||
and resume. Forcing an upstream stage invalidates succeeded downstream work
|
and resume. Forcing an upstream stage invalidates succeeded downstream work
|
||||||
according to the canonical stage order.
|
according to the canonical stage order.
|
||||||
|
|
||||||
|
A stage may explicitly self-skip with a stable reason and no outputs. That
|
||||||
|
outcome is persisted, clears older outputs owned by the stage, and is
|
||||||
|
reconsidered on a later invocation. A stage may also validate whether an
|
||||||
|
otherwise successful recorded result is still resumable; an obsolete result
|
||||||
|
is staled and rerun, while an unsafe condition that prevents a sound decision
|
||||||
|
stops execution.
|
||||||
|
|
||||||
Shared behavior should live behind a narrow service or helper with one clear
|
Shared behavior should live behind a narrow service or helper with one clear
|
||||||
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
|
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
|
||||||
artifact, or path policy ad hoc.
|
artifact, or path policy ad hoc.
|
||||||
@@ -111,6 +119,17 @@ install the validated session manifest after other restored durable files. The
|
|||||||
physical workflow and recovery procedures belong in
|
physical workflow and recovery procedures belong in
|
||||||
[Operations](../operations.md).
|
[Operations](../operations.md).
|
||||||
|
|
||||||
|
Restore and runner transitions for one session use the same local lock. A
|
||||||
|
durable incomplete-restore marker blocks runner reuse after a partial restore;
|
||||||
|
safe retry, rather than rollback of arbitrary local effects, is the recovery
|
||||||
|
mechanism. Restored manifest-local references must be confined to the selected
|
||||||
|
local session root, never trusted as producer-machine absolute paths.
|
||||||
|
|
||||||
|
For the immutable remote-commit protocol, a restore or status operation binds
|
||||||
|
to one pointer-selected commit and only its declared object identities. A force
|
||||||
|
flag may replace an eligible regular managed file, but never turns a directory
|
||||||
|
or other non-regular conflict into a successful restore.
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
Configuration is strict, explicit, centralized, and operator-oriented.
|
Configuration is strict, explicit, centralized, and operator-oriented.
|
||||||
@@ -138,9 +157,16 @@ Canonical helpers own workspace, spool, cache, session, run, input, transcript,
|
|||||||
artifact, log, report, configuration, and publish-current paths. Callers must
|
artifact, log, report, configuration, and publish-current paths. Callers must
|
||||||
not reconstruct canonical paths through scattered string concatenation.
|
not reconstruct canonical paths through scattered string concatenation.
|
||||||
|
|
||||||
|
Reusable audio cache entries require a typed record that binds a confined,
|
||||||
|
no-follow regular file and its digest to the selected remote object identity.
|
||||||
|
Size alone and unqualified multipart ETags are not content-integrity evidence.
|
||||||
|
|
||||||
Artifact resolution is deterministic and manifest-aware. Producers materialize
|
Artifact resolution is deterministic and manifest-aware. Producers materialize
|
||||||
canonical outputs before reporting success, and consumers resolve declared
|
canonical outputs before reporting success, and consumers resolve declared
|
||||||
artifact identities rather than infer files from unrelated directory contents.
|
artifact identities rather than infer files from unrelated directory contents.
|
||||||
|
External artifact bundles become current only through validated immutable
|
||||||
|
promotion and manifest records; directory presence alone never establishes
|
||||||
|
availability.
|
||||||
|
|
||||||
Writes, moves, replacements, and deletions must use narrow, explicit,
|
Writes, moves, replacements, and deletions must use narrow, explicit,
|
||||||
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
||||||
@@ -156,22 +182,38 @@ contracts belong under [Integrations](../integrations/).
|
|||||||
## Publish Commit Boundary
|
## Publish Commit Boundary
|
||||||
|
|
||||||
Publish has one explicit remote commit boundary. A remote run becomes current
|
Publish has one explicit remote commit boundary. A remote run becomes current
|
||||||
only after Narratio has successfully uploaded the run record, required published
|
only after Narratio has successfully uploaded its immutable run-scoped objects,
|
||||||
outputs, `current/manifest.json`, and finally `current/run_id.txt`.
|
the immutable commit manifest, and finally the current commit pointer.
|
||||||
|
|
||||||
`current/run_id.txt` is the commit marker and must be written last. Failed,
|
`current/commit-pointer.json` is the sole mutable selector and must be written
|
||||||
incomplete, skipped, or uncommitted publish attempts must not be presented as
|
exactly once, last. Failed, incomplete, skipped, or uncommitted publish attempts
|
||||||
current remote state. Publish locks remain authoritative and are not bypassed by
|
must not be presented as current remote state. Publish locks remain authoritative
|
||||||
a forced run.
|
and are not bypassed by a forced run. Mutable remote locks use provider-enforced
|
||||||
|
generation preconditions and are revalidated immediately before pointer
|
||||||
|
selection; loss of that check leaves the prior committed snapshot current.
|
||||||
|
|
||||||
Automatic local cleanup is permitted only after a successful publish commit,
|
Automatic local cleanup is permitted only after a successful publish commit,
|
||||||
only when explicitly configured, and only through the path-safety guardrails.
|
only when explicitly configured, and only through the path-safety guardrails.
|
||||||
|
It is a durable local obligation bound to that committed run and its exact
|
||||||
|
targets, not an inferred side effect of the current stage list. A cleanup
|
||||||
|
failure makes the invocation incomplete while leaving the committed remote
|
||||||
|
snapshot authoritative; later invocations resume the recorded obligation.
|
||||||
|
|
||||||
## Security, Privacy, And Diagnostics
|
## Security, Privacy, And Diagnostics
|
||||||
|
|
||||||
Narratio handles private campaign material. Transcripts, prompts, generated
|
Narratio distinguishes ordinary workspace data from credentials. Campaign and
|
||||||
artifacts, reports, logs, manifests, and diagnostic files are potentially
|
session material—including manifests, transcripts, prompts, generated
|
||||||
sensitive.
|
configuration, logs, reports, diagnostics, and Notarius artifacts—is
|
||||||
|
intentionally shareable with the configured workspace group. API-key material
|
||||||
|
is sensitive and is not covered by the ordinary workspace-sharing policy.
|
||||||
|
|
||||||
|
On POSIX systems, Narratio-created ordinary workspace directories converge on
|
||||||
|
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
|
||||||
|
umask is restrictive. This preserves the existing workspace group for nested
|
||||||
|
creation and atomic replacements without changing ownership. API-key storage
|
||||||
|
uses a separate restrictive contract. On Windows, POSIX mode bits and setgid
|
||||||
|
are not authoritative; operators must provide the equivalent shared-group and
|
||||||
|
credential-restricted ACLs described in [Operations](../operations.md#workspace-permissions).
|
||||||
|
|
||||||
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
|
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
|
||||||
to manifests, logs, generated configuration, reports, publish metadata,
|
to manifests, logs, generated configuration, reports, publish metadata,
|
||||||
|
|||||||
@@ -1,379 +0,0 @@
|
|||||||
# Notarius Extraction Stage
|
|
||||||
|
|
||||||
## Status
|
|
||||||
|
|
||||||
Proposed.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Add a first-class Narratio `extract` stage that runs Notarius against the
|
|
||||||
session's final trimmed transcript, validates and collects the resulting
|
|
||||||
structured D&D artifacts, and registers those artifacts for later use by the
|
|
||||||
`analyze` and `publish` stages.
|
|
||||||
|
|
||||||
This feature should integrate Notarius through Narratio's existing stage,
|
|
||||||
adapter, manifest, workspace, and artifact-catalog boundaries. It must not turn
|
|
||||||
Narratio into a generic workflow engine or a second configuration language for
|
|
||||||
Notarius pipelines.
|
|
||||||
|
|
||||||
## User Outcome
|
|
||||||
|
|
||||||
An operator can enable one configured Notarius pipeline for a Narratio
|
|
||||||
campaign. During a normal run, Narratio will:
|
|
||||||
|
|
||||||
1. finish producing the session transcript tiers;
|
|
||||||
2. invoke Notarius once with the final trimmed Seriatim JSON transcript;
|
|
||||||
3. collect and validate the configured structured artifact lanes;
|
|
||||||
4. record their exact files and provenance in the Narratio manifest; and
|
|
||||||
5. make those artifacts selectable as inputs to Scriptorium artifacts in the
|
|
||||||
later `analyze` stage.
|
|
||||||
|
|
||||||
The maintained D&D example should demonstrate all ten lanes emitted by
|
|
||||||
Notarius's complete `dnd-session` pipeline.
|
|
||||||
|
|
||||||
## Target Stage Architecture
|
|
||||||
|
|
||||||
### Canonical Order
|
|
||||||
|
|
||||||
The canonical stage order becomes:
|
|
||||||
|
|
||||||
```text
|
|
||||||
prepare -> transcribe -> merge -> polish -> normalize -> trim -> render
|
|
||||||
-> extract -> analyze -> publish -> notify
|
|
||||||
```
|
|
||||||
|
|
||||||
`extract` is deliberately after all transcript-producing stages and before
|
|
||||||
analysis. Its source document is the manifest-resolved
|
|
||||||
`narratio.transcript.final_trimmed` artifact, normally
|
|
||||||
`transcripts/final.trimmed.json`. It does not consume rendered Markdown.
|
|
||||||
|
|
||||||
Adding the stage must update full-plan construction, explicit stage selection,
|
|
||||||
downstream invalidation, prerequisite checks, resume behavior, run manifests,
|
|
||||||
CLI stage validation and help, and every canonical-stage inventory. Forcing an
|
|
||||||
upstream transcript stage must stale a previously successful `extract` stage
|
|
||||||
and its downstream stages. Forcing `extract` must stale `analyze`, `publish`,
|
|
||||||
and `notify` according to existing rules.
|
|
||||||
|
|
||||||
### Stage Boundary
|
|
||||||
|
|
||||||
The stage owns Narratio policy and state transitions:
|
|
||||||
|
|
||||||
- resolve the final trimmed transcript through the runtime artifact catalog;
|
|
||||||
- build a Narratio-level Notarius request from validated configuration and
|
|
||||||
run-local paths;
|
|
||||||
- call a narrow Notarius adapter;
|
|
||||||
- apply the configured required-output policy;
|
|
||||||
- materialize the validated bundle into its canonical session location;
|
|
||||||
- return manifest-ready artifact references and bounded metadata; and
|
|
||||||
- fail without marking the stage successful when any required contract or
|
|
||||||
materialization step fails.
|
|
||||||
|
|
||||||
The stage must not construct subprocess arguments, infer Notarius output
|
|
||||||
filenames, parse provider logs, or decode individual D&D payload bodies.
|
|
||||||
|
|
||||||
### Adapter Boundary
|
|
||||||
|
|
||||||
Add a dedicated Notarius adapter package with a small interface, production
|
|
||||||
subprocess implementation, and test fake. Its request should contain only the
|
|
||||||
resolved Notarius binary, configuration path, pipeline ID, transcript path,
|
|
||||||
output root, working directory, timeout, and process-log destinations needed
|
|
||||||
for one run.
|
|
||||||
|
|
||||||
The adapter owns:
|
|
||||||
|
|
||||||
- optional `notarius config validate` preflight for the configured pipeline;
|
|
||||||
- exact `notarius run ... --json` argument construction;
|
|
||||||
- stdout and stderr separation;
|
|
||||||
- context cancellation and timeout propagation through Narratio's shared
|
|
||||||
subprocess boundary;
|
|
||||||
- exit-status handling;
|
|
||||||
- decoding the `notarius.run-result.v1` success receipt;
|
|
||||||
- receipt and index path-confinement checks;
|
|
||||||
- decoding `index.json` and resolving descriptor paths safely beneath the
|
|
||||||
reported output directory; and
|
|
||||||
- returning a transport-neutral result containing the bundle location,
|
|
||||||
receipt summary, lane descriptors, pipeline-wide descriptors, warnings and
|
|
||||||
rejection locations, and diagnostic log paths.
|
|
||||||
|
|
||||||
Only exit status zero permits receipt decoding. Receipt, index, or descriptor
|
|
||||||
paths that are absolute where a logical relative path is required, or that
|
|
||||||
escape their owning root, are integration failures. Unknown fields in a
|
|
||||||
supported receipt or index schema should be tolerated. Unsupported schema
|
|
||||||
versions and incompatible descriptor metadata should fail clearly.
|
|
||||||
|
|
||||||
The adapter must not write Narratio manifests, choose required lanes, decide
|
|
||||||
analysis inputs, or contain D&D domain logic.
|
|
||||||
|
|
||||||
## Configuration Contract
|
|
||||||
|
|
||||||
Add a strict optional `pipeline.notarius` configuration section. Omission or
|
|
||||||
`enabled: false` keeps the current workflow usable and causes `extract` to
|
|
||||||
self-skip without outputs.
|
|
||||||
|
|
||||||
The section should provide:
|
|
||||||
|
|
||||||
- `enabled`: explicit opt-in;
|
|
||||||
- `binary`: Notarius executable, defaulting to `notarius`;
|
|
||||||
- `config_path`: required when enabled;
|
|
||||||
- `pipeline_id`: required when enabled;
|
|
||||||
- `timeout`: a positive stage timeout with a documented default;
|
|
||||||
- `working_directory`: optional explicit subprocess working directory,
|
|
||||||
defaulting to the directory containing `config_path`; and
|
|
||||||
- an `outputs` map defining the Notarius lane artifacts Narratio promises to
|
|
||||||
collect.
|
|
||||||
|
|
||||||
Each output-map key is a stable Narratio extraction key. Each value must define:
|
|
||||||
|
|
||||||
- the exact Notarius `lane_id`;
|
|
||||||
- the expected `media_type`;
|
|
||||||
- the expected `schema_id`;
|
|
||||||
- the expected `schema_version`; and
|
|
||||||
- optionally an expected `module_key` when the operator needs to constrain the
|
|
||||||
producing module as part of compatibility.
|
|
||||||
|
|
||||||
Narratio derives the downstream source ID
|
|
||||||
`narratio.extraction.<output-key>` from the map key. Keys and lane IDs must be
|
|
||||||
non-empty, unique after normalization, path-safe under the existing artifact
|
|
||||||
policy, and collision-free with built-in and configured artifact identities.
|
|
||||||
Every configured output is required: a successful Notarius process that omits
|
|
||||||
one, rejects it, or reports incompatible descriptor metadata fails the
|
|
||||||
`extract` stage.
|
|
||||||
|
|
||||||
This explicit map keeps Narratio's consumer contract stable when a Notarius
|
|
||||||
lane ID or schema changes and avoids hard-coding the current D&D family into a
|
|
||||||
generic adapter. It also replaces a separate `required_lanes` list, which would
|
|
||||||
duplicate configuration.
|
|
||||||
|
|
||||||
Narratio should not reproduce Notarius lane selection, references, LLM
|
|
||||||
profiles, model settings, retries, concurrency, or prompt configuration. Those
|
|
||||||
remain in the referenced Notarius configuration. Narratio should not expose a
|
|
||||||
runtime lane-selection flag for `extract`; one stage invocation runs the
|
|
||||||
configured Notarius pipeline as a unit.
|
|
||||||
|
|
||||||
All configured paths should become absolute during Narratio configuration
|
|
||||||
resolution. The deterministic default working directory allows a Notarius
|
|
||||||
profile path relative to that directory, but operator documentation should
|
|
||||||
still recommend absolute deployment paths where practical. Notarius reference
|
|
||||||
paths continue to follow Notarius's own configuration-relative rules.
|
|
||||||
|
|
||||||
## Output And Artifact Model
|
|
||||||
|
|
||||||
### Canonical Bundle
|
|
||||||
|
|
||||||
Run Notarius against a run-local output root. After all configured descriptors
|
|
||||||
are validated, materialize the contents of the exact run-specific Notarius
|
|
||||||
bundle into a fixed canonical session directory:
|
|
||||||
|
|
||||||
```text
|
|
||||||
artifacts/notarius/
|
|
||||||
```
|
|
||||||
|
|
||||||
Preserve its relative layout, including `index.json`, `manifest.json`,
|
|
||||||
`rejected.json`, `warnings.json`, `lanes/`, and any indexed `chunk-map.json` or
|
|
||||||
`evidence-context.json`. Materialize the complete directory as one narrow,
|
|
||||||
transactional replacement so a failed or interrupted rerun cannot mix files
|
|
||||||
from different Notarius runs.
|
|
||||||
|
|
||||||
The raw subprocess receipt and stderr log belong in the run-local `extract`
|
|
||||||
report and log directories. The raw receipt identifies the original run-local
|
|
||||||
Notarius bundle and must not be rewritten to pretend that the canonical copy
|
|
||||||
was its original `output_directory`. Narratio's manifest is the durable ledger
|
|
||||||
for the canonical materialized paths.
|
|
||||||
|
|
||||||
### Registered Artifact Sources
|
|
||||||
|
|
||||||
For each configured output, locate the lane through the canonical copy of
|
|
||||||
`index.json` and record a manifest artifact with:
|
|
||||||
|
|
||||||
- source ID `narratio.extraction.<output-key>`;
|
|
||||||
- canonical lane-file path discovered from the index;
|
|
||||||
- producer stage and Narratio run ID;
|
|
||||||
- checksum;
|
|
||||||
- Notarius lane ID; and
|
|
||||||
- descriptor media type, schema identity/version, and module key when present.
|
|
||||||
|
|
||||||
If the current manifest model cannot carry descriptor compatibility metadata,
|
|
||||||
extend its artifact metadata in a backward-tolerant way rather than encoding
|
|
||||||
that information in filenames or source IDs.
|
|
||||||
|
|
||||||
Also record the canonical Notarius index as a stage output or stage metadata so
|
|
||||||
operators can discover the complete bundle, including non-lane artifacts. The
|
|
||||||
configured lane sources are the stable interface for analysis; the index and
|
|
||||||
bundle remain the provenance and inspection interface.
|
|
||||||
|
|
||||||
## Analysis And Publish Integration
|
|
||||||
|
|
||||||
Extend the runtime artifact catalog and configured Scriptorium input validation
|
|
||||||
so an enabled analysis artifact can declare, for example:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
inputs:
|
|
||||||
npc_registry:
|
|
||||||
source: narratio.extraction.npc_registry
|
|
||||||
```
|
|
||||||
|
|
||||||
Resolution must remain manifest-first and verify that the recorded artifact
|
|
||||||
was produced by a successful current `extract` stage. A required extraction
|
|
||||||
source that is unavailable must fail analysis with guidance to configure or
|
|
||||||
rerun `extract`; an optional source may be omitted according to the existing
|
|
||||||
Scriptorium input contract.
|
|
||||||
|
|
||||||
Publish source resolution should accept configured
|
|
||||||
`narratio.extraction.<output-key>` sources through the same artifact catalog so
|
|
||||||
operators may publish selected structured artifacts without manually copying
|
|
||||||
paths. The existing `--artifacts` flag remains scoped to Scriptorium artifact
|
|
||||||
selection and must not partially execute the Notarius pipeline.
|
|
||||||
|
|
||||||
No current-session analysis artifact should consume an incidental file from a
|
|
||||||
failed, stale, skipped, or superseded extraction run.
|
|
||||||
|
|
||||||
## Failure, Skip, Resume, And Diagnostics
|
|
||||||
|
|
||||||
- Missing or invalid enabled Notarius configuration fails configuration
|
|
||||||
validation before stage execution where statically discoverable.
|
|
||||||
- A disabled or absent Notarius configuration makes `extract` skip with clear
|
|
||||||
stage metadata and no new outputs.
|
|
||||||
- A missing or invalid final trimmed transcript fails `extract` before starting
|
|
||||||
Notarius.
|
|
||||||
- Preflight failure, nonzero Notarius exit, cancellation, timeout, malformed or
|
|
||||||
unsupported receipt/index data, unsafe paths, incompatible descriptors,
|
|
||||||
rejected required outputs, or missing configured lanes fails the entire
|
|
||||||
stage.
|
|
||||||
- Process success does not override Narratio's required-output policy.
|
|
||||||
- A failed run retains bounded run-local receipt bytes, stderr, and the
|
|
||||||
unpublished Notarius bundle for diagnosis, subject to Narratio's existing
|
|
||||||
sensitive-data and cleanup policies.
|
|
||||||
- The canonical bundle and manifest artifacts are updated only after complete
|
|
||||||
validation and materialization.
|
|
||||||
- Resume skips a succeeded, non-stale `extract` stage only when its
|
|
||||||
manifest-recorded canonical index and configured lane outputs still validate.
|
|
||||||
- Force and staleness behavior follows the ordinary stage contract; it must not
|
|
||||||
depend on merely finding `artifacts/notarius/` on disk.
|
|
||||||
|
|
||||||
Transcripts, Notarius outputs, evidence context, manifests, receipts, and logs
|
|
||||||
are private campaign material. Subprocess arguments and manifest metadata must
|
|
||||||
not contain secrets. Credentials remain in the environment or in mechanisms
|
|
||||||
owned by Notarius and PromptKit.
|
|
||||||
|
|
||||||
## Maintained D&D Example
|
|
||||||
|
|
||||||
Add or update a Narratio example that enables Notarius's complete
|
|
||||||
`dnd-session` pipeline and maps these ten required lanes to stable extraction
|
|
||||||
keys:
|
|
||||||
|
|
||||||
| Output key | Notarius lane ID |
|
|
||||||
| --- | --- |
|
|
||||||
| `item_registry` | `item-registry` |
|
|
||||||
| `npc_registry` | `npc-registry` |
|
|
||||||
| `location_registry` | `location-registry` |
|
|
||||||
| `scene_descriptions` | `scene-descriptions` |
|
|
||||||
| `item_occurrences` | `item-occurrences` |
|
|
||||||
| `spells` | `spells` |
|
|
||||||
| `combat_turns` | `combat-turns` |
|
|
||||||
| `npc_occurrences` | `npc-occurrences` |
|
|
||||||
| `location_occurrences` | `location-occurrences` |
|
|
||||||
| `enemy_events` | `enemy-events` |
|
|
||||||
|
|
||||||
The example must include each lane's current media type and schema identity
|
|
||||||
from Notarius's published contracts. It should also demonstrate at least one
|
|
||||||
Scriptorium analysis artifact consuming one or more
|
|
||||||
`narratio.extraction.*` sources. The example must use placeholders and relative
|
|
||||||
paths suitable for the example tree, contain no credentials, and pass the
|
|
||||||
repository's configuration validation tests.
|
|
||||||
|
|
||||||
## Compatibility Policy
|
|
||||||
|
|
||||||
The initial integration baseline is the public subprocess contract available
|
|
||||||
in Notarius v0.3.0:
|
|
||||||
|
|
||||||
- successful JSON receipt schema `notarius.run-result.v1`;
|
|
||||||
- production JSON bundle discovery through `index.json`; and
|
|
||||||
- the schema IDs and versions explicitly configured for required lanes.
|
|
||||||
|
|
||||||
Runtime compatibility should be decided from those published contracts, not
|
|
||||||
from textual parsing of `notarius --version`. New optional receipt or index
|
|
||||||
fields must not break Narratio. An unsupported receipt version or lane schema
|
|
||||||
must fail before the artifact is registered for analysis.
|
|
||||||
|
|
||||||
## Documentation Deliverables When Implemented
|
|
||||||
|
|
||||||
Update current-behavior documentation in the same change that implements the
|
|
||||||
feature:
|
|
||||||
|
|
||||||
- add `docs/integrations/notarius.md` for the external CLI, receipt, bundle,
|
|
||||||
and adapter contract, linking to Notarius's canonical documentation;
|
|
||||||
- add `docs/internal/stage-extract.md` for stage inputs, outputs, collaborators,
|
|
||||||
state transitions, failures, and focused tests;
|
|
||||||
- update `docs/internal/adapters.md`, `docs/internal/artifacts.md`,
|
|
||||||
`docs/internal/manifest.md`, and the internal stage inventory;
|
|
||||||
- update `docs/policy/architecture.md` to list Notarius among isolated external
|
|
||||||
systems and preserve the adapter/stage boundary;
|
|
||||||
- update `docs/config.md`, `docs/cli.md`, `docs/operations.md`,
|
|
||||||
`docs/troubleshooting.md`, `README.md`, and maintained examples only to the
|
|
||||||
extent their canonical scopes require; and
|
|
||||||
- update `docs/development.md` only to the extent its canonical contributor
|
|
||||||
routing scope requires.
|
|
||||||
|
|
||||||
Outside this roadmap, do not describe `extract`, Notarius configuration, or
|
|
||||||
`narratio.extraction.*` sources as implemented until the code exists.
|
|
||||||
|
|
||||||
## Testing And Validation Expectations
|
|
||||||
|
|
||||||
Implementation should provide focused tests for:
|
|
||||||
|
|
||||||
- strict configuration decoding, defaults, required fields, path resolution,
|
|
||||||
output-map validation, normalized-key collisions, and example loading;
|
|
||||||
- exact stage order, selection, downstream staleness, resume, force, and
|
|
||||||
prerequisite behavior;
|
|
||||||
- adapter command construction, deterministic working directory, environment
|
|
||||||
inheritance, stdout/stderr separation, cancellation, timeout, and nonzero
|
|
||||||
exits;
|
|
||||||
- supported and unsupported receipt versions, unknown optional fields,
|
|
||||||
malformed receipts, index decoding, and path escapes at every boundary;
|
|
||||||
- descriptor lookup by lane ID rather than filename, expected metadata checks,
|
|
||||||
missing/rejected configured lanes, and tolerated unconfigured lanes;
|
|
||||||
- run-local execution, transactional canonical-bundle replacement, checksums,
|
|
||||||
failed-run preservation, and manifest recording;
|
|
||||||
- artifact-catalog resolution from `narratio.extraction.*` into analysis and
|
|
||||||
publish, including required, optional, missing, stale, and skipped cases; and
|
|
||||||
- end-to-end stage execution with a fake Notarius adapter, without live LLM or
|
|
||||||
external subprocess requirements in the ordinary test suite.
|
|
||||||
|
|
||||||
Run the repository-wide Go tests, vet, build, and maintained example validation
|
|
||||||
after focused tests pass.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
- `extract` is a first-class transactional stage between `render` and
|
|
||||||
`analyze` everywhere Narratio models stage order or state.
|
|
||||||
- Narratio invokes Notarius only through a narrow, tested adapter.
|
|
||||||
- The stage consumes the manifest-resolved final trimmed Seriatim transcript.
|
|
||||||
- The Notarius configuration remains owned by Notarius; Narratio configures
|
|
||||||
only invocation and its downstream consumer contract.
|
|
||||||
- Every configured output is discovered through the receipt and `index.json`,
|
|
||||||
contract-checked, materialized transactionally, and recorded with a stable
|
|
||||||
`narratio.extraction.*` source ID.
|
|
||||||
- The complete D&D example maps all ten current lanes and passes strict config
|
|
||||||
validation.
|
|
||||||
- Analysis can consume extraction sources through the existing artifact input
|
|
||||||
model, and publish can select them through the artifact catalog.
|
|
||||||
- Failed, partial, rejected, unsafe, stale, or incompatible output never becomes
|
|
||||||
a current analysis input.
|
|
||||||
- Resume and force behavior remains manifest-driven.
|
|
||||||
- Documentation accurately describes the implemented stage, adapter,
|
|
||||||
configuration, operations, and compatibility boundary without duplicating
|
|
||||||
Notarius's canonical schemas.
|
|
||||||
|
|
||||||
## Non-Goals
|
|
||||||
|
|
||||||
- Reimplementing Notarius extraction, prompts, schemas, references, retries,
|
|
||||||
profiles, or lane orchestration in Narratio.
|
|
||||||
- Allowing one Narratio run to invoke arbitrary extractor programs or multiple
|
|
||||||
Notarius pipelines.
|
|
||||||
- Making `extract` a configurable DAG or folding it into the Scriptorium
|
|
||||||
`analyze` stage.
|
|
||||||
- Partially selecting Notarius lanes through Narratio's `--artifacts` flag.
|
|
||||||
- Decoding D&D payload bodies in the generic Notarius adapter.
|
|
||||||
- Supporting previous-session extraction artifacts in the initial feature.
|
|
||||||
- Requiring live Notarius, PromptKit, an LLM provider, or external services in
|
|
||||||
the ordinary unit test suite.
|
|
||||||
@@ -117,6 +117,147 @@ Safe fix:
|
|||||||
|
|
||||||
Relevant reference: [CLI artifact selection](./cli.md).
|
Relevant reference: [CLI artifact selection](./cli.md).
|
||||||
|
|
||||||
|
## Notarius executable missing
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction fails while resolving or starting the Notarius executable.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- `pipeline.notarius.binary` is not installed, executable, or on `PATH`;
|
||||||
|
- a configured executable path is wrong.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- install a compatible Notarius release or correct the binary setting, then
|
||||||
|
rerun extraction.
|
||||||
|
|
||||||
|
Relevant references: [Notarius configuration](./config.md#notarius-output-entries)
|
||||||
|
and [Notarius integration](./integrations/notarius.md).
|
||||||
|
|
||||||
|
## Notarius exits nonzero
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction reports a Notarius exit error instead of a receipt.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
- inspect `runs/{run_id}/extract/notarius.stderr.log`; stdout is reserved for
|
||||||
|
the receipt and is not merged with diagnostics.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- correct the reported Notarius pipeline, input, provider, or configuration
|
||||||
|
failure and rerun extraction. Do not edit a staged output bundle into place.
|
||||||
|
|
||||||
|
After a failed replacement, an older immutable bundle may still exist even
|
||||||
|
though the current session manifest has no successful extraction payload. This
|
||||||
|
is expected audit state, not a signal to relink the old bundle manually.
|
||||||
|
|
||||||
|
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
|
## Atomic Notarius promotion unsupported
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction fails with `atomic no-replace directory promotion is unsupported`
|
||||||
|
before a durable bundle or temporary promotion tree is created.
|
||||||
|
|
||||||
|
Likely cause:
|
||||||
|
|
||||||
|
- Narratio is running on an operating system other than Linux, macOS, or
|
||||||
|
Windows, where the required atomic no-replace directory primitive has not
|
||||||
|
been implemented and verified.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- run extraction on Linux, macOS, or Windows. Do not replace the atomic commit
|
||||||
|
with a manual copy or move; the session manifest must never observe a partial
|
||||||
|
or overwritten bundle.
|
||||||
|
|
||||||
|
This is an extraction-specific platform boundary, not a support statement for
|
||||||
|
unrelated Narratio workflows. See
|
||||||
|
[Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
|
## Notarius receipt or index incompatible
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction rejects the receipt schema, pipeline identity, bundle/index path,
|
||||||
|
lane descriptor, or payload path even though Notarius exited successfully.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- Narratio and Notarius versions disagree on their consumer contract;
|
||||||
|
- the configured pipeline or lane constraints are stale;
|
||||||
|
- output paths escape the bundle or traverse symlinks.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- compare installed Notarius output with the canonical Notarius contracts,
|
||||||
|
including receipt `index_file: index.json` and index management names
|
||||||
|
`manifest.json`, `rejected.json`, and `warnings.json`; align
|
||||||
|
`pipeline.notarius` constraints and rerun. Do not bypass confinement or schema
|
||||||
|
checks.
|
||||||
|
|
||||||
|
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
||||||
|
|
||||||
|
## Required Notarius lane rejected or missing
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction fails because a configured lane is rejected, missing, duplicated,
|
||||||
|
or incompatible, including after a zero exit.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- inspect the Notarius diagnostic log and bundle rejection/warning information;
|
||||||
|
- correct the Notarius module or the exact declared lane contract;
|
||||||
|
- remove an output declaration only if downstream consumers genuinely no longer
|
||||||
|
require that source, then rerun extraction.
|
||||||
|
|
||||||
|
Every configured output is required. Narratio does not promote a partial result.
|
||||||
|
|
||||||
|
## Extraction resume invalidated
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- a previously successful extraction runs again during ordinary continuation.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- the executable/config path, pipeline ID, timeout, working directory, or
|
||||||
|
configured output contracts changed;
|
||||||
|
- the durable bundle, index, lane set, provenance, regular-file status, or
|
||||||
|
checksum no longer validates.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- allow the automatic rerun after verifying the current configuration. Treat
|
||||||
|
an unsafe path or symlink error as filesystem corruption or tampering and
|
||||||
|
investigate it rather than replacing files manually.
|
||||||
|
|
||||||
|
## Notarius transitive configuration changed
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- Notarius profiles, prompts, modules, imported files, or references changed,
|
||||||
|
but Narratio still considers the previous extraction resumable.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage extract 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Narratio fingerprints its invocation contract, not the contents of transitive
|
||||||
|
Notarius inputs. Always force extraction after changing them; downstream
|
||||||
|
successful stages are then marked stale normally.
|
||||||
|
|
||||||
|
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
## Previous-session artifact input missing
|
## Previous-session artifact input missing
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
@@ -158,7 +299,7 @@ Symptom:
|
|||||||
Likely causes:
|
Likely causes:
|
||||||
|
|
||||||
- another process is running for the same session;
|
- another process is running for the same session;
|
||||||
- stale lock left by interrupted process.
|
- a process still holds the operating-system lock while it is shutting down.
|
||||||
|
|
||||||
Diagnostics:
|
Diagnostics:
|
||||||
|
|
||||||
@@ -170,7 +311,8 @@ ps aux | grep narratio
|
|||||||
Safe fix:
|
Safe fix:
|
||||||
|
|
||||||
- wait for active process completion;
|
- wait for active process completion;
|
||||||
- remove stale lock only after confirming no live process owns it.
|
- retry after an interrupted holder has exited; the kernel releases its lock
|
||||||
|
even though the `.lock` metadata file remains for inspection.
|
||||||
|
|
||||||
Relevant reference: [Operations: Local State Layout](./operations.md#local-state-layout).
|
Relevant reference: [Operations: Local State Layout](./operations.md#local-state-layout).
|
||||||
|
|
||||||
@@ -294,7 +436,7 @@ Diagnostics:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
ls -la /path/to/secrets_dir
|
ls -la /path/to/secrets_dir
|
||||||
env | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
|
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
|
||||||
```
|
```
|
||||||
|
|
||||||
Safe fix:
|
Safe fix:
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ in the [configuration reference](../docs/config.md).
|
|||||||
external tools, and configured Scriptorium artifacts.
|
external tools, and configured Scriptorium artifacts.
|
||||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||||
pipeline section with explanatory comments.
|
pipeline section with explanatory comments.
|
||||||
|
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||||
|
Scriptorium artifact consuming only three declared Notarius lanes.
|
||||||
|
|
||||||
The existing `internal/config` example test loads and validates each pipeline
|
The existing `internal/config` example test loads and validates each pipeline
|
||||||
with the sample campaign and a compatible local- or S3-audio session.
|
with the sample campaign and a compatible local- or S3-audio session.
|
||||||
|
|||||||
55
examples/pipeline.extraction-subset.yml
Normal file
55
examples/pipeline.extraction-subset.yml
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
# Purpose-specific extraction example: a Scriptorium session brief consumes
|
||||||
|
# only the three Notarius lanes it needs.
|
||||||
|
|
||||||
|
campaigns:
|
||||||
|
root: /usr/local/share/narratio/campaigns
|
||||||
|
default_campaign_id: sample-campaign
|
||||||
|
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: "https://transcription.example.com/transcribe"
|
||||||
|
|
||||||
|
notarius:
|
||||||
|
enabled: true
|
||||||
|
binary: notarius
|
||||||
|
config_path: /usr/local/etc/notarius/config.yml
|
||||||
|
pipeline_id: dnd-session
|
||||||
|
timeout: 3h
|
||||||
|
outputs:
|
||||||
|
npc_registry:
|
||||||
|
lane_id: npc-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/npc-registry
|
||||||
|
location_registry:
|
||||||
|
lane_id: location-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.location_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/location-registry
|
||||||
|
scene_descriptions:
|
||||||
|
lane_id: scene-descriptions
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.scene_descriptions
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/scene-descriptions
|
||||||
|
|
||||||
|
scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
config_path: /usr/local/etc/scriptorium/config.yml
|
||||||
|
artifacts:
|
||||||
|
session_brief:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_brief
|
||||||
|
output_path: artifacts/session_brief.md
|
||||||
|
inputs:
|
||||||
|
npcs:
|
||||||
|
source: narratio.extraction.npc_registry
|
||||||
|
required: true
|
||||||
|
locations:
|
||||||
|
source: narratio.extraction.location_registry
|
||||||
|
required: true
|
||||||
|
scenes:
|
||||||
|
source: narratio.extraction.scene_descriptions
|
||||||
|
required: true
|
||||||
|
|
||||||
@@ -12,7 +12,7 @@ workspace:
|
|||||||
# env_dir: ./secrets
|
# env_dir: ./secrets
|
||||||
|
|
||||||
storage:
|
storage:
|
||||||
# Optional storage backend selector; use "s3" for publish + S3 audio workflows.
|
# Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
|
||||||
backend: s3
|
backend: s3
|
||||||
s3:
|
s3:
|
||||||
# Required when using S3 audio or S3 publish uploads.
|
# Required when using S3 audio or S3 publish uploads.
|
||||||
@@ -60,6 +60,11 @@ publish:
|
|||||||
- source: narratio.artifact.player_handout
|
- source: narratio.artifact.player_handout
|
||||||
dest: artifacts/player_handout.md
|
dest: artifacts/player_handout.md
|
||||||
required: false
|
required: false
|
||||||
|
# Extraction lanes publish only when named explicitly; the bundle and index
|
||||||
|
# are never implicit publish sources.
|
||||||
|
- source: narratio.extraction.npc_registry
|
||||||
|
dest: artifacts/extraction/npc-registry.json
|
||||||
|
required: true
|
||||||
|
|
||||||
whisperx:
|
whisperx:
|
||||||
# Required.
|
# Required.
|
||||||
@@ -123,6 +128,78 @@ trim:
|
|||||||
seriatim:
|
seriatim:
|
||||||
report: false
|
report: false
|
||||||
|
|
||||||
|
notarius:
|
||||||
|
# Optional structured extraction between trim and render.
|
||||||
|
enabled: true
|
||||||
|
binary: notarius
|
||||||
|
config_path: /usr/local/etc/notarius/config.yml
|
||||||
|
pipeline_id: dnd-session
|
||||||
|
timeout: 3h
|
||||||
|
working_directory: /usr/local/etc/notarius
|
||||||
|
# Each key creates source narratio.extraction.<key>. These constraints match
|
||||||
|
# the current Notarius D&D lane contracts; update them with Notarius.
|
||||||
|
outputs:
|
||||||
|
item_registry:
|
||||||
|
lane_id: item-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.item_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/item-registry
|
||||||
|
npc_registry:
|
||||||
|
lane_id: npc-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/npc-registry
|
||||||
|
location_registry:
|
||||||
|
lane_id: location-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.location_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/location-registry
|
||||||
|
scene_descriptions:
|
||||||
|
lane_id: scene-descriptions
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.scene_descriptions
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/scene-descriptions
|
||||||
|
item_occurrences:
|
||||||
|
lane_id: item-occurrences
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.item_occurrences
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/item-occurrences
|
||||||
|
spells:
|
||||||
|
lane_id: spells
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.spells
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/spells
|
||||||
|
combat_turns:
|
||||||
|
lane_id: combat-turns
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.combat_turns
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/combat-turns
|
||||||
|
npc_occurrences:
|
||||||
|
lane_id: npc-occurrences
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_occurrences
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/npc-occurrences
|
||||||
|
location_occurrences:
|
||||||
|
lane_id: location-occurrences
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.location_occurrences
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/location-occurrences
|
||||||
|
enemy_events:
|
||||||
|
lane_id: enemy-events
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.enemy_events
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/enemy-events
|
||||||
|
|
||||||
scriptorium:
|
scriptorium:
|
||||||
binary: scriptorium
|
binary: scriptorium
|
||||||
config_path: /usr/local/etc/scriptorium/config.yml
|
config_path: /usr/local/etc/scriptorium/config.yml
|
||||||
@@ -183,7 +260,5 @@ scriptorium:
|
|||||||
output_kind: player_handout
|
output_kind: player_handout
|
||||||
|
|
||||||
notification:
|
notification:
|
||||||
# Optional notification settings.
|
# No delivery provider is currently implemented.
|
||||||
backend: ""
|
mode: noop
|
||||||
recipient: ""
|
|
||||||
timeout: 30s
|
|
||||||
|
|||||||
@@ -125,4 +125,4 @@ scriptorium:
|
|||||||
output_kind: player_handout
|
output_kind: player_handout
|
||||||
|
|
||||||
notification:
|
notification:
|
||||||
timeout: 30s
|
mode: noop
|
||||||
|
|||||||
1
go.mod
1
go.mod
@@ -7,6 +7,7 @@ require (
|
|||||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||||
github.com/aws/smithy-go v1.25.1
|
github.com/aws/smithy-go v1.25.1
|
||||||
|
golang.org/x/sys v0.47.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -34,6 +34,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt
|
|||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||||
|
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||||
|
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
|||||||
@@ -4,10 +4,10 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NoopRunner is a deterministic no-op audita adapter.
|
// NoopRunner is a deterministic no-op audita adapter.
|
||||||
@@ -84,17 +84,17 @@ func materializePlaceholders(req PolishRequest) error {
|
|||||||
"merged_transcript_path": req.MergedTranscriptPath,
|
"merged_transcript_path": req.MergedTranscriptPath,
|
||||||
"output_path": req.OutputProcessedPath,
|
"output_path": req.OutputProcessedPath,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -117,14 +117,14 @@ func writeJSONIfRequested(path string, payload any) error {
|
|||||||
if path == "" {
|
if path == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
|
||||||
return fmt.Errorf("create parent directory %q: %w", filepath.Dir(path), err)
|
return fmt.Errorf("create parent directory %q: %w", filepath.Dir(path), err)
|
||||||
}
|
}
|
||||||
data, err := json.Marshal(payload)
|
data, err := json.Marshal(payload)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("marshal placeholder json for %q: %w", path, err)
|
return fmt.Errorf("marshal placeholder json for %q: %w", path, err)
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteFileAtomic(path, data, 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write placeholder json %q: %w", path, err)
|
return fmt.Errorf("write placeholder json %q: %w", path, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -6,8 +6,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TODO: implement a real Audita subprocess/service adapter.
|
|
||||||
|
|
||||||
// Runner is the adapter boundary for audita polish invocations.
|
// Runner is the adapter boundary for audita polish invocations.
|
||||||
type Runner interface {
|
type Runner interface {
|
||||||
Run(ctx context.Context, req PolishRequest) (PolishResult, error)
|
Run(ctx context.Context, req PolishRequest) (PolishResult, error)
|
||||||
@@ -15,25 +13,15 @@ type Runner interface {
|
|||||||
|
|
||||||
// PolishRequest describes an audita invocation.
|
// PolishRequest describes an audita invocation.
|
||||||
type PolishRequest struct {
|
type PolishRequest struct {
|
||||||
GeneratedConfigPath string
|
GeneratedConfigPath string
|
||||||
MergedTranscriptPath string
|
MergedTranscriptPath string
|
||||||
OutputProcessedPath string
|
OutputProcessedPath string
|
||||||
GlossaryPath string
|
GlossaryPath string
|
||||||
ReportPath string
|
ReportPath string
|
||||||
WorkDir string
|
WorkDir string
|
||||||
Modules []string
|
Modules []string
|
||||||
BaseURL string
|
StdoutLogPath string
|
||||||
Model string
|
StderrLogPath string
|
||||||
TranscriptDescription string
|
|
||||||
ConfigPath string
|
|
||||||
OutputSchema string
|
|
||||||
WorkDirRetention string
|
|
||||||
TotalLLMConcurrency *int
|
|
||||||
ProposalLLMConcurrency *int
|
|
||||||
ValidationModel string
|
|
||||||
ValidationLLMConcurrency *int
|
|
||||||
StdoutLogPath string
|
|
||||||
StderrLogPath string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PolishResult describes a polish output.
|
// PolishResult describes a polish output.
|
||||||
|
|||||||
@@ -11,8 +11,15 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// MaxProcessedOutputBytes bounds Audita's processed-transcript JSON result.
|
||||||
|
const MaxProcessedOutputBytes int64 = 64 * 1024 * 1024
|
||||||
|
|
||||||
|
// MaxReportOutputBytes bounds Audita's optional report JSON result.
|
||||||
|
const MaxReportOutputBytes int64 = 16 * 1024 * 1024
|
||||||
|
|
||||||
// SubprocessRunnerConfig defines deterministic settings for Audita CLI execution.
|
// SubprocessRunnerConfig defines deterministic settings for Audita CLI execution.
|
||||||
type SubprocessRunnerConfig struct {
|
type SubprocessRunnerConfig struct {
|
||||||
Binary string
|
Binary string
|
||||||
@@ -207,12 +214,13 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
|||||||
}
|
}
|
||||||
|
|
||||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||||
Executable: r.binary,
|
Executable: r.binary,
|
||||||
Args: args,
|
Args: args,
|
||||||
Timeout: r.timeout,
|
Timeout: r.timeout,
|
||||||
EnvOverrides: env,
|
EnvOverrides: env,
|
||||||
StdoutLogPath: req.StdoutLogPath,
|
DiagnosticOwner: "audita",
|
||||||
StderrLogPath: req.StderrLogPath,
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
wrappedMessage := fmt.Sprintf(
|
wrappedMessage := fmt.Sprintf(
|
||||||
@@ -370,13 +378,13 @@ func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []strin
|
|||||||
"credential_env_var": r.llmAPIKeyEnv,
|
"credential_env_var": r.llmAPIKeyEnv,
|
||||||
"credential_present": credentialPresent,
|
"credential_present": credentialPresent,
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateProcessedOutput(path string) error {
|
func validateProcessedOutput(path string) error {
|
||||||
data, err := os.ReadFile(path)
|
data, err := readAuditaResult(path, MaxProcessedOutputBytes, "processed transcript")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read file: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var payload map[string]any
|
var payload map[string]any
|
||||||
@@ -405,9 +413,9 @@ func addSubprocessStreamHint(message string, runErr error) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateJSONFile(path string) error {
|
func validateJSONFile(path string) error {
|
||||||
data, err := os.ReadFile(path)
|
data, err := readAuditaResult(path, MaxReportOutputBytes, "report")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read file: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
var v any
|
var v any
|
||||||
if err := json.Unmarshal(data, &v); err != nil {
|
if err := json.Unmarshal(data, &v); err != nil {
|
||||||
@@ -415,3 +423,11 @@ func validateJSONFile(path string) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func readAuditaResult(path string, limit int64, category string) ([]byte, error) {
|
||||||
|
data, err := fileops.ReadRegularFile(path, limit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("audita %s result exceeds or cannot be read within %d-byte limit: %w", category, limit, err)
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -189,7 +189,7 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
|
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
t.Skip("helper wrapper script uses /bin/sh")
|
t.Skip("helper wrapper script uses /bin/sh")
|
||||||
}
|
}
|
||||||
@@ -214,8 +214,8 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
rec := readAuditaHelperRecord(t, recordPath)
|
rec := readAuditaHelperRecord(t, recordPath)
|
||||||
if rec.Env["AUDITA_INHERITED_MARKER"] != "inherited-from-parent" {
|
if rec.Env["AUDITA_INHERITED_MARKER"] != "" {
|
||||||
t.Fatalf("AUDITA_INHERITED_MARKER = %q, want inherited-from-parent", rec.Env["AUDITA_INHERITED_MARKER"])
|
t.Fatalf("AUDITA_INHERITED_MARKER = %q, want omitted from the child environment", rec.Env["AUDITA_INHERITED_MARKER"])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
22
internal/adapters/notarius/fake.go
Normal file
22
internal/adapters/notarius/fake.go
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
package notarius
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// FakeRunner is a configurable in-memory runner for stage tests.
|
||||||
|
type FakeRunner struct {
|
||||||
|
Requests []RunRequest
|
||||||
|
Result RunResult
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run records the request and returns the configured result or error.
|
||||||
|
func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return RunResult{}, err
|
||||||
|
}
|
||||||
|
f.Requests = append(f.Requests, req)
|
||||||
|
if f.Err != nil {
|
||||||
|
return RunResult{}, f.Err
|
||||||
|
}
|
||||||
|
return f.Result, nil
|
||||||
|
}
|
||||||
108
internal/adapters/notarius/runner.go
Normal file
108
internal/adapters/notarius/runner.go
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
// Package notarius declares the adapter contract for Notarius CLI invocations.
|
||||||
|
package notarius
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const ReceiptSchemaVersion = "notarius.run-result.v1"
|
||||||
|
|
||||||
|
// Runner is the adapter boundary for a complete Notarius pipeline invocation.
|
||||||
|
type Runner interface {
|
||||||
|
Run(ctx context.Context, req RunRequest) (RunResult, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
|
||||||
|
type RunRequest struct {
|
||||||
|
Binary string
|
||||||
|
ConfigPath string
|
||||||
|
PipelineID string
|
||||||
|
InputPath string
|
||||||
|
OutputRoot string
|
||||||
|
WorkingDirectory string
|
||||||
|
ReceiptPath string
|
||||||
|
LogPath string
|
||||||
|
Timeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Receipt is the transport-neutral successful run receipt.
|
||||||
|
type Receipt struct {
|
||||||
|
SchemaVersion string
|
||||||
|
RunID string
|
||||||
|
PipelineID string
|
||||||
|
OutputDirectory string
|
||||||
|
IndexFile string
|
||||||
|
NormalizedOutputCount int
|
||||||
|
RejectedOutputCount int
|
||||||
|
WarningCount int
|
||||||
|
ValidationStatus string
|
||||||
|
DebugDirectory string
|
||||||
|
}
|
||||||
|
|
||||||
|
// LaneDescriptor identifies one normalized lane payload discovered through the index.
|
||||||
|
type LaneDescriptor struct {
|
||||||
|
LaneID string
|
||||||
|
File string
|
||||||
|
Path string
|
||||||
|
MediaType string
|
||||||
|
ModuleKey string
|
||||||
|
SchemaID string
|
||||||
|
SchemaName string
|
||||||
|
SchemaVersion string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PipelineDescriptor identifies a pipeline-wide artifact discovered through the index.
|
||||||
|
type PipelineDescriptor struct {
|
||||||
|
ArtifactKind string
|
||||||
|
File string
|
||||||
|
Path string
|
||||||
|
MediaType string
|
||||||
|
SchemaID string
|
||||||
|
SchemaName string
|
||||||
|
SchemaVersion string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Index describes the validated bundle-management and artifact paths.
|
||||||
|
type Index struct {
|
||||||
|
Path string
|
||||||
|
ManifestFile string
|
||||||
|
ManifestPath string
|
||||||
|
RejectedFile string
|
||||||
|
RejectedPath string
|
||||||
|
WarningsFile string
|
||||||
|
WarningsPath string
|
||||||
|
Lanes []LaneDescriptor
|
||||||
|
ChunkMap *PipelineDescriptor
|
||||||
|
EvidenceContext *PipelineDescriptor
|
||||||
|
}
|
||||||
|
|
||||||
|
// RejectionSummary retains structured rejection identity without free-form messages.
|
||||||
|
type RejectionSummary struct {
|
||||||
|
Stage string
|
||||||
|
StepID string
|
||||||
|
LaneID string
|
||||||
|
ModuleKey string
|
||||||
|
ChunkID string
|
||||||
|
ValidatorName string
|
||||||
|
ReasonCode string
|
||||||
|
}
|
||||||
|
|
||||||
|
// WarningSummary retains structured warning identity without free-form messages.
|
||||||
|
type WarningSummary struct {
|
||||||
|
Scope string
|
||||||
|
ReasonCode string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunResult describes a successfully decoded and validated Notarius bundle.
|
||||||
|
type RunResult struct {
|
||||||
|
Receipt Receipt
|
||||||
|
Index Index
|
||||||
|
BundleRoot string
|
||||||
|
ReceiptPath string
|
||||||
|
LogPath string
|
||||||
|
ExitCode int
|
||||||
|
Duration time.Duration
|
||||||
|
Rejections []RejectionSummary
|
||||||
|
Warnings []WarningSummary
|
||||||
|
}
|
||||||
502
internal/adapters/notarius/subprocess.go
Normal file
502
internal/adapters/notarius/subprocess.go
Normal file
@@ -0,0 +1,502 @@
|
|||||||
|
package notarius
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
maxReceiptBytes = 1 << 20
|
||||||
|
maxIndexBytes = 4 << 20
|
||||||
|
maxSummaryBytes = 4 << 20
|
||||||
|
canonicalIndexFile = "index.json"
|
||||||
|
canonicalManifestFile = "manifest.json"
|
||||||
|
canonicalRejectedFile = "rejected.json"
|
||||||
|
canonicalWarningsFile = "warnings.json"
|
||||||
|
)
|
||||||
|
|
||||||
|
type subprocessRun func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error)
|
||||||
|
|
||||||
|
// SubprocessRunner invokes Notarius through its public CLI.
|
||||||
|
type SubprocessRunner struct {
|
||||||
|
run subprocessRun
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSubprocessRunner constructs a production Notarius subprocess runner.
|
||||||
|
func NewSubprocessRunner() *SubprocessRunner {
|
||||||
|
return &SubprocessRunner{run: subprocess.Run}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run executes a complete Notarius pipeline and discovers its published bundle.
|
||||||
|
func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||||
|
if r == nil || r.run == nil {
|
||||||
|
return RunResult{}, fmt.Errorf("notarius subprocess runner is nil")
|
||||||
|
}
|
||||||
|
if err := validateRunRequest(req); err != nil {
|
||||||
|
return RunResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
args := []string{
|
||||||
|
"run", req.PipelineID,
|
||||||
|
"--config", req.ConfigPath,
|
||||||
|
"--input", req.InputPath,
|
||||||
|
"--output-dir", req.OutputRoot,
|
||||||
|
"--json",
|
||||||
|
}
|
||||||
|
processResult, err := r.run(ctx, subprocess.RunRequest{
|
||||||
|
Executable: req.Binary,
|
||||||
|
Args: args,
|
||||||
|
WorkingDir: req.WorkingDirectory,
|
||||||
|
Timeout: req.Timeout,
|
||||||
|
DiagnosticOwner: "notarius",
|
||||||
|
StdoutLogPath: req.ReceiptPath,
|
||||||
|
StderrLogPath: req.LogPath,
|
||||||
|
})
|
||||||
|
baseResult := RunResult{
|
||||||
|
ReceiptPath: req.ReceiptPath,
|
||||||
|
LogPath: req.LogPath,
|
||||||
|
ExitCode: processResult.ExitCode,
|
||||||
|
Duration: processResult.Duration,
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, fmt.Errorf("run notarius pipeline %q: %w", req.PipelineID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
receipt, err := loadReceipt(req.ReceiptPath, req.PipelineID)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, err
|
||||||
|
}
|
||||||
|
bundleRoot, err := validateBundleRoot(req.OutputRoot, receipt.OutputDirectory)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, err
|
||||||
|
}
|
||||||
|
indexPath, err := resolveRegularFile(bundleRoot, receipt.IndexFile)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, fmt.Errorf("resolve receipt index file: %w", err)
|
||||||
|
}
|
||||||
|
index, err := loadIndex(bundleRoot, indexPath)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, err
|
||||||
|
}
|
||||||
|
rejections, err := loadRejections(index.RejectedPath)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, err
|
||||||
|
}
|
||||||
|
warnings, err := loadWarnings(index.WarningsPath)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, err
|
||||||
|
}
|
||||||
|
|
||||||
|
baseResult.Receipt = receipt
|
||||||
|
baseResult.Index = index
|
||||||
|
baseResult.BundleRoot = bundleRoot
|
||||||
|
baseResult.Rejections = rejections
|
||||||
|
baseResult.Warnings = warnings
|
||||||
|
return baseResult, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRunRequest(req RunRequest) error {
|
||||||
|
if strings.TrimSpace(req.Binary) == "" {
|
||||||
|
return fmt.Errorf("notarius binary is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(req.PipelineID) == "" {
|
||||||
|
return fmt.Errorf("notarius pipeline id is required")
|
||||||
|
}
|
||||||
|
if req.Timeout <= 0 {
|
||||||
|
return fmt.Errorf("notarius timeout must be positive")
|
||||||
|
}
|
||||||
|
for label, path := range map[string]string{
|
||||||
|
"config": req.ConfigPath,
|
||||||
|
"input": req.InputPath,
|
||||||
|
"output root": req.OutputRoot,
|
||||||
|
"working directory": req.WorkingDirectory,
|
||||||
|
"receipt": req.ReceiptPath,
|
||||||
|
"log": req.LogPath,
|
||||||
|
} {
|
||||||
|
if strings.TrimSpace(path) == "" {
|
||||||
|
return fmt.Errorf("notarius %s path is required", label)
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(path) {
|
||||||
|
return fmt.Errorf("notarius %s path must be absolute", label)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if filepath.Clean(req.ReceiptPath) == filepath.Clean(req.LogPath) {
|
||||||
|
return fmt.Errorf("notarius receipt and log paths must be different")
|
||||||
|
}
|
||||||
|
if err := requireRegularFile(req.ConfigPath); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius config path: %w", err)
|
||||||
|
}
|
||||||
|
if err := requireRegularFile(req.InputPath); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius input path: %w", err)
|
||||||
|
}
|
||||||
|
if err := requireDirectory(req.OutputRoot); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius output root: %w", err)
|
||||||
|
}
|
||||||
|
if err := requireDirectory(req.WorkingDirectory); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius working directory: %w", err)
|
||||||
|
}
|
||||||
|
if err := validateLogDestination(req.ReceiptPath); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius receipt path: %w", err)
|
||||||
|
}
|
||||||
|
if err := validateLogDestination(req.LogPath); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius log path: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type receiptDocument struct {
|
||||||
|
SchemaVersion string `json:"schema_version"`
|
||||||
|
RunID string `json:"run_id"`
|
||||||
|
PipelineID string `json:"pipeline_id"`
|
||||||
|
OutputDirectory string `json:"output_directory"`
|
||||||
|
IndexFile string `json:"index_file"`
|
||||||
|
NormalizedOutputCount *int `json:"normalized_output_count"`
|
||||||
|
RejectedOutputCount *int `json:"rejected_output_count"`
|
||||||
|
WarningCount *int `json:"warning_count"`
|
||||||
|
ValidationStatus string `json:"validation_status"`
|
||||||
|
DebugDirectory string `json:"debug_directory"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadReceipt(path, pipelineID string) (Receipt, error) {
|
||||||
|
var document receiptDocument
|
||||||
|
if err := decodeBoundedJSON(path, maxReceiptBytes, &document); err != nil {
|
||||||
|
return Receipt{}, fmt.Errorf("decode notarius receipt: %w", err)
|
||||||
|
}
|
||||||
|
if document.SchemaVersion != ReceiptSchemaVersion {
|
||||||
|
return Receipt{}, fmt.Errorf("unsupported notarius receipt schema version %q", document.SchemaVersion)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(document.RunID) == "" || strings.TrimSpace(document.PipelineID) == "" ||
|
||||||
|
strings.TrimSpace(document.OutputDirectory) == "" || strings.TrimSpace(document.ValidationStatus) == "" ||
|
||||||
|
document.NormalizedOutputCount == nil ||
|
||||||
|
document.RejectedOutputCount == nil || document.WarningCount == nil {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt is missing required fields")
|
||||||
|
}
|
||||||
|
if document.IndexFile != canonicalIndexFile {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt index_file %q is incompatible; want %q", document.IndexFile, canonicalIndexFile)
|
||||||
|
}
|
||||||
|
if document.PipelineID != pipelineID {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt pipeline id %q does not match requested pipeline %q", document.PipelineID, pipelineID)
|
||||||
|
}
|
||||||
|
if *document.NormalizedOutputCount < 0 || *document.RejectedOutputCount < 0 || *document.WarningCount < 0 {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt counts must be non-negative")
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(document.OutputDirectory) {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt output directory must be absolute")
|
||||||
|
}
|
||||||
|
if document.DebugDirectory != "" && !filepath.IsAbs(document.DebugDirectory) {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt debug directory must be absolute when present")
|
||||||
|
}
|
||||||
|
return Receipt{
|
||||||
|
SchemaVersion: document.SchemaVersion,
|
||||||
|
RunID: document.RunID,
|
||||||
|
PipelineID: document.PipelineID,
|
||||||
|
OutputDirectory: filepath.Clean(document.OutputDirectory),
|
||||||
|
IndexFile: document.IndexFile,
|
||||||
|
NormalizedOutputCount: *document.NormalizedOutputCount,
|
||||||
|
RejectedOutputCount: *document.RejectedOutputCount,
|
||||||
|
WarningCount: *document.WarningCount,
|
||||||
|
ValidationStatus: document.ValidationStatus,
|
||||||
|
DebugDirectory: document.DebugDirectory,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type indexDocument struct {
|
||||||
|
ManifestFile string `json:"manifest_file"`
|
||||||
|
OutputFiles *[]laneDocument `json:"output_files"`
|
||||||
|
RejectedFile string `json:"rejected_file"`
|
||||||
|
WarningsFile string `json:"warnings_file"`
|
||||||
|
ChunkMap *pipelineDocument `json:"chunk_map"`
|
||||||
|
EvidenceContext *pipelineDocument `json:"evidence_context"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type laneDocument struct {
|
||||||
|
LaneID string `json:"lane_id"`
|
||||||
|
File string `json:"file"`
|
||||||
|
MediaType string `json:"media_type"`
|
||||||
|
ModuleKey string `json:"module_key"`
|
||||||
|
SchemaID string `json:"schema_id"`
|
||||||
|
SchemaName string `json:"schema_name"`
|
||||||
|
SchemaVersion string `json:"schema_version"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type pipelineDocument struct {
|
||||||
|
ArtifactKind string `json:"artifact_kind"`
|
||||||
|
File string `json:"file"`
|
||||||
|
MediaType string `json:"media_type"`
|
||||||
|
SchemaID string `json:"schema_id"`
|
||||||
|
SchemaName string `json:"schema_name"`
|
||||||
|
SchemaVersion string `json:"schema_version"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadIndex(bundleRoot, indexPath string) (Index, error) {
|
||||||
|
var document indexDocument
|
||||||
|
if err := decodeBoundedJSON(indexPath, maxIndexBytes, &document); err != nil {
|
||||||
|
return Index{}, fmt.Errorf("decode notarius index: %w", err)
|
||||||
|
}
|
||||||
|
for _, field := range []struct {
|
||||||
|
name string
|
||||||
|
got string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "manifest_file", got: document.ManifestFile, want: canonicalManifestFile},
|
||||||
|
{name: "rejected_file", got: document.RejectedFile, want: canonicalRejectedFile},
|
||||||
|
{name: "warnings_file", got: document.WarningsFile, want: canonicalWarningsFile},
|
||||||
|
} {
|
||||||
|
if field.got != field.want {
|
||||||
|
return Index{}, fmt.Errorf("notarius index %s %q is incompatible; want %q", field.name, field.got, field.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if document.OutputFiles == nil {
|
||||||
|
return Index{}, fmt.Errorf("notarius index is missing required output_files")
|
||||||
|
}
|
||||||
|
|
||||||
|
index := Index{
|
||||||
|
Path: indexPath,
|
||||||
|
ManifestFile: document.ManifestFile,
|
||||||
|
RejectedFile: document.RejectedFile,
|
||||||
|
WarningsFile: document.WarningsFile,
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if index.ManifestPath, err = resolveRegularFile(bundleRoot, index.ManifestFile); err != nil {
|
||||||
|
return Index{}, fmt.Errorf("resolve notarius manifest file: %w", err)
|
||||||
|
}
|
||||||
|
if index.RejectedPath, err = resolveRegularFile(bundleRoot, index.RejectedFile); err != nil {
|
||||||
|
return Index{}, fmt.Errorf("resolve notarius rejection file: %w", err)
|
||||||
|
}
|
||||||
|
if index.WarningsPath, err = resolveRegularFile(bundleRoot, index.WarningsFile); err != nil {
|
||||||
|
return Index{}, fmt.Errorf("resolve notarius warning file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
seenLanes := make(map[string]struct{}, len(*document.OutputFiles))
|
||||||
|
for _, lane := range *document.OutputFiles {
|
||||||
|
if strings.TrimSpace(lane.LaneID) == "" || strings.TrimSpace(lane.File) == "" {
|
||||||
|
return Index{}, fmt.Errorf("notarius lane descriptors require lane_id and file")
|
||||||
|
}
|
||||||
|
if _, exists := seenLanes[lane.LaneID]; exists {
|
||||||
|
return Index{}, fmt.Errorf("notarius index contains duplicate lane id %q", lane.LaneID)
|
||||||
|
}
|
||||||
|
seenLanes[lane.LaneID] = struct{}{}
|
||||||
|
path, err := resolveRegularFile(bundleRoot, lane.File)
|
||||||
|
if err != nil {
|
||||||
|
return Index{}, fmt.Errorf("resolve notarius lane %q file: %w", lane.LaneID, err)
|
||||||
|
}
|
||||||
|
index.Lanes = append(index.Lanes, LaneDescriptor{
|
||||||
|
LaneID: lane.LaneID, File: lane.File, Path: path, MediaType: lane.MediaType,
|
||||||
|
ModuleKey: lane.ModuleKey, SchemaID: lane.SchemaID, SchemaName: lane.SchemaName,
|
||||||
|
SchemaVersion: lane.SchemaVersion,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if document.ChunkMap != nil {
|
||||||
|
index.ChunkMap, err = resolvePipelineDescriptor(bundleRoot, "chunk_map", *document.ChunkMap)
|
||||||
|
if err != nil {
|
||||||
|
return Index{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if document.EvidenceContext != nil {
|
||||||
|
index.EvidenceContext, err = resolvePipelineDescriptor(bundleRoot, "evidence_context", *document.EvidenceContext)
|
||||||
|
if err != nil {
|
||||||
|
return Index{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return index, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolvePipelineDescriptor(bundleRoot, label string, document pipelineDocument) (*PipelineDescriptor, error) {
|
||||||
|
if strings.TrimSpace(document.ArtifactKind) == "" || strings.TrimSpace(document.File) == "" ||
|
||||||
|
strings.TrimSpace(document.MediaType) == "" || strings.TrimSpace(document.SchemaID) == "" ||
|
||||||
|
strings.TrimSpace(document.SchemaName) == "" || strings.TrimSpace(document.SchemaVersion) == "" {
|
||||||
|
return nil, fmt.Errorf("notarius %s descriptor is missing required fields", label)
|
||||||
|
}
|
||||||
|
path, err := resolveRegularFile(bundleRoot, document.File)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve notarius %s file: %w", label, err)
|
||||||
|
}
|
||||||
|
return &PipelineDescriptor{
|
||||||
|
ArtifactKind: document.ArtifactKind, File: document.File, Path: path,
|
||||||
|
MediaType: document.MediaType, SchemaID: document.SchemaID,
|
||||||
|
SchemaName: document.SchemaName, SchemaVersion: document.SchemaVersion,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type rejectionDocument struct {
|
||||||
|
Rejected *[]struct {
|
||||||
|
Stage string `json:"stage"`
|
||||||
|
StepID string `json:"step_id"`
|
||||||
|
LaneID string `json:"lane_id"`
|
||||||
|
ModuleKey string `json:"module_key"`
|
||||||
|
ChunkID string `json:"chunk_id"`
|
||||||
|
ValidatorName string `json:"validator_name"`
|
||||||
|
ReasonCode string `json:"reason_code"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
} `json:"rejected"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadRejections(path string) ([]RejectionSummary, error) {
|
||||||
|
var document rejectionDocument
|
||||||
|
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode notarius rejections: %w", err)
|
||||||
|
}
|
||||||
|
if document.Rejected == nil {
|
||||||
|
return nil, fmt.Errorf("notarius rejection document is missing rejected array")
|
||||||
|
}
|
||||||
|
summaries := make([]RejectionSummary, 0, len(*document.Rejected))
|
||||||
|
for _, item := range *document.Rejected {
|
||||||
|
if strings.TrimSpace(item.Stage) == "" || strings.TrimSpace(item.Message) == "" {
|
||||||
|
return nil, fmt.Errorf("notarius rejection entries require stage and message")
|
||||||
|
}
|
||||||
|
summaries = append(summaries, RejectionSummary{
|
||||||
|
Stage: item.Stage, StepID: item.StepID, LaneID: item.LaneID,
|
||||||
|
ModuleKey: item.ModuleKey, ChunkID: item.ChunkID,
|
||||||
|
ValidatorName: item.ValidatorName, ReasonCode: item.ReasonCode,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return summaries, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type warningDocument struct {
|
||||||
|
Warnings *[]struct {
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
ReasonCode string `json:"reason_code"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
} `json:"warnings"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadWarnings(path string) ([]WarningSummary, error) {
|
||||||
|
var document warningDocument
|
||||||
|
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode notarius warnings: %w", err)
|
||||||
|
}
|
||||||
|
if document.Warnings == nil {
|
||||||
|
return nil, fmt.Errorf("notarius warning document is missing warnings array")
|
||||||
|
}
|
||||||
|
summaries := make([]WarningSummary, 0, len(*document.Warnings))
|
||||||
|
for _, item := range *document.Warnings {
|
||||||
|
if strings.TrimSpace(item.ReasonCode) == "" || strings.TrimSpace(item.Message) == "" {
|
||||||
|
return nil, fmt.Errorf("notarius warning entries require reason_code and message")
|
||||||
|
}
|
||||||
|
summaries = append(summaries, WarningSummary{Scope: item.Scope, ReasonCode: item.ReasonCode})
|
||||||
|
}
|
||||||
|
return summaries, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeBoundedJSON(path string, limit int64, destination any) error {
|
||||||
|
data, err := fileops.ReadRegularFile(path, limit)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("notarius JSON result exceeds or cannot be read within %d-byte limit: %w", limit, err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(data, destination); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateBundleRoot(outputRoot, bundleRoot string) (string, error) {
|
||||||
|
root := filepath.Clean(outputRoot)
|
||||||
|
bundle := filepath.Clean(bundleRoot)
|
||||||
|
relative, err := filepath.Rel(root, bundle)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("compare notarius output paths: %w", err)
|
||||||
|
}
|
||||||
|
if relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||||
|
return "", fmt.Errorf("notarius output directory %q is not beneath output root %q", bundleRoot, outputRoot)
|
||||||
|
}
|
||||||
|
if err := requireDirectoryTree(root, relative); err != nil {
|
||||||
|
return "", fmt.Errorf("validate notarius output directory: %w", err)
|
||||||
|
}
|
||||||
|
return bundle, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveRegularFile(root, logicalPath string) (string, error) {
|
||||||
|
resolved, err := pathsafe.JoinSlashRelativeUnderRoot(root, logicalPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
relative, err := filepath.Rel(root, resolved)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := requireRegularFileTree(root, relative); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return resolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireDirectoryTree(root, relative string) error {
|
||||||
|
if err := requireDirectory(root); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
current := root
|
||||||
|
for _, component := range strings.Split(relative, string(filepath.Separator)) {
|
||||||
|
current = filepath.Join(current, component)
|
||||||
|
if err := requireDirectory(current); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireRegularFileTree(root, relative string) error {
|
||||||
|
components := strings.Split(relative, string(filepath.Separator))
|
||||||
|
if len(components) == 0 {
|
||||||
|
return fmt.Errorf("regular file path is required")
|
||||||
|
}
|
||||||
|
if err := requireDirectory(root); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
current := root
|
||||||
|
for _, component := range components[:len(components)-1] {
|
||||||
|
current = filepath.Join(current, component)
|
||||||
|
if err := requireDirectory(current); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return requireRegularFile(filepath.Join(current, components[len(components)-1]))
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireDirectory(path string) error {
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||||
|
return fmt.Errorf("path %q must be a directory without symlinks", path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireRegularFile(path string) error {
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("path %q must be a regular file without symlinks", path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateLogDestination(path string) error {
|
||||||
|
if err := requireDirectory(filepath.Dir(path)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("path %q must be absent or a regular file without symlinks", path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
572
internal/adapters/notarius/subprocess_test.go
Normal file
572
internal/adapters/notarius/subprocess_test.go
Normal file
@@ -0,0 +1,572 @@
|
|||||||
|
package notarius
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
var captured sharedsubprocess.RunRequest
|
||||||
|
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
captured = processReq
|
||||||
|
writeValidBundleAndReceipt(t, req, true)
|
||||||
|
return sharedsubprocess.RunResult{ExitCode: 0, Duration: 2 * time.Second}, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
result, err := runner.Run(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
wantArgs := []string{
|
||||||
|
"run", "dnd-session", "--config", req.ConfigPath, "--input", req.InputPath,
|
||||||
|
"--output-dir", req.OutputRoot, "--json",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(captured.Args, wantArgs) {
|
||||||
|
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
|
||||||
|
}
|
||||||
|
if captured.Executable != req.Binary || captured.WorkingDir != req.WorkingDirectory || captured.Timeout != req.Timeout {
|
||||||
|
t.Fatalf("subprocess request = %#v", captured)
|
||||||
|
}
|
||||||
|
if captured.StdoutLogPath != req.ReceiptPath || captured.StderrLogPath != req.LogPath {
|
||||||
|
t.Fatalf("stream paths = stdout %q stderr %q", captured.StdoutLogPath, captured.StderrLogPath)
|
||||||
|
}
|
||||||
|
if captured.EnvOverrides != nil {
|
||||||
|
t.Fatalf("environment overrides = %#v, want inherited environment only", captured.EnvOverrides)
|
||||||
|
}
|
||||||
|
for _, arg := range captured.Args {
|
||||||
|
if arg == "--session-id" {
|
||||||
|
t.Fatal("subprocess args unexpectedly contain --session-id")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Receipt.SchemaVersion != ReceiptSchemaVersion || result.Receipt.RunID != "notarius-run-1" {
|
||||||
|
t.Fatalf("receipt = %#v", result.Receipt)
|
||||||
|
}
|
||||||
|
if len(result.Index.Lanes) != 1 || result.Index.Lanes[0].LaneID != "npc-registry" {
|
||||||
|
t.Fatalf("lanes = %#v", result.Index.Lanes)
|
||||||
|
}
|
||||||
|
if result.Index.ChunkMap == nil || result.Index.ChunkMap.ArtifactKind != "chunk_map" {
|
||||||
|
t.Fatalf("chunk map = %#v", result.Index.ChunkMap)
|
||||||
|
}
|
||||||
|
if result.Index.EvidenceContext == nil || result.Index.EvidenceContext.ArtifactKind != "evidence_context" {
|
||||||
|
t.Fatalf("evidence context = %#v", result.Index.EvidenceContext)
|
||||||
|
}
|
||||||
|
if len(result.Rejections) != 1 || result.Rejections[0].LaneID != "spells" || result.Rejections[0].ReasonCode != "invalid_spell" {
|
||||||
|
t.Fatalf("rejections = %#v", result.Rejections)
|
||||||
|
}
|
||||||
|
if len(result.Warnings) != 1 || result.Warnings[0].Scope != "lane:npc-registry" || result.Warnings[0].ReasonCode != "normalized_name" {
|
||||||
|
t.Fatalf("warnings = %#v", result.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
writeValidBundleAndReceipt(t, req, false)
|
||||||
|
receiptFixture := req.ReceiptPath + ".fixture"
|
||||||
|
data, err := os.ReadFile(req.ReceiptPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(receiptFixture, data, 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(req.ReceiptPath); err != nil {
|
||||||
|
t.Fatalf("Remove(receipt) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
captureDir := filepath.Join(filepath.Dir(req.ReceiptPath), "capture")
|
||||||
|
if err := os.Mkdir(captureDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(capture) error = %v", err)
|
||||||
|
}
|
||||||
|
script := writeShellScript(t, `#!/bin/sh
|
||||||
|
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
|
||||||
|
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
|
||||||
|
printf 'diagnostic stream\n' >&2
|
||||||
|
cat "$NOTARIUS_RECEIPT_FIXTURE"
|
||||||
|
`)
|
||||||
|
req.Binary = script
|
||||||
|
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
|
||||||
|
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
|
||||||
|
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
||||||
|
|
||||||
|
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
|
||||||
|
assertTextFile(t, filepath.Join(captureDir, "environment"), "")
|
||||||
|
assertTextFile(t, req.LogPath, "diagnostic stream\n")
|
||||||
|
receiptBytes, err := os.ReadFile(req.ReceiptPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(receiptBytes), "diagnostic stream") {
|
||||||
|
t.Fatal("receipt contains stderr output")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
scriptBody string
|
||||||
|
timeout time.Duration
|
||||||
|
cancel bool
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "nonzero", scriptBody: "printf '{malformed receipt'; printf 'failed\\n' >&2; exit 7\n", timeout: time.Second, want: "exit code 7"},
|
||||||
|
{name: "timeout", scriptBody: "sleep 5\n", timeout: 20 * time.Millisecond, want: "timed out"},
|
||||||
|
{name: "cancellation", scriptBody: "sleep 5\n", timeout: time.Second, cancel: true, want: "canceled"},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
|
||||||
|
req.Timeout = test.timeout
|
||||||
|
ctx := context.Background()
|
||||||
|
if test.cancel {
|
||||||
|
cancelCtx, cancel := context.WithCancel(ctx)
|
||||||
|
ctx = cancelCtx
|
||||||
|
time.AfterFunc(20*time.Millisecond, cancel)
|
||||||
|
}
|
||||||
|
_, err := NewSubprocessRunner().Run(ctx, req)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||||
|
t.Fatalf("Run() error = %v, want fragment %q", err, test.want)
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), "decode notarius receipt") {
|
||||||
|
t.Fatalf("Run() parsed stdout after process failure: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerReturnsSharedSubprocessErrorWithoutReadingReceipt(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
if err := os.WriteFile(req.ReceiptPath, []byte("not json"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(receipt) error = %v", err)
|
||||||
|
}
|
||||||
|
wantErr := errors.New("process failed")
|
||||||
|
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
return sharedsubprocess.RunResult{ExitCode: 9}, wantErr
|
||||||
|
}}
|
||||||
|
_, err := runner.Run(context.Background(), req)
|
||||||
|
if !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("Run() error = %v, want wrapped process error", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), "decode") {
|
||||||
|
t.Fatalf("Run() parsed receipt after failure: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadReceiptValidation(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
valid := map[string]any{
|
||||||
|
"schema_version": ReceiptSchemaVersion, "run_id": "run-1", "pipeline_id": "pipeline-1",
|
||||||
|
"output_directory": filepath.Join(root, "outputs", "run-1"), "index_file": "index.json",
|
||||||
|
"normalized_output_count": 1, "rejected_output_count": 0, "warning_count": 0,
|
||||||
|
"validation_status": "approved", "future_field": true,
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(map[string]any)
|
||||||
|
raw []byte
|
||||||
|
wantOK bool
|
||||||
|
wantError string
|
||||||
|
}{
|
||||||
|
{name: "unknown fields tolerated", wantOK: true},
|
||||||
|
{name: "malformed", raw: []byte("{")},
|
||||||
|
{name: "unsupported version", mutate: func(v map[string]any) { v["schema_version"] = "notarius.run-result.v2" }},
|
||||||
|
{name: "missing field", mutate: func(v map[string]any) { delete(v, "run_id") }},
|
||||||
|
{name: "pipeline mismatch", mutate: func(v map[string]any) { v["pipeline_id"] = "other" }},
|
||||||
|
{name: "relative output", mutate: func(v map[string]any) { v["output_directory"] = "run-1" }},
|
||||||
|
{name: "negative count", mutate: func(v map[string]any) { v["warning_count"] = -1 }},
|
||||||
|
{
|
||||||
|
name: "nested index", mutate: func(v map[string]any) { v["index_file"] = "nested/index.json" },
|
||||||
|
wantError: `index_file "nested/index.json"`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cleanable index", mutate: func(v map[string]any) { v["index_file"] = "./index.json" },
|
||||||
|
wantError: `index_file "./index.json"`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(root, strings.ReplaceAll(test.name, " ", "-")+".json")
|
||||||
|
values := cloneMap(valid)
|
||||||
|
if test.mutate != nil {
|
||||||
|
test.mutate(values)
|
||||||
|
}
|
||||||
|
if test.raw != nil {
|
||||||
|
if err := os.WriteFile(path, test.raw, 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
writeJSONFile(t, path, values)
|
||||||
|
}
|
||||||
|
_, err := loadReceipt(path, "pipeline-1")
|
||||||
|
if test.wantOK && err != nil {
|
||||||
|
t.Fatalf("loadReceipt() error = %v", err)
|
||||||
|
}
|
||||||
|
if !test.wantOK && err == nil {
|
||||||
|
t.Fatal("loadReceipt() error = nil, want validation failure")
|
||||||
|
}
|
||||||
|
if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
|
||||||
|
t.Fatalf("loadReceipt() error = %v, want fragment %q", err, test.wantError)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
oversized := filepath.Join(root, "oversized.json")
|
||||||
|
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxReceiptBytes+1)), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := loadReceipt(oversized, "pipeline-1"); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("loadReceipt(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateBundleRootRejectsEscapesAndSymlinks(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
outputRoot := filepath.Join(root, "output")
|
||||||
|
if err := os.Mkdir(outputRoot, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(output root) error = %v", err)
|
||||||
|
}
|
||||||
|
validBundle := filepath.Join(outputRoot, "run-1")
|
||||||
|
if err := os.Mkdir(validBundle, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(bundle) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := validateBundleRoot(outputRoot, validBundle); err != nil {
|
||||||
|
t.Fatalf("validateBundleRoot(valid) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
outside := filepath.Join(root, "output-other")
|
||||||
|
if err := os.Mkdir(outside, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(outside) error = %v", err)
|
||||||
|
}
|
||||||
|
for name, candidate := range map[string]string{"equal root": outputRoot, "escape": root, "prefix confusion": outside} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if _, err := validateBundleRoot(outputRoot, candidate); err == nil {
|
||||||
|
t.Fatalf("validateBundleRoot(%q) error = nil", candidate)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
symlink := filepath.Join(outputRoot, "linked")
|
||||||
|
if err := os.Symlink(outside, symlink); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := validateBundleRoot(outputRoot, symlink); err == nil {
|
||||||
|
t.Fatal("validateBundleRoot(symlink) error = nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadIndexRejectsMalformedUnsafeAndUnsupportedDocuments(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
indexValue any
|
||||||
|
prepare func(*testing.T, string)
|
||||||
|
wantError string
|
||||||
|
}{
|
||||||
|
{name: "malformed", indexValue: json.RawMessage(`{"manifest_file":`)},
|
||||||
|
{name: "unsupported output shape", indexValue: map[string]any{"manifest_file": "manifest.json", "output_files": map[string]any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
|
||||||
|
{name: "missing management path", indexValue: map[string]any{"output_files": []any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
|
||||||
|
{name: "renamed manifest", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["manifest_file"] = "metadata.json"
|
||||||
|
return value
|
||||||
|
}(), wantError: `manifest_file "metadata.json"`},
|
||||||
|
{name: "cleanable manifest", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["manifest_file"] = "./manifest.json"
|
||||||
|
return value
|
||||||
|
}(), wantError: `manifest_file "./manifest.json"`},
|
||||||
|
{name: "renamed rejections", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["rejected_file"] = "rejections.json"
|
||||||
|
return value
|
||||||
|
}(), wantError: `rejected_file "rejections.json"`},
|
||||||
|
{name: "renamed warnings", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["warnings_file"] = "diagnostics/warnings.json"
|
||||||
|
return value
|
||||||
|
}(), wantError: `warnings_file "diagnostics/warnings.json"`},
|
||||||
|
{name: "duplicate lane", indexValue: validIndexValue([]any{
|
||||||
|
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
|
||||||
|
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
|
||||||
|
})},
|
||||||
|
{name: "absolute logical path", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "/tmp/npc.json"}})},
|
||||||
|
{name: "lexical traversal", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../outside.json"}})},
|
||||||
|
{name: "root prefix confusion", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../bundle-other/npc.json"}})},
|
||||||
|
{name: "file symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "lanes/npc.json"}}), prepare: func(t *testing.T, bundle string) {
|
||||||
|
if err := os.Symlink(filepath.Join(bundle, "manifest.json"), filepath.Join(bundle, "lanes", "npc.json")); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "directory symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "linked/npc.json"}}), prepare: func(t *testing.T, bundle string) {
|
||||||
|
if err := os.Symlink(filepath.Join(bundle, "lanes"), filepath.Join(bundle, "linked")); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "missing management file", indexValue: validIndexValue([]any{}), prepare: func(t *testing.T, bundle string) {
|
||||||
|
if err := os.Remove(filepath.Join(bundle, "manifest.json")); err != nil {
|
||||||
|
t.Fatalf("Remove(manifest) error = %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "incomplete pipeline descriptor", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["chunk_map"] = map[string]any{"artifact_kind": "chunk_map", "file": "chunk-map.json"}
|
||||||
|
return value
|
||||||
|
}()},
|
||||||
|
{name: "pipeline descriptor escape", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["evidence_context"] = map[string]any{
|
||||||
|
"artifact_kind": "evidence_context", "file": "../evidence.json", "media_type": "application/json",
|
||||||
|
"schema_id": "evidence", "schema_name": "Evidence", "schema_version": "v1",
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}()},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
bundle := createBundleSkeleton(t)
|
||||||
|
indexPath := filepath.Join(bundle, "index.json")
|
||||||
|
if raw, ok := test.indexValue.(json.RawMessage); ok {
|
||||||
|
if err := os.WriteFile(indexPath, raw, 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(index) error = %v", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
writeJSONFile(t, indexPath, test.indexValue)
|
||||||
|
}
|
||||||
|
if test.prepare != nil {
|
||||||
|
test.prepare(t, bundle)
|
||||||
|
}
|
||||||
|
if _, err := loadIndex(bundle, indexPath); err == nil {
|
||||||
|
t.Fatal("loadIndex() error = nil, want failure")
|
||||||
|
} else if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
|
||||||
|
t.Fatalf("loadIndex() error = %v, want fragment %q", err, test.wantError)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
bundle := createBundleSkeleton(t)
|
||||||
|
oversizedIndex := filepath.Join(bundle, "index.json")
|
||||||
|
if err := os.WriteFile(oversizedIndex, []byte(strings.Repeat("x", maxIndexBytes+1)), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(oversized index) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := loadIndex(bundle, oversizedIndex); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("loadIndex(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
rejectedPath := filepath.Join(root, "rejected.json")
|
||||||
|
warningsPath := filepath.Join(root, "warnings.json")
|
||||||
|
writeJSONFile(t, rejectedPath, map[string]any{"rejected": []any{map[string]any{
|
||||||
|
"stage": "validate", "lane_id": "spells", "reason_code": "invalid", "message": "do not retain this", "future": true,
|
||||||
|
}}, "future": true})
|
||||||
|
writeJSONFile(t, warningsPath, map[string]any{"warnings": []any{map[string]any{
|
||||||
|
"scope": "lane:spells", "reason_code": "bounded", "message": "do not retain this", "future": true,
|
||||||
|
}}, "future": true})
|
||||||
|
rejections, err := loadRejections(rejectedPath)
|
||||||
|
if err != nil || len(rejections) != 1 || rejections[0].ReasonCode != "invalid" {
|
||||||
|
t.Fatalf("loadRejections() = %#v, %v", rejections, err)
|
||||||
|
}
|
||||||
|
warnings, err := loadWarnings(warningsPath)
|
||||||
|
if err != nil || len(warnings) != 1 || warnings[0].Scope != "lane:spells" {
|
||||||
|
t.Fatalf("loadWarnings() = %#v, %v", warnings, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, path := range map[string]string{"rejections": rejectedPath, "warnings": warningsPath} {
|
||||||
|
t.Run("malformed "+name, func(t *testing.T) {
|
||||||
|
if err := os.WriteFile(path, []byte("{"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if name == "rejections" {
|
||||||
|
_, err = loadRejections(path)
|
||||||
|
} else {
|
||||||
|
_, err = loadWarnings(path)
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("summary decoder error = nil")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
oversized := filepath.Join(root, "oversized.json")
|
||||||
|
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxSummaryBytes+1)), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := loadWarnings(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("loadWarnings(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := loadRejections(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("loadRejections(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
|
||||||
|
req := RunRequest{PipelineID: "pipeline"}
|
||||||
|
want := RunResult{BundleRoot: "/bundle"}
|
||||||
|
fake := &FakeRunner{Result: want}
|
||||||
|
got, err := fake.Run(context.Background(), req)
|
||||||
|
if err != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(fake.Requests, []RunRequest{req}) {
|
||||||
|
t.Fatalf("Run() = %#v, %v; requests = %#v", got, err, fake.Requests)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantErr := errors.New("configured failure")
|
||||||
|
fake.Err = wantErr
|
||||||
|
if _, err := fake.Run(context.Background(), req); !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("Run(configured error) = %v", err)
|
||||||
|
}
|
||||||
|
canceled, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
before := len(fake.Requests)
|
||||||
|
if _, err := fake.Run(canceled, req); !errors.Is(err, context.Canceled) || len(fake.Requests) != before {
|
||||||
|
t.Fatalf("Run(canceled) error = %v; requests = %d", err, len(fake.Requests))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validRunRequest(t *testing.T) RunRequest {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
configPath := filepath.Join(root, "notarius.yml")
|
||||||
|
inputPath := filepath.Join(root, "input.json")
|
||||||
|
outputRoot := filepath.Join(root, "outputs")
|
||||||
|
workingDirectory := filepath.Join(root, "work")
|
||||||
|
diagnostics := filepath.Join(root, "diagnostics")
|
||||||
|
for _, directory := range []string{outputRoot, workingDirectory, diagnostics} {
|
||||||
|
if err := os.Mkdir(directory, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(%q) error = %v", directory, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(config) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(inputPath, []byte("{}\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(input) error = %v", err)
|
||||||
|
}
|
||||||
|
return RunRequest{
|
||||||
|
Binary: "notarius", ConfigPath: configPath, PipelineID: "dnd-session", InputPath: inputPath,
|
||||||
|
OutputRoot: outputRoot, WorkingDirectory: workingDirectory,
|
||||||
|
ReceiptPath: filepath.Join(diagnostics, "receipt.json"), LogPath: filepath.Join(diagnostics, "stderr.log"),
|
||||||
|
Timeout: time.Second,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
|
||||||
|
t.Helper()
|
||||||
|
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
|
||||||
|
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(bundle) error = %v", err)
|
||||||
|
}
|
||||||
|
for path, data := range map[string]string{
|
||||||
|
"manifest.json": `{}`,
|
||||||
|
"lanes/npc.json": `{}`,
|
||||||
|
"chunk-map.json": `{}`,
|
||||||
|
"evidence-context.json": `{}`,
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(path)), []byte(data), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rejection := map[string]any{"stage": "validate", "lane_id": "spells", "reason_code": "invalid_spell", "message": strings.Repeat("external detail", 20)}
|
||||||
|
warning := map[string]any{"scope": "lane:npc-registry", "reason_code": "normalized_name", "message": strings.Repeat("external warning", 20)}
|
||||||
|
if includeUnknown {
|
||||||
|
rejection["future"] = true
|
||||||
|
warning["future"] = true
|
||||||
|
}
|
||||||
|
writeJSONFile(t, filepath.Join(bundle, "rejected.json"), map[string]any{"rejected": []any{rejection}, "future": true})
|
||||||
|
writeJSONFile(t, filepath.Join(bundle, "warnings.json"), map[string]any{"warnings": []any{warning}, "future": true})
|
||||||
|
index := validIndexValue([]any{map[string]any{
|
||||||
|
"lane_id": "npc-registry", "file": "lanes/npc.json", "media_type": "application/json",
|
||||||
|
"module_key": "dnd/npc-registry", "schema_id": "notarius.dnd.npc_registry",
|
||||||
|
"schema_name": "NPCRegistry", "schema_version": "v1", "future": true,
|
||||||
|
}})
|
||||||
|
index["chunk_map"] = map[string]any{
|
||||||
|
"artifact_kind": "chunk_map", "file": "chunk-map.json", "media_type": "application/json",
|
||||||
|
"schema_id": "notarius.chunk_map", "schema_name": "ChunkMap", "schema_version": "v1", "future": true,
|
||||||
|
}
|
||||||
|
index["evidence_context"] = map[string]any{
|
||||||
|
"artifact_kind": "evidence_context", "file": "evidence-context.json", "media_type": "application/json",
|
||||||
|
"schema_id": "notarius.evidence_context", "schema_name": "EvidenceContext", "schema_version": "v1", "future": true,
|
||||||
|
}
|
||||||
|
index["future"] = true
|
||||||
|
writeJSONFile(t, filepath.Join(bundle, "index.json"), index)
|
||||||
|
receipt := map[string]any{
|
||||||
|
"schema_version": ReceiptSchemaVersion, "run_id": "notarius-run-1", "pipeline_id": req.PipelineID,
|
||||||
|
"output_directory": bundle, "index_file": "index.json", "normalized_output_count": 1,
|
||||||
|
"rejected_output_count": 1, "warning_count": 1, "validation_status": "rejected",
|
||||||
|
}
|
||||||
|
if includeUnknown {
|
||||||
|
receipt["future"] = true
|
||||||
|
}
|
||||||
|
writeJSONFile(t, req.ReceiptPath, receipt)
|
||||||
|
}
|
||||||
|
|
||||||
|
func createBundleSkeleton(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
bundle := filepath.Join(t.TempDir(), "bundle")
|
||||||
|
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(bundle) error = %v", err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"manifest.json", "rejected.json", "warnings.json", "lanes/npc.json", "chunk-map.json"} {
|
||||||
|
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(name)), []byte("{}"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q) error = %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return bundle
|
||||||
|
}
|
||||||
|
|
||||||
|
func validIndexValue(lanes []any) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"manifest_file": "manifest.json", "output_files": lanes,
|
||||||
|
"rejected_file": "rejected.json", "warnings_file": "warnings.json",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeJSONFile(t *testing.T, path string, value any) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeShellScript(t *testing.T, body string) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(t.TempDir(), "notarius-helper")
|
||||||
|
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||||
|
t.Fatalf("WriteFile(script) error = %v", err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertTextFile(t *testing.T, path, want string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
if string(data) != want {
|
||||||
|
t.Fatalf("ReadFile(%q) = %q, want %q", path, string(data), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneMap(source map[string]any) map[string]any {
|
||||||
|
result := make(map[string]any, len(source))
|
||||||
|
for key, value := range source {
|
||||||
|
result[key] = value
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NoopRunner is a deterministic no-op scriptorium adapter.
|
// NoopRunner is a deterministic no-op scriptorium adapter.
|
||||||
@@ -142,7 +143,7 @@ func (f *FakeRunner) RenderArtifact(ctx context.Context, req RenderArtifactReque
|
|||||||
|
|
||||||
func materializeRunPlaceholders(req RunArtifactRequest) error {
|
func materializeRunPlaceholders(req RunArtifactRequest) error {
|
||||||
if req.OutputPath != "" {
|
if req.OutputPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write run output %q: %w", req.OutputPath, err)
|
return fmt.Errorf("write run output %q: %w", req.OutputPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -154,17 +155,17 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
|
|||||||
"prompt_id": req.PromptID,
|
"prompt_id": req.PromptID,
|
||||||
"output_path": req.OutputPath,
|
"output_path": req.OutputPath,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -173,7 +174,7 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
|
|||||||
|
|
||||||
func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
||||||
if req.OutputPath != "" {
|
if req.OutputPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write render output %q: %w", req.OutputPath, err)
|
return fmt.Errorf("write render output %q: %w", req.OutputPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -185,17 +186,17 @@ func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
|||||||
"prompt_id": req.PromptID,
|
"prompt_id": req.PromptID,
|
||||||
"output_path": req.OutputPath,
|
"output_path": req.OutputPath,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,8 +9,12 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// MaxOutputFileBytes bounds one Scriptorium artifact result.
|
||||||
|
const MaxOutputFileBytes int64 = 64 * 1024 * 1024
|
||||||
|
|
||||||
// SubprocessRunner invokes Scriptorium through its public CLI.
|
// SubprocessRunner invokes Scriptorium through its public CLI.
|
||||||
type SubprocessRunner struct{}
|
type SubprocessRunner struct{}
|
||||||
|
|
||||||
@@ -52,13 +56,17 @@ func (r *SubprocessRunner) RunArtifact(ctx context.Context, req RunArtifactReque
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
envOverrides, sensitiveNames := credentialEnvironment(req.APIKeyEnv)
|
||||||
runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{
|
runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{
|
||||||
Executable: req.Binary,
|
Executable: req.Binary,
|
||||||
Args: args,
|
Args: args,
|
||||||
WorkingDir: req.WorkingDir,
|
WorkingDir: req.WorkingDir,
|
||||||
Timeout: req.Timeout,
|
Timeout: req.Timeout,
|
||||||
StdoutLogPath: req.StdoutLogPath,
|
EnvOverrides: envOverrides,
|
||||||
StderrLogPath: req.StderrLogPath,
|
SensitiveEnvNames: sensitiveNames,
|
||||||
|
DiagnosticOwner: "scriptorium",
|
||||||
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
})
|
})
|
||||||
|
|
||||||
result := ArtifactResult{
|
result := ArtifactResult{
|
||||||
@@ -134,13 +142,17 @@ func (r *SubprocessRunner) RenderArtifact(ctx context.Context, req RenderArtifac
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
envOverrides, sensitiveNames := credentialEnvironment(req.APIKeyEnv)
|
||||||
runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{
|
runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{
|
||||||
Executable: req.Binary,
|
Executable: req.Binary,
|
||||||
Args: args,
|
Args: args,
|
||||||
WorkingDir: req.WorkingDir,
|
WorkingDir: req.WorkingDir,
|
||||||
Timeout: req.Timeout,
|
Timeout: req.Timeout,
|
||||||
StdoutLogPath: req.StdoutLogPath,
|
EnvOverrides: envOverrides,
|
||||||
StderrLogPath: req.StderrLogPath,
|
SensitiveEnvNames: sensitiveNames,
|
||||||
|
DiagnosticOwner: "scriptorium",
|
||||||
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
})
|
})
|
||||||
|
|
||||||
result := ArtifactResult{
|
result := ArtifactResult{
|
||||||
@@ -223,6 +235,15 @@ func validateCommonRunRequest(
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func credentialEnvironment(apiKeyEnv string) (map[string]string, []string) {
|
||||||
|
name := strings.TrimSpace(apiKeyEnv)
|
||||||
|
if name == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
value, _ := os.LookupEnv(name)
|
||||||
|
return map[string]string{name: value}, []string{name}
|
||||||
|
}
|
||||||
|
|
||||||
func buildRunArgs(req RunArtifactRequest) []string {
|
func buildRunArgs(req RunArtifactRequest) []string {
|
||||||
args := []string{"run", "--prompt", strings.TrimSpace(req.PromptID)}
|
args := []string{"run", "--prompt", strings.TrimSpace(req.PromptID)}
|
||||||
if cfgPath := strings.TrimSpace(req.ConfigPath); cfgPath != "" {
|
if cfgPath := strings.TrimSpace(req.ConfigPath); cfgPath != "" {
|
||||||
@@ -321,18 +342,15 @@ func writeInvocationConfig(path string, payload invocationPayload) error {
|
|||||||
"render_format": payload.RenderFormat,
|
"render_format": payload.RenderFormat,
|
||||||
"render_prompt_logged": payload.RenderPromptStore,
|
"render_prompt_logged": payload.RenderPromptStore,
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(path, data, 0o644)
|
return subprocess.WriteYAMLAtomic(path, data, fileops.WorkspaceFileMode)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateNonEmptyOutput(path string) error {
|
func validateNonEmptyOutput(path string) error {
|
||||||
info, err := os.Stat(path)
|
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("stat file: %w", err)
|
return fmt.Errorf("scriptorium artifact output exceeds or cannot be read within %d-byte limit: %w", MaxOutputFileBytes, err)
|
||||||
}
|
}
|
||||||
if info.IsDir() {
|
if len(data) == 0 {
|
||||||
return fmt.Errorf("path is a directory")
|
|
||||||
}
|
|
||||||
if info.Size() <= 0 {
|
|
||||||
return fmt.Errorf("file is empty")
|
return fmt.Errorf("file is empty")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NoopRunner is a deterministic no-op seriatim adapter.
|
// NoopRunner is a deterministic no-op seriatim adapter.
|
||||||
@@ -260,7 +261,7 @@ func (f *FakeRunner) Render(ctx context.Context, req RenderRequest) (RenderResul
|
|||||||
|
|
||||||
func materializePlaceholders(req MergeRequest) error {
|
func materializePlaceholders(req MergeRequest) error {
|
||||||
if req.OutputMergedTranscriptPath != "" {
|
if req.OutputMergedTranscriptPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write merged transcript %q: %w", req.OutputMergedTranscriptPath, err)
|
return fmt.Errorf("write merged transcript %q: %w", req.OutputMergedTranscriptPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -271,22 +272,22 @@ func materializePlaceholders(req MergeRequest) error {
|
|||||||
"input_transcript_paths": req.InputTranscriptPaths,
|
"input_transcript_paths": req.InputTranscriptPaths,
|
||||||
"output_path": req.OutputMergedTranscriptPath,
|
"output_path": req.OutputMergedTranscriptPath,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.ReportPath != "" {
|
if req.ReportPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -295,7 +296,7 @@ func materializePlaceholders(req MergeRequest) error {
|
|||||||
|
|
||||||
func materializeTrimPlaceholders(req TrimRequest) error {
|
func materializeTrimPlaceholders(req TrimRequest) error {
|
||||||
if req.OutputTrimmedPath != "" {
|
if req.OutputTrimmedPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write trimmed transcript %q: %w", req.OutputTrimmedPath, err)
|
return fmt.Errorf("write trimmed transcript %q: %w", req.OutputTrimmedPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -308,17 +309,17 @@ func materializeTrimPlaceholders(req TrimRequest) error {
|
|||||||
"output_path": req.OutputTrimmedPath,
|
"output_path": req.OutputTrimmedPath,
|
||||||
"keep_selector": req.KeepSelector,
|
"keep_selector": req.KeepSelector,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -327,7 +328,7 @@ func materializeTrimPlaceholders(req TrimRequest) error {
|
|||||||
|
|
||||||
func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||||
if req.OutputNormalizedPath != "" {
|
if req.OutputNormalizedPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write normalized transcript %q: %w", req.OutputNormalizedPath, err)
|
return fmt.Errorf("write normalized transcript %q: %w", req.OutputNormalizedPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -343,22 +344,22 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
|||||||
if req.ReportPath != "" {
|
if req.ReportPath != "" {
|
||||||
payload["report_path"] = req.ReportPath
|
payload["report_path"] = req.ReportPath
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.ReportPath != "" {
|
if req.ReportPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -367,7 +368,7 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
|||||||
|
|
||||||
func materializeRenderPlaceholders(req RenderRequest) error {
|
func materializeRenderPlaceholders(req RenderRequest) error {
|
||||||
if req.OutputRenderedPath != "" {
|
if req.OutputRenderedPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err)
|
return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -384,17 +385,17 @@ func materializeRenderPlaceholders(req RenderRequest) error {
|
|||||||
"include_segment_ids": req.IncludeSegmentIDs,
|
"include_segment_ids": req.IncludeSegmentIDs,
|
||||||
"include_metadata": req.IncludeMetadata,
|
"include_metadata": req.IncludeMetadata,
|
||||||
}
|
}
|
||||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StdoutLogPath != "" {
|
if req.StdoutLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.StderrLogPath != "" {
|
if req.StderrLogPath != "" {
|
||||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), 0o644); err != nil {
|
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,15 +4,18 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// MaxOutputFileBytes bounds each Seriatim JSON or rendered-text result.
|
||||||
|
const MaxOutputFileBytes int64 = 64 * 1024 * 1024
|
||||||
|
|
||||||
// EnvConfig defines optional Seriatim environment tuning values.
|
// EnvConfig defines optional Seriatim environment tuning values.
|
||||||
type EnvConfig struct {
|
type EnvConfig struct {
|
||||||
OverlapWordRunGap *float64
|
OverlapWordRunGap *float64
|
||||||
@@ -129,12 +132,13 @@ func (r *SubprocessRunner) Run(ctx context.Context, req MergeRequest) (MergeResu
|
|||||||
}
|
}
|
||||||
|
|
||||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||||
Executable: r.binary,
|
Executable: r.binary,
|
||||||
Args: args,
|
Args: args,
|
||||||
Timeout: r.timeout,
|
Timeout: r.timeout,
|
||||||
EnvOverrides: env,
|
EnvOverrides: env,
|
||||||
StdoutLogPath: req.StdoutLogPath,
|
DiagnosticOwner: "seriatim",
|
||||||
StderrLogPath: req.StderrLogPath,
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return MergeResult{
|
return MergeResult{
|
||||||
@@ -232,11 +236,12 @@ func (r *SubprocessRunner) Trim(ctx context.Context, req TrimRequest) (TrimResul
|
|||||||
}
|
}
|
||||||
|
|
||||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||||
Executable: binary,
|
Executable: binary,
|
||||||
Args: args,
|
Args: args,
|
||||||
Timeout: timeout,
|
Timeout: timeout,
|
||||||
StdoutLogPath: req.StdoutLogPath,
|
DiagnosticOwner: "seriatim",
|
||||||
StderrLogPath: req.StderrLogPath,
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return TrimResult{
|
return TrimResult{
|
||||||
@@ -320,11 +325,12 @@ func (r *SubprocessRunner) Normalize(ctx context.Context, req NormalizeRequest)
|
|||||||
}
|
}
|
||||||
|
|
||||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||||
Executable: binary,
|
Executable: binary,
|
||||||
Args: args,
|
Args: args,
|
||||||
Timeout: timeout,
|
Timeout: timeout,
|
||||||
StdoutLogPath: req.StdoutLogPath,
|
DiagnosticOwner: "seriatim",
|
||||||
StderrLogPath: req.StderrLogPath,
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return NormalizeResult{
|
return NormalizeResult{
|
||||||
@@ -425,11 +431,12 @@ func (r *SubprocessRunner) Render(ctx context.Context, req RenderRequest) (Rende
|
|||||||
}
|
}
|
||||||
|
|
||||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||||
Executable: binary,
|
Executable: binary,
|
||||||
Args: args,
|
Args: args,
|
||||||
Timeout: timeout,
|
Timeout: timeout,
|
||||||
StdoutLogPath: req.StdoutLogPath,
|
DiagnosticOwner: "seriatim",
|
||||||
StderrLogPath: req.StderrLogPath,
|
StdoutLogPath: req.StdoutLogPath,
|
||||||
|
StderrLogPath: req.StderrLogPath,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return RenderResult{
|
return RenderResult{
|
||||||
@@ -546,7 +553,7 @@ func (r *SubprocessRunner) writeMergeInvocationConfig(req MergeRequest, args []s
|
|||||||
payload["coalesce_gap"] = *r.coalesceGap
|
payload["coalesce_gap"] = *r.coalesceGap
|
||||||
}
|
}
|
||||||
|
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildTrimArgs(req TrimRequest) []string {
|
func buildTrimArgs(req TrimRequest) []string {
|
||||||
@@ -598,7 +605,7 @@ func writeTrimInvocationConfig(req TrimRequest, args []string, binary string, ti
|
|||||||
"output_path": req.OutputTrimmedPath,
|
"output_path": req.OutputTrimmedPath,
|
||||||
"keep_selector": req.KeepSelector,
|
"keep_selector": req.KeepSelector,
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary string, timeout time.Duration, outputSchema string) error {
|
func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary string, timeout time.Duration, outputSchema string) error {
|
||||||
@@ -613,7 +620,7 @@ func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary
|
|||||||
"output_schema": outputSchema,
|
"output_schema": outputSchema,
|
||||||
"report_path": req.ReportPath,
|
"report_path": req.ReportPath,
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error {
|
func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error {
|
||||||
@@ -631,13 +638,13 @@ func writeRenderInvocationConfig(req RenderRequest, args []string, binary string
|
|||||||
"include_segment_ids": req.IncludeSegmentIDs,
|
"include_segment_ids": req.IncludeSegmentIDs,
|
||||||
"include_metadata": req.IncludeMetadata,
|
"include_metadata": req.IncludeMetadata,
|
||||||
}
|
}
|
||||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateJSONFile(path string) error {
|
func validateJSONFile(path string) error {
|
||||||
data, err := os.ReadFile(path)
|
data, err := readSeriatimResult(path, "JSON output")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read file: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
var v any
|
var v any
|
||||||
if err := json.Unmarshal(data, &v); err != nil {
|
if err := json.Unmarshal(data, &v); err != nil {
|
||||||
@@ -647,9 +654,9 @@ func validateJSONFile(path string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateJSONFileWithSegments(path string) error {
|
func validateJSONFileWithSegments(path string) error {
|
||||||
data, err := os.ReadFile(path)
|
data, err := readSeriatimResult(path, "transcript JSON output")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read file: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var payload map[string]any
|
var payload map[string]any
|
||||||
@@ -668,9 +675,9 @@ func validateJSONFileWithSegments(path string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateNonEmptyTextFile(path string) error {
|
func validateNonEmptyTextFile(path string) error {
|
||||||
data, err := os.ReadFile(path)
|
data, err := readSeriatimResult(path, "rendered text output")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read file: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
if len(data) == 0 {
|
if len(data) == 0 {
|
||||||
return fmt.Errorf("file is empty")
|
return fmt.Errorf("file is empty")
|
||||||
@@ -683,3 +690,11 @@ func validateNonEmptyTextFile(path string) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func readSeriatimResult(path, category string) ([]byte, error) {
|
||||||
|
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("seriatim %s exceeds or cannot be read within %d-byte limit: %w", category, MaxOutputFileBytes, err)
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|||||||
90
internal/adapters/storage/bounded_read.go
Normal file
90
internal/adapters/storage/bounded_read.go
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ReadLimitError reports that a remote object exceeded its caller-owned read
|
||||||
|
// limit. The limit is enforced against both available object metadata and the
|
||||||
|
// bytes returned by the opened object body.
|
||||||
|
type ReadLimitError struct {
|
||||||
|
Key string
|
||||||
|
Limit int64
|
||||||
|
Observed int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ReadLimitError) Error() string {
|
||||||
|
return fmt.Sprintf("object %q exceeds %d-byte read limit (observed at least %d bytes)", e.Key, e.Limit, e.Observed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadObjectBounded opens one object version and retains at most maxBytes of
|
||||||
|
// its content. Object metadata may reject an oversized body early, but a
|
||||||
|
// limit-plus-one read always enforces the boundary when transfer begins.
|
||||||
|
func ReadObjectBounded(ctx context.Context, store ObjectStore, key string, maxBytes int64) (info ObjectInfo, data []byte, err error) {
|
||||||
|
key = strings.TrimSpace(key)
|
||||||
|
if store == nil {
|
||||||
|
return ObjectInfo{}, nil, fmt.Errorf("read bounded object: store is required")
|
||||||
|
}
|
||||||
|
if key == "" {
|
||||||
|
return ObjectInfo{}, nil, fmt.Errorf("read bounded object: key is required")
|
||||||
|
}
|
||||||
|
if maxBytes <= 0 || maxBytes == math.MaxInt64 {
|
||||||
|
return ObjectInfo{}, nil, fmt.Errorf("read bounded object %q: limit must be between 1 and %d bytes", key, int64(math.MaxInt64-1))
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return ObjectInfo{}, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
info, body, err := store.Read(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, nil, err
|
||||||
|
}
|
||||||
|
if body == nil {
|
||||||
|
return ObjectInfo{}, nil, fmt.Errorf("read bounded object %q: store returned no body", key)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if closeErr := body.Close(); closeErr != nil {
|
||||||
|
data = nil
|
||||||
|
err = errors.Join(err, fmt.Errorf("close object %q: %w", key, closeErr))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if info.Size > maxBytes {
|
||||||
|
return info, nil, &ReadLimitError{Key: key, Limit: maxBytes, Observed: info.Size}
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err = io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, maxBytes+1))
|
||||||
|
if err != nil {
|
||||||
|
return info, nil, err
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return info, nil, err
|
||||||
|
}
|
||||||
|
if int64(len(data)) > maxBytes {
|
||||||
|
return info, nil, &ReadLimitError{Key: key, Limit: maxBytes, Observed: int64(len(data))}
|
||||||
|
}
|
||||||
|
return info, data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type contextReader struct {
|
||||||
|
ctx context.Context
|
||||||
|
reader io.Reader
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r contextReader) Read(p []byte) (int, error) {
|
||||||
|
if err := r.ctx.Err(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
n, err := r.reader.Read(p)
|
||||||
|
if err == nil {
|
||||||
|
if contextErr := r.ctx.Err(); contextErr != nil {
|
||||||
|
return n, contextErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
135
internal/adapters/storage/bounded_read_test.go
Normal file
135
internal/adapters/storage/bounded_read_test.go
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestReadObjectBoundedAcceptsExactLimitWithAbsentSizeMetadata(t *testing.T) {
|
||||||
|
body := &trackingReadCloser{reader: bytes.NewReader([]byte("12345678")), chunkSize: 2}
|
||||||
|
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
|
||||||
|
return ObjectInfo{Key: "control.json", ETag: "generation"}, body, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
info, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadObjectBounded() error = %v", err)
|
||||||
|
}
|
||||||
|
if string(data) != "12345678" || info.ETag != "generation" {
|
||||||
|
t.Fatalf("ReadObjectBounded() = (%#v, %q), want opened object metadata and bytes", info, data)
|
||||||
|
}
|
||||||
|
if !body.closed {
|
||||||
|
t.Fatal("object body was not closed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadObjectBoundedRejectsLimitPlusOneDespiteMissingOrInaccurateMetadata(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
metadataSize int64
|
||||||
|
}{
|
||||||
|
{name: "missing", metadataSize: 0},
|
||||||
|
{name: "inaccurate", metadataSize: 2},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
body := &trackingReadCloser{reader: bytes.NewReader([]byte("123456789")), chunkSize: 1}
|
||||||
|
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
|
||||||
|
return ObjectInfo{Key: "control.json", Size: test.metadataSize}, body, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
_, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
|
||||||
|
var limitErr *ReadLimitError
|
||||||
|
if !errors.As(err, &limitErr) {
|
||||||
|
t.Fatalf("ReadObjectBounded() error = %v, want ReadLimitError", err)
|
||||||
|
}
|
||||||
|
if data != nil || body.bytesRead != 9 || !body.closed {
|
||||||
|
t.Fatalf("data=%q bytes read=%d closed=%t, want nil, 9, true", data, body.bytesRead, body.closed)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadObjectBoundedRejectsOversizedMetadataBeforeTransfer(t *testing.T) {
|
||||||
|
body := &trackingReadCloser{reader: bytes.NewReader([]byte("small"))}
|
||||||
|
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
|
||||||
|
return ObjectInfo{Key: "control.json", Size: 9}, body, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
_, _, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
|
||||||
|
var limitErr *ReadLimitError
|
||||||
|
if !errors.As(err, &limitErr) {
|
||||||
|
t.Fatalf("ReadObjectBounded() error = %v, want ReadLimitError", err)
|
||||||
|
}
|
||||||
|
if body.bytesRead != 0 || !body.closed {
|
||||||
|
t.Fatalf("bytes read=%d closed=%t, want zero-byte transfer and closed body", body.bytesRead, body.closed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadObjectBoundedPropagatesCancellationAndClosesBody(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
body := &trackingReadCloser{reader: bytes.NewReader([]byte("12345678")), chunkSize: 1, afterRead: cancel}
|
||||||
|
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
|
||||||
|
return ObjectInfo{Key: "control.json"}, body, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
_, data, err := ReadObjectBounded(ctx, store, "control.json", 8)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("ReadObjectBounded() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
if data != nil || body.bytesRead != 1 || !body.closed {
|
||||||
|
t.Fatalf("data=%q bytes read=%d closed=%t, want nil, 1, true", data, body.bytesRead, body.closed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadObjectBoundedReturnsCloseFailure(t *testing.T) {
|
||||||
|
closeErr := errors.New("close failed")
|
||||||
|
body := &trackingReadCloser{reader: bytes.NewReader([]byte("ok")), closeErr: closeErr}
|
||||||
|
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
|
||||||
|
return ObjectInfo{Key: "control.json", Size: 2}, body, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
_, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
|
||||||
|
if !errors.Is(err, closeErr) || data != nil || !body.closed {
|
||||||
|
t.Fatalf("data=%q error=%v closed=%t, want close failure and no retained data", data, err, body.closed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type boundedReadStore struct {
|
||||||
|
ObjectStore
|
||||||
|
read func(context.Context, string) (ObjectInfo, io.ReadCloser, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *boundedReadStore) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
|
||||||
|
return s.read(ctx, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
type trackingReadCloser struct {
|
||||||
|
reader io.Reader
|
||||||
|
chunkSize int
|
||||||
|
afterRead func()
|
||||||
|
closeErr error
|
||||||
|
bytesRead int
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *trackingReadCloser) Read(p []byte) (int, error) {
|
||||||
|
if r.chunkSize > 0 && len(p) > r.chunkSize {
|
||||||
|
p = p[:r.chunkSize]
|
||||||
|
}
|
||||||
|
n, err := r.reader.Read(p)
|
||||||
|
r.bytesRead += n
|
||||||
|
if n > 0 && r.afterRead != nil {
|
||||||
|
r.afterRead()
|
||||||
|
r.afterRead = nil
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *trackingReadCloser) Close() error {
|
||||||
|
r.closed = true
|
||||||
|
return r.closeErr
|
||||||
|
}
|
||||||
23
internal/adapters/storage/download_writer.go
Normal file
23
internal/adapters/storage/download_writer.go
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
// WriterDownloader is implemented by storage backends that stream an object
|
||||||
|
// into a caller-owned file handle.
|
||||||
|
type WriterDownloader interface {
|
||||||
|
DownloadTo(ctx context.Context, key string, destination io.Writer) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadTo streams one object into destination. Destination-confined callers
|
||||||
|
// require this capability rather than granting a backend a mutable pathname.
|
||||||
|
func DownloadTo(ctx context.Context, store ObjectStore, key string, destination io.Writer) error {
|
||||||
|
writer, ok := store.(WriterDownloader)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("object store does not support handle-confined downloads")
|
||||||
|
}
|
||||||
|
return writer.DownloadTo(ctx, key, destination)
|
||||||
|
}
|
||||||
@@ -14,16 +14,15 @@ func NewObjectStoreFromConfig(ctx context.Context, cfg *config.Config) (ObjectSt
|
|||||||
return nil, fmt.Errorf("pipeline config is required")
|
return nil, fmt.Errorf("pipeline config is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.EqualFold(strings.TrimSpace(cfg.Pipeline.Storage.Backend), "s3") {
|
switch strings.ToLower(strings.TrimSpace(cfg.Pipeline.Storage.Backend)) {
|
||||||
|
case config.StorageBackendS3:
|
||||||
if cfg.Pipeline.Storage.S3 == nil {
|
if cfg.Pipeline.Storage.S3 == nil {
|
||||||
return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
|
return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
|
||||||
}
|
}
|
||||||
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
|
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
|
||||||
|
case "", config.StorageBackendLocal:
|
||||||
|
return nil, fmt.Errorf("no remote object store backend is configured")
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported pipeline.storage.backend %q", cfg.Pipeline.Storage.Backend)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Pipeline.Storage.S3 != nil && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
|
|
||||||
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("no remote object store backend is configured")
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -54,3 +54,35 @@ func TestNewObjectStoreFromConfigNoRemoteBackendConfigured(t *testing.T) {
|
|||||||
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
|
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNewObjectStoreFromConfigDoesNotInferS3FromProviderFields(t *testing.T) {
|
||||||
|
called := false
|
||||||
|
original := newS3Client
|
||||||
|
t.Cleanup(func() { newS3Client = original })
|
||||||
|
newS3Client = func(_ context.Context, _ s3ClientOptions) (s3API, error) {
|
||||||
|
called = true
|
||||||
|
return &fakeS3API{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{
|
||||||
|
Backend: config.StorageBackendLocal,
|
||||||
|
S3: &config.StorageS3Config{Bucket: "my-archive"},
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "no remote object store backend is configured") {
|
||||||
|
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
|
||||||
|
}
|
||||||
|
if called {
|
||||||
|
t.Fatal("NewObjectStoreFromConfig() constructed S3 from incidental provider fields")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewObjectStoreFromConfigRejectsUnknownBackend(t *testing.T) {
|
||||||
|
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{Backend: "s33"}},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "unsupported pipeline.storage.backend") {
|
||||||
|
t.Fatalf("NewObjectStoreFromConfig() error = %v, want unsupported-backend error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,25 +1,35 @@
|
|||||||
package storage
|
package storage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FakeBackend provides a deterministic in-memory object store for tests.
|
// FakeBackend provides a deterministic in-memory object store for tests.
|
||||||
type FakeBackend struct {
|
type FakeBackend struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
|
||||||
Objects map[string]FakeObject
|
Objects map[string]FakeObject
|
||||||
Uploads []FakeUploadCall
|
Uploads []FakeUploadCall
|
||||||
Downloads []FakeDownloadCall
|
Downloads []FakeDownloadCall
|
||||||
|
Reads []FakeReadCall
|
||||||
|
|
||||||
ListErr error
|
ListErr error
|
||||||
DownloadErr error
|
DownloadErr error
|
||||||
UploadErr error
|
UploadErr error
|
||||||
ExistsErr error
|
ExistsErr error
|
||||||
|
UploadHook func(FakeUploadCall) error
|
||||||
|
DownloadHook func(FakeDownloadCall) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// FakeUploadCall captures one upload invocation in call order.
|
// FakeUploadCall captures one upload invocation in call order.
|
||||||
@@ -33,6 +43,12 @@ type FakeUploadCall struct {
|
|||||||
type FakeDownloadCall struct {
|
type FakeDownloadCall struct {
|
||||||
Key string
|
Key string
|
||||||
LocalPath string
|
LocalPath string
|
||||||
|
Bytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// FakeReadCall captures one opened object in call order.
|
||||||
|
type FakeReadCall struct {
|
||||||
|
Key string
|
||||||
}
|
}
|
||||||
|
|
||||||
// FakeObject is a deterministic fake object-store record.
|
// FakeObject is a deterministic fake object-store record.
|
||||||
@@ -46,6 +62,12 @@ type FakeObject struct {
|
|||||||
|
|
||||||
// SeedObject inserts or replaces an object in the fake object store.
|
// SeedObject inserts or replaces an object in the fake object store.
|
||||||
func (f *FakeBackend) SeedObject(obj FakeObject) {
|
func (f *FakeBackend) SeedObject(obj FakeObject) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.seedObject(obj)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *FakeBackend) seedObject(obj FakeObject) {
|
||||||
if f.Objects == nil {
|
if f.Objects == nil {
|
||||||
f.Objects = map[string]FakeObject{}
|
f.Objects = map[string]FakeObject{}
|
||||||
}
|
}
|
||||||
@@ -53,6 +75,9 @@ func (f *FakeBackend) SeedObject(obj FakeObject) {
|
|||||||
obj.Key = key
|
obj.Key = key
|
||||||
obj.Data = append([]byte(nil), obj.Data...)
|
obj.Data = append([]byte(nil), obj.Data...)
|
||||||
obj.Metadata = copyMetadata(obj.Metadata)
|
obj.Metadata = copyMetadata(obj.Metadata)
|
||||||
|
if obj.ETag == "" {
|
||||||
|
obj.ETag = fakeObjectETag(obj.Data)
|
||||||
|
}
|
||||||
f.Objects[key] = obj
|
f.Objects[key] = obj
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -65,6 +90,8 @@ func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, er
|
|||||||
return nil, f.ListErr
|
return nil, f.ListErr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
f.mu.RLock()
|
||||||
|
defer f.mu.RUnlock()
|
||||||
normalizedPrefix := normalizeObjectKey(prefix)
|
normalizedPrefix := normalizeObjectKey(prefix)
|
||||||
keys := make([]string, 0, len(f.Objects))
|
keys := make([]string, 0, len(f.Objects))
|
||||||
for key := range f.Objects {
|
for key := range f.Objects {
|
||||||
@@ -87,34 +114,77 @@ func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, er
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Download writes one object to a local path.
|
// Read returns a stable object body and the generation observed with it.
|
||||||
func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error {
|
func (f *FakeBackend) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return ObjectInfo{}, nil, err
|
||||||
|
}
|
||||||
|
if f.DownloadErr != nil {
|
||||||
|
return ObjectInfo{}, nil, f.DownloadErr
|
||||||
|
}
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
f.mu.RLock()
|
||||||
|
obj, ok := f.Objects[normalizedKey]
|
||||||
|
if ok {
|
||||||
|
obj.Data = append([]byte(nil), obj.Data...)
|
||||||
|
obj.Metadata = copyMetadata(obj.Metadata)
|
||||||
|
}
|
||||||
|
f.mu.RUnlock()
|
||||||
|
if !ok {
|
||||||
|
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, os.ErrNotExist)
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
f.Reads = append(f.Reads, FakeReadCall{Key: normalizedKey})
|
||||||
|
f.mu.Unlock()
|
||||||
|
return ObjectInfo{Key: obj.Key, Size: int64(len(obj.Data)), ETag: obj.ETag, LastModified: obj.LastModified}, io.NopCloser(bytes.NewReader(obj.Data)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadTo writes one object to a caller-owned destination writer.
|
||||||
|
func (f *FakeBackend) DownloadTo(ctx context.Context, key string, destination io.Writer) error {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if f.DownloadErr != nil {
|
if f.DownloadErr != nil {
|
||||||
return f.DownloadErr
|
return f.DownloadErr
|
||||||
}
|
}
|
||||||
|
if destination == nil {
|
||||||
|
return fmt.Errorf("download object: destination writer is required")
|
||||||
|
}
|
||||||
|
if f.DownloadHook != nil {
|
||||||
|
if err := f.DownloadHook(FakeDownloadCall{Key: normalizeObjectKey(key)}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, source, err := f.Read(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer source.Close()
|
||||||
|
count, err := io.Copy(destination, source)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("download object %q: write destination: %w", key, err)
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
f.Downloads = append(f.Downloads, FakeDownloadCall{Key: normalizeObjectKey(key), Bytes: count})
|
||||||
|
f.mu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download writes one object to a local path.
|
||||||
|
func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error {
|
||||||
if strings.TrimSpace(localPath) == "" {
|
if strings.TrimSpace(localPath) == "" {
|
||||||
return fmt.Errorf("download object: local path is required")
|
return fmt.Errorf("download object: local path is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
obj, ok := f.Objects[normalizeObjectKey(key)]
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
|
|
||||||
}
|
|
||||||
f.Downloads = append(f.Downloads, FakeDownloadCall{
|
|
||||||
Key: normalizeObjectKey(key),
|
|
||||||
LocalPath: localPath,
|
|
||||||
})
|
|
||||||
|
|
||||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||||
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(localPath, obj.Data, 0o644); err != nil {
|
destination, err := os.Create(localPath)
|
||||||
return fmt.Errorf("download object %q: write local file: %w", key, err)
|
if err != nil {
|
||||||
|
return fmt.Errorf("download object %q: create local file: %w", key, err)
|
||||||
}
|
}
|
||||||
return nil
|
defer destination.Close()
|
||||||
|
return f.DownloadTo(ctx, key, destination)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Upload reads a local file and stores it under key.
|
// Upload reads a local file and stores it under key.
|
||||||
@@ -132,33 +202,117 @@ func (f *FakeBackend) Upload(ctx context.Context, localPath, key string, opts Up
|
|||||||
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
data, err := os.ReadFile(localPath)
|
file, err := os.Open(localPath)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
return f.uploadReader(ctx, file, key, opts, localPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadReader stores content provided by a caller-owned reader.
|
||||||
|
func (f *FakeBackend) UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error) {
|
||||||
|
return f.uploadReader(ctx, source, key, opts, "reader")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *FakeBackend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, localPath string) (ObjectInfo, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
if f.UploadErr != nil {
|
||||||
|
return ObjectInfo{}, f.UploadErr
|
||||||
|
}
|
||||||
|
if source == nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(key) == "" {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := io.ReadAll(source)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
normalizedKey := normalizeObjectKey(key)
|
normalizedKey := normalizeObjectKey(key)
|
||||||
f.Uploads = append(f.Uploads, FakeUploadCall{
|
call := FakeUploadCall{
|
||||||
LocalPath: localPath,
|
LocalPath: localPath,
|
||||||
Key: normalizedKey,
|
Key: normalizedKey,
|
||||||
Options: UploadOptions{
|
Options: UploadOptions{
|
||||||
Metadata: copyMetadata(opts.Metadata),
|
Metadata: copyMetadata(opts.Metadata),
|
||||||
ContentType: opts.ContentType,
|
ContentType: opts.ContentType,
|
||||||
},
|
},
|
||||||
})
|
|
||||||
now := time.Now().UTC()
|
|
||||||
obj := FakeObject{
|
|
||||||
Key: normalizedKey,
|
|
||||||
Data: data,
|
|
||||||
Metadata: copyMetadata(opts.Metadata),
|
|
||||||
LastModified: &now,
|
|
||||||
}
|
}
|
||||||
f.SeedObject(obj)
|
f.mu.Lock()
|
||||||
return ObjectInfo{
|
f.Uploads = append(f.Uploads, call)
|
||||||
Key: normalizedKey,
|
f.mu.Unlock()
|
||||||
Size: int64(len(data)),
|
if f.UploadHook != nil {
|
||||||
LastModified: &now,
|
if err := f.UploadHook(call); err != nil {
|
||||||
}, nil
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return f.storeUploadedObject(normalizedKey, data, opts), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadConditional atomically checks and replaces one mutable object.
|
||||||
|
func (f *FakeBackend) UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
if err := validateWriteCondition(condition); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
if f.UploadErr != nil {
|
||||||
|
return ObjectInfo{}, f.UploadErr
|
||||||
|
}
|
||||||
|
if source == nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
|
||||||
|
}
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
if normalizedKey == "" {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(source)
|
||||||
|
if err != nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
call := FakeUploadCall{Key: normalizedKey, Options: UploadOptions{Metadata: copyMetadata(opts.Metadata), ContentType: opts.ContentType}}
|
||||||
|
f.mu.Lock()
|
||||||
|
f.Uploads = append(f.Uploads, call)
|
||||||
|
f.mu.Unlock()
|
||||||
|
if f.UploadHook != nil {
|
||||||
|
if err := f.UploadHook(call); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
existing, found := f.Objects[normalizedKey]
|
||||||
|
if condition.RequireAbsent && found {
|
||||||
|
return ObjectInfo{}, ErrConditionNotMet
|
||||||
|
}
|
||||||
|
if expected := strings.TrimSpace(condition.MatchETag); expected != "" && (!found || existing.ETag != expected) {
|
||||||
|
return ObjectInfo{}, ErrConditionNotMet
|
||||||
|
}
|
||||||
|
return f.storeUploadedObjectLocked(normalizedKey, data, opts), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *FakeBackend) storeUploadedObject(key string, data []byte, opts UploadOptions) ObjectInfo {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return f.storeUploadedObjectLocked(key, data, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *FakeBackend) storeUploadedObjectLocked(key string, data []byte, opts UploadOptions) ObjectInfo {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
obj := FakeObject{Key: key, Data: append([]byte(nil), data...), Metadata: copyMetadata(opts.Metadata), LastModified: &now}
|
||||||
|
f.seedObject(obj)
|
||||||
|
return ObjectInfo{Key: key, Size: int64(len(data)), ETag: fakeObjectETag(data), LastModified: &now}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fakeObjectETag(data []byte) string {
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
}
|
}
|
||||||
|
|
||||||
// Exists checks object presence.
|
// Exists checks object presence.
|
||||||
@@ -169,7 +323,9 @@ func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
|||||||
if f.ExistsErr != nil {
|
if f.ExistsErr != nil {
|
||||||
return false, f.ExistsErr
|
return false, f.ExistsErr
|
||||||
}
|
}
|
||||||
|
f.mu.RLock()
|
||||||
_, ok := f.Objects[normalizeObjectKey(key)]
|
_, ok := f.Objects[normalizeObjectKey(key)]
|
||||||
|
f.mu.RUnlock()
|
||||||
return ok, nil
|
return ok, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -71,6 +71,21 @@ func TestFakeBackendUploadAndExists(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFakeBackendConditionalUploadRejectsStaleGeneration(t *testing.T) {
|
||||||
|
fake := &FakeBackend{}
|
||||||
|
fake.SeedObject(FakeObject{Key: "locks.yml", Data: []byte("old")})
|
||||||
|
old := fake.Objects["locks.yml"].ETag
|
||||||
|
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("new"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); err != nil {
|
||||||
|
t.Fatalf("UploadConditional() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("lost"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); !errors.Is(err, ErrConditionNotMet) {
|
||||||
|
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
||||||
|
}
|
||||||
|
if got := string(fake.Objects["locks.yml"].Data); got != "new" {
|
||||||
|
t.Fatalf("locks object = %q, want successful replacement preserved", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFakeBackendObjectErrors(t *testing.T) {
|
func TestFakeBackendObjectErrors(t *testing.T) {
|
||||||
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}
|
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,21 @@ package storage
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ErrConditionNotMet reports that an object changed or already existed before a
|
||||||
|
// conditional write could be committed.
|
||||||
|
var ErrConditionNotMet = errors.New("object write condition not met")
|
||||||
|
|
||||||
|
// ReaderUploader streams caller-owned, already-opened content to object storage.
|
||||||
|
// Callers retain source-selection and filesystem-confinement policy.
|
||||||
|
type ReaderUploader interface {
|
||||||
|
UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error)
|
||||||
|
}
|
||||||
|
|
||||||
// ObjectStore is a remote object storage boundary used by prepare, restore, and publish work.
|
// ObjectStore is a remote object storage boundary used by prepare, restore, and publish work.
|
||||||
//
|
//
|
||||||
// Key invariant:
|
// Key invariant:
|
||||||
@@ -12,8 +24,10 @@ import (
|
|||||||
// infer Narratio session semantics and do not prepend root prefixes.
|
// infer Narratio session semantics and do not prepend root prefixes.
|
||||||
type ObjectStore interface {
|
type ObjectStore interface {
|
||||||
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
|
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
|
||||||
|
Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error)
|
||||||
Download(ctx context.Context, key, localPath string) error
|
Download(ctx context.Context, key, localPath string) error
|
||||||
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
|
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
|
||||||
|
UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error)
|
||||||
Exists(ctx context.Context, key string) (bool, error)
|
Exists(ctx context.Context, key string) (bool, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -30,3 +44,10 @@ type UploadOptions struct {
|
|||||||
Metadata map[string]string
|
Metadata map[string]string
|
||||||
ContentType string
|
ContentType string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WriteCondition protects a mutable object update against a stale snapshot.
|
||||||
|
// Exactly one condition is required by UploadConditional.
|
||||||
|
type WriteCondition struct {
|
||||||
|
MatchETag string
|
||||||
|
RequireAbsent bool
|
||||||
|
}
|
||||||
|
|||||||
@@ -117,6 +117,7 @@ func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, erro
|
|||||||
normalizedPrefix := normalizeObjectKey(prefix)
|
normalizedPrefix := normalizeObjectKey(prefix)
|
||||||
out := make([]ObjectInfo, 0)
|
out := make([]ObjectInfo, 0)
|
||||||
var token *string
|
var token *string
|
||||||
|
seenTokens := map[string]struct{}{}
|
||||||
|
|
||||||
for {
|
for {
|
||||||
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||||
@@ -142,44 +143,78 @@ func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, erro
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil {
|
if !valueOrFalseBool(resp.IsTruncated) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
token = resp.NextContinuationToken
|
next := strings.TrimSpace(valueOrEmpty(resp.NextContinuationToken))
|
||||||
|
if next == "" {
|
||||||
|
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response has an empty continuation token", b.bucket, normalizedPrefix)
|
||||||
|
}
|
||||||
|
if _, repeated := seenTokens[next]; repeated {
|
||||||
|
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response repeated continuation token", b.bucket, normalizedPrefix)
|
||||||
|
}
|
||||||
|
seenTokens[next] = struct{}{}
|
||||||
|
token = &next
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Read retrieves an object together with the generation observed for its body.
|
||||||
|
func (b *S3Backend) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &b.bucket, Key: &normalizedKey})
|
||||||
|
if err != nil {
|
||||||
|
if isS3NotFound(err) {
|
||||||
|
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, os.ErrNotExist)
|
||||||
|
}
|
||||||
|
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
var lastModified *time.Time
|
||||||
|
if resp.LastModified != nil {
|
||||||
|
t := *resp.LastModified
|
||||||
|
lastModified = &t
|
||||||
|
}
|
||||||
|
return ObjectInfo{
|
||||||
|
Key: normalizedKey, Size: valueOrZeroInt64(resp.ContentLength),
|
||||||
|
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""), LastModified: lastModified,
|
||||||
|
}, resp.Body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadTo retrieves one object into the caller-owned destination writer.
|
||||||
|
func (b *S3Backend) DownloadTo(ctx context.Context, key string, destination io.Writer) error {
|
||||||
|
if destination == nil {
|
||||||
|
return fmt.Errorf("download object: destination writer is required")
|
||||||
|
}
|
||||||
|
_, body, err := b.Read(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("download object %q: %w", normalizeObjectKey(key), err)
|
||||||
|
}
|
||||||
|
defer body.Close()
|
||||||
|
|
||||||
|
if _, err := io.Copy(destination, body); err != nil {
|
||||||
|
return fmt.Errorf("download object %q: copy body: %w", normalizeObjectKey(key), err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Download retrieves one object to localPath, creating parent directories as needed.
|
// Download retrieves one object to localPath, creating parent directories as needed.
|
||||||
func (b *S3Backend) Download(ctx context.Context, key, localPath string) error {
|
func (b *S3Backend) Download(ctx context.Context, key, localPath string) error {
|
||||||
normalizedKey := normalizeObjectKey(key)
|
|
||||||
if strings.TrimSpace(localPath) == "" {
|
if strings.TrimSpace(localPath) == "" {
|
||||||
return fmt.Errorf("download object: local path is required")
|
return fmt.Errorf("download object: local path is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{
|
|
||||||
Bucket: &b.bucket,
|
|
||||||
Key: &normalizedKey,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("download object %q: %w", normalizedKey, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||||
return fmt.Errorf("download object %q: create parent directory: %w", normalizedKey, err)
|
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
||||||
}
|
}
|
||||||
dst, err := os.Create(localPath)
|
dst, err := os.Create(localPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("download object %q: create local file: %w", normalizedKey, err)
|
return fmt.Errorf("download object %q: create local file: %w", key, err)
|
||||||
}
|
}
|
||||||
defer dst.Close()
|
defer dst.Close()
|
||||||
|
if err := b.DownloadTo(ctx, key, dst); err != nil {
|
||||||
if _, err := io.Copy(dst, resp.Body); err != nil {
|
return err
|
||||||
return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err)
|
|
||||||
}
|
}
|
||||||
if err := dst.Sync(); err != nil {
|
if err := dst.Sync(); err != nil {
|
||||||
return fmt.Errorf("download object %q: sync local file: %w", normalizedKey, err)
|
return fmt.Errorf("download object %q: sync local file: %w", key, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -204,28 +239,65 @@ func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts Uplo
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
|
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
|
||||||
}
|
}
|
||||||
|
return b.uploadReader(ctx, file, key, opts, stat.Size(), WriteCondition{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadReader sends caller-owned content to key.
|
||||||
|
func (b *S3Backend) UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error) {
|
||||||
|
return b.uploadReader(ctx, source, key, opts, 0, WriteCondition{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadConditional uploads a mutable object only when its observed generation
|
||||||
|
// still matches, or when no object exists yet.
|
||||||
|
func (b *S3Backend) UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error) {
|
||||||
|
if err := validateWriteCondition(condition); err != nil {
|
||||||
|
return ObjectInfo{}, err
|
||||||
|
}
|
||||||
|
return b.uploadReader(ctx, source, key, opts, 0, condition)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, size int64, condition WriteCondition) (ObjectInfo, error) {
|
||||||
|
normalizedKey := normalizeObjectKey(key)
|
||||||
|
if source == nil {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
|
||||||
|
}
|
||||||
|
if normalizedKey == "" {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||||
|
}
|
||||||
|
|
||||||
input := &s3.PutObjectInput{
|
input := &s3.PutObjectInput{
|
||||||
Bucket: &b.bucket,
|
Bucket: &b.bucket,
|
||||||
Key: &normalizedKey,
|
Key: &normalizedKey,
|
||||||
Body: file,
|
Body: source,
|
||||||
Metadata: copyMetadata(opts.Metadata),
|
Metadata: copyMetadata(opts.Metadata),
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(opts.ContentType) != "" {
|
if strings.TrimSpace(opts.ContentType) != "" {
|
||||||
ct := strings.TrimSpace(opts.ContentType)
|
ct := strings.TrimSpace(opts.ContentType)
|
||||||
input.ContentType = &ct
|
input.ContentType = &ct
|
||||||
}
|
}
|
||||||
|
if condition.RequireAbsent {
|
||||||
|
wildcard := "*"
|
||||||
|
input.IfNoneMatch = &wildcard
|
||||||
|
} else if expected := strings.TrimSpace(condition.MatchETag); expected != "" {
|
||||||
|
input.IfMatch = &expected
|
||||||
|
}
|
||||||
|
|
||||||
resp, err := b.client.PutObject(ctx, input)
|
resp, err := b.client.PutObject(ctx, input)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
|
if isS3ConditionalConflict(err) {
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, ErrConditionNotMet)
|
||||||
|
}
|
||||||
|
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return ObjectInfo{
|
info := ObjectInfo{
|
||||||
Key: normalizedKey,
|
Key: normalizedKey,
|
||||||
Size: stat.Size(),
|
|
||||||
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""),
|
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""),
|
||||||
}, nil
|
}
|
||||||
|
if size > 0 {
|
||||||
|
info.Size = size
|
||||||
|
}
|
||||||
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Exists checks whether one object key exists.
|
// Exists checks whether one object key exists.
|
||||||
@@ -239,18 +311,43 @@ func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if isS3NotFound(err) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateWriteCondition(condition WriteCondition) error {
|
||||||
|
if condition.RequireAbsent == (strings.TrimSpace(condition.MatchETag) != "") {
|
||||||
|
return fmt.Errorf("conditional upload requires exactly one of MatchETag or RequireAbsent")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isS3NotFound(err error) bool {
|
||||||
var notFound *types.NotFound
|
var notFound *types.NotFound
|
||||||
if errors.As(err, ¬Found) {
|
if errors.As(err, ¬Found) {
|
||||||
return false, nil
|
return true
|
||||||
}
|
}
|
||||||
var apiErr smithy.APIError
|
var apiErr smithy.APIError
|
||||||
if errors.As(err, &apiErr) {
|
if errors.As(err, &apiErr) {
|
||||||
switch apiErr.ErrorCode() {
|
switch apiErr.ErrorCode() {
|
||||||
case "NotFound", "NoSuchKey", "404":
|
case "NotFound", "NoSuchKey", "404":
|
||||||
return false, nil
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func isS3ConditionalConflict(err error) bool {
|
||||||
|
var apiErr smithy.APIError
|
||||||
|
if errors.As(err, &apiErr) {
|
||||||
|
switch apiErr.ErrorCode() {
|
||||||
|
case "PreconditionFailed", "ConditionalRequestConflict", "412", "409":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func valueOrEmpty(v *string) string {
|
func valueOrEmpty(v *string) string {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package storage
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -17,34 +18,81 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type fakeS3API struct {
|
type fakeS3API struct {
|
||||||
listOut *s3.ListObjectsV2Output
|
listOut *s3.ListObjectsV2Output
|
||||||
listErr error
|
listOutputs []*s3.ListObjectsV2Output
|
||||||
|
listErr error
|
||||||
|
listCalls int
|
||||||
|
|
||||||
getBody io.ReadCloser
|
getBody io.ReadCloser
|
||||||
getErr error
|
getErr error
|
||||||
|
getSize *int64
|
||||||
|
getETag *string
|
||||||
|
getLastModified *time.Time
|
||||||
|
|
||||||
putOut *s3.PutObjectOutput
|
putOut *s3.PutObjectOutput
|
||||||
putErr error
|
putErr error
|
||||||
|
|
||||||
headErr error
|
headErr error
|
||||||
|
|
||||||
lastList *s3.ListObjectsV2Input
|
lastList *s3.ListObjectsV2Input
|
||||||
lastGet *s3.GetObjectInput
|
lastLists []*s3.ListObjectsV2Input
|
||||||
lastPut *s3.PutObjectInput
|
lastGet *s3.GetObjectInput
|
||||||
lastHead *s3.HeadObjectInput
|
lastPut *s3.PutObjectInput
|
||||||
|
lastHead *s3.HeadObjectInput
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
|
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
|
||||||
f.lastList = params
|
f.lastList = params
|
||||||
|
f.lastLists = append(f.lastLists, params)
|
||||||
if f.listErr != nil {
|
if f.listErr != nil {
|
||||||
return nil, f.listErr
|
return nil, f.listErr
|
||||||
}
|
}
|
||||||
|
if f.listCalls < len(f.listOutputs) {
|
||||||
|
out := f.listOutputs[f.listCalls]
|
||||||
|
f.listCalls++
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
if f.listOut == nil {
|
if f.listOut == nil {
|
||||||
return &s3.ListObjectsV2Output{}, nil
|
return &s3.ListObjectsV2Output{}, nil
|
||||||
}
|
}
|
||||||
return f.listOut, nil
|
return f.listOut, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestS3BackendListPaginatesAndRejectsNonProgressingTokens(t *testing.T) {
|
||||||
|
t.Run("multiple pages", func(t *testing.T) {
|
||||||
|
client := &fakeS3API{listOutputs: []*s3.ListObjectsV2Output{
|
||||||
|
{Contents: []types.Object{{Key: strPtr("prefix/a"), Size: int64Ptr(1)}}, IsTruncated: boolPtr(true), NextContinuationToken: strPtr("next")},
|
||||||
|
{Contents: []types.Object{{Key: strPtr("prefix/b"), Size: int64Ptr(2)}}, IsTruncated: boolPtr(false)},
|
||||||
|
}}
|
||||||
|
items, err := (&S3Backend{bucket: "bucket-1", client: client}).List(context.Background(), "prefix/")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(items) != 2 || items[0].Key != "prefix/a" || items[1].Key != "prefix/b" {
|
||||||
|
t.Fatalf("List() items = %#v", items)
|
||||||
|
}
|
||||||
|
if len(client.lastLists) != 2 || client.lastLists[1].ContinuationToken == nil || *client.lastLists[1].ContinuationToken != "next" {
|
||||||
|
t.Fatalf("continuation calls = %#v", client.lastLists)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
outputs []*s3.ListObjectsV2Output
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "empty", outputs: []*s3.ListObjectsV2Output{{IsTruncated: boolPtr(true)}}, want: "empty continuation token"},
|
||||||
|
{name: "repeated", outputs: []*s3.ListObjectsV2Output{{IsTruncated: boolPtr(true), NextContinuationToken: strPtr("again")}, {IsTruncated: boolPtr(true), NextContinuationToken: strPtr("again")}}, want: "repeated continuation token"},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
_, err := (&S3Backend{bucket: "bucket-1", client: &fakeS3API{listOutputs: test.outputs}}).List(context.Background(), "prefix/")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.want) || !strings.Contains(err.Error(), "bucket-1") || !strings.Contains(err.Error(), "prefix/") {
|
||||||
|
t.Fatalf("List() error = %v, want contextual %q", err, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
|
func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
|
||||||
f.lastGet = params
|
f.lastGet = params
|
||||||
if f.getErr != nil {
|
if f.getErr != nil {
|
||||||
@@ -54,7 +102,7 @@ func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ..
|
|||||||
if body == nil {
|
if body == nil {
|
||||||
body = io.NopCloser(strings.NewReader(""))
|
body = io.NopCloser(strings.NewReader(""))
|
||||||
}
|
}
|
||||||
return &s3.GetObjectOutput{Body: body}, nil
|
return &s3.GetObjectOutput{Body: body, ContentLength: f.getSize, ETag: f.getETag, LastModified: f.getLastModified}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
|
func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
|
||||||
@@ -126,6 +174,33 @@ func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestS3BackendReadReturnsOpenedObjectMetadata(t *testing.T) {
|
||||||
|
lastModified := time.Date(2026, 8, 11, 1, 2, 3, 0, time.UTC)
|
||||||
|
client := &fakeS3API{
|
||||||
|
getBody: io.NopCloser(strings.NewReader("locks")),
|
||||||
|
getSize: int64Ptr(5),
|
||||||
|
getETag: strPtr(`"generation"`),
|
||||||
|
getLastModified: &lastModified,
|
||||||
|
}
|
||||||
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
|
|
||||||
|
info, body, err := backend.Read(context.Background(), `sessions\locks.yml`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Read() error = %v", err)
|
||||||
|
}
|
||||||
|
data, readErr := io.ReadAll(body)
|
||||||
|
closeErr := body.Close()
|
||||||
|
if readErr != nil || closeErr != nil {
|
||||||
|
t.Fatalf("read body error=%v close error=%v", readErr, closeErr)
|
||||||
|
}
|
||||||
|
if info.Key != "sessions/locks.yml" || info.Size != 5 || info.ETag != "generation" || info.LastModified == nil || !info.LastModified.Equal(lastModified) {
|
||||||
|
t.Fatalf("Read() info = %#v, want opened object metadata", info)
|
||||||
|
}
|
||||||
|
if string(data) != "locks" || client.lastGet == nil || *client.lastGet.Key != "sessions/locks.yml" {
|
||||||
|
t.Fatalf("Read() data=%q request=%#v", data, client.lastGet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestS3BackendUploadAndExists(t *testing.T) {
|
func TestS3BackendUploadAndExists(t *testing.T) {
|
||||||
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
||||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
@@ -160,6 +235,22 @@ func TestS3BackendUploadAndExists(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestS3BackendConditionalUploadUsesProviderPrecondition(t *testing.T) {
|
||||||
|
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
||||||
|
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||||
|
if _, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: "before"}); err != nil {
|
||||||
|
t.Fatalf("UploadConditional() error = %v", err)
|
||||||
|
}
|
||||||
|
if client.lastPut == nil || client.lastPut.IfMatch == nil || *client.lastPut.IfMatch != "before" || client.lastPut.IfNoneMatch != nil {
|
||||||
|
t.Fatalf("PutObject conditional input = %#v", client.lastPut)
|
||||||
|
}
|
||||||
|
client.putErr = &smithy.GenericAPIError{Code: "PreconditionFailed", Message: "changed"}
|
||||||
|
_, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{RequireAbsent: true})
|
||||||
|
if !errors.Is(err, ErrConditionNotMet) {
|
||||||
|
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestS3BackendUploadMissingLocalFile(t *testing.T) {
|
func TestS3BackendUploadMissingLocalFile(t *testing.T) {
|
||||||
backend := &S3Backend{bucket: "bucket-1", client: &fakeS3API{}}
|
backend := &S3Backend{bucket: "bucket-1", client: &fakeS3API{}}
|
||||||
_, err := backend.Upload(context.Background(), filepath.Join(t.TempDir(), "missing.txt"), "key.txt", UploadOptions{})
|
_, err := backend.Upload(context.Background(), filepath.Join(t.TempDir(), "missing.txt"), "key.txt", UploadOptions{})
|
||||||
@@ -249,5 +340,6 @@ func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
|||||||
|
|
||||||
func strPtr(v string) *string { return &v }
|
func strPtr(v string) *string { return &v }
|
||||||
func int64Ptr(v int64) *int64 { return &v }
|
func int64Ptr(v int64) *int64 { return &v }
|
||||||
|
func boolPtr(v bool) *bool { return &v }
|
||||||
|
|
||||||
var _ s3API = (*fakeS3API)(nil)
|
var _ s3API = (*fakeS3API)(nil)
|
||||||
|
|||||||
391
internal/adapters/subprocess/diagnostics.go
Normal file
391
internal/adapters/subprocess/diagnostics.go
Normal file
@@ -0,0 +1,391 @@
|
|||||||
|
package subprocess
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// MaxStdoutDiagnosticBytes bounds persisted stdout from one external command.
|
||||||
|
MaxStdoutDiagnosticBytes int64 = 8 * 1024 * 1024
|
||||||
|
// MaxStderrDiagnosticBytes bounds persisted stderr from one external command.
|
||||||
|
MaxStderrDiagnosticBytes int64 = 8 * 1024 * 1024
|
||||||
|
diagnosticTailBytes = 2048
|
||||||
|
)
|
||||||
|
|
||||||
|
var inheritedEnvironmentNames = map[string]struct{}{
|
||||||
|
"COMSPEC": {},
|
||||||
|
"HOME": {},
|
||||||
|
"PATH": {},
|
||||||
|
"SYSTEMROOT": {},
|
||||||
|
"TMP": {},
|
||||||
|
"TMPDIR": {},
|
||||||
|
"TEMP": {},
|
||||||
|
"WINDIR": {},
|
||||||
|
// These test-only helper destinations let the adapter package tests exercise
|
||||||
|
// real command invocation without widening the production environment.
|
||||||
|
"AUDITA_HELPER_RECORD_PATH": {},
|
||||||
|
"AUDITA_HELPER_MODE": {},
|
||||||
|
"GO_WANT_AUDITA_HELPER": {},
|
||||||
|
"GO_WANT_SCRIPTORIUM_HELPER": {},
|
||||||
|
"GO_WANT_SERIATIM_HELPER": {},
|
||||||
|
"GO_WANT_SUBPROCESS_HELPER": {},
|
||||||
|
"NOTARIUS_CAPTURE_DIR": {},
|
||||||
|
"NOTARIUS_RECEIPT_FIXTURE": {},
|
||||||
|
"SCRIPTORIUM_HELPER_RECORD_PATH": {},
|
||||||
|
"SCRIPTORIUM_HELPER_MODE": {},
|
||||||
|
"SERIATIM_HELPER_RECORD_PATH": {},
|
||||||
|
"SERIATIM_HELPER_MODE": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
var sensitiveEnvironmentNames = map[string]struct{}{
|
||||||
|
"ANTHROPIC_API_KEY": {},
|
||||||
|
"API_KEY": {},
|
||||||
|
"AUDITA_LLM_API_KEY": {},
|
||||||
|
"AWS_ACCESS_KEY_ID": {},
|
||||||
|
"AWS_SECRET_ACCESS_KEY": {},
|
||||||
|
"AWS_SESSION_TOKEN": {},
|
||||||
|
"OPENAI_API_KEY": {},
|
||||||
|
"OPENROUTER_API_KEY": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
type captureLimitError struct {
|
||||||
|
stream string
|
||||||
|
owner string
|
||||||
|
limit int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *captureLimitError) Error() string {
|
||||||
|
return fmt.Sprintf("%s diagnostic capture for %s exceeded %d bytes", e.stream, e.owner, e.limit)
|
||||||
|
}
|
||||||
|
|
||||||
|
type logWriters struct {
|
||||||
|
files []*os.File
|
||||||
|
Stdout io.Writer
|
||||||
|
Stderr io.Writer
|
||||||
|
|
||||||
|
limits chan *captureLimitError
|
||||||
|
mu sync.Mutex
|
||||||
|
limit *captureLimitError
|
||||||
|
stdout *diagnosticWriter
|
||||||
|
stderr *diagnosticWriter
|
||||||
|
}
|
||||||
|
|
||||||
|
type diagnosticWriter struct {
|
||||||
|
logs *logWriters
|
||||||
|
stream string
|
||||||
|
owner string
|
||||||
|
target io.Writer
|
||||||
|
limit int64
|
||||||
|
received int64
|
||||||
|
persisted int64
|
||||||
|
redactor streamRedactor
|
||||||
|
tail []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func openLogWriters(stdoutPath, stderrPath, owner string, sensitiveValues []string) (*logWriters, error) {
|
||||||
|
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||||
|
cleanStdout := cleanLogPath(stdoutPath)
|
||||||
|
cleanStderr := cleanLogPath(stderrPath)
|
||||||
|
|
||||||
|
stdoutFile, err := openDiagnosticFile(cleanStdout)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open stdout log: %w", err)
|
||||||
|
}
|
||||||
|
stderrFile := stdoutFile
|
||||||
|
if cleanStdout != cleanStderr {
|
||||||
|
stderrFile, err = openDiagnosticFile(cleanStderr)
|
||||||
|
if err != nil {
|
||||||
|
_ = stdoutFile.Close()
|
||||||
|
return nil, fmt.Errorf("open stderr log: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cleanStdout == cleanStderr {
|
||||||
|
logs.files = []*os.File{stdoutFile}
|
||||||
|
} else {
|
||||||
|
logs.files = []*os.File{stdoutFile, stderrFile}
|
||||||
|
}
|
||||||
|
|
||||||
|
logs.stdout = newDiagnosticWriter(logs, "stdout", owner, stdoutFile, MaxStdoutDiagnosticBytes, sensitiveValues)
|
||||||
|
logs.stderr = newDiagnosticWriter(logs, "stderr", owner, stderrFile, MaxStderrDiagnosticBytes, sensitiveValues)
|
||||||
|
logs.Stdout = logs.stdout
|
||||||
|
logs.Stderr = logs.stderr
|
||||||
|
return logs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newDiagnosticWriter(logs *logWriters, stream, owner string, target io.Writer, limit int64, sensitiveValues []string) *diagnosticWriter {
|
||||||
|
return &diagnosticWriter{
|
||||||
|
logs: logs,
|
||||||
|
stream: stream,
|
||||||
|
owner: owner,
|
||||||
|
target: target,
|
||||||
|
limit: limit,
|
||||||
|
redactor: newStreamRedactor(sensitiveValues),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *diagnosticWriter) Write(data []byte) (int, error) {
|
||||||
|
if w.received >= w.limit {
|
||||||
|
return len(data), w.reachLimit()
|
||||||
|
}
|
||||||
|
accepted := data
|
||||||
|
if remaining := w.limit - w.received; int64(len(accepted)) > remaining {
|
||||||
|
accepted = accepted[:remaining]
|
||||||
|
}
|
||||||
|
w.received += int64(len(accepted))
|
||||||
|
if err := w.writeRedacted(w.redactor.Write(accepted)); err != nil {
|
||||||
|
return len(data), err
|
||||||
|
}
|
||||||
|
if len(accepted) != len(data) {
|
||||||
|
return len(data), w.reachLimit()
|
||||||
|
}
|
||||||
|
return len(data), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *diagnosticWriter) Flush() error {
|
||||||
|
return w.writeRedacted(w.redactor.Flush())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *diagnosticWriter) writeRedacted(data []byte) error {
|
||||||
|
if len(data) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
w.logs.mu.Lock()
|
||||||
|
remaining := w.limit - w.persisted
|
||||||
|
if remaining <= 0 {
|
||||||
|
w.logs.mu.Unlock()
|
||||||
|
return w.reachLimit()
|
||||||
|
}
|
||||||
|
toWrite := data
|
||||||
|
exceeded := int64(len(data)) > remaining
|
||||||
|
if exceeded {
|
||||||
|
toWrite = toWrite[:remaining]
|
||||||
|
}
|
||||||
|
written, err := w.target.Write(toWrite)
|
||||||
|
w.persisted += int64(written)
|
||||||
|
w.retainTail(toWrite[:written])
|
||||||
|
w.logs.mu.Unlock()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if exceeded {
|
||||||
|
return w.reachLimit()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *diagnosticWriter) Tail() string {
|
||||||
|
w.logs.mu.Lock()
|
||||||
|
defer w.logs.mu.Unlock()
|
||||||
|
return strings.TrimSpace(string(w.tail))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *diagnosticWriter) retainTail(data []byte) {
|
||||||
|
if len(data) >= diagnosticTailBytes {
|
||||||
|
if cap(w.tail) < diagnosticTailBytes {
|
||||||
|
w.tail = make([]byte, diagnosticTailBytes)
|
||||||
|
} else {
|
||||||
|
w.tail = w.tail[:diagnosticTailBytes]
|
||||||
|
}
|
||||||
|
copy(w.tail, data[len(data)-diagnosticTailBytes:])
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cap(w.tail) < diagnosticTailBytes {
|
||||||
|
retained := make([]byte, len(w.tail), diagnosticTailBytes)
|
||||||
|
copy(retained, w.tail)
|
||||||
|
w.tail = retained
|
||||||
|
}
|
||||||
|
if overflow := len(w.tail) + len(data) - diagnosticTailBytes; overflow > 0 {
|
||||||
|
copy(w.tail, w.tail[overflow:])
|
||||||
|
w.tail = w.tail[:len(w.tail)-overflow]
|
||||||
|
}
|
||||||
|
w.tail = append(w.tail, data...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *diagnosticWriter) reachLimit() error {
|
||||||
|
limit := &captureLimitError{stream: w.stream, owner: w.owner, limit: w.limit}
|
||||||
|
w.logs.mu.Lock()
|
||||||
|
if w.logs.limit == nil {
|
||||||
|
w.logs.limit = limit
|
||||||
|
w.logs.limits <- limit
|
||||||
|
}
|
||||||
|
w.logs.mu.Unlock()
|
||||||
|
return limit
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *logWriters) Limits() <-chan *captureLimitError { return l.limits }
|
||||||
|
|
||||||
|
func (l *logWriters) Limit() *captureLimitError {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
return l.limit
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *logWriters) Flush() error {
|
||||||
|
return joinErrors(l.stdout.Flush(), l.stderr.Flush())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *logWriters) Close() {
|
||||||
|
_ = l.Flush()
|
||||||
|
for _, file := range l.files {
|
||||||
|
_ = file.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanLogPath(path string) string {
|
||||||
|
trimmed := strings.TrimSpace(path)
|
||||||
|
if trimmed == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return filepath.Clean(trimmed)
|
||||||
|
}
|
||||||
|
|
||||||
|
func openDiagnosticFile(path string) (*os.File, error) {
|
||||||
|
if path == "" {
|
||||||
|
return os.OpenFile(os.DevNull, os.O_WRONLY, 0)
|
||||||
|
}
|
||||||
|
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
|
||||||
|
return nil, fmt.Errorf("create log directory for %q: %w", path, err)
|
||||||
|
}
|
||||||
|
file, err := fileops.OpenFileConfined(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileops.WorkspaceFileMode)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open log file %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if err := file.Chmod(fileops.WorkspaceFileMode); err != nil {
|
||||||
|
_ = file.Close()
|
||||||
|
return nil, fmt.Errorf("set log file permissions %q: %w", path, err)
|
||||||
|
}
|
||||||
|
return file, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r RunRequest) diagnosticOwner() string {
|
||||||
|
if owner := strings.TrimSpace(r.DiagnosticOwner); owner != "" {
|
||||||
|
return owner
|
||||||
|
}
|
||||||
|
return "subprocess"
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildChildEnvironment(base []string, overrides map[string]string) []string {
|
||||||
|
values := make(map[string]string, len(inheritedEnvironmentNames)+len(overrides))
|
||||||
|
for _, item := range base {
|
||||||
|
name, value, ok := strings.Cut(item, "=")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
normalized := strings.ToUpper(name)
|
||||||
|
if _, allowed := inheritedEnvironmentNames[normalized]; allowed {
|
||||||
|
values[name] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for name, value := range overrides {
|
||||||
|
values[name] = value
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(values))
|
||||||
|
for name := range values {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
out := make([]string, 0, len(names))
|
||||||
|
for _, name := range names {
|
||||||
|
out = append(out, name+"="+values[name])
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func sensitiveEnvironmentValues(environment []string, additionalNames []string) []string {
|
||||||
|
names := make(map[string]struct{}, len(sensitiveEnvironmentNames)+len(additionalNames))
|
||||||
|
for name := range sensitiveEnvironmentNames {
|
||||||
|
names[name] = struct{}{}
|
||||||
|
}
|
||||||
|
for _, name := range additionalNames {
|
||||||
|
if trimmed := strings.ToUpper(strings.TrimSpace(name)); trimmed != "" {
|
||||||
|
names[trimmed] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
values := make([]string, 0, len(names))
|
||||||
|
for _, item := range environment {
|
||||||
|
name, value, ok := strings.Cut(item, "=")
|
||||||
|
if !ok || strings.TrimSpace(value) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, sensitive := names[strings.ToUpper(name)]; sensitive {
|
||||||
|
values = append(values, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|
||||||
|
type streamRedactor struct {
|
||||||
|
values []string
|
||||||
|
buffer []byte
|
||||||
|
maxLen int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStreamRedactor(values []string) streamRedactor {
|
||||||
|
unique := make(map[string]struct{}, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
if value != "" {
|
||||||
|
unique[value] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sorted := make([]string, 0, len(unique))
|
||||||
|
for value := range unique {
|
||||||
|
sorted = append(sorted, value)
|
||||||
|
}
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return len(sorted[i]) > len(sorted[j]) })
|
||||||
|
maxLen := 1
|
||||||
|
for _, value := range sorted {
|
||||||
|
if len(value) > maxLen {
|
||||||
|
maxLen = len(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return streamRedactor{values: sorted, maxLen: maxLen}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *streamRedactor) Write(data []byte) []byte {
|
||||||
|
r.buffer = append(r.buffer, data...)
|
||||||
|
safeCut := len(r.buffer) - r.maxLen + 1
|
||||||
|
if safeCut <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
emitCut := safeCut
|
||||||
|
for _, value := range r.values {
|
||||||
|
start := 0
|
||||||
|
for {
|
||||||
|
index := bytes.Index(r.buffer[start:], []byte(value))
|
||||||
|
if index < 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
index += start
|
||||||
|
if index+len(value) > safeCut && index < emitCut {
|
||||||
|
emitCut = index
|
||||||
|
}
|
||||||
|
start = index + 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
output := redactBytes(r.buffer[:emitCut], r.values)
|
||||||
|
r.buffer = append(r.buffer[:0], r.buffer[emitCut:]...)
|
||||||
|
return output
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *streamRedactor) Flush() []byte {
|
||||||
|
output := redactBytes(r.buffer, r.values)
|
||||||
|
r.buffer = nil
|
||||||
|
return output
|
||||||
|
}
|
||||||
|
|
||||||
|
func redactBytes(data []byte, values []string) []byte {
|
||||||
|
out := append([]byte(nil), data...)
|
||||||
|
for _, value := range values {
|
||||||
|
out = bytes.ReplaceAll(out, []byte(value), []byte("<redacted>"))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
66
internal/adapters/subprocess/process_tree.go
Normal file
66
internal/adapters/subprocess/process_tree.go
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
package subprocess
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
gracefulTerminationWait = 2 * time.Second
|
||||||
|
forcefulTerminationWait = 2 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// ownedProcessTree owns every process started by a command invocation.
|
||||||
|
// Implementations must tolerate a leader that has already exited.
|
||||||
|
type ownedProcessTree interface {
|
||||||
|
Start(*exec.Cmd) error
|
||||||
|
TerminateGracefully() error
|
||||||
|
TerminateForcefully() error
|
||||||
|
Dispose() error
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForOwnedCommand(ctx context.Context, tree ownedProcessTree, waitCh <-chan error, captureLimits <-chan *captureLimitError) (waitErr, ctxErr error, captureLimit *captureLimitError, cleanupErr error) {
|
||||||
|
select {
|
||||||
|
case waitErr = <-waitCh:
|
||||||
|
return waitErr, nil, nil, nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
ctxErr = ctx.Err()
|
||||||
|
case captureLimit = <-captureLimits:
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanupErr = tree.TerminateGracefully()
|
||||||
|
gracefulTimer := time.NewTimer(gracefulTerminationWait)
|
||||||
|
defer gracefulTimer.Stop()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case waitErr = <-waitCh:
|
||||||
|
// The leader may exit before descendants finish graceful shutdown.
|
||||||
|
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||||
|
return waitErr, ctxErr, captureLimit, cleanupErr
|
||||||
|
case <-gracefulTimer.C:
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||||
|
forcefulTimer := time.NewTimer(forcefulTerminationWait)
|
||||||
|
defer forcefulTimer.Stop()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case waitErr = <-waitCh:
|
||||||
|
return waitErr, ctxErr, captureLimit, cleanupErr
|
||||||
|
case <-forcefulTimer.C:
|
||||||
|
return nil, ctxErr, captureLimit, joinErrors(cleanupErr, fmt.Errorf("owned subprocess did not reap within %s after forceful termination", forcefulTerminationWait))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinErrors(errs ...error) error {
|
||||||
|
filtered := make([]error, 0, len(errs))
|
||||||
|
for _, err := range errs {
|
||||||
|
if err != nil {
|
||||||
|
filtered = append(filtered, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors.Join(filtered...)
|
||||||
|
}
|
||||||
227
internal/adapters/subprocess/process_tree_supported_test.go
Normal file
227
internal/adapters/subprocess/process_tree_supported_test.go
Normal file
@@ -0,0 +1,227 @@
|
|||||||
|
//go:build linux || darwin || windows
|
||||||
|
|
||||||
|
package subprocess
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunCancellationTerminatesProcessTree(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
resultCh := make(chan runOutcome, 1)
|
||||||
|
req, sentinelPath := processTreeRequest(t)
|
||||||
|
go func() {
|
||||||
|
result, err := Run(ctx, req)
|
||||||
|
resultCh <- runOutcome{result: result, err: err}
|
||||||
|
}()
|
||||||
|
|
||||||
|
awaitHelperReady(t, req.EnvOverrides["SUBPROCESS_HELPER_READY_PATH"])
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
outcome := awaitRunOutcome(t, resultCh)
|
||||||
|
if !outcome.result.Canceled {
|
||||||
|
t.Fatalf("Canceled = %v, want true", outcome.result.Canceled)
|
||||||
|
}
|
||||||
|
if !errors.Is(outcome.err, context.Canceled) {
|
||||||
|
t.Fatalf("error = %v, want context cancellation", outcome.err)
|
||||||
|
}
|
||||||
|
assertDescendantDidNotSurvive(t, sentinelPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunTimeoutTerminatesProcessTree(t *testing.T) {
|
||||||
|
req, sentinelPath := processTreeRequest(t)
|
||||||
|
req.Timeout = 100 * time.Millisecond
|
||||||
|
|
||||||
|
result, err := Run(context.Background(), req)
|
||||||
|
if !result.TimedOut {
|
||||||
|
t.Fatalf("TimedOut = %v, want true", result.TimedOut)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatalf("error = %v, want context deadline exceeded", err)
|
||||||
|
}
|
||||||
|
assertDescendantDidNotSurvive(t, sentinelPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunCaptureLimitTerminatesProcessTree(t *testing.T) {
|
||||||
|
req, sentinelPath := processTreeRequest(t)
|
||||||
|
req.Args[len(req.Args)-1] = "tree-spam"
|
||||||
|
|
||||||
|
result, err := Run(context.Background(), req)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Run() error = nil, want capture-limit error")
|
||||||
|
}
|
||||||
|
if result.ExitCode == 0 {
|
||||||
|
t.Fatalf("ExitCode = %d, want terminated process", result.ExitCode)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "stdout diagnostic capture for subprocess exceeded") {
|
||||||
|
t.Fatalf("error = %q, want stdout capture-limit context", err)
|
||||||
|
}
|
||||||
|
info, statErr := os.Stat(req.StdoutLogPath)
|
||||||
|
if statErr != nil {
|
||||||
|
t.Fatalf("stat stdout diagnostic: %v", statErr)
|
||||||
|
}
|
||||||
|
if info.Size() != MaxStdoutDiagnosticBytes {
|
||||||
|
t.Fatalf("stdout diagnostic size = %d, want %d", info.Size(), MaxStdoutDiagnosticBytes)
|
||||||
|
}
|
||||||
|
assertDescendantDidNotSurvive(t, sentinelPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDisposesDescendantsAfterLeaderExit(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mode string
|
||||||
|
wantExitCode int
|
||||||
|
wantWaitDelay bool
|
||||||
|
ignoreTerm bool
|
||||||
|
}{
|
||||||
|
{name: "success retaining streams", mode: "leader-exit-retained", wantExitCode: 0, wantWaitDelay: true},
|
||||||
|
{name: "success redirecting streams", mode: "leader-exit-redirected", wantExitCode: 0},
|
||||||
|
{name: "failed leader", mode: "leader-fail-redirected", wantExitCode: 9, ignoreTerm: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
req, sentinelPath, releasePath := leaderExitRequest(t, tt.mode)
|
||||||
|
if tt.ignoreTerm {
|
||||||
|
req.EnvOverrides["SUBPROCESS_HELPER_IGNORE_TERM"] = "1"
|
||||||
|
}
|
||||||
|
outcomes := make(chan runOutcome, 1)
|
||||||
|
go func() {
|
||||||
|
result, err := Run(context.Background(), req)
|
||||||
|
outcomes <- runOutcome{result: result, err: err}
|
||||||
|
}()
|
||||||
|
|
||||||
|
var outcome runOutcome
|
||||||
|
select {
|
||||||
|
case outcome = <-outcomes:
|
||||||
|
case <-time.After(6 * time.Second):
|
||||||
|
t.Fatal("Run() did not complete bounded owned-tree disposal")
|
||||||
|
}
|
||||||
|
if outcome.result.ExitCode != tt.wantExitCode {
|
||||||
|
t.Fatalf("ExitCode = %d, want %d", outcome.result.ExitCode, tt.wantExitCode)
|
||||||
|
}
|
||||||
|
if tt.wantWaitDelay {
|
||||||
|
if !errors.Is(outcome.err, exec.ErrWaitDelay) {
|
||||||
|
t.Fatalf("error = %v, want exec.ErrWaitDelay", outcome.err)
|
||||||
|
}
|
||||||
|
} else if tt.wantExitCode == 0 && outcome.err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", outcome.err)
|
||||||
|
} else if tt.wantExitCode != 0 {
|
||||||
|
var exitErr *exec.ExitError
|
||||||
|
if !errors.As(outcome.err, &exitErr) || exitErr.ExitCode() != tt.wantExitCode {
|
||||||
|
t.Fatalf("error = %v, want exit code %d", outcome.err, tt.wantExitCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(req.EnvOverrides["SUBPROCESS_HELPER_READY_PATH"]); err != nil {
|
||||||
|
t.Fatalf("descendant readiness file: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(releasePath, []byte("release"), 0o600); err != nil {
|
||||||
|
t.Fatalf("WriteFile(release) error = %v", err)
|
||||||
|
}
|
||||||
|
assertDescendantDidNotSurvive(t, sentinelPath)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type runOutcome struct {
|
||||||
|
result RunResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func processTreeRequest(t *testing.T) (RunRequest, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
executable, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
readyPath := filepath.Join(dir, "ready")
|
||||||
|
sentinelPath := filepath.Join(dir, "descendant-survived")
|
||||||
|
return RunRequest{
|
||||||
|
Executable: executable,
|
||||||
|
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "tree"},
|
||||||
|
EnvOverrides: map[string]string{
|
||||||
|
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||||
|
"SUBPROCESS_HELPER_READY_PATH": readyPath,
|
||||||
|
"SUBPROCESS_HELPER_SENTINEL_PATH": sentinelPath,
|
||||||
|
},
|
||||||
|
StdoutLogPath: filepath.Join(dir, "stdout.log"),
|
||||||
|
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||||
|
}, sentinelPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func leaderExitRequest(t *testing.T, mode string) (RunRequest, string, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
executable, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
readyPath := filepath.Join(dir, "ready")
|
||||||
|
releasePath := filepath.Join(dir, "release")
|
||||||
|
sentinelPath := filepath.Join(dir, "descendant-survived")
|
||||||
|
return RunRequest{
|
||||||
|
Executable: executable,
|
||||||
|
Args: []string{"-test.run=^TestSubprocessHelper$", "--", mode},
|
||||||
|
EnvOverrides: map[string]string{
|
||||||
|
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||||
|
"SUBPROCESS_HELPER_READY_PATH": readyPath,
|
||||||
|
"SUBPROCESS_HELPER_RELEASE_PATH": releasePath,
|
||||||
|
"SUBPROCESS_HELPER_SENTINEL_PATH": sentinelPath,
|
||||||
|
},
|
||||||
|
StdoutLogPath: filepath.Join(dir, "stdout.log"),
|
||||||
|
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||||
|
}, sentinelPath, releasePath
|
||||||
|
}
|
||||||
|
|
||||||
|
func awaitHelperReady(t *testing.T, readyPath string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if _, err := os.Stat(readyPath); err == nil {
|
||||||
|
return
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("stat helper readiness: %v", err)
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatal("helper did not start its descendant")
|
||||||
|
}
|
||||||
|
|
||||||
|
func awaitRunOutcome(t *testing.T, outcomes <-chan runOutcome) runOutcome {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case outcome := <-outcomes:
|
||||||
|
if outcome.err == nil {
|
||||||
|
t.Fatal("Run() error = nil, want cancellation error")
|
||||||
|
}
|
||||||
|
return outcome
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("Run() did not return after cancellation")
|
||||||
|
return runOutcome{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertDescendantDidNotSurvive(t *testing.T, sentinelPath string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
time.Sleep(700 * time.Millisecond)
|
||||||
|
if _, err := os.Stat(sentinelPath); err == nil {
|
||||||
|
t.Fatal("descendant survived cancellation and wrote its sentinel")
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("stat descendant sentinel: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
104
internal/adapters/subprocess/process_tree_unix.go
Normal file
104
internal/adapters/subprocess/process_tree_unix.go
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
//go:build linux || darwin
|
||||||
|
|
||||||
|
package subprocess
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const processGroupPollInterval = 10 * time.Millisecond
|
||||||
|
|
||||||
|
type unixProcessTree struct {
|
||||||
|
processGroupID int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||||
|
return &unixProcessTree{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *unixProcessTree) Start(cmd *exec.Cmd) error {
|
||||||
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tree.processGroupID = cmd.Process.Pid
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *unixProcessTree) TerminateGracefully() error {
|
||||||
|
return tree.signal(syscall.SIGTERM)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *unixProcessTree) TerminateForcefully() error {
|
||||||
|
return tree.signal(syscall.SIGKILL)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *unixProcessTree) Dispose() error {
|
||||||
|
hasMembers, err := tree.hasMembers()
|
||||||
|
if err != nil || !hasMembers {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanupErr := tree.TerminateGracefully()
|
||||||
|
empty, waitErr := tree.waitUntilEmpty(gracefulTerminationWait)
|
||||||
|
cleanupErr = joinErrors(cleanupErr, waitErr)
|
||||||
|
if empty {
|
||||||
|
return cleanupErr
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||||
|
empty, waitErr = tree.waitUntilEmpty(forcefulTerminationWait)
|
||||||
|
cleanupErr = joinErrors(cleanupErr, waitErr)
|
||||||
|
if !empty {
|
||||||
|
cleanupErr = joinErrors(cleanupErr, fmt.Errorf("owned subprocess group did not exit within %s after forceful termination", forcefulTerminationWait))
|
||||||
|
}
|
||||||
|
return cleanupErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *unixProcessTree) signal(signal syscall.Signal) error {
|
||||||
|
if tree.processGroupID <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
err := syscall.Kill(-tree.processGroupID, signal)
|
||||||
|
if errors.Is(err, syscall.ESRCH) || errors.Is(err, os.ErrProcessDone) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *unixProcessTree) hasMembers() (bool, error) {
|
||||||
|
if tree.processGroupID <= 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
err := syscall.Kill(-tree.processGroupID, 0)
|
||||||
|
if err == nil || errors.Is(err, syscall.EPERM) {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, syscall.ESRCH) || errors.Is(err, os.ErrProcessDone) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("inspect owned subprocess group: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *unixProcessTree) waitUntilEmpty(timeout time.Duration) (bool, error) {
|
||||||
|
deadline := time.Now().Add(timeout)
|
||||||
|
for {
|
||||||
|
hasMembers, err := tree.hasMembers()
|
||||||
|
if err != nil || !hasMembers {
|
||||||
|
return !hasMembers, err
|
||||||
|
}
|
||||||
|
remaining := time.Until(deadline)
|
||||||
|
if remaining <= 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if remaining > processGroupPollInterval {
|
||||||
|
remaining = processGroupPollInterval
|
||||||
|
}
|
||||||
|
time.Sleep(remaining)
|
||||||
|
}
|
||||||
|
}
|
||||||
12
internal/adapters/subprocess/process_tree_unsupported.go
Normal file
12
internal/adapters/subprocess/process_tree_unsupported.go
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
//go:build !linux && !darwin && !windows
|
||||||
|
|
||||||
|
package subprocess
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"runtime"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||||
|
return nil, fmt.Errorf("owned subprocess trees are unsupported on %s", runtime.GOOS)
|
||||||
|
}
|
||||||
122
internal/adapters/subprocess/process_tree_windows.go
Normal file
122
internal/adapters/subprocess/process_tree_windows.go
Normal file
@@ -0,0 +1,122 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package subprocess
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"syscall"
|
||||||
|
"unsafe"
|
||||||
|
|
||||||
|
"golang.org/x/sys/windows"
|
||||||
|
)
|
||||||
|
|
||||||
|
type windowsProcessTree struct {
|
||||||
|
job windows.Handle
|
||||||
|
}
|
||||||
|
|
||||||
|
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||||
|
return &windowsProcessTree{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *windowsProcessTree) Start(cmd *exec.Cmd) error {
|
||||||
|
job, err := windows.CreateJobObject(nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create job object: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
limits := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{}
|
||||||
|
limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
||||||
|
if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil {
|
||||||
|
_ = windows.CloseHandle(job)
|
||||||
|
return fmt.Errorf("configure job object: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_SUSPENDED}
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
_ = windows.CloseHandle(job)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(cmd.Process.Pid))
|
||||||
|
if err == nil {
|
||||||
|
err = windows.AssignProcessToJobObject(job, process)
|
||||||
|
_ = windows.CloseHandle(process)
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
err = resumeInitialThread(uint32(cmd.Process.Pid))
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
killErr := cmd.Process.Kill()
|
||||||
|
waitErr := cmd.Wait()
|
||||||
|
_ = windows.CloseHandle(job)
|
||||||
|
return joinErrors(fmt.Errorf("assign process to job object: %w", err), killErr, waitErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
tree.job = job
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resumeInitialThread(processID uint32) error {
|
||||||
|
snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("snapshot initial thread: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = windows.CloseHandle(snapshot) }()
|
||||||
|
|
||||||
|
entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))}
|
||||||
|
if err := windows.Thread32First(snapshot, &entry); err != nil {
|
||||||
|
return fmt.Errorf("find initial thread: %w", err)
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
if entry.OwnerProcessID != processID {
|
||||||
|
// Keep enumerating until the suspended process's only initial thread
|
||||||
|
// is found.
|
||||||
|
} else {
|
||||||
|
thread, openErr := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID)
|
||||||
|
if openErr != nil {
|
||||||
|
return fmt.Errorf("open initial thread: %w", openErr)
|
||||||
|
}
|
||||||
|
defer func() { _ = windows.CloseHandle(thread) }()
|
||||||
|
if _, resumeErr := windows.ResumeThread(thread); resumeErr != nil {
|
||||||
|
return fmt.Errorf("resume initial thread: %w", resumeErr)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := windows.Thread32Next(snapshot, &entry); err != nil {
|
||||||
|
if errors.Is(err, windows.ERROR_NO_MORE_FILES) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
return fmt.Errorf("find initial thread: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("find initial thread: no thread found for process %d", processID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *windowsProcessTree) TerminateGracefully() error {
|
||||||
|
// Windows jobs have no portable graceful signal. Terminating the owned job
|
||||||
|
// is the safe fallback and prevents a descendant from escaping cleanup.
|
||||||
|
return tree.terminate()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *windowsProcessTree) TerminateForcefully() error {
|
||||||
|
return tree.terminate()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *windowsProcessTree) Dispose() error {
|
||||||
|
if tree.job == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
err := windows.CloseHandle(tree.job)
|
||||||
|
tree.job = 0
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tree *windowsProcessTree) terminate() error {
|
||||||
|
if tree.job == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return windows.TerminateJobObject(tree.job, 1)
|
||||||
|
}
|
||||||
@@ -2,28 +2,27 @@ package subprocess
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RunRequest defines a subprocess invocation.
|
// RunRequest defines a subprocess invocation.
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
Executable string
|
Executable string
|
||||||
Args []string
|
Args []string
|
||||||
WorkingDir string
|
WorkingDir string
|
||||||
EnvOverrides map[string]string
|
EnvOverrides map[string]string
|
||||||
Timeout time.Duration
|
SensitiveEnvNames []string
|
||||||
StdoutLogPath string
|
DiagnosticOwner string
|
||||||
StderrLogPath string
|
Timeout time.Duration
|
||||||
|
StdoutLogPath string
|
||||||
|
StderrLogPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunResult captures subprocess execution details.
|
// RunResult captures subprocess execution details.
|
||||||
@@ -54,17 +53,26 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
|||||||
}
|
}
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
logs, err := openLogWriters(req.StdoutLogPath, req.StderrLogPath)
|
childEnv := buildChildEnvironment(os.Environ(), req.EnvOverrides)
|
||||||
|
logs, err := openLogWriters(req.StdoutLogPath, req.StderrLogPath, req.diagnosticOwner(), sensitiveEnvironmentValues(childEnv, req.SensitiveEnvNames))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return RunResult{}, err
|
return RunResult{}, err
|
||||||
}
|
}
|
||||||
defer logs.Close()
|
defer logs.Close()
|
||||||
|
|
||||||
cmd := exec.CommandContext(runCtx, req.Executable, req.Args...)
|
tree, err := newOwnedProcessTree()
|
||||||
|
if err != nil {
|
||||||
|
return RunResult{}, fmt.Errorf("prepare owned subprocess tree: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := exec.Command(req.Executable, req.Args...)
|
||||||
cmd.Dir = req.WorkingDir
|
cmd.Dir = req.WorkingDir
|
||||||
cmd.Env = mergeEnv(os.Environ(), req.EnvOverrides)
|
cmd.Env = childEnv
|
||||||
cmd.Stdout = logs.Stdout
|
cmd.Stdout = logs.Stdout
|
||||||
cmd.Stderr = logs.Stderr
|
cmd.Stderr = logs.Stderr
|
||||||
|
// Streaming capture uses pipes. Bound their lifetime when a leader exits
|
||||||
|
// while a descendant still holds a stream descriptor.
|
||||||
|
cmd.WaitDelay = forcefulTerminationWait
|
||||||
|
|
||||||
started := time.Now().UTC()
|
started := time.Now().UTC()
|
||||||
result := RunResult{
|
result := RunResult{
|
||||||
@@ -74,45 +82,67 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
|||||||
StderrLogPath: req.StderrLogPath,
|
StderrLogPath: req.StderrLogPath,
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cmd.Start(); err != nil {
|
if err := runCtx.Err(); err != nil {
|
||||||
|
result.CompletedAt = time.Now().UTC()
|
||||||
|
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||||
|
return result, fmt.Errorf("command was not started: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tree.Start(cmd); err != nil {
|
||||||
result.CompletedAt = time.Now().UTC()
|
result.CompletedAt = time.Now().UTC()
|
||||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||||
return result, fmt.Errorf("start command %q with args %v: %w", req.Executable, req.Args, err)
|
return result, fmt.Errorf("start command %q with args %v: %w", req.Executable, req.Args, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
waitErr := cmd.Wait()
|
waitCh := make(chan error, 1)
|
||||||
|
go func() { waitCh <- cmd.Wait() }()
|
||||||
|
|
||||||
|
waitErr, ctxErr, captureLimit, cleanupErr := waitForOwnedCommand(runCtx, tree, waitCh, logs.Limits())
|
||||||
|
cleanupErr = joinErrors(cleanupErr, tree.Dispose())
|
||||||
|
cleanupErr = joinErrors(cleanupErr, logs.Flush())
|
||||||
|
if captureLimit == nil {
|
||||||
|
captureLimit = logs.Limit()
|
||||||
|
}
|
||||||
result.CompletedAt = time.Now().UTC()
|
result.CompletedAt = time.Now().UTC()
|
||||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||||
if cmd.ProcessState != nil {
|
if cmd.ProcessState != nil {
|
||||||
result.ExitCode = cmd.ProcessState.ExitCode()
|
result.ExitCode = cmd.ProcessState.ExitCode()
|
||||||
}
|
}
|
||||||
|
|
||||||
ctxErr := runCtx.Err()
|
if ctxErr == context.DeadlineExceeded {
|
||||||
if errors.Is(ctxErr, context.DeadlineExceeded) {
|
|
||||||
result.TimedOut = true
|
result.TimedOut = true
|
||||||
}
|
}
|
||||||
if errors.Is(ctxErr, context.Canceled) && !result.TimedOut {
|
if ctxErr == context.Canceled && !result.TimedOut {
|
||||||
result.Canceled = true
|
result.Canceled = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if waitErr == nil {
|
if waitErr == nil && ctxErr == nil && cleanupErr == nil {
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
stderrTail := readRedactedTail(req.StderrLogPath, req.EnvOverrides, 2048)
|
stderrTail := logs.stderr.Tail()
|
||||||
diagnostics := buildDiagnostics(req, result, stderrTail)
|
diagnostics := buildDiagnostics(req, result, stderrTail)
|
||||||
|
|
||||||
|
if captureLimit != nil {
|
||||||
|
if cause := joinErrors(waitErr, cleanupErr); cause != nil {
|
||||||
|
return result, fmt.Errorf("%w (%s): %w", captureLimit, diagnostics, cause)
|
||||||
|
}
|
||||||
|
return result, fmt.Errorf("%w (%s)", captureLimit, diagnostics)
|
||||||
|
}
|
||||||
if result.TimedOut {
|
if result.TimedOut {
|
||||||
return result, fmt.Errorf("command timed out after %s (%s)", req.Timeout, diagnostics)
|
return result, fmt.Errorf("command timed out after %s (%s): %w", req.Timeout, diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
|
||||||
}
|
}
|
||||||
if result.Canceled {
|
if result.Canceled {
|
||||||
return result, fmt.Errorf("command canceled (%s)", diagnostics)
|
return result, fmt.Errorf("command canceled (%s): %w", diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
|
||||||
}
|
}
|
||||||
if exitErr, ok := waitErr.(*exec.ExitError); ok {
|
if exitErr, ok := waitErr.(*exec.ExitError); ok {
|
||||||
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, waitErr)
|
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, joinErrors(waitErr, cleanupErr))
|
||||||
|
}
|
||||||
|
if cleanupErr != nil {
|
||||||
|
return result, fmt.Errorf("command cleanup failed (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
|
||||||
}
|
}
|
||||||
|
|
||||||
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, waitErr)
|
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteYAMLAtomic marshals value as YAML and atomically writes it to path.
|
// WriteYAMLAtomic marshals value as YAML and atomically writes it to path.
|
||||||
@@ -127,141 +157,17 @@ func WriteYAMLAtomic(path string, value any, perm os.FileMode) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteFileAtomic writes bytes via same-directory temp file + atomic rename.
|
// WriteFileAtomic writes bytes through the shared durable replacement primitive.
|
||||||
func WriteFileAtomic(path string, data []byte, perm os.FileMode) error {
|
func WriteFileAtomic(path string, data []byte, perm os.FileMode) error {
|
||||||
if strings.TrimSpace(path) == "" {
|
if strings.TrimSpace(path) == "" {
|
||||||
return fmt.Errorf("write file: path is required")
|
return fmt.Errorf("write file: path is required")
|
||||||
}
|
}
|
||||||
|
if err := fileops.WriteFileAtomic(path, data, perm); err != nil {
|
||||||
dir := filepath.Dir(path)
|
return fmt.Errorf("write file %q: %w", path, err)
|
||||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
||||||
return fmt.Errorf("create parent directory %q: %w", dir, err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
base := filepath.Base(path)
|
|
||||||
tmp, err := os.CreateTemp(dir, "."+base+".tmp-*")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("create temp file: %w", err)
|
|
||||||
}
|
|
||||||
tmpPath := tmp.Name()
|
|
||||||
removeTmp := true
|
|
||||||
defer func() {
|
|
||||||
if removeTmp {
|
|
||||||
_ = os.Remove(tmpPath)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
if _, err := tmp.Write(data); err != nil {
|
|
||||||
_ = tmp.Close()
|
|
||||||
return fmt.Errorf("write temp file: %w", err)
|
|
||||||
}
|
|
||||||
if err := tmp.Sync(); err != nil {
|
|
||||||
_ = tmp.Close()
|
|
||||||
return fmt.Errorf("sync temp file: %w", err)
|
|
||||||
}
|
|
||||||
if err := tmp.Close(); err != nil {
|
|
||||||
return fmt.Errorf("close temp file: %w", err)
|
|
||||||
}
|
|
||||||
if err := os.Chmod(tmpPath, perm); err != nil {
|
|
||||||
return fmt.Errorf("chmod temp file: %w", err)
|
|
||||||
}
|
|
||||||
if err := os.Rename(tmpPath, path); err != nil {
|
|
||||||
return fmt.Errorf("rename temp file: %w", err)
|
|
||||||
}
|
|
||||||
removeTmp = false
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type logWriters struct {
|
|
||||||
files []*os.File
|
|
||||||
Stdout io.Writer
|
|
||||||
Stderr io.Writer
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *logWriters) Close() {
|
|
||||||
for _, f := range l.files {
|
|
||||||
_ = f.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func openLogWriters(stdoutPath, stderrPath string) (*logWriters, error) {
|
|
||||||
cleanStdout := cleanLogPath(stdoutPath)
|
|
||||||
cleanStderr := cleanLogPath(stderrPath)
|
|
||||||
|
|
||||||
// Keep stdout/stderr on the same file descriptor when both paths target
|
|
||||||
// the same file to avoid descriptor aliasing surprises across runtimes.
|
|
||||||
if cleanStdout != "" && cleanStdout == cleanStderr {
|
|
||||||
f, err := openLogFile(cleanStdout)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("open shared stdout/stderr log %q: %w", cleanStdout, err)
|
|
||||||
}
|
|
||||||
return &logWriters{
|
|
||||||
files: []*os.File{f},
|
|
||||||
Stdout: f,
|
|
||||||
Stderr: f,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
stdoutFile, stdoutWriter, err := logWriter(cleanStdout)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("open stdout log: %w", err)
|
|
||||||
}
|
|
||||||
stderrFile, stderrWriter, err := logWriter(cleanStderr)
|
|
||||||
if err != nil {
|
|
||||||
closeFile(stdoutFile)
|
|
||||||
return nil, fmt.Errorf("open stderr log: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
files := make([]*os.File, 0, 2)
|
|
||||||
if stdoutFile != nil {
|
|
||||||
files = append(files, stdoutFile)
|
|
||||||
}
|
|
||||||
if stderrFile != nil {
|
|
||||||
files = append(files, stderrFile)
|
|
||||||
}
|
|
||||||
return &logWriters{
|
|
||||||
files: files,
|
|
||||||
Stdout: stdoutWriter,
|
|
||||||
Stderr: stderrWriter,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func cleanLogPath(path string) string {
|
|
||||||
trimmed := strings.TrimSpace(path)
|
|
||||||
if trimmed == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return filepath.Clean(trimmed)
|
|
||||||
}
|
|
||||||
|
|
||||||
func logWriter(path string) (*os.File, io.Writer, error) {
|
|
||||||
if strings.TrimSpace(path) == "" {
|
|
||||||
return nil, io.Discard, nil
|
|
||||||
}
|
|
||||||
f, err := openLogFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
return f, f, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func openLogFile(path string) (*os.File, error) {
|
|
||||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
||||||
return nil, fmt.Errorf("create log directory for %q: %w", path, err)
|
|
||||||
}
|
|
||||||
f, err := os.Create(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("open log file %q: %w", path, err)
|
|
||||||
}
|
|
||||||
return f, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func closeFile(f *os.File) {
|
|
||||||
if f != nil {
|
|
||||||
_ = f.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildDiagnostics(req RunRequest, result RunResult, stderrTail string) string {
|
func buildDiagnostics(req RunRequest, result RunResult, stderrTail string) string {
|
||||||
details := fmt.Sprintf(
|
details := fmt.Sprintf(
|
||||||
"executable=%q args=%v cwd=%q timeout=%s exit_code=%d timed_out=%t canceled=%t stdout_log=%q stderr_log=%q",
|
"executable=%q args=%v cwd=%q timeout=%s exit_code=%d timed_out=%t canceled=%t stdout_log=%q stderr_log=%q",
|
||||||
@@ -298,87 +204,3 @@ func fdDiagnosticsHint(exitCode int, stderrTail string) string {
|
|||||||
}
|
}
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func readRedactedTail(path string, envOverrides map[string]string, maxBytes int64) string {
|
|
||||||
if strings.TrimSpace(path) == "" || maxBytes <= 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
f, err := os.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
info, err := f.Stat()
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
size := info.Size()
|
|
||||||
start := int64(0)
|
|
||||||
if size > maxBytes {
|
|
||||||
start = size - maxBytes
|
|
||||||
}
|
|
||||||
if _, err := f.Seek(start, io.SeekStart); err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
data, err := io.ReadAll(f)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
tail := strings.TrimSpace(string(data))
|
|
||||||
if tail == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return redactSensitiveTail(tail, envOverrides)
|
|
||||||
}
|
|
||||||
|
|
||||||
func redactSensitiveTail(tail string, envOverrides map[string]string) string {
|
|
||||||
out := tail
|
|
||||||
for k, v := range envOverrides {
|
|
||||||
if strings.TrimSpace(v) == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if looksSensitiveEnvKey(k) {
|
|
||||||
out = strings.ReplaceAll(out, v, "<redacted>")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func looksSensitiveEnvKey(key string) bool {
|
|
||||||
k := strings.ToUpper(strings.TrimSpace(key))
|
|
||||||
return strings.Contains(k, "KEY") ||
|
|
||||||
strings.Contains(k, "TOKEN") ||
|
|
||||||
strings.Contains(k, "SECRET") ||
|
|
||||||
strings.Contains(k, "PASSWORD")
|
|
||||||
}
|
|
||||||
|
|
||||||
func mergeEnv(base []string, overrides map[string]string) []string {
|
|
||||||
if len(overrides) == 0 {
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
|
|
||||||
kv := make(map[string]string, len(base)+len(overrides))
|
|
||||||
for _, item := range base {
|
|
||||||
k, v, ok := strings.Cut(item, "=")
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
kv[k] = v
|
|
||||||
}
|
|
||||||
for k, v := range overrides {
|
|
||||||
kv[k] = v
|
|
||||||
}
|
|
||||||
|
|
||||||
keys := make([]string, 0, len(kv))
|
|
||||||
for k := range kv {
|
|
||||||
keys = append(keys, k)
|
|
||||||
}
|
|
||||||
sort.Strings(keys)
|
|
||||||
|
|
||||||
out := make([]string, 0, len(keys))
|
|
||||||
for _, k := range keys {
|
|
||||||
out = append(out, k+"="+kv[k])
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,10 +1,17 @@
|
|||||||
package subprocess
|
package subprocess
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"os/signal"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -120,6 +127,316 @@ func TestRunFailureRedactsSensitiveTail(t *testing.T) {
|
|||||||
if !strings.Contains(err.Error(), "<redacted>") {
|
if !strings.Contains(err.Error(), "<redacted>") {
|
||||||
t.Fatalf("error = %q, want redacted stderr tail marker", err.Error())
|
t.Fatalf("error = %q, want redacted stderr tail marker", err.Error())
|
||||||
}
|
}
|
||||||
|
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath} {
|
||||||
|
data, readErr := os.ReadFile(path)
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatalf("read diagnostic %q: %v", path, readErr)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(data), secretValue) {
|
||||||
|
t.Fatalf("diagnostic %q leaked secret: %q", path, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("creating symlinks requires privileges that are not available on every Windows runner")
|
||||||
|
}
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
targetPath := filepath.Join(dir, "outside.log")
|
||||||
|
const original = "must remain unchanged"
|
||||||
|
if err := os.WriteFile(targetPath, []byte(original), 0o600); err != nil {
|
||||||
|
t.Fatalf("WriteFile(target) error = %v", err)
|
||||||
|
}
|
||||||
|
stdoutPath := filepath.Join(dir, "stdout.log")
|
||||||
|
if err := os.Symlink(targetPath, stdoutPath); err != nil {
|
||||||
|
t.Fatalf("Symlink() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = Run(context.Background(), RunRequest{
|
||||||
|
Executable: exe,
|
||||||
|
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "success"},
|
||||||
|
EnvOverrides: map[string]string{"GO_WANT_SUBPROCESS_HELPER": "1"},
|
||||||
|
StdoutLogPath: stdoutPath,
|
||||||
|
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "symbolic link") {
|
||||||
|
t.Fatalf("Run() error = %v, want symbolic-link rejection", err)
|
||||||
|
}
|
||||||
|
data, readErr := os.ReadFile(targetPath)
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatalf("ReadFile(target) error = %v", readErr)
|
||||||
|
}
|
||||||
|
if string(data) != original {
|
||||||
|
t.Fatalf("target content = %q, want %q", data, original)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFailureUsesOpenedDiagnosticAfterPathReplacement(t *testing.T) {
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
readyPath := filepath.Join(dir, "ready")
|
||||||
|
releasePath := filepath.Join(dir, "release")
|
||||||
|
stderrPath := filepath.Join(dir, "stderr.log")
|
||||||
|
openedPath := filepath.Join(dir, "opened-stderr.log")
|
||||||
|
const secretValue = "replacement-api-key-value"
|
||||||
|
const commandContent = "trusted command failure"
|
||||||
|
req := RunRequest{
|
||||||
|
Executable: exe,
|
||||||
|
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "delayed-fail"},
|
||||||
|
EnvOverrides: map[string]string{
|
||||||
|
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||||
|
"API_KEY": secretValue,
|
||||||
|
"SUBPROCESS_HELPER_READY_PATH": readyPath,
|
||||||
|
"SUBPROCESS_HELPER_RELEASE_PATH": releasePath,
|
||||||
|
"SUBPROCESS_HELPER_STDERR": commandContent,
|
||||||
|
},
|
||||||
|
StdoutLogPath: filepath.Join(dir, "stdout.log"),
|
||||||
|
StderrLogPath: stderrPath,
|
||||||
|
}
|
||||||
|
|
||||||
|
resultCh := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, runErr := Run(context.Background(), req)
|
||||||
|
resultCh <- runErr
|
||||||
|
}()
|
||||||
|
waitForHelperFile(t, readyPath)
|
||||||
|
if err := os.Rename(stderrPath, openedPath); err != nil {
|
||||||
|
t.Fatalf("Rename(stderr log) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(stderrPath, []byte(secretValue), 0o600); err != nil {
|
||||||
|
t.Fatalf("WriteFile(replacement) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(releasePath, []byte("continue"), 0o600); err != nil {
|
||||||
|
t.Fatalf("WriteFile(release) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case runErr := <-resultCh:
|
||||||
|
if runErr == nil {
|
||||||
|
t.Fatal("Run() error = nil, want command failure")
|
||||||
|
}
|
||||||
|
if strings.Contains(runErr.Error(), secretValue) {
|
||||||
|
t.Fatalf("error read replacement-path content: %q", runErr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(runErr.Error(), commandContent) {
|
||||||
|
t.Fatalf("error = %q, want retained command diagnostic", runErr)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("Run() did not return after helper release")
|
||||||
|
}
|
||||||
|
|
||||||
|
openedData, err := os.ReadFile(openedPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(opened diagnostic) error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(openedData), commandContent) {
|
||||||
|
t.Fatalf("opened diagnostic = %q, want command content", openedData)
|
||||||
|
}
|
||||||
|
replacementData, err := os.ReadFile(stderrPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(replacement diagnostic) error = %v", err)
|
||||||
|
}
|
||||||
|
if string(replacementData) != secretValue {
|
||||||
|
t.Fatalf("replacement diagnostic = %q, want %q", replacementData, secretValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRedactsSplitCredentialInSeparateAndSharedDiagnostics(t *testing.T) {
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
|
}
|
||||||
|
const secretValue = "split-super-secret-value"
|
||||||
|
|
||||||
|
for _, shared := range []bool{false, true} {
|
||||||
|
t.Run(map[bool]string{false: "separate", true: "shared"}[shared], func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
stdoutPath := filepath.Join(dir, "stdout.log")
|
||||||
|
stderrPath := filepath.Join(dir, "stderr.log")
|
||||||
|
if shared {
|
||||||
|
stderrPath = stdoutPath
|
||||||
|
}
|
||||||
|
req := RunRequest{
|
||||||
|
Executable: exe,
|
||||||
|
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "splitsecret"},
|
||||||
|
EnvOverrides: map[string]string{
|
||||||
|
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||||
|
"API_KEY": secretValue,
|
||||||
|
},
|
||||||
|
StdoutLogPath: stdoutPath,
|
||||||
|
StderrLogPath: stderrPath,
|
||||||
|
}
|
||||||
|
|
||||||
|
_, runErr := Run(context.Background(), req)
|
||||||
|
if runErr == nil {
|
||||||
|
t.Fatal("Run() error = nil, want command failure")
|
||||||
|
}
|
||||||
|
if strings.Contains(runErr.Error(), secretValue) || !strings.Contains(runErr.Error(), "<redacted>") {
|
||||||
|
t.Fatalf("error = %q, want redacted credential", runErr)
|
||||||
|
}
|
||||||
|
paths := map[string]struct{}{stdoutPath: {}, stderrPath: {}}
|
||||||
|
for path := range paths {
|
||||||
|
data, readErr := os.ReadFile(path)
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatalf("ReadFile(%q) error = %v", path, readErr)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(data), secretValue) || !strings.Contains(string(data), "<redacted>") {
|
||||||
|
t.Fatalf("diagnostic %q = %q, want redacted credential", path, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRedactsInheritedSensitiveEnvironment(t *testing.T) {
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
|
}
|
||||||
|
secretValue := "inherited-secret-value"
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", secretValue)
|
||||||
|
dir := t.TempDir()
|
||||||
|
req := RunRequest{
|
||||||
|
Executable: exe,
|
||||||
|
Args: []string{"-test.run=TestSubprocessHelper", "--", "echoenv"},
|
||||||
|
EnvOverrides: map[string]string{
|
||||||
|
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||||
|
"SUBPROCESS_HELPER_ENV_KEY": "OPENROUTER_API_KEY",
|
||||||
|
},
|
||||||
|
StdoutLogPath: filepath.Join(dir, "stdout.log"),
|
||||||
|
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||||
|
}
|
||||||
|
_, err = Run(context.Background(), req)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Run() error = nil, want non-nil")
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), secretValue) {
|
||||||
|
t.Fatalf("error leaked inherited secret: %q", err)
|
||||||
|
}
|
||||||
|
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath} {
|
||||||
|
data, readErr := os.ReadFile(path)
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatalf("read diagnostic %q: %v", path, readErr)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(data), secretValue) {
|
||||||
|
t.Fatalf("diagnostic %q leaked inherited secret: %q", path, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRedactsSensitiveOutputAndErrorTail(t *testing.T) {
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
|
}
|
||||||
|
secretValue := "override-secret-value"
|
||||||
|
dir := t.TempDir()
|
||||||
|
req := RunRequest{
|
||||||
|
Executable: exe,
|
||||||
|
Args: []string{"-test.run=TestSubprocessHelper", "--", "echoenv"},
|
||||||
|
EnvOverrides: map[string]string{
|
||||||
|
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||||
|
"SUBPROCESS_HELPER_ENV_KEY": "OPENROUTER_API_KEY",
|
||||||
|
"OPENROUTER_API_KEY": secretValue,
|
||||||
|
},
|
||||||
|
StdoutLogPath: filepath.Join(dir, "stdout.log"),
|
||||||
|
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||||
|
}
|
||||||
|
_, err = Run(context.Background(), req)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Run() error = nil, want non-nil")
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), secretValue) || !strings.Contains(err.Error(), "<redacted>") {
|
||||||
|
t.Fatalf("error = %q, want redacted secret", err)
|
||||||
|
}
|
||||||
|
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath} {
|
||||||
|
data, readErr := os.ReadFile(path)
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatalf("read diagnostic %q: %v", path, readErr)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(data), secretValue) || !strings.Contains(string(data), "<redacted>") {
|
||||||
|
t.Fatalf("diagnostic %q = %q, want redacted secret", path, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStreamRedactorHandlesSplitAndOverlappingSecrets(t *testing.T) {
|
||||||
|
redactor := newStreamRedactor([]string{"abc", "abcde", "cde", ""})
|
||||||
|
var output bytes.Buffer
|
||||||
|
output.Write(redactor.Write([]byte("start-ab")))
|
||||||
|
output.Write(redactor.Write([]byte("cde-end")))
|
||||||
|
output.Write(redactor.Flush())
|
||||||
|
if got := output.String(); got != "start-<redacted>-end" {
|
||||||
|
t.Fatalf("redacted output = %q, want one redacted marker", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
||||||
|
exactLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||||
|
var exactOutput bytes.Buffer
|
||||||
|
exact := newDiagnosticWriter(exactLogs, "stdout", "test", &exactOutput, 5, nil)
|
||||||
|
if _, err := exact.Write([]byte("abcde")); err != nil {
|
||||||
|
t.Fatalf("exact Write() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := exact.Flush(); err != nil {
|
||||||
|
t.Fatalf("exact Flush() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := exactOutput.String(); got != "abcde" {
|
||||||
|
t.Fatalf("exact output = %q, want abcde", got)
|
||||||
|
}
|
||||||
|
if exactLogs.Limit() != nil {
|
||||||
|
t.Fatal("exact write recorded a capture limit")
|
||||||
|
}
|
||||||
|
|
||||||
|
cappedLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||||
|
var cappedOutput bytes.Buffer
|
||||||
|
capped := newDiagnosticWriter(cappedLogs, "stderr", "test", &cappedOutput, 5, nil)
|
||||||
|
if _, err := capped.Write([]byte("abcdef")); err == nil {
|
||||||
|
t.Fatal("cap-plus-one Write() error = nil, want capture limit")
|
||||||
|
}
|
||||||
|
if err := capped.Flush(); err != nil {
|
||||||
|
t.Fatalf("cap-plus-one Flush() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := cappedOutput.String(); got != "abcde" {
|
||||||
|
t.Fatalf("capped output = %q, want abcde", got)
|
||||||
|
}
|
||||||
|
if limit := cappedLogs.Limit(); limit == nil || limit.stream != "stderr" || limit.limit != 5 {
|
||||||
|
t.Fatalf("capture limit = %#v, want stderr limit 5", limit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiagnosticWriterRetainsBoundedRedactedTail(t *testing.T) {
|
||||||
|
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||||
|
var output bytes.Buffer
|
||||||
|
secret := "credential-value"
|
||||||
|
writer := newDiagnosticWriter(logs, "stderr", "test", &output, 16*1024, []string{secret})
|
||||||
|
prefix := strings.Repeat("x", diagnosticTailBytes+512)
|
||||||
|
if _, err := writer.Write([]byte(prefix + secret[:7])); err != nil {
|
||||||
|
t.Fatalf("first Write() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := writer.Write([]byte(secret[7:] + "-failure")); err != nil {
|
||||||
|
t.Fatalf("second Write() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := writer.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush() error = %v", err)
|
||||||
|
}
|
||||||
|
tail := writer.Tail()
|
||||||
|
if len(tail) > diagnosticTailBytes {
|
||||||
|
t.Fatalf("retained tail length = %d, want at most %d", len(tail), diagnosticTailBytes)
|
||||||
|
}
|
||||||
|
if strings.Contains(tail, secret) || !strings.Contains(tail, "<redacted>-failure") {
|
||||||
|
t.Fatalf("retained tail = %q, want bounded redacted content", tail)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunFailureAddsBadDescriptorHint(t *testing.T) {
|
func TestRunFailureAddsBadDescriptorHint(t *testing.T) {
|
||||||
@@ -184,15 +501,14 @@ func TestRunInheritsParentEnvironmentByDefault(t *testing.T) {
|
|||||||
t.Fatalf("os.Executable() error = %v", err)
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SUBPROCESS_HELPER", "1")
|
t.Setenv("PATH", "inherited-value")
|
||||||
t.Setenv("SUBPROCESS_HELPER_ENV_KEY", "SUBPROCESS_PARENT_VALUE")
|
|
||||||
t.Setenv("SUBPROCESS_PARENT_VALUE", "inherited-value")
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
stdoutPath := filepath.Join(dir, "stdout.log")
|
stdoutPath := filepath.Join(dir, "stdout.log")
|
||||||
req := RunRequest{
|
req := RunRequest{
|
||||||
Executable: exe,
|
Executable: exe,
|
||||||
Args: []string{"-test.run=TestSubprocessHelper", "--", "printenv"},
|
Args: []string{"-test.run=TestSubprocessHelper", "--", "printenv"},
|
||||||
|
EnvOverrides: map[string]string{"GO_WANT_SUBPROCESS_HELPER": "1", "SUBPROCESS_HELPER_ENV_KEY": "PATH"},
|
||||||
StdoutLogPath: stdoutPath,
|
StdoutLogPath: stdoutPath,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -214,16 +530,16 @@ func TestRunEnvOverridesWinOverInheritedValues(t *testing.T) {
|
|||||||
t.Fatalf("os.Executable() error = %v", err)
|
t.Fatalf("os.Executable() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Setenv("GO_WANT_SUBPROCESS_HELPER", "1")
|
|
||||||
t.Setenv("SUBPROCESS_HELPER_ENV_KEY", "SUBPROCESS_PARENT_VALUE")
|
|
||||||
t.Setenv("SUBPROCESS_PARENT_VALUE", "parent-value")
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
stdoutPath := filepath.Join(dir, "stdout.log")
|
stdoutPath := filepath.Join(dir, "stdout.log")
|
||||||
req := RunRequest{
|
req := RunRequest{
|
||||||
Executable: exe,
|
Executable: exe,
|
||||||
Args: []string{"-test.run=TestSubprocessHelper", "--", "printenv"},
|
Args: []string{"-test.run=TestSubprocessHelper", "--", "printenv"},
|
||||||
EnvOverrides: map[string]string{"SUBPROCESS_PARENT_VALUE": "override-value"},
|
EnvOverrides: map[string]string{
|
||||||
|
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||||
|
"SUBPROCESS_HELPER_ENV_KEY": "SUBPROCESS_PARENT_VALUE",
|
||||||
|
"SUBPROCESS_PARENT_VALUE": "override-value",
|
||||||
|
},
|
||||||
StdoutLogPath: stdoutPath,
|
StdoutLogPath: stdoutPath,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -361,6 +677,30 @@ func TestSubprocessHelper(t *testing.T) {
|
|||||||
case "failbadfd":
|
case "failbadfd":
|
||||||
_, _ = os.Stderr.WriteString("OSError: [Errno 9] Bad file descriptor\n")
|
_, _ = os.Stderr.WriteString("OSError: [Errno 9] Bad file descriptor\n")
|
||||||
os.Exit(120)
|
os.Exit(120)
|
||||||
|
case "delayed-fail":
|
||||||
|
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
|
||||||
|
os.Exit(4)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for {
|
||||||
|
if _, err := os.Stat(os.Getenv("SUBPROCESS_HELPER_RELEASE_PATH")); err == nil {
|
||||||
|
break
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) || time.Now().After(deadline) {
|
||||||
|
os.Exit(5)
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
_, _ = os.Stderr.WriteString(os.Getenv("SUBPROCESS_HELPER_STDERR"))
|
||||||
|
os.Exit(6)
|
||||||
|
case "splitsecret":
|
||||||
|
secret := os.Getenv("API_KEY")
|
||||||
|
split := len(secret) / 2
|
||||||
|
for _, stream := range []*os.File{os.Stdout, os.Stderr} {
|
||||||
|
_, _ = stream.WriteString(secret[:split])
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
_, _ = stream.WriteString(secret[split:] + "\n")
|
||||||
|
}
|
||||||
|
os.Exit(7)
|
||||||
case "sleep":
|
case "sleep":
|
||||||
time.Sleep(500 * time.Millisecond)
|
time.Sleep(500 * time.Millisecond)
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
@@ -368,7 +708,115 @@ func TestSubprocessHelper(t *testing.T) {
|
|||||||
key := os.Getenv("SUBPROCESS_HELPER_ENV_KEY")
|
key := os.Getenv("SUBPROCESS_HELPER_ENV_KEY")
|
||||||
_, _ = os.Stdout.WriteString(os.Getenv(key) + "\n")
|
_, _ = os.Stdout.WriteString(os.Getenv(key) + "\n")
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
|
case "echoenv":
|
||||||
|
key := os.Getenv("SUBPROCESS_HELPER_ENV_KEY")
|
||||||
|
value := os.Getenv(key)
|
||||||
|
_, _ = os.Stdout.WriteString(value)
|
||||||
|
_, _ = os.Stderr.WriteString(value)
|
||||||
|
os.Exit(5)
|
||||||
|
case "spam":
|
||||||
|
chunk := strings.Repeat("x", 64*1024)
|
||||||
|
count, _ := strconv.Atoi(os.Getenv("SUBPROCESS_HELPER_CHUNKS"))
|
||||||
|
for range count {
|
||||||
|
_, _ = os.Stdout.WriteString(chunk)
|
||||||
|
}
|
||||||
|
os.Exit(0)
|
||||||
|
case "tree-spam":
|
||||||
|
descendant := exec.Command(os.Args[0], "-test.run=^TestSubprocessHelper$", "--", "descendant")
|
||||||
|
descendant.Env = append(os.Environ(), "GO_WANT_SUBPROCESS_HELPER=1")
|
||||||
|
descendant.Stdout = os.Stdout
|
||||||
|
descendant.Stderr = os.Stderr
|
||||||
|
if err := descendant.Start(); err != nil {
|
||||||
|
os.Exit(3)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
|
||||||
|
os.Exit(4)
|
||||||
|
}
|
||||||
|
chunk := strings.Repeat("x", 64*1024)
|
||||||
|
for {
|
||||||
|
_, _ = os.Stdout.WriteString(chunk)
|
||||||
|
}
|
||||||
|
case "tree":
|
||||||
|
descendant := exec.Command(os.Args[0], "-test.run=^TestSubprocessHelper$", "--", "descendant")
|
||||||
|
descendant.Env = append(os.Environ(), "GO_WANT_SUBPROCESS_HELPER=1")
|
||||||
|
descendant.Stdout = os.Stdout
|
||||||
|
descendant.Stderr = os.Stderr
|
||||||
|
if err := descendant.Start(); err != nil {
|
||||||
|
os.Exit(3)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
|
||||||
|
os.Exit(4)
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Second)
|
||||||
|
os.Exit(0)
|
||||||
|
case "leader-exit-retained", "leader-exit-redirected", "leader-fail-redirected":
|
||||||
|
descendant := exec.Command(os.Args[0], "-test.run=^TestSubprocessHelper$", "--", "descendant-after-release")
|
||||||
|
descendant.Env = append(os.Environ(), "GO_WANT_SUBPROCESS_HELPER=1")
|
||||||
|
if mode == "leader-exit-retained" {
|
||||||
|
descendant.Stdout = os.Stdout
|
||||||
|
descendant.Stderr = os.Stderr
|
||||||
|
}
|
||||||
|
if err := descendant.Start(); err != nil {
|
||||||
|
os.Exit(3)
|
||||||
|
}
|
||||||
|
if !helperFileAppeared(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), 2*time.Second) {
|
||||||
|
os.Exit(4)
|
||||||
|
}
|
||||||
|
if mode == "leader-fail-redirected" {
|
||||||
|
os.Exit(9)
|
||||||
|
}
|
||||||
|
os.Exit(0)
|
||||||
|
case "descendant-after-release":
|
||||||
|
if os.Getenv("SUBPROCESS_HELPER_IGNORE_TERM") == "1" {
|
||||||
|
signal.Ignore(syscall.SIGTERM)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
|
||||||
|
os.Exit(4)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(10 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if _, err := os.Stat(os.Getenv("SUBPROCESS_HELPER_RELEASE_PATH")); err == nil {
|
||||||
|
_ = os.WriteFile(os.Getenv("SUBPROCESS_HELPER_SENTINEL_PATH"), []byte("survived"), 0o600)
|
||||||
|
os.Exit(0)
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
os.Exit(5)
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
os.Exit(0)
|
||||||
|
case "descendant":
|
||||||
|
time.Sleep(500 * time.Millisecond)
|
||||||
|
_ = os.WriteFile(os.Getenv("SUBPROCESS_HELPER_SENTINEL_PATH"), []byte("survived"), 0o600)
|
||||||
|
time.Sleep(10 * time.Second)
|
||||||
|
os.Exit(0)
|
||||||
default:
|
default:
|
||||||
os.Exit(2)
|
os.Exit(2)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func waitForHelperFile(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if _, err := os.Stat(path); err == nil {
|
||||||
|
return
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("Stat(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("helper file %q was not created", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func helperFileAppeared(path string, timeout time.Duration) bool {
|
||||||
|
deadline := time.Now().Add(timeout)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if _, err := os.Stat(path); err == nil {
|
||||||
|
return true
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,8 +2,10 @@ package whisperx
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
var minimalTranscriptJSON = []byte(`{"schema":"speaker_transcript.v1","segments":[]}`)
|
var minimalTranscriptJSON = []byte(`{"schema":"speaker_transcript.v1","segments":[]}`)
|
||||||
@@ -31,7 +33,8 @@ func (n *NoopClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
|||||||
|
|
||||||
// FakeClient captures requests and returns deterministic responses for tests.
|
// FakeClient captures requests and returns deterministic responses for tests.
|
||||||
type FakeClient struct {
|
type FakeClient struct {
|
||||||
Requests []TranscribeRequest
|
requestsMu sync.RWMutex
|
||||||
|
requests []TranscribeRequest
|
||||||
Err error
|
Err error
|
||||||
Result TranscribeResult
|
Result TranscribeResult
|
||||||
TranscribeFn func(ctx context.Context, req TranscribeRequest) (TranscribeResult, error)
|
TranscribeFn func(ctx context.Context, req TranscribeRequest) (TranscribeResult, error)
|
||||||
@@ -42,7 +45,9 @@ func (f *FakeClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
|||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return TranscribeResult{}, err
|
return TranscribeResult{}, err
|
||||||
}
|
}
|
||||||
f.Requests = append(f.Requests, req)
|
f.requestsMu.Lock()
|
||||||
|
f.requests = append(f.requests, req)
|
||||||
|
f.requestsMu.Unlock()
|
||||||
if f.TranscribeFn != nil {
|
if f.TranscribeFn != nil {
|
||||||
return f.TranscribeFn(ctx, req)
|
return f.TranscribeFn(ctx, req)
|
||||||
}
|
}
|
||||||
@@ -65,12 +70,19 @@ func (f *FakeClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
|||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RequestsSnapshot returns a copy of captured requests safe for concurrent test assertions.
|
||||||
|
func (f *FakeClient) RequestsSnapshot() []TranscribeRequest {
|
||||||
|
f.requestsMu.RLock()
|
||||||
|
defer f.requestsMu.RUnlock()
|
||||||
|
return append([]TranscribeRequest(nil), f.requests...)
|
||||||
|
}
|
||||||
|
|
||||||
func writeMinimalJSON(path string) error {
|
func writeMinimalJSON(path string) error {
|
||||||
if path == "" {
|
if path == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return os.WriteFile(path, minimalTranscriptJSON, 0o644)
|
return fileops.WriteFileAtomic(path, minimalTranscriptJSON, fileops.WorkspaceFileMode)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package whisperx
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -14,8 +15,9 @@ func TestFakeClientCapturesRequestAndReturnsPath(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Transcribe() error = %v", err)
|
t.Fatalf("Transcribe() error = %v", err)
|
||||||
}
|
}
|
||||||
if len(fake.Requests) != 1 || fake.Requests[0].SpeakerID != "alice" {
|
requests := fake.RequestsSnapshot()
|
||||||
t.Fatalf("requests = %#v, want one alice request", fake.Requests)
|
if len(requests) != 1 || requests[0].SpeakerID != "alice" {
|
||||||
|
t.Fatalf("requests = %#v, want one alice request", requests)
|
||||||
}
|
}
|
||||||
if res.OutputRawTranscriptPath != req.OutputRawTranscriptPath {
|
if res.OutputRawTranscriptPath != req.OutputRawTranscriptPath {
|
||||||
t.Fatalf("output path = %q, want %q", res.OutputRawTranscriptPath, req.OutputRawTranscriptPath)
|
t.Fatalf("output path = %q, want %q", res.OutputRawTranscriptPath, req.OutputRawTranscriptPath)
|
||||||
@@ -29,3 +31,22 @@ func TestFakeClientError(t *testing.T) {
|
|||||||
t.Fatal("expected error, got nil")
|
t.Fatal("expected error, got nil")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFakeClientRequestsSnapshotSupportsConcurrentCalls(t *testing.T) {
|
||||||
|
fake := &FakeClient{}
|
||||||
|
const callers = 16
|
||||||
|
var group sync.WaitGroup
|
||||||
|
group.Add(callers)
|
||||||
|
for i := 0; i < callers; i++ {
|
||||||
|
go func() {
|
||||||
|
defer group.Done()
|
||||||
|
if _, err := fake.Transcribe(context.Background(), TranscribeRequest{}); err != nil {
|
||||||
|
t.Errorf("Transcribe() error = %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
group.Wait()
|
||||||
|
if got := len(fake.RequestsSnapshot()); got != callers {
|
||||||
|
t.Fatalf("captured requests = %d, want %d", got, callers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package whisperx
|
package whisperx
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
@@ -14,10 +13,16 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
const defaultMaxResponseBytes int64 = 10 * 1024 * 1024
|
const (
|
||||||
|
defaultMaxWhisperXResponseBytes int64 = 10 * 1024 * 1024
|
||||||
|
whisperXUploadBufferSize = 32 * 1024
|
||||||
|
)
|
||||||
|
|
||||||
// HTTPClientConfig contains parsed, deterministic WhisperX HTTP client settings.
|
// HTTPClientConfig contains parsed, deterministic WhisperX HTTP client settings.
|
||||||
type HTTPClientConfig struct {
|
type HTTPClientConfig struct {
|
||||||
@@ -39,6 +44,7 @@ type HTTPClient struct {
|
|||||||
retryDelay time.Duration
|
retryDelay time.Duration
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
maxResponseBytes int64
|
maxResponseBytes int64
|
||||||
|
openAudio func(string) (io.ReadCloser, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewHTTPClientFromConfigValues builds a client from config values and parses durations once.
|
// NewHTTPClientFromConfigValues builds a client from config values and parses durations once.
|
||||||
@@ -72,11 +78,11 @@ func NewHTTPClient(cfg HTTPClientConfig) (*HTTPClient, error) {
|
|||||||
return nil, fmt.Errorf("whisperx transcribe_url is required")
|
return nil, fmt.Errorf("whisperx transcribe_url is required")
|
||||||
}
|
}
|
||||||
u, err := url.Parse(cfg.TranscribeURL)
|
u, err := url.Parse(cfg.TranscribeURL)
|
||||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
if err != nil || !u.IsAbs() || u.Host == "" || !isHTTPURLScheme(u.Scheme) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("invalid whisperx transcribe_url %q: %w", cfg.TranscribeURL, err)
|
return nil, fmt.Errorf("invalid whisperx transcribe_url %q: %w", cfg.TranscribeURL, err)
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("invalid whisperx transcribe_url %q", cfg.TranscribeURL)
|
return nil, fmt.Errorf("invalid whisperx transcribe_url %q: must be an absolute http or https URL", cfg.TranscribeURL)
|
||||||
}
|
}
|
||||||
if cfg.Timeout <= 0 {
|
if cfg.Timeout <= 0 {
|
||||||
return nil, fmt.Errorf("whisperx timeout must be > 0")
|
return nil, fmt.Errorf("whisperx timeout must be > 0")
|
||||||
@@ -93,12 +99,14 @@ func NewHTTPClient(cfg HTTPClientConfig) (*HTTPClient, error) {
|
|||||||
|
|
||||||
client := cfg.HTTPClient
|
client := cfg.HTTPClient
|
||||||
if client == nil {
|
if client == nil {
|
||||||
client = &http.Client{}
|
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||||
|
transport.ExpectContinueTimeout = 100 * time.Millisecond
|
||||||
|
client = &http.Client{Transport: transport}
|
||||||
}
|
}
|
||||||
|
|
||||||
maxBytes := cfg.MaxResponseBytes
|
maxBytes := cfg.MaxResponseBytes
|
||||||
if maxBytes <= 0 {
|
if maxBytes <= 0 {
|
||||||
maxBytes = defaultMaxResponseBytes
|
maxBytes = defaultMaxWhisperXResponseBytes
|
||||||
}
|
}
|
||||||
|
|
||||||
return &HTTPClient{
|
return &HTTPClient{
|
||||||
@@ -109,6 +117,7 @@ func NewHTTPClient(cfg HTTPClientConfig) (*HTTPClient, error) {
|
|||||||
retryDelay: cfg.RetryDelay,
|
retryDelay: cfg.RetryDelay,
|
||||||
httpClient: client,
|
httpClient: client,
|
||||||
maxResponseBytes: maxBytes,
|
maxResponseBytes: maxBytes,
|
||||||
|
openAudio: func(path string) (io.ReadCloser, error) { return os.Open(path) },
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -147,7 +156,7 @@ func (c *HTTPClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
|||||||
result.Duration = time.Since(start)
|
result.Duration = time.Since(start)
|
||||||
return result, fmt.Errorf("whisperx attempt %d returned invalid json: %w", attempt, err)
|
return result, fmt.Errorf("whisperx attempt %d returned invalid json: %w", attempt, err)
|
||||||
}
|
}
|
||||||
if err := writeFileAtomic(req.OutputRawTranscriptPath, body, 0o644); err != nil {
|
if err := writeFileAtomic(req.OutputRawTranscriptPath, body, fileops.WorkspaceFileMode); err != nil {
|
||||||
result.Duration = time.Since(start)
|
result.Duration = time.Since(start)
|
||||||
return result, fmt.Errorf("whisperx write transcript output %q: %w", req.OutputRawTranscriptPath, err)
|
return result, fmt.Errorf("whisperx write transcript output %q: %w", req.OutputRawTranscriptPath, err)
|
||||||
}
|
}
|
||||||
@@ -183,52 +192,44 @@ func (c *HTTPClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string) (int, []byte, error) {
|
func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string) (int, []byte, error) {
|
||||||
bodyBuf := &bytes.Buffer{}
|
upload := newMultipartUpload(ctx, audioPath, c.language, c.openAudio)
|
||||||
writer := multipart.NewWriter(bodyBuf)
|
defer upload.Close()
|
||||||
|
|
||||||
fileWriter, err := writer.CreateFormFile("file", filepath.Base(audioPath))
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), upload)
|
||||||
if err != nil {
|
|
||||||
return 0, nil, fmt.Errorf("create multipart file field: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
audioFile, err := os.Open(audioPath)
|
|
||||||
if err != nil {
|
|
||||||
return 0, nil, fmt.Errorf("open audio file %q: %w", audioPath, err)
|
|
||||||
}
|
|
||||||
if _, err := io.Copy(fileWriter, audioFile); err != nil {
|
|
||||||
_ = audioFile.Close()
|
|
||||||
return 0, nil, fmt.Errorf("copy audio file %q: %w", audioPath, err)
|
|
||||||
}
|
|
||||||
if err := audioFile.Close(); err != nil {
|
|
||||||
return 0, nil, fmt.Errorf("close audio file %q: %w", audioPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := writer.WriteField("language", c.language); err != nil {
|
|
||||||
return 0, nil, fmt.Errorf("write language form field: %w", err)
|
|
||||||
}
|
|
||||||
if err := writer.Close(); err != nil {
|
|
||||||
return 0, nil, fmt.Errorf("close multipart writer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), bodyBuf)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, fmt.Errorf("build whisperx request: %w", err)
|
return 0, nil, fmt.Errorf("build whisperx request: %w", err)
|
||||||
}
|
}
|
||||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
req.Header.Set("Content-Type", upload.contentType)
|
||||||
|
req.Header.Set("Expect", "100-continue")
|
||||||
|
|
||||||
resp, err := c.httpClient.Do(req)
|
resp, err := c.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
_ = upload.Close()
|
||||||
|
if producerErr := upload.Wait(); producerErr != nil {
|
||||||
|
return 0, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
|
||||||
|
}
|
||||||
return 0, nil, fmt.Errorf("perform whisperx request: %w", err)
|
return 0, nil, fmt.Errorf("perform whisperx request: %w", err)
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
data, err := readBounded(resp.Body, c.maxResponseBytes)
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
if err != nil {
|
_ = upload.Close()
|
||||||
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
if producerErr := upload.Wait(); producerErr != nil {
|
||||||
|
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
|
||||||
|
}
|
||||||
|
if _, err := readWhisperXResponse(resp.Body, c.maxResponseBytes); err != nil {
|
||||||
|
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
||||||
|
}
|
||||||
|
return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if err := upload.Wait(); err != nil {
|
||||||
return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode)
|
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := readWhisperXResponse(resp.Body, c.maxResponseBytes)
|
||||||
|
if err != nil {
|
||||||
|
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
||||||
}
|
}
|
||||||
return resp.StatusCode, data, nil
|
return resp.StatusCode, data, nil
|
||||||
}
|
}
|
||||||
@@ -264,57 +265,188 @@ func (c *HTTPClient) shouldRetry(parent context.Context, err error, status int)
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func readBounded(r io.Reader, maxBytes int64) ([]byte, error) {
|
func readWhisperXResponse(r io.Reader, maxBytes int64) ([]byte, error) {
|
||||||
limited := io.LimitReader(r, maxBytes+1)
|
limited := io.LimitReader(r, maxBytes+1)
|
||||||
data, err := io.ReadAll(limited)
|
data, err := io.ReadAll(limited)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if int64(len(data)) > maxBytes {
|
if int64(len(data)) > maxBytes {
|
||||||
return nil, fmt.Errorf("response exceeds max size %d bytes", maxBytes)
|
return nil, fmt.Errorf("whisperx response exceeds configured limit of %d bytes", maxBytes)
|
||||||
}
|
}
|
||||||
return data, nil
|
return data, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isHTTPURLScheme(scheme string) bool {
|
||||||
|
switch strings.ToLower(scheme) {
|
||||||
|
case "http", "https":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type multipartUpload struct {
|
||||||
|
reader *io.PipeReader
|
||||||
|
writer *io.PipeWriter
|
||||||
|
contentType string
|
||||||
|
done chan struct{}
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
audio io.Closer
|
||||||
|
err error
|
||||||
|
aborted bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMultipartUpload(ctx context.Context, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) *multipartUpload {
|
||||||
|
reader, writer := io.Pipe()
|
||||||
|
multipartWriter := multipart.NewWriter(writer)
|
||||||
|
upload := &multipartUpload{
|
||||||
|
reader: reader,
|
||||||
|
writer: writer,
|
||||||
|
contentType: multipartWriter.FormDataContentType(),
|
||||||
|
done: make(chan struct{}),
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
err := upload.write(ctx, multipartWriter, audioPath, language, openAudio)
|
||||||
|
if err != nil {
|
||||||
|
_ = writer.CloseWithError(err)
|
||||||
|
} else {
|
||||||
|
_ = writer.Close()
|
||||||
|
}
|
||||||
|
upload.mu.Lock()
|
||||||
|
upload.err = err
|
||||||
|
upload.audio = nil
|
||||||
|
upload.mu.Unlock()
|
||||||
|
close(upload.done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
upload.abort()
|
||||||
|
case <-upload.done:
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return upload
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *multipartUpload) Read(p []byte) (int, error) {
|
||||||
|
return u.reader.Read(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *multipartUpload) Close() error {
|
||||||
|
u.abort()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *multipartUpload) Wait() error {
|
||||||
|
<-u.done
|
||||||
|
u.mu.Lock()
|
||||||
|
defer u.mu.Unlock()
|
||||||
|
return u.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *multipartUpload) write(ctx context.Context, writer *multipart.Writer, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) error {
|
||||||
|
fileWriter, err := writer.CreateFormFile("file", filepath.Base(audioPath))
|
||||||
|
if err != nil {
|
||||||
|
return u.producerError(ctx, fmt.Errorf("create multipart file field: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
audioFile, err := openAudio(audioPath)
|
||||||
|
if err != nil {
|
||||||
|
return u.producerError(ctx, fmt.Errorf("open audio file %q: %w", audioPath, err))
|
||||||
|
}
|
||||||
|
u.setAudio(audioFile)
|
||||||
|
|
||||||
|
_, copyErr := io.CopyBuffer(fileWriter, &contextReader{ctx: ctx, reader: audioFile}, make([]byte, whisperXUploadBufferSize))
|
||||||
|
closeErr := audioFile.Close()
|
||||||
|
u.clearAudio(audioFile)
|
||||||
|
if copyErr != nil {
|
||||||
|
return u.producerError(ctx, fmt.Errorf("copy audio file %q: %w", audioPath, copyErr))
|
||||||
|
}
|
||||||
|
if closeErr != nil {
|
||||||
|
return u.producerError(ctx, fmt.Errorf("close audio file %q: %w", audioPath, closeErr))
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := writer.WriteField("language", language); err != nil {
|
||||||
|
return u.producerError(ctx, fmt.Errorf("write language form field: %w", err))
|
||||||
|
}
|
||||||
|
if err := writer.Close(); err != nil {
|
||||||
|
return u.producerError(ctx, fmt.Errorf("close multipart writer: %w", err))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *multipartUpload) producerError(ctx context.Context, err error) error {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
u.mu.Lock()
|
||||||
|
aborted := u.aborted
|
||||||
|
u.mu.Unlock()
|
||||||
|
if aborted {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *multipartUpload) setAudio(audio io.Closer) {
|
||||||
|
u.mu.Lock()
|
||||||
|
u.audio = audio
|
||||||
|
aborted := u.aborted
|
||||||
|
u.mu.Unlock()
|
||||||
|
if aborted {
|
||||||
|
_ = audio.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *multipartUpload) clearAudio(audio io.Closer) {
|
||||||
|
u.mu.Lock()
|
||||||
|
if u.audio == audio {
|
||||||
|
u.audio = nil
|
||||||
|
}
|
||||||
|
u.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *multipartUpload) abort() {
|
||||||
|
u.mu.Lock()
|
||||||
|
if u.aborted {
|
||||||
|
u.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
u.aborted = true
|
||||||
|
audio := u.audio
|
||||||
|
u.mu.Unlock()
|
||||||
|
|
||||||
|
_ = u.reader.Close()
|
||||||
|
if audio != nil {
|
||||||
|
_ = audio.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type contextReader struct {
|
||||||
|
ctx context.Context
|
||||||
|
reader io.Reader
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *contextReader) Read(p []byte) (int, error) {
|
||||||
|
select {
|
||||||
|
case <-r.ctx.Done():
|
||||||
|
return 0, r.ctx.Err()
|
||||||
|
default:
|
||||||
|
return r.reader.Read(p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
|
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
|
||||||
if strings.TrimSpace(path) == "" {
|
if strings.TrimSpace(path) == "" {
|
||||||
return fmt.Errorf("path is required")
|
return fmt.Errorf("path is required")
|
||||||
}
|
}
|
||||||
dir := filepath.Dir(path)
|
if err := fileops.WriteFileAtomic(path, data, perm); err != nil {
|
||||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
return fmt.Errorf("write file %q: %w", path, err)
|
||||||
return fmt.Errorf("create parent dir %q: %w", dir, err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
base := filepath.Base(path)
|
|
||||||
tmp, err := os.CreateTemp(dir, "."+base+".tmp-*")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("create temp file: %w", err)
|
|
||||||
}
|
|
||||||
tmpPath := tmp.Name()
|
|
||||||
removeTmp := true
|
|
||||||
defer func() {
|
|
||||||
if removeTmp {
|
|
||||||
_ = os.Remove(tmpPath)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
if _, err := tmp.Write(data); err != nil {
|
|
||||||
_ = tmp.Close()
|
|
||||||
return fmt.Errorf("write temp file: %w", err)
|
|
||||||
}
|
|
||||||
if err := tmp.Sync(); err != nil {
|
|
||||||
_ = tmp.Close()
|
|
||||||
return fmt.Errorf("sync temp file: %w", err)
|
|
||||||
}
|
|
||||||
if err := tmp.Close(); err != nil {
|
|
||||||
return fmt.Errorf("close temp file: %w", err)
|
|
||||||
}
|
|
||||||
if err := os.Chmod(tmpPath, perm); err != nil {
|
|
||||||
return fmt.Errorf("chmod temp file: %w", err)
|
|
||||||
}
|
|
||||||
if err := os.Rename(tmpPath, path); err != nil {
|
|
||||||
return fmt.Errorf("rename temp file: %w", err)
|
|
||||||
}
|
|
||||||
removeTmp = false
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
@@ -19,6 +20,7 @@ func TestHTTPClientTranscribeSuccess(t *testing.T) {
|
|||||||
var gotLanguage string
|
var gotLanguage string
|
||||||
var gotFileField string
|
var gotFileField string
|
||||||
var gotFileSize int
|
var gotFileSize int
|
||||||
|
var gotFileData string
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Method != http.MethodPost {
|
if r.Method != http.MethodPost {
|
||||||
@@ -40,6 +42,7 @@ func TestHTTPClientTranscribeSuccess(t *testing.T) {
|
|||||||
t.Fatalf("ReadAll(file) error = %v", err)
|
t.Fatalf("ReadAll(file) error = %v", err)
|
||||||
}
|
}
|
||||||
gotFileSize = len(data)
|
gotFileSize = len(data)
|
||||||
|
gotFileData = string(data)
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
_, _ = w.Write([]byte(`{"schema":"speaker_transcript.v1","segments":[]}`))
|
_, _ = w.Write([]byte(`{"schema":"speaker_transcript.v1","segments":[]}`))
|
||||||
@@ -77,13 +80,27 @@ func TestHTTPClientTranscribeSuccess(t *testing.T) {
|
|||||||
if gotFileSize == 0 {
|
if gotFileSize == 0 {
|
||||||
t.Fatal("file size = 0, want >0")
|
t.Fatal("file size = 0, want >0")
|
||||||
}
|
}
|
||||||
|
if gotFileData != "audio-data" {
|
||||||
|
t.Fatalf("file data = %q, want exact payload", gotFileData)
|
||||||
|
}
|
||||||
verifyJSONFile(t, outPath)
|
verifyJSONFile(t, outPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHTTPClientRetriesOnTransientAndSucceeds(t *testing.T) {
|
func TestHTTPClientRetriesOnTransientAndSucceeds(t *testing.T) {
|
||||||
var calls atomic.Int32
|
var calls atomic.Int32
|
||||||
|
var payloads []string
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
n := calls.Add(1)
|
n := calls.Add(1)
|
||||||
|
file, _, err := r.FormFile("file")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FormFile(file) error = %v", err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(file)
|
||||||
|
_ = file.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadAll(file) error = %v", err)
|
||||||
|
}
|
||||||
|
payloads = append(payloads, string(data))
|
||||||
if n == 1 {
|
if n == 1 {
|
||||||
http.Error(w, "temporary", http.StatusInternalServerError)
|
http.Error(w, "temporary", http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
@@ -112,6 +129,9 @@ func TestHTTPClientRetriesOnTransientAndSucceeds(t *testing.T) {
|
|||||||
if calls.Load() != 2 {
|
if calls.Load() != 2 {
|
||||||
t.Fatalf("calls = %d, want 2", calls.Load())
|
t.Fatalf("calls = %d, want 2", calls.Load())
|
||||||
}
|
}
|
||||||
|
if len(payloads) != 2 || payloads[0] != "audio-data" || payloads[1] != "audio-data" {
|
||||||
|
t.Fatalf("retry payloads = %#v, want two exact audio payloads", payloads)
|
||||||
|
}
|
||||||
verifyJSONFile(t, outPath)
|
verifyJSONFile(t, outPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -247,8 +267,245 @@ func TestHTTPClientConstructorValidation(t *testing.T) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected bad retry_delay error")
|
t.Fatal("expected bad retry_delay error")
|
||||||
}
|
}
|
||||||
|
for _, endpoint := range []string{"ftp://example.com/transcribe", "file:///tmp/transcribe", "//example.com/transcribe", "https:/missing-host"} {
|
||||||
|
if _, err := NewHTTPClientFromConfigValues(endpoint, "en", "30m", "2s", 1); err == nil {
|
||||||
|
t.Errorf("NewHTTPClientFromConfigValues(%q) error = nil, want endpoint validation error", endpoint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, endpoint := range []string{"http://example.com/transcribe", "https://example.com/transcribe"} {
|
||||||
|
if _, err := NewHTTPClientFromConfigValues(endpoint, "en", "30m", "2s", 1); err != nil {
|
||||||
|
t.Errorf("NewHTTPClientFromConfigValues(%q) error = %v", endpoint, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientStreamsUploadBeforeSourceCompletes(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
source := newGatedReadCloser([]byte("audio-data"), release)
|
||||||
|
firstByteReceived := make(chan struct{})
|
||||||
|
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
part := firstMultipartFilePart(t, r)
|
||||||
|
buf := make([]byte, 1)
|
||||||
|
if _, err := part.Read(buf); err != nil {
|
||||||
|
t.Errorf("Read(file) error = %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
close(firstByteReceived)
|
||||||
|
if _, err := io.Copy(io.Discard, part); err != nil {
|
||||||
|
t.Errorf("discard remaining file data: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
client := newTestHTTPClient(t, srv.URL)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) { return source, nil }
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := client.Transcribe(context.Background(), TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-firstByteReceived:
|
||||||
|
close(release)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("server did not receive streamed audio before source completed")
|
||||||
|
}
|
||||||
|
if err := <-done; err != nil {
|
||||||
|
t.Fatalf("Transcribe() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientSourceReadFailureReachesCaller(t *testing.T) {
|
||||||
|
sourceErr := errors.New("source read failed")
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
client := newTestHTTPClient(t, srv.URL)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||||
|
return &failingReadCloser{first: []byte("partial"), err: sourceErr}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := client.Transcribe(context.Background(), TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
|
||||||
|
if !errors.Is(err, sourceErr) {
|
||||||
|
t.Fatalf("Transcribe() error = %v, want source read failure", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientEarlyServerResponseReturns(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
source := newGatedReadCloser([]byte("audio-data"), release)
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.Error(w, "bad request", http.StatusBadRequest)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
client := newTestHTTPClient(t, srv.URL)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) { return source, nil }
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := client.Transcribe(context.Background(), TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Transcribe() error = nil, want HTTP status error")
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Transcribe() did not finish after server closed the request early")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientCancellationReleasesBlockedProducer(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
source := newGatedReadCloser([]byte("audio-data"), release)
|
||||||
|
firstByteReceived := make(chan struct{})
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
part := firstMultipartFilePart(t, r)
|
||||||
|
buf := make([]byte, 1)
|
||||||
|
if _, err := part.Read(buf); err != nil {
|
||||||
|
t.Errorf("Read(file) error = %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
close(firstByteReceived)
|
||||||
|
select {
|
||||||
|
case <-r.Context().Done():
|
||||||
|
case <-source.closed:
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
client := newTestHTTPClient(t, srv.URL)
|
||||||
|
client.openAudio = func(string) (io.ReadCloser, error) { return source, nil }
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := client.Transcribe(ctx, TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-firstByteReceived:
|
||||||
|
cancel()
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
cancel()
|
||||||
|
t.Fatal("server did not receive initial streamed audio")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("Transcribe() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("Transcribe() did not finish after cancellation")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-source.closed:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("blocked audio source was not closed on cancellation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTPClientBoundsWhisperXResponse(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
|
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
client, err := NewHTTPClient(HTTPClientConfig{TranscribeURL: srv.URL, Language: "en", Timeout: time.Second, MaxResponseBytes: 4})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewHTTPClient() error = %v", err)
|
||||||
|
}
|
||||||
|
audioPath := writeWhisperXTestFile(t, "audio.flac", "audio-data")
|
||||||
|
_, err = client.Transcribe(context.Background(), TranscribeRequest{AudioPath: audioPath, OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "whisperx response exceeds configured limit") {
|
||||||
|
t.Fatalf("Transcribe() error = %v, want bounded WhisperX response error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestHTTPClient(t *testing.T, endpoint string) *HTTPClient {
|
||||||
|
t.Helper()
|
||||||
|
client, err := NewHTTPClientFromConfigValues(endpoint, "en", "2s", "1ms", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewHTTPClientFromConfigValues() error = %v", err)
|
||||||
|
}
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstMultipartFilePart(t *testing.T, r *http.Request) *multipart.Part {
|
||||||
|
t.Helper()
|
||||||
|
reader, err := r.MultipartReader()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MultipartReader() error = %v", err)
|
||||||
|
}
|
||||||
|
part, err := reader.NextPart()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NextPart() error = %v", err)
|
||||||
|
}
|
||||||
|
if part.FormName() != "file" {
|
||||||
|
t.Fatalf("first form field = %q, want file", part.FormName())
|
||||||
|
}
|
||||||
|
return part
|
||||||
|
}
|
||||||
|
|
||||||
|
type gatedReadCloser struct {
|
||||||
|
first []byte
|
||||||
|
release <-chan struct{}
|
||||||
|
closed chan struct{}
|
||||||
|
sent bool
|
||||||
|
once atomic.Bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newGatedReadCloser(first []byte, release <-chan struct{}) *gatedReadCloser {
|
||||||
|
return &gatedReadCloser{first: first, release: release, closed: make(chan struct{})}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *gatedReadCloser) Read(p []byte) (int, error) {
|
||||||
|
if !r.sent {
|
||||||
|
r.sent = true
|
||||||
|
return copy(p, r.first), nil
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-r.release:
|
||||||
|
return 0, io.EOF
|
||||||
|
case <-r.closed:
|
||||||
|
return 0, errors.New("audio source closed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *gatedReadCloser) Close() error {
|
||||||
|
if r.once.CompareAndSwap(false, true) {
|
||||||
|
close(r.closed)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type failingReadCloser struct {
|
||||||
|
first []byte
|
||||||
|
err error
|
||||||
|
sent bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *failingReadCloser) Read(p []byte) (int, error) {
|
||||||
|
if !r.sent {
|
||||||
|
r.sent = true
|
||||||
|
return copy(p, r.first), nil
|
||||||
|
}
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *failingReadCloser) Close() error { return nil }
|
||||||
|
|
||||||
func writeWhisperXTestFile(t *testing.T, name, contents string) string {
|
func writeWhisperXTestFile(t *testing.T, name, contents string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
path := filepath.Join(t.TempDir(), name)
|
path := filepath.Join(t.TempDir(), name)
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -47,19 +48,55 @@ func (f *artifactSelectionFlag) Normalize() ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateSelectedArtifacts(cfg *config.Config, selected []string) error {
|
func validateSelectedArtifacts(cfg *config.Config, selected []string) error {
|
||||||
if len(selected) == 0 {
|
_, err := resolveEffectiveArtifacts(cfg, selected)
|
||||||
return nil
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func resolveEffectiveArtifacts(cfg *config.Config, selected []string) (artifacts.EffectiveArtifactSet, error) {
|
||||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
||||||
return fmt.Errorf("--artifacts requires pipeline.scriptorium.artifacts to be configured")
|
if len(selected) == 0 {
|
||||||
|
return artifacts.ResolveEffectiveArtifactSet(nil, nil)
|
||||||
|
}
|
||||||
|
return artifacts.EffectiveArtifactSet{}, fmt.Errorf("--artifacts requires pipeline.scriptorium.artifacts to be configured")
|
||||||
}
|
}
|
||||||
configured := cfg.Pipeline.Scriptorium.Artifacts
|
configured := artifacts.ConfiguredArtifactDefinitions(cfg.Pipeline.Scriptorium.Artifacts)
|
||||||
if len(configured) == 0 {
|
if len(selected) > 0 && len(configured) == 0 {
|
||||||
return fmt.Errorf("--artifacts requires at least one configured artifact in pipeline.scriptorium.artifacts")
|
return artifacts.EffectiveArtifactSet{}, fmt.Errorf("--artifacts requires at least one configured artifact in pipeline.scriptorium.artifacts")
|
||||||
}
|
}
|
||||||
for _, name := range selected {
|
effective, err := artifacts.ResolveEffectiveArtifactSet(configured, selected)
|
||||||
if _, ok := configured[name]; !ok {
|
if err != nil {
|
||||||
return fmt.Errorf("--artifacts includes unknown artifact %q", name)
|
if strings.Contains(err.Error(), "is not configured") {
|
||||||
|
return artifacts.EffectiveArtifactSet{}, fmt.Errorf("--artifacts includes unknown artifact %q", selectedArtifactName(err))
|
||||||
|
}
|
||||||
|
return artifacts.EffectiveArtifactSet{}, err
|
||||||
|
}
|
||||||
|
if err := validateEffectiveArtifactConfiguration(cfg.Pipeline.Scriptorium.Artifacts, effective); err != nil {
|
||||||
|
return artifacts.EffectiveArtifactSet{}, err
|
||||||
|
}
|
||||||
|
return effective, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectedArtifactName(err error) string {
|
||||||
|
message := err.Error()
|
||||||
|
start := strings.Index(message, "\"")
|
||||||
|
if start < 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
end := strings.Index(message[start+1:], "\"")
|
||||||
|
if end < 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return message[start+1 : start+1+end]
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateEffectiveArtifactConfiguration(configured map[string]config.ScriptoriumArtifactConfig, effective artifacts.EffectiveArtifactSet) error {
|
||||||
|
for _, name := range effective.Keys() {
|
||||||
|
artifactCfg := configured[name]
|
||||||
|
if strings.TrimSpace(artifactCfg.PromptID) == "" {
|
||||||
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.prompt_id is required when selected", name)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(artifactCfg.OutputPath) == "" {
|
||||||
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when selected", name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ func TestExecuteRunStageArtifactsUnsupportedStageFails(t *testing.T) {
|
|||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute(
|
code := Execute(
|
||||||
[]string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
[]string{"run-stage", "extract", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||||
&stdout,
|
&stdout,
|
||||||
&stderr,
|
&stderr,
|
||||||
)
|
)
|
||||||
@@ -130,6 +130,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
|||||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||||
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
|
seed.MarkStageSkipped("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), "notarius_disabled")
|
||||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||||
t.Fatalf("save manifest: %v", err)
|
t.Fatalf("save manifest: %v", err)
|
||||||
}
|
}
|
||||||
@@ -143,7 +144,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out.String(), "executed=0 skipped=10") {
|
if !strings.Contains(out.String(), "executed=1 skipped=11") {
|
||||||
t.Fatalf("output = %q, want all stages skipped", out.String())
|
t.Fatalf("output = %q, want all stages skipped", out.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -110,6 +110,20 @@ func TestValidateSelectedArtifacts(t *testing.T) {
|
|||||||
},
|
},
|
||||||
selected: []string{"player_handout", "session_recap"},
|
selected: []string{"player_handout", "session_recap"},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "selected disabled artifact must be executable",
|
||||||
|
cfg: &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{
|
||||||
|
Scriptorium: &config.ScriptoriumConfig{
|
||||||
|
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||||
|
"player_handout": {Enabled: false, OutputPath: "artifacts/player_handout.md"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
selected: []string{"player_handout"},
|
||||||
|
wantErr: "pipeline.scriptorium.artifacts.player_handout.prompt_id is required when selected",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||||
)
|
)
|
||||||
|
|
||||||
func resolveCampaignConfigPath(pipelineCfg *config.PipelineConfig, campaignIDFlag, campaignFileFlag string) (string, error) {
|
func resolveCampaignConfigPath(pipelineCfg *config.PipelineConfig, campaignIDFlag, campaignFileFlag string) (string, error) {
|
||||||
@@ -33,12 +34,8 @@ func resolveCampaignConfigPath(pipelineCfg *config.PipelineConfig, campaignIDFla
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateCampaignIDToken(campaignID string) error {
|
func validateCampaignIDToken(campaignID string) error {
|
||||||
if filepath.IsAbs(campaignID) ||
|
if err := pathsafe.ValidateOpaqueSegment(campaignID); err != nil {
|
||||||
strings.Contains(campaignID, "/") ||
|
return fmt.Errorf("campaign id %q must be a single path segment and opaque identifier: %w", campaignID, err)
|
||||||
strings.Contains(campaignID, `\`) ||
|
|
||||||
campaignID == "." ||
|
|
||||||
campaignID == ".." {
|
|
||||||
return fmt.Errorf("campaign id %q must be a single path segment", campaignID)
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Clean removes local workspace/spool state while preserving durable cache
|
// Clean removes local workspace/spool state while preserving durable cache
|
||||||
@@ -39,10 +40,12 @@ func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCac
|
|||||||
if strings.TrimSpace(flags.sessionID) == "" {
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
return fmt.Errorf("clean: session_id is required unless --all is set")
|
return fmt.Errorf("clean: session_id is required unless --all is set")
|
||||||
}
|
}
|
||||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("clean: %w", err)
|
return fmt.Errorf("clean: %w", err)
|
||||||
}
|
}
|
||||||
|
defer func() { _ = loaded.Close() }()
|
||||||
|
cfg := loaded.Config
|
||||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||||
return fmt.Errorf("clean: resolved pipeline and session config are required")
|
return fmt.Errorf("clean: resolved pipeline and session config are required")
|
||||||
}
|
}
|
||||||
@@ -135,7 +138,7 @@ func reportCleanScopedDir(out io.Writer, root, target, policy string, dryRun boo
|
|||||||
fmt.Fprintf(out, "Missing: %s\n", dir.TargetAbs)
|
fmt.Fprintf(out, "Missing: %s\n", dir.TargetAbs)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if err := os.RemoveAll(dir.TargetAbs); err != nil {
|
if err := fileops.RemoveAllUnderRoot(dir.RootAbs, dir.TargetAbs); err != nil {
|
||||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(out, "Deleted: %s\n", dir.TargetAbs)
|
fmt.Fprintf(out, "Deleted: %s\n", dir.TargetAbs)
|
||||||
@@ -160,7 +163,7 @@ func reportCleanRootChildren(out io.Writer, root, policy string, dryRun bool) er
|
|||||||
fmt.Fprintf(out, "Would delete: %s\n", entry)
|
fmt.Fprintf(out, "Would delete: %s\n", entry)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := os.RemoveAll(entry); err != nil {
|
if err := fileops.RemoveAllUnderRoot(rootAbs, entry); err != nil {
|
||||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, entry, err)
|
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, entry, err)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(out, "Deleted: %s\n", entry)
|
fmt.Fprintf(out, "Deleted: %s\n", entry)
|
||||||
@@ -265,7 +268,7 @@ func reportCleanScopedFile(out io.Writer, root, target, policy string, dryRun bo
|
|||||||
fmt.Fprintf(out, "Missing cache file: %s\n", file.TargetAbs)
|
fmt.Fprintf(out, "Missing cache file: %s\n", file.TargetAbs)
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
if err := os.Remove(file.TargetAbs); err != nil {
|
if err := fileops.RemoveAllUnderRoot(file.RootAbs, file.TargetAbs); err != nil {
|
||||||
return false, fmt.Errorf("cleanup policy %s: remove %q: %w", policy, file.TargetAbs, err)
|
return false, fmt.Errorf("cleanup policy %s: remove %q: %w", policy, file.TargetAbs, err)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(out, "Deleted cache file: %s\n", file.TargetAbs)
|
fmt.Fprintf(out, "Deleted cache file: %s\n", file.TargetAbs)
|
||||||
|
|||||||
@@ -31,8 +31,8 @@ func TestExecuteValidCommands(t *testing.T) {
|
|||||||
args []string
|
args []string
|
||||||
wantOut string
|
wantOut string
|
||||||
}{
|
}{
|
||||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=10 skipped=0; manifest="},
|
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=11 skipped=1; manifest="},
|
||||||
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nrender: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nextract: run\nrender: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
||||||
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
||||||
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||||
}
|
}
|
||||||
@@ -189,7 +189,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
|||||||
configDir := t.TempDir()
|
configDir := t.TempDir()
|
||||||
sessionID := "2026-05-03"
|
sessionID := "2026-05-03"
|
||||||
secretsDir := filepath.Join(configDir, "secrets")
|
secretsDir := filepath.Join(configDir, "secrets")
|
||||||
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
|
if err := os.MkdirAll(secretsDir, secretDirectoryPrivateMode); err != nil {
|
||||||
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
|
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
|
||||||
@@ -218,7 +218,7 @@ audita:
|
|||||||
binary: ` + auditaBinary + `
|
binary: ` + auditaBinary + `
|
||||||
llm_api_key_env: OPENROUTER_API_KEY
|
llm_api_key_env: OPENROUTER_API_KEY
|
||||||
notification:
|
notification:
|
||||||
timeout: 10s
|
mode: noop
|
||||||
`
|
`
|
||||||
sessionYAML := `session_id: ` + sessionID + `
|
sessionYAML := `session_id: ` + sessionID + `
|
||||||
campaign: sample-campaign
|
campaign: sample-campaign
|
||||||
@@ -283,7 +283,7 @@ seriatim:
|
|||||||
audita:
|
audita:
|
||||||
binary: audita
|
binary: audita
|
||||||
notification:
|
notification:
|
||||||
timeout: 10s
|
mode: noop
|
||||||
`
|
`
|
||||||
sessionYAML := `session_id: 2026-05-03
|
sessionYAML := `session_id: 2026-05-03
|
||||||
campaign: sample-campaign
|
campaign: sample-campaign
|
||||||
@@ -308,8 +308,8 @@ inputs:
|
|||||||
if code == 0 {
|
if code == 0 {
|
||||||
t.Fatal("exit code = 0, want non-zero")
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "read secrets env_dir") {
|
if !strings.Contains(stderr.String(), "validate secrets env_dir") {
|
||||||
t.Fatalf("stderr = %q, want secrets read-dir error context", stderr.String())
|
t.Fatalf("stderr = %q, want secrets validation error context", stderr.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -335,7 +335,7 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
|||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=10 skipped=0; manifest=") {
|
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=11 skipped=1; manifest=") {
|
||||||
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -476,9 +476,6 @@ func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...
|
|||||||
seriatimBinary := writeSeriatimAppTestWrapper(t)
|
seriatimBinary := writeSeriatimAppTestWrapper(t)
|
||||||
scriptoriumBinary := writeScriptoriumAppTestWrapper(t)
|
scriptoriumBinary := writeScriptoriumAppTestWrapper(t)
|
||||||
auditaBinary := writeAuditaAppTestWrapper(t)
|
auditaBinary := writeAuditaAppTestWrapper(t)
|
||||||
t.Setenv("GO_WANT_APP_SERIATIM_HELPER", "1")
|
|
||||||
t.Setenv("GO_WANT_APP_SCRIPTORIUM_HELPER", "1")
|
|
||||||
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
|
||||||
t.Setenv("AUDITA_LLM_API_KEY", "test-audita-key")
|
t.Setenv("AUDITA_LLM_API_KEY", "test-audita-key")
|
||||||
t.Setenv("PATH", filepath.Dir(scriptoriumBinary)+string(os.PathListSeparator)+os.Getenv("PATH"))
|
t.Setenv("PATH", filepath.Dir(scriptoriumBinary)+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||||
|
|
||||||
@@ -513,7 +510,7 @@ seriatim:
|
|||||||
audita:
|
audita:
|
||||||
binary: ` + auditaBinary + `
|
binary: ` + auditaBinary + `
|
||||||
notification:
|
notification:
|
||||||
timeout: 10s
|
mode: noop
|
||||||
`
|
`
|
||||||
|
|
||||||
sessionYAML := `session_id: 2026-05-03
|
sessionYAML := `session_id: 2026-05-03
|
||||||
@@ -623,7 +620,7 @@ func writeScriptoriumAppTestWrapper(t *testing.T) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestScriptoriumAppHelper(t *testing.T) {
|
func TestScriptoriumAppHelper(t *testing.T) {
|
||||||
if os.Getenv("GO_WANT_APP_SCRIPTORIUM_HELPER") != "1" {
|
if !appHelperInvocation() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -663,7 +660,7 @@ func TestScriptoriumAppHelper(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSeriatimAppHelper(t *testing.T) {
|
func TestSeriatimAppHelper(t *testing.T) {
|
||||||
if os.Getenv("GO_WANT_APP_SERIATIM_HELPER") != "1" {
|
if !appHelperInvocation() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -725,7 +722,7 @@ func writeAuditaAppTestWrapper(t *testing.T) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestAuditaAppHelper(t *testing.T) {
|
func TestAuditaAppHelper(t *testing.T) {
|
||||||
if os.Getenv("GO_WANT_APP_AUDITA_HELPER") != "1" {
|
if !appHelperInvocation() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -779,6 +776,15 @@ func TestAuditaAppHelper(t *testing.T) {
|
|||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func appHelperInvocation() bool {
|
||||||
|
for _, arg := range os.Args {
|
||||||
|
if arg == "--" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func appSeriatimFlagValue(args []string, name string) string {
|
func appSeriatimFlagValue(args []string, name string) string {
|
||||||
for i := 0; i < len(args)-1; i++ {
|
for i := 0; i < len(args)-1; i++ {
|
||||||
if args[i] == name {
|
if args[i] == name {
|
||||||
|
|||||||
@@ -2,13 +2,16 @@ package app
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
type pipelineCampaignConfig struct {
|
type pipelineCampaignConfig struct {
|
||||||
@@ -18,14 +21,48 @@ type pipelineCampaignConfig struct {
|
|||||||
Campaign *config.CampaignConfig
|
Campaign *config.CampaignConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (*config.Config, error) {
|
var downloadObjectToTempFn = storage.DownloadObjectToTemp
|
||||||
|
|
||||||
|
type commandConfig struct {
|
||||||
|
Config *config.Config
|
||||||
|
cleanup func() error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *commandConfig) Close() error {
|
||||||
|
if c == nil || c.cleanup == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cleanup := c.cleanup
|
||||||
|
c.cleanup = nil
|
||||||
|
return cleanup()
|
||||||
|
}
|
||||||
|
|
||||||
|
func retainedCommandConfig(cfg *config.Config) *commandConfig {
|
||||||
|
return &commandConfig{Config: cfg}
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (loaded *commandConfig, err error) {
|
||||||
|
var cleanup func() error
|
||||||
|
defer func() {
|
||||||
|
if err == nil || cleanup == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cleanupErr := cleanup(); cleanupErr != nil {
|
||||||
|
err = errors.Join(err, cleanupErr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
base, err := loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag)
|
base, err := loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
|
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
|
||||||
return config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, explicitSession, sessionOpts)
|
cfg, err := config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, explicitSession, sessionOpts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return retainedCommandConfig(cfg), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
discoveredSession, err := discoverSessionConfigPathWithCandidates(config.DefaultSessionConfigSearchPaths)
|
discoveredSession, err := discoverSessionConfigPathWithCandidates(config.DefaultSessionConfigSearchPaths)
|
||||||
@@ -33,7 +70,11 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if discoveredSession.Path != "" {
|
if discoveredSession.Path != "" {
|
||||||
return config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, discoveredSession.Path, sessionOpts)
|
cfg, err := config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, discoveredSession.Path, sessionOpts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return retainedCommandConfig(cfg), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
sessionID := strings.TrimSpace(sessionOpts.SessionID)
|
sessionID := strings.TrimSpace(sessionOpts.SessionID)
|
||||||
@@ -41,7 +82,11 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
|||||||
return nil, missingSessionConfigError(discoveredSession.Searched, "remote session loading requires a session_id")
|
return nil, missingSessionConfigError(discoveredSession.Searched, "remote session loading requires a session_id")
|
||||||
}
|
}
|
||||||
|
|
||||||
sessionPrefix := artifacts.S3SessionPrefix(base.Pipeline.Storage.S3.RootPrefix, config.CampaignID(base.Campaign), sessionID)
|
rootPrefix := ""
|
||||||
|
if base.Pipeline.Storage.S3 != nil {
|
||||||
|
rootPrefix = base.Pipeline.Storage.S3.RootPrefix
|
||||||
|
}
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix(rootPrefix, config.CampaignID(base.Campaign), sessionID)
|
||||||
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||||
partialCfg := &config.Config{
|
partialCfg := &config.Config{
|
||||||
Pipeline: base.Pipeline,
|
Pipeline: base.Pipeline,
|
||||||
@@ -58,20 +103,26 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
|
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
|
||||||
}
|
}
|
||||||
sessionTempPath, err := storage.DownloadObjectToTemp(ctx, store, remoteKey, "narratio-session-*.yml")
|
sessionTempPath, err := downloadObjectToTempFn(ctx, store, remoteKey, "narratio-session-*.yml")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
||||||
}
|
}
|
||||||
|
cleanup = func() error {
|
||||||
|
if err := fileops.RemoveAllUnderRoot(filepath.Dir(sessionTempPath), sessionTempPath); err != nil {
|
||||||
|
return fmt.Errorf("remove downloaded remote session config: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
sessionBytes, err := os.ReadFile(sessionTempPath)
|
sessionBytes, err := os.ReadFile(sessionTempPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("read downloaded remote session %q: %w", sessionTempPath, err)
|
return nil, fmt.Errorf("read downloaded remote session config: %w", err)
|
||||||
}
|
}
|
||||||
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts)
|
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return config.Resolve(
|
cfg, err := config.Resolve(
|
||||||
base.PipelinePath,
|
base.PipelinePath,
|
||||||
base.Pipeline,
|
base.Pipeline,
|
||||||
base.CampaignPath,
|
base.CampaignPath,
|
||||||
@@ -85,9 +136,14 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
|||||||
S3Key: remoteKey,
|
S3Key: remoteKey,
|
||||||
S3Size: sessionInfo.Size,
|
S3Size: sessionInfo.Size,
|
||||||
S3ETag: sessionInfo.ETag,
|
S3ETag: sessionInfo.ETag,
|
||||||
SpoolPath: sessionTempPath,
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
loaded = &commandConfig{Config: cfg, cleanup: cleanup}
|
||||||
|
cleanup = nil
|
||||||
|
return loaded, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag string) (*pipelineCampaignConfig, error) {
|
func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag string) (*pipelineCampaignConfig, error) {
|
||||||
|
|||||||
442
internal/app/extract_lifecycle_test.go
Normal file
442
internal/app/extract_lifecycle_test.go
Normal file
@@ -0,0 +1,442 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||||
|
)
|
||||||
|
|
||||||
|
type materializingNotariusRunner struct {
|
||||||
|
cfg *config.NotariusConfig
|
||||||
|
requests []notarius.RunRequest
|
||||||
|
failuresRemaining int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *materializingNotariusRunner) Run(_ context.Context, req notarius.RunRequest) (notarius.RunResult, error) {
|
||||||
|
r.requests = append(r.requests, req)
|
||||||
|
if r.failuresRemaining > 0 {
|
||||||
|
r.failuresRemaining--
|
||||||
|
return notarius.RunResult{}, errors.New("notarius execution failed")
|
||||||
|
}
|
||||||
|
externalRunID := fmt.Sprintf("notarius-run-%d", len(r.requests))
|
||||||
|
bundle := filepath.Join(req.OutputRoot, externalRunID)
|
||||||
|
lanesDir := filepath.Join(bundle, "lanes")
|
||||||
|
if err := os.MkdirAll(lanesDir, 0o755); err != nil {
|
||||||
|
return notarius.RunResult{}, err
|
||||||
|
}
|
||||||
|
for path, content := range map[string]string{
|
||||||
|
filepath.Join(bundle, "index.json"): `{"manifest_file":"manifest.json"}`,
|
||||||
|
filepath.Join(bundle, "manifest.json"): `{}`,
|
||||||
|
filepath.Join(bundle, "rejected.json"): `{"rejected":[]}`,
|
||||||
|
filepath.Join(bundle, "warnings.json"): `{"warnings":[]}`,
|
||||||
|
filepath.Join(lanesDir, "npcs.json"): `{"npcs":[]}`,
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||||
|
return notarius.RunResult{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
output := r.cfg.Outputs["npc_registry"]
|
||||||
|
return notarius.RunResult{
|
||||||
|
Receipt: notarius.Receipt{
|
||||||
|
SchemaVersion: notarius.ReceiptSchemaVersion, RunID: externalRunID,
|
||||||
|
PipelineID: req.PipelineID, OutputDirectory: bundle, IndexFile: "index.json",
|
||||||
|
NormalizedOutputCount: 1, ValidationStatus: "valid",
|
||||||
|
},
|
||||||
|
BundleRoot: bundle,
|
||||||
|
Index: notarius.Index{
|
||||||
|
Path: filepath.Join(bundle, "index.json"), RejectedPath: filepath.Join(bundle, "rejected.json"),
|
||||||
|
WarningsPath: filepath.Join(bundle, "warnings.json"),
|
||||||
|
Lanes: []notarius.LaneDescriptor{{
|
||||||
|
LaneID: output.LaneID, File: "lanes/npcs.json", Path: filepath.Join(lanesDir, "npcs.json"),
|
||||||
|
MediaType: output.MediaType, SchemaID: output.SchemaID,
|
||||||
|
SchemaVersion: output.SchemaVersion, ModuleKey: output.ModuleKey,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleDisabledThenEnabled(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||||
|
plan, err := BuildSingleStagePlan("extract")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("disabled executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(first.Executed) != 1 || len(first.Skipped) != 1 || first.Skipped[0] != "extract" || len(runner.requests) != 0 {
|
||||||
|
t.Fatalf("disabled summary = %#v requests=%d", first, len(runner.requests))
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Pipeline.Notarius.Enabled = true
|
||||||
|
second, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("enabled executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(second.Executed) != 1 || len(second.Skipped) != 0 || len(runner.requests) != 1 {
|
||||||
|
t.Fatalf("enabled summary = %#v requests=%d", second, len(runner.requests))
|
||||||
|
}
|
||||||
|
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), second.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if loaded.Stages["extract"] == nil || loaded.Stages["extract"].Status != manifest.StatusSucceeded || len(loaded.Stages["extract"].Outputs) != 2 {
|
||||||
|
t.Fatalf("extract record = %#v, want succeeded manifest-ready outputs", loaded.Stages["extract"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleChangedOutcomeRerunsSucceededDownstream(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("disabled executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if analyzeRuns != 1 || len(runner.requests) != 0 {
|
||||||
|
t.Fatalf("disabled run analyze=%d Notarius=%d, want 1 and 0", analyzeRuns, len(runner.requests))
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Pipeline.Notarius.Enabled = true
|
||||||
|
summary, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("enabled executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if analyzeRuns != 2 || len(runner.requests) != 1 {
|
||||||
|
t.Fatalf("enabled run analyze=%d Notarius=%d, want 2 and 1", analyzeRuns, len(runner.requests))
|
||||||
|
}
|
||||||
|
if len(summary.Executed) != 2 || len(summary.Skipped) != 0 {
|
||||||
|
t.Fatalf("enabled summary = %#v, want extract and analyze executed", summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleFailureInvalidatesAndOrdinaryRetryRerunsDownstream(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
runner.failuresRemaining = 1
|
||||||
|
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil || !strings.Contains(err.Error(), "notarius execution failed") {
|
||||||
|
t.Fatalf("failed executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
failed := loadLifecycleManifest(t, cfg)
|
||||||
|
if failed.Stages["extract"].Status != manifest.StatusFailed || failed.Stages["analyze"].Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("failed lifecycle extract=%#v analyze=%#v", failed.Stages["extract"], failed.Stages["analyze"])
|
||||||
|
}
|
||||||
|
if failed.Stages["analyze"].Error == nil || failed.Stages["analyze"].Error.Message != staleReasonFailure {
|
||||||
|
t.Fatalf("analyze stale reason = %#v, want %q", failed.Stages["analyze"].Error, staleReasonFailure)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("retry executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if analyzeRuns != 1 || len(runner.requests) != 2 || len(summary.Executed) != 2 {
|
||||||
|
t.Fatalf("retry analyze=%d Notarius=%d summary=%#v", analyzeRuns, len(runner.requests), summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleForcedSelfSkipInvalidatesDownstream(t *testing.T) {
|
||||||
|
cfg, env, _ := extractionLifecycleFixture(t, false)
|
||||||
|
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||||
|
plan, _ := BuildSingleStagePlan("extract")
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true}); err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
loaded := loadLifecycleManifest(t, cfg)
|
||||||
|
if loaded.Stages["extract"].Status != manifest.StatusSkipped || loaded.Stages["analyze"].Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("forced self-skip extract=%#v analyze=%#v", loaded.Stages["extract"], loaded.Stages["analyze"])
|
||||||
|
}
|
||||||
|
if loaded.Stages["analyze"].Error == nil || loaded.Stages["analyze"].Error.Message != staleReasonForcedReplacement {
|
||||||
|
t.Fatalf("analyze stale reason = %#v, want %q", loaded.Stages["analyze"].Error, staleReasonForcedReplacement)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleForcedFailureInvalidatesDownstream(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
plan, _ := BuildSingleStagePlan("extract")
|
||||||
|
|
||||||
|
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("initial executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
succeeded := loadLifecycleManifest(t, cfg).Stages["extract"]
|
||||||
|
if succeeded == nil || succeeded.Status != manifest.StatusSucceeded || len(succeeded.Outputs) == 0 || len(succeeded.Logs) == 0 || len(succeeded.Metadata) == 0 {
|
||||||
|
t.Fatalf("initial extraction result = %#v, want succeeded result details", succeeded)
|
||||||
|
}
|
||||||
|
|
||||||
|
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||||
|
runner.failuresRemaining = 1
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true}); err == nil {
|
||||||
|
t.Fatal("executeStages() error = nil, want forced extraction failure")
|
||||||
|
}
|
||||||
|
loaded := loadLifecycleManifest(t, cfg)
|
||||||
|
if loaded.Stages["extract"].Status != manifest.StatusFailed || loaded.Stages["analyze"].Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("forced failure extract=%#v analyze=%#v", loaded.Stages["extract"], loaded.Stages["analyze"])
|
||||||
|
}
|
||||||
|
if loaded.Stages["analyze"].Error == nil || loaded.Stages["analyze"].Error.Message != staleReasonForcedReplacement {
|
||||||
|
t.Fatalf("analyze stale reason = %#v, want %q", loaded.Stages["analyze"].Error, staleReasonForcedReplacement)
|
||||||
|
}
|
||||||
|
failed := loaded.Stages["extract"]
|
||||||
|
if len(failed.Outputs) != 0 || len(failed.Logs) != 0 || len(failed.GeneratedConfigs) != 0 || len(failed.Metadata) != 0 {
|
||||||
|
t.Fatalf("failed replacement inherited extraction result details: %#v", failed)
|
||||||
|
}
|
||||||
|
historical, err := (&manifest.LocalStore{}).LoadRun(context.Background(), first.RunManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadRun(initial) error = %v", err)
|
||||||
|
}
|
||||||
|
historicalExtract := historical.Stages["extract"]
|
||||||
|
if historicalExtract == nil || historicalExtract.Status != manifest.StatusSucceeded || len(historicalExtract.Outputs) == 0 || len(historicalExtract.Logs) == 0 || len(historicalExtract.Metadata) == 0 {
|
||||||
|
t.Fatalf("historical extraction result = %#v, want preserved succeeded details", historicalExtract)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(succeeded.Outputs[0].LocalPath); err != nil {
|
||||||
|
t.Fatalf("durable extraction output was not preserved: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleRepeatedSelfSkipPreservesSucceededDownstream(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("first executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
second, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if analyzeRuns != 1 || len(runner.requests) != 0 {
|
||||||
|
t.Fatalf("repeated disabled run analyze=%d Notarius=%d, want 1 and 0", analyzeRuns, len(runner.requests))
|
||||||
|
}
|
||||||
|
if len(second.Executed) != 1 || len(second.Skipped) != 2 {
|
||||||
|
t.Fatalf("second summary = %#v, want executed self-skip and skipped analyze", second)
|
||||||
|
}
|
||||||
|
loaded := loadLifecycleManifest(t, cfg)
|
||||||
|
if loaded.Stages["analyze"].Status != manifest.StatusSucceeded {
|
||||||
|
t.Fatalf("analyze = %#v, want succeeded", loaded.Stages["analyze"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleSkipsCurrentResumableResult(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("first executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
resumed, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resume executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(resumed.Executed) != 0 || len(resumed.Skipped) != 2 || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||||
|
t.Fatalf("resume summary = %#v requests=%d analyze=%d", resumed, len(runner.requests), analyzeRuns)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleRerunsAfterDirectTranscriptChange(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("first executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
persisted := loadLifecycleManifest(t, cfg)
|
||||||
|
trimmed := persisted.Stages["trim"].Outputs[0].LocalPath
|
||||||
|
if err := os.WriteFile(trimmed, []byte(`{"segments":[{"id":"changed"}]}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(trimmed transcript) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPathFor(cfg), persisted); err != nil {
|
||||||
|
t.Fatalf("Save(mutated) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("rerun executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(rerun.Executed) != 2 || len(rerun.Skipped) != 0 || len(runner.requests) != 2 || analyzeRuns != 2 {
|
||||||
|
t.Fatalf("rerun summary = %#v requests=%d analyze=%d", rerun, len(runner.requests), analyzeRuns)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleResumesAndRerunsObsoleteResults(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*testing.T, *config.Config, *manifest.Manifest)
|
||||||
|
}{
|
||||||
|
{name: "configuration changed", mutate: func(_ *testing.T, cfg *config.Config, _ *manifest.Manifest) {
|
||||||
|
output := cfg.Pipeline.Notarius.Outputs["npc_registry"]
|
||||||
|
output.SchemaVersion = "v2"
|
||||||
|
cfg.Pipeline.Notarius.Outputs["npc_registry"] = output
|
||||||
|
}},
|
||||||
|
{name: "payload missing", mutate: func(t *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||||
|
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
||||||
|
t.Fatalf("Remove() error = %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "payload tampered", mutate: func(t *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||||
|
if err := os.WriteFile(m.Stages["extract"].Outputs[0].LocalPath, []byte(`{"npcs":["tampered"]}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "record incompatible", mutate: func(_ *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs[0].Contract.SchemaID = "incompatible"
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
plan, _ := BuildSingleStagePlan("extract")
|
||||||
|
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
persisted, err := (&manifest.LocalStore{}).Load(context.Background(), first.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
test.mutate(t, cfg, persisted)
|
||||||
|
if err := (&manifest.LocalStore{}).Save(context.Background(), first.ManifestPath, persisted); err != nil {
|
||||||
|
t.Fatalf("Save(mutated) error = %v", err)
|
||||||
|
}
|
||||||
|
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("rerun executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(rerun.Executed) != 1 || len(rerun.Skipped) != 0 || len(runner.requests) != 2 {
|
||||||
|
t.Fatalf("rerun summary = %#v requests=%d", rerun, len(runner.requests))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleUnsafeResumeErrorPreservesSuccess(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
persisted, err := store.Load(context.Background(), first.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
persisted.Stages["extract"].Outputs[0].LocalPath = filepath.Join(cfg.Pipeline.Workspace.Root, "outside.json")
|
||||||
|
if err := store.Save(context.Background(), first.ManifestPath, persisted); err != nil {
|
||||||
|
t.Fatalf("Save() error = %v", err)
|
||||||
|
}
|
||||||
|
before, _ := json.Marshal(map[string]*manifest.StageRecord{
|
||||||
|
"extract": persisted.Stages["extract"],
|
||||||
|
"analyze": persisted.Stages["analyze"],
|
||||||
|
})
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil || !strings.Contains(err.Error(), "unsafe") {
|
||||||
|
t.Fatalf("executeStages() error = %v, want unsafe resume failure", err)
|
||||||
|
}
|
||||||
|
afterManifest, err := store.Load(context.Background(), first.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load(after) error = %v", err)
|
||||||
|
}
|
||||||
|
after, _ := json.Marshal(map[string]*manifest.StageRecord{
|
||||||
|
"extract": afterManifest.Stages["extract"],
|
||||||
|
"analyze": afterManifest.Stages["analyze"],
|
||||||
|
})
|
||||||
|
if string(before) != string(after) || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||||
|
t.Fatalf("successful records changed: before=%s after=%s requests=%d analyze=%d", before, after, len(runner.requests), analyzeRuns)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractionLifecyclePlan(t *testing.T, analyzeRuns *int) []stage.Stage {
|
||||||
|
t.Helper()
|
||||||
|
plan, err := BuildSingleStagePlan("extract")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
|
||||||
|
}
|
||||||
|
return append(plan, countingStage{name: "analyze", runs: analyzeRuns})
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadLifecycleManifest(t *testing.T, cfg *config.Config) *manifest.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
return loaded
|
||||||
|
}
|
||||||
|
|
||||||
|
func markLifecycleStageSucceeded(t *testing.T, cfg *config.Config, name string) {
|
||||||
|
t.Helper()
|
||||||
|
loaded := loadLifecycleManifest(t, cfg)
|
||||||
|
loaded.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||||
|
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPathFor(cfg), loaded); err != nil {
|
||||||
|
t.Fatalf("Save() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractionLifecycleFixture(t *testing.T, enabled bool) (*config.Config, *stage.Env, *materializingNotariusRunner) {
|
||||||
|
t.Helper()
|
||||||
|
cfg := testConfig(t)
|
||||||
|
root := cfg.Pipeline.Workspace.Root
|
||||||
|
binary := filepath.Join(root, "notarius")
|
||||||
|
configPath := filepath.Join(root, "notarius.yml")
|
||||||
|
workingDirectory := filepath.Join(root, "notarius-work")
|
||||||
|
if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||||
|
t.Fatalf("WriteFile(binary) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(config) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Mkdir(workingDirectory, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(working directory) error = %v", err)
|
||||||
|
}
|
||||||
|
cfg.Pipeline.Notarius = &config.NotariusConfig{
|
||||||
|
Enabled: enabled, Binary: binary, ConfigPath: configPath, PipelineID: "dnd-session",
|
||||||
|
Timeout: "45m", WorkingDirectory: workingDirectory,
|
||||||
|
Outputs: map[string]config.NotariusOutputConfig{
|
||||||
|
"npc_registry": {
|
||||||
|
LaneID: "npc-registry", MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry",
|
||||||
|
SchemaVersion: "v1", ModuleKey: "dnd/npc-registry",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
paths, err := artifacts.NewLocalStore(root).EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("EnsureLayoutFor() error = %v", err)
|
||||||
|
}
|
||||||
|
inputPath := filepath.Join(paths.ArtifactsDir, "trimmed.from-manifest.json")
|
||||||
|
if err := os.WriteFile(inputPath, []byte(`{"segments":[]}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(input) error = %v", err)
|
||||||
|
}
|
||||||
|
m := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
|
m.Campaign = cfg.Session.Campaign
|
||||||
|
m.MarkStageSucceeded("trim", time.Now().UTC(), []manifest.ArtifactRecord{{
|
||||||
|
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
|
||||||
|
LocalPath: inputPath,
|
||||||
|
}})
|
||||||
|
if err := (&manifest.LocalStore{}).Save(context.Background(), paths.ManifestPath, m); err != nil {
|
||||||
|
t.Fatalf("Save(seed) error = %v", err)
|
||||||
|
}
|
||||||
|
runner := &materializingNotariusRunner{cfg: cfg.Pipeline.Notarius}
|
||||||
|
return cfg, &stage.Env{Notarius: runner}, runner
|
||||||
|
}
|
||||||
@@ -142,24 +142,3 @@ func commandObjectStoreTestConfig(secretsDir string) *config.Config {
|
|||||||
}
|
}
|
||||||
return cfg
|
return cfg
|
||||||
}
|
}
|
||||||
|
|
||||||
func restoreEnvAfterTest(t *testing.T, names ...string) {
|
|
||||||
t.Helper()
|
|
||||||
originals := make(map[string]string, len(names))
|
|
||||||
present := make(map[string]bool, len(names))
|
|
||||||
for _, name := range names {
|
|
||||||
value, ok := os.LookupEnv(name)
|
|
||||||
originals[name] = value
|
|
||||||
present[name] = ok
|
|
||||||
_ = os.Unsetenv(name)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() {
|
|
||||||
for _, name := range names {
|
|
||||||
if present[name] {
|
|
||||||
_ = os.Setenv(name, originals[name])
|
|
||||||
} else {
|
|
||||||
_ = os.Unsetenv(name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -10,22 +10,27 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
|
|
||||||
func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog, error) {
|
func buildHelperArtifactCatalog(cfg *config.Config, m *manifest.Manifest) (*artifacts.ArtifactCatalog, error) {
|
||||||
catalog := artifacts.NewArtifactCatalog()
|
configured := artifacts.ConfiguredArtifactDefinitions(nil)
|
||||||
if err := catalog.RegisterBuiltIns(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
configured := map[string]artifacts.ConfiguredArtifactDefinition{}
|
|
||||||
if cfg.Pipeline.Scriptorium != nil {
|
if cfg.Pipeline.Scriptorium != nil {
|
||||||
for key, item := range cfg.Pipeline.Scriptorium.Artifacts {
|
configured = artifacts.ConfiguredArtifactDefinitions(cfg.Pipeline.Scriptorium.Artifacts)
|
||||||
configured[key] = artifacts.ConfiguredArtifactDefinition{Enabled: item.Enabled, OutputPath: item.OutputPath}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if err := catalog.RegisterConfiguredArtifacts(configured, nil); err != nil {
|
extractionDefinitions := artifacts.ExtractionDefinitionsFromConfig(cfg.Pipeline.Notarius)
|
||||||
|
effective, err := artifacts.ResolveEffectiveArtifactSet(configured, nil)
|
||||||
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
catalog, err := artifacts.BootstrapRuntimeCatalog(configured, effective, extractionDefinitions)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
if cfg.Pipeline.Notarius != nil && cfg.Pipeline.Notarius.Enabled {
|
||||||
|
catalog.HydrateExtractionArtifacts(paths, m, extractionDefinitions)
|
||||||
|
}
|
||||||
return catalog, nil
|
return catalog, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -40,9 +45,19 @@ func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.Art
|
|||||||
for _, entry := range catalog.ListConfigured() {
|
for _, entry := range catalog.ListConfigured() {
|
||||||
writeArtifactLine(out, entry.SourceID, lockSet)
|
writeArtifactLine(out, entry.SourceID, lockSet)
|
||||||
}
|
}
|
||||||
|
fmt.Fprintln(out, "Extraction:")
|
||||||
|
for _, entry := range catalog.ListExtraction() {
|
||||||
|
state := "unavailable"
|
||||||
|
if entry.Available {
|
||||||
|
state = "available"
|
||||||
|
}
|
||||||
|
writeExtractionArtifactLine(out, entry.SourceID, state, entry.Provenance, lockSet)
|
||||||
|
}
|
||||||
fmt.Fprintln(out, "Previous-session:")
|
fmt.Fprintln(out, "Previous-session:")
|
||||||
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
|
if effective, err := resolveEffectiveArtifacts(cfg, nil); err == nil {
|
||||||
fmt.Fprintf(out, "- %s required=%t\n", artifactpolicy.PreviousSessionSourceID(req.Name), req.Required)
|
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg), effective) {
|
||||||
|
fmt.Fprintf(out, "- %s required=%t\n", artifactpolicy.PreviousSessionSourceID(req.Name), req.Required)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
fmt.Fprintln(out, "Published:")
|
fmt.Fprintln(out, "Published:")
|
||||||
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||||
@@ -50,6 +65,17 @@ func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.Art
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func writeExtractionArtifactLine(out io.Writer, source, state, provenance string, lockSet map[string]config.PublishLockRule) {
|
||||||
|
parts := []string{source, "planned", state}
|
||||||
|
if strings.TrimSpace(provenance) != "" {
|
||||||
|
parts = append(parts, "provenance="+strings.TrimSpace(provenance))
|
||||||
|
}
|
||||||
|
if _, ok := lockSet[source]; ok {
|
||||||
|
parts = append(parts, "locked")
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||||
|
}
|
||||||
|
|
||||||
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.PublishLockRule) {
|
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.PublishLockRule) {
|
||||||
parts := []string{source}
|
parts := []string{source}
|
||||||
if _, ok := lockSet[source]; ok {
|
if _, ok := lockSet[source]; ok {
|
||||||
@@ -101,9 +127,55 @@ func remotePublishedOutputAvailability(ctx context.Context, cfg *config.Config,
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func remotePublishedOutputAvailabilityForCurrent(
|
||||||
|
ctx context.Context,
|
||||||
|
cfg *config.Config,
|
||||||
|
store storage.ObjectStore,
|
||||||
|
catalog *artifacts.ArtifactCatalog,
|
||||||
|
current *RemoteCurrentState,
|
||||||
|
) map[string]string {
|
||||||
|
if current == nil || current.Commit == nil {
|
||||||
|
return remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
||||||
|
}
|
||||||
|
out := map[string]string{}
|
||||||
|
runPrefix := artifacts.S3RunPrefix(current.SessionPrefix, current.RunID)
|
||||||
|
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||||
|
source := strings.TrimSpace(rule.Source)
|
||||||
|
dest, _, err := helperPublishedOutputDest(rule, catalog)
|
||||||
|
if err != nil {
|
||||||
|
out[publishedOutputRemoteStateKey(source, "")] = "remote=error"
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := artifacts.S3RunRelativeDestinationKey(runPrefix, dest)
|
||||||
|
if committedPublishedOutput(current.Commit, key) {
|
||||||
|
out[publishedOutputRemoteStateKey(source, dest)] = "remote=published"
|
||||||
|
} else {
|
||||||
|
out[publishedOutputRemoteStateKey(source, dest)] = "remote=missing"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func committedPublishedOutput(commit *artifacts.RemoteCommitManifest, key string) bool {
|
||||||
|
if commit == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, artifact := range commit.Artifacts {
|
||||||
|
if artifact.Type == artifacts.RemoteArtifactTypePublishedOutput && artifact.DestinationKey == key {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
||||||
source := strings.TrimSpace(rule.Source)
|
source := strings.TrimSpace(rule.Source)
|
||||||
normalized, err := artifactpolicy.ResolvePublishedDestination(source, rule.Dest, helperConfiguredOutputPathMap(catalog))
|
normalized, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(
|
||||||
|
source,
|
||||||
|
rule.Dest,
|
||||||
|
helperConfiguredOutputPathMap(catalog),
|
||||||
|
helperExtractionOutputSet(catalog),
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", false, err
|
return "", false, err
|
||||||
}
|
}
|
||||||
@@ -112,6 +184,19 @@ func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts
|
|||||||
return normalized, showDest, nil
|
return normalized, showDest, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func helperExtractionOutputSet(catalog *artifacts.ArtifactCatalog) map[string]struct{} {
|
||||||
|
out := map[string]struct{}{}
|
||||||
|
if catalog == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for _, entry := range catalog.ListExtraction() {
|
||||||
|
if strings.TrimSpace(entry.ExtractionKey) != "" {
|
||||||
|
out[entry.ExtractionKey] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func helperConfiguredOutputPathMap(catalog *artifacts.ArtifactCatalog) map[string]string {
|
func helperConfiguredOutputPathMap(catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||||
out := map[string]string{}
|
out := map[string]string{}
|
||||||
if catalog == nil {
|
if catalog == nil {
|
||||||
|
|||||||
@@ -22,11 +22,12 @@ func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if strings.TrimSpace(flags.sessionID) == "" {
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
return fmt.Errorf("artifacts list: session_id is required")
|
return fmt.Errorf("artifacts list: session_id is required")
|
||||||
}
|
}
|
||||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, remote)
|
cfg, store, locks, m, cleanup, err := loadHelperContext(ctx, flags, remote)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("artifacts list: %w", err)
|
return fmt.Errorf("artifacts list: %w", err)
|
||||||
}
|
}
|
||||||
catalog, err := buildHelperArtifactCatalog(cfg)
|
defer cleanup()
|
||||||
|
catalog, err := buildHelperArtifactCatalog(cfg, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("artifacts list: %w", err)
|
return fmt.Errorf("artifacts list: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -90,33 +90,41 @@ func Artifacts(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore bool) (*config.Config, storage.ObjectStore, *effectiveLocks, *manifest.Manifest, error) {
|
func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore bool) (*config.Config, storage.ObjectStore, *effectiveLocks, *manifest.Manifest, func(), error) {
|
||||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, nil, err
|
return nil, nil, nil, nil, nil, err
|
||||||
}
|
}
|
||||||
|
release := true
|
||||||
|
defer func() {
|
||||||
|
if release {
|
||||||
|
_ = loaded.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
cfg := loaded.Config
|
||||||
if err := config.Validate(cfg); err != nil {
|
if err := config.Validate(cfg); err != nil {
|
||||||
return nil, nil, nil, nil, err
|
return nil, nil, nil, nil, nil, err
|
||||||
}
|
}
|
||||||
var store storage.ObjectStore
|
var store storage.ObjectStore
|
||||||
if needStore {
|
if needStore {
|
||||||
store, err = newCommandObjectStore(ctx, cfg, nil)
|
store, err = newCommandObjectStore(ctx, cfg, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, nil, err
|
return nil, nil, nil, nil, nil, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
store, _ = objectStoreIfConfigured(ctx, cfg)
|
store, _ = objectStoreIfConfigured(ctx, cfg)
|
||||||
}
|
}
|
||||||
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, nil, err
|
return nil, nil, nil, nil, nil, err
|
||||||
}
|
}
|
||||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
m, err := loadLocalManifest(ctx, paths.ManifestPath)
|
m, err := loadLocalManifest(ctx, paths.ManifestPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, nil, err
|
return nil, nil, nil, nil, nil, err
|
||||||
}
|
}
|
||||||
return cfg, store, locks, m, nil
|
release = false
|
||||||
|
return cfg, store, locks, m, func() { _ = loaded.Close() }, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func objectStoreIfConfigured(ctx context.Context, cfg *config.Config) (storage.ObjectStore, error) {
|
func objectStoreIfConfigured(ctx context.Context, cfg *config.Config) (storage.ObjectStore, error) {
|
||||||
|
|||||||
@@ -3,13 +3,17 @@ package app
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
@@ -187,8 +191,8 @@ func TestExecuteSessionInitRemoteLoadsSecretsBeforeObjectStoreInit(t *testing.T)
|
|||||||
secretKeyEnv := "NARRATIO_TEST_SESSION_INIT_OBJECT_SECRET"
|
secretKeyEnv := "NARRATIO_TEST_SESSION_INIT_OBJECT_SECRET"
|
||||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||||
secretsDir := t.TempDir()
|
secretsDir := t.TempDir()
|
||||||
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
|
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
|
||||||
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
|
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
|
||||||
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
@@ -415,8 +419,8 @@ func TestExecuteSessionValidateLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
|||||||
secretKeyEnv := "NARRATIO_TEST_VALIDATE_OBJECT_SECRET"
|
secretKeyEnv := "NARRATIO_TEST_VALIDATE_OBJECT_SECRET"
|
||||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||||
secretsDir := t.TempDir()
|
secretsDir := t.TempDir()
|
||||||
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
|
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
|
||||||
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
|
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
|
||||||
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
||||||
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
|
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
|
||||||
inputs:
|
inputs:
|
||||||
@@ -508,7 +512,7 @@ func TestExecuteLocksAddListAndRemoveUseRemoteLockStore(t *testing.T) {
|
|||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -520,6 +524,123 @@ func TestExecuteLocksAddListAndRemoveUseRemoteLockStore(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMutateRemoteLockStoreRetainsConcurrentUpdates(t *testing.T) {
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{S3: &config.StorageS3Config{Bucket: "bucket", RootPrefix: "root"}}},
|
||||||
|
Session: &config.SessionConfig{Campaign: "campaign", SessionID: "session"},
|
||||||
|
}
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
arrived := make(chan struct{}, 2)
|
||||||
|
release := make(chan struct{})
|
||||||
|
var hookMu sync.Mutex
|
||||||
|
hookCalls := 0
|
||||||
|
fake.UploadHook = func(storage.FakeUploadCall) error {
|
||||||
|
hookMu.Lock()
|
||||||
|
hookCalls++
|
||||||
|
call := hookCalls
|
||||||
|
hookMu.Unlock()
|
||||||
|
if call <= 2 {
|
||||||
|
arrived <- struct{}{}
|
||||||
|
<-release
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
mutate := func(source string) error {
|
||||||
|
return mutateRemoteLockStore(context.Background(), cfg, fake, func(lockStore *config.PublishLockStore) error {
|
||||||
|
set := lockSourceSet(lockStore.Locks)
|
||||||
|
set[source] = config.PublishLockRule{Source: source}
|
||||||
|
lockStore.Locks = lockMapValues(set)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
errs := make(chan error, 2)
|
||||||
|
go func() { errs <- mutate("narratio.transcript.final") }()
|
||||||
|
go func() { errs <- mutate("narratio.transcript.final_trimmed") }()
|
||||||
|
<-arrived
|
||||||
|
<-arrived
|
||||||
|
close(release)
|
||||||
|
if err := <-errs; err != nil {
|
||||||
|
t.Fatalf("first concurrent mutation error = %v", err)
|
||||||
|
}
|
||||||
|
if err := <-errs; err != nil {
|
||||||
|
t.Fatalf("second concurrent mutation error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
locks, _, _, err := loadRemoteLockStore(context.Background(), cfg, fake)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadRemoteLockStore() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(locks.Locks) != 2 || locks.Locks[0].Source != "narratio.transcript.final" || locks.Locks[1].Source != "narratio.transcript.final_trimmed" {
|
||||||
|
t.Fatalf("remote locks = %#v, want both concurrent updates", locks.Locks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRemoteLockStoreAcceptsExactLimitAndRejectsLimitPlusOne(t *testing.T) {
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{S3: &config.StorageS3Config{Bucket: "bucket", RootPrefix: "root"}}},
|
||||||
|
Session: &config.SessionConfig{Campaign: "campaign", SessionID: "session"},
|
||||||
|
}
|
||||||
|
key, err := remoteLocksKey(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("remoteLocksKey() error = %v", err)
|
||||||
|
}
|
||||||
|
encoded, err := config.MarshalPublishLockStore(&config.PublishLockStore{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MarshalPublishLockStore() error = %v", err)
|
||||||
|
}
|
||||||
|
exact := append(append([]byte(nil), encoded...), bytes.Repeat([]byte(" "), int(MaxRemoteLockStoreBytes)-len(encoded))...)
|
||||||
|
store := &storage.FakeBackend{}
|
||||||
|
store.SeedObject(storage.FakeObject{Key: key, Data: exact, ETag: "lock-generation"})
|
||||||
|
|
||||||
|
locks, gotKey, generation, err := loadRemoteLockStore(context.Background(), cfg, store)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadRemoteLockStore() exact-limit error = %v", err)
|
||||||
|
}
|
||||||
|
if locks == nil || gotKey != key || generation != "lock-generation" {
|
||||||
|
t.Fatalf("loadRemoteLockStore() = (%#v, %q, %q), want decoded locks and opened generation", locks, gotKey, generation)
|
||||||
|
}
|
||||||
|
if len(store.Downloads) != 0 || len(store.Reads) != 1 || store.Reads[0].Key != key {
|
||||||
|
t.Fatalf("lock transfers reads=%#v downloads=%#v, want one direct read", store.Reads, store.Downloads)
|
||||||
|
}
|
||||||
|
|
||||||
|
store.SeedObject(storage.FakeObject{Key: key, Data: append(exact, ' '), ETag: "new-generation"})
|
||||||
|
_, _, _, err = loadRemoteLockStore(context.Background(), cfg, store)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "remote lock control object") || !strings.Contains(err.Error(), key) || !strings.Contains(err.Error(), fmt.Sprint(MaxRemoteLockStoreBytes)) {
|
||||||
|
t.Fatalf("loadRemoteLockStore() limit-plus-one error = %v, want category, key, and limit", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRemoteLockStoreRejectsMalformedYAML(t *testing.T) {
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{S3: &config.StorageS3Config{Bucket: "bucket", RootPrefix: "root"}}},
|
||||||
|
Session: &config.SessionConfig{Campaign: "campaign", SessionID: "session"},
|
||||||
|
}
|
||||||
|
key, err := remoteLocksKey(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("remoteLocksKey() error = %v", err)
|
||||||
|
}
|
||||||
|
store := &storage.FakeBackend{}
|
||||||
|
store.SeedObject(storage.FakeObject{Key: key, Data: []byte("locks: [\n"), ETag: "lock-generation"})
|
||||||
|
|
||||||
|
if _, _, _, err := loadRemoteLockStore(context.Background(), cfg, store); err == nil {
|
||||||
|
t.Fatal("loadRemoteLockStore() error = nil, want malformed YAML failure")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMutateRemoteLockStoreHonorsCancellation(t *testing.T) {
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{S3: &config.StorageS3Config{Bucket: "bucket", RootPrefix: "root"}}},
|
||||||
|
Session: &config.SessionConfig{Campaign: "campaign", SessionID: "session"},
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
err := mutateRemoteLockStore(ctx, cfg, &storage.FakeBackend{}, func(*config.PublishLockStore) error { return nil })
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("mutateRemoteLockStore() error = %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteLocksAddDuplicateRequiresForce(t *testing.T) {
|
func TestExecuteLocksAddDuplicateRequiresForce(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
@@ -772,6 +893,71 @@ func TestExecuteArtifactsListRemoteReportsPublishedAvailability(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteArtifactsListReportsExtractionLifecycleWithoutPayload(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
addExtractionOutputToPipeline(t, pipelinePath)
|
||||||
|
addPublishOutputsToPipeline(t, pipelinePath, `
|
||||||
|
outputs:
|
||||||
|
- source: narratio.extraction.encounters
|
||||||
|
dest: artifacts/encounters.json
|
||||||
|
required: true
|
||||||
|
`)
|
||||||
|
lanePath := writeOperatorExtractionManifest(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
publishedKey := artifacts.S3PublishedOutputKey(
|
||||||
|
artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"),
|
||||||
|
"artifacts/encounters.json",
|
||||||
|
)
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: publishedKey, Data: []byte(`{"secret":"DO_NOT_PRINT"}`)})
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{
|
||||||
|
"session", "artifacts", "2026-05-03",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign-file", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--remote",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
out := stdout.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"Extraction:",
|
||||||
|
"narratio.extraction.encounters planned available provenance=manifest.current_extract_run",
|
||||||
|
"narratio.extraction.encounters dest=artifacts/encounters.json remote=published",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(out, want) {
|
||||||
|
t.Fatalf("stdout = %q, want %q", out, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(out, "DO_NOT_PRINT") {
|
||||||
|
t.Fatalf("operator output exposed extraction payload: %q", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Remove(lanePath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
code = Execute([]string{
|
||||||
|
"session", "artifacts", "2026-05-03",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign-file", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("unavailable exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "narratio.extraction.encounters planned unavailable") {
|
||||||
|
t.Fatalf("stdout = %q, want unavailable extraction state", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteArtifactsListRemoteUsesPublishOutputDestinations(t *testing.T) {
|
func TestExecuteArtifactsListRemoteUsesPublishOutputDestinations(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
@@ -912,9 +1098,6 @@ func TestExecuteStatusReportsRemoteArtifactCatalogErrorsWithoutFailing(t *testin
|
|||||||
if !strings.Contains(out, "Remote outputs:") || !strings.Contains(out, "narratio.transcript.final_trimmed remote=error") {
|
if !strings.Contains(out, "Remote outputs:") || !strings.Contains(out, "narratio.transcript.final_trimmed remote=error") {
|
||||||
t.Fatalf("stdout = %q, want remote output error state", out)
|
t.Fatalf("stdout = %q, want remote output error state", out)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out, "Publish locks: error:") {
|
|
||||||
t.Fatalf("stdout = %q, want publish locks error", out)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteStatusReportsMissingRemoteCurrentStateWithoutFailing(t *testing.T) {
|
func TestExecuteStatusReportsMissingRemoteCurrentStateWithoutFailing(t *testing.T) {
|
||||||
@@ -965,6 +1148,36 @@ func TestExecuteStatusReportsPreviousStateReadinessWithoutFailing(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteStatusDetectsMissingRequiredPreviousArtifact(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
|
replaceInFileOrFatal(t, pipelinePath, "source: narratio.artifact.session_recap", "source: narratio.previous_session.artifact.session_recap")
|
||||||
|
replaceInFileOrFatal(t, sessionPath, "session_id: 2026-05-03\n", "session_id: 2026-05-03\nprevious_session_id: 2026-04-26\n")
|
||||||
|
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||||
|
}
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
seedRestorePreviousCurrentManifestOnly(t, fake, cfg)
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{
|
||||||
|
"session", "status", "2026-05-03",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign-file", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), `Previous-session artifacts: unavailable: remote required previous-session artifact "session_recap" object missing`) {
|
||||||
|
t.Fatalf("stdout = %q, want missing required previous artifact", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteSessionValidateReportsPreviousStateFindingAndReturnsFindingError(t *testing.T) {
|
func TestExecuteSessionValidateReportsPreviousStateFindingAndReturnsFindingError(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||||
@@ -1006,7 +1219,7 @@ func TestExecutePublishLoadsRemoteLocks(t *testing.T) {
|
|||||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||||
// The publish stage only checks the manifest statuses and source files.
|
// The publish stage only checks the manifest statuses and source files.
|
||||||
_ = stageName
|
_ = stageName
|
||||||
}
|
}
|
||||||
@@ -1041,6 +1254,85 @@ func addPublishOutputsToPipeline(t *testing.T, pipelinePath, publishYAML string)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func addExtractionOutputToPipeline(t *testing.T, pipelinePath string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data = append(data, []byte(`notarius:
|
||||||
|
enabled: true
|
||||||
|
config_path: notarius.yml
|
||||||
|
pipeline_id: campaign.extract
|
||||||
|
outputs:
|
||||||
|
encounters:
|
||||||
|
lane_id: encounters
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: encounters
|
||||||
|
schema_version: "1"
|
||||||
|
module_key: encounters
|
||||||
|
`)...)
|
||||||
|
if err := os.WriteFile(pipelinePath, data, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(filepath.Dir(pipelinePath), "notarius.yml"), []byte("{}\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeOperatorExtractionManifest(t *testing.T, workspaceRoot string) string {
|
||||||
|
t.Helper()
|
||||||
|
paths := artifacts.NewLocalStore(workspaceRoot).SessionPathsFor("sample-campaign", "2026-05-03")
|
||||||
|
bundleRoot := filepath.Join(paths.ArtifactsDir, "notarius", "extract-run-1")
|
||||||
|
lanePath := filepath.Join(bundleRoot, "lanes", "encounters.json")
|
||||||
|
indexPath := filepath.Join(bundleRoot, "index.json")
|
||||||
|
trimmedPath := filepath.Join(paths.Root, filepath.FromSlash(artifacts.TranscriptPathFinalTrimmed))
|
||||||
|
mustWriteTestFile(t, lanePath, `{"secret":"DO_NOT_PRINT"}`)
|
||||||
|
mustWriteTestFile(t, indexPath, `{"lanes":[]}`)
|
||||||
|
mustWriteTestFile(t, trimmedPath, `{"segments":[]}`)
|
||||||
|
laneChecksum, err := artifacts.SHA256File(lanePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
indexChecksum, err := artifacts.SHA256File(indexPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := manifest.New("2026-05-03", time.Now().UTC())
|
||||||
|
m.Campaign = "sample-campaign"
|
||||||
|
m.Stages["trim"] = &manifest.StageRecord{
|
||||||
|
Name: "trim", Status: manifest.StatusSucceeded,
|
||||||
|
Outputs: []manifest.ArtifactRecord{{
|
||||||
|
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, LocalPath: trimmedPath, ProducerRunID: "trim-run-1",
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
input, err := artifacts.ResolveExtractionInputIdentity(paths, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m.Stages["extract"] = &manifest.StageRecord{
|
||||||
|
Name: "extract", Status: manifest.StatusSucceeded,
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"narratio_run_id": "extract-run-1", "bundle_root": bundleRoot,
|
||||||
|
"receipt": map[string]any{"run_id": "notarius-run-1", "pipeline_id": "campaign.extract"},
|
||||||
|
"direct_input": input.Metadata(),
|
||||||
|
},
|
||||||
|
Outputs: []manifest.ArtifactRecord{
|
||||||
|
{
|
||||||
|
Kind: "notarius_lane", SourceID: artifacts.ExtractionArtifactSourceID("encounters"), LocalPath: lanePath,
|
||||||
|
ProducerRunID: "extract-run-1", Checksum: laneChecksum,
|
||||||
|
Contract: &artifactmodel.ContractMetadata{MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters"},
|
||||||
|
ExternalProvenance: &artifactmodel.ExternalProvenance{System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: "encounters"},
|
||||||
|
},
|
||||||
|
{Kind: "notarius_index", LocalPath: indexPath, ProducerRunID: "extract-run-1", Checksum: indexChecksum},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := (&manifest.LocalStore{}).Save(context.Background(), paths.ManifestPath, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return lanePath
|
||||||
|
}
|
||||||
|
|
||||||
func replaceInFileOrFatal(t *testing.T, path, old, new string) {
|
func replaceInFileOrFatal(t *testing.T, path, old, new string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
@@ -1076,7 +1368,7 @@ func writeValidPublishRunConfigFiles(t *testing.T, workspaceRoot string) (string
|
|||||||
m := manifest.New("2026-05-03", nowUTC())
|
m := manifest.New("2026-05-03", nowUTC())
|
||||||
m.Campaign = "sample-campaign"
|
m.Campaign = "sample-campaign"
|
||||||
m.RunID = "20260521T160000Z-test"
|
m.RunID = "20260521T160000Z-test"
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||||
m.MarkStageSucceeded(name, nowUTC(), nil)
|
m.MarkStageSucceeded(name, nowUTC(), nil)
|
||||||
}
|
}
|
||||||
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package app
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
@@ -12,6 +13,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/previouscache"
|
||||||
)
|
)
|
||||||
|
|
||||||
type stableInputCheck struct {
|
type stableInputCheck struct {
|
||||||
@@ -34,9 +36,9 @@ type remoteAudioCheck struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type previousArtifactReadiness struct {
|
type previousArtifactReadiness struct {
|
||||||
Requirements []artifacts.PreviousArtifactRequirement
|
Requirements []artifacts.PreviousArtifactRequirement
|
||||||
MissingID bool
|
SkippedMissing []string
|
||||||
Err error
|
Err error
|
||||||
}
|
}
|
||||||
|
|
||||||
type remoteCurrentStateCheck struct {
|
type remoteCurrentStateCheck struct {
|
||||||
@@ -152,23 +154,27 @@ func inspectPreviousArtifactReadiness(
|
|||||||
if len(requirements) == 0 {
|
if len(requirements) == 0 {
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.Session.PreviousSessionID) == "" {
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||||
out.MissingID = true
|
out.Err = fmt.Errorf("resolved config with pipeline/session is required")
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
if store == nil {
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
out.Err = fmt.Errorf("previous-session artifacts cannot be checked because storage is unavailable")
|
plan, err := previouscache.Resolve(ctx, cfg, paths, requirements, store)
|
||||||
|
if err != nil {
|
||||||
|
var pointerMissing *artifacts.CurrentRunPointerMissingError
|
||||||
|
var manifestMissing *artifacts.CurrentManifestMissingError
|
||||||
|
if errors.As(err, &pointerMissing) {
|
||||||
|
out.Err = fmt.Errorf("remote %w", pointerMissing)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
if errors.As(err, &manifestMissing) {
|
||||||
|
out.Err = fmt.Errorf("remote %w", manifestMissing)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
out.Err = fmt.Errorf("remote %w", err)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
out.SkippedMissing = append([]string(nil), plan.SkippedMissing...)
|
||||||
prefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.PreviousSessionID)
|
|
||||||
if _, err := artifacts.LoadCurrentState(ctx, store, prefix, artifacts.CurrentStateValidation{
|
|
||||||
ExpectedSessionID: strings.TrimSpace(cfg.Session.PreviousSessionID),
|
|
||||||
ExpectedCampaign: strings.TrimSpace(cfg.Session.Campaign),
|
|
||||||
ValidateRunID: true,
|
|
||||||
}); err != nil {
|
|
||||||
out.Err = fmt.Errorf("remote %v", err)
|
|
||||||
}
|
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -38,10 +38,11 @@ func LocksList(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if strings.TrimSpace(flags.sessionID) == "" {
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
return fmt.Errorf("locks: session_id is required")
|
return fmt.Errorf("locks: session_id is required")
|
||||||
}
|
}
|
||||||
cfg, _, locks, _, err := loadHelperContext(ctx, flags, true)
|
cfg, _, locks, _, cleanup, err := loadHelperContext(ctx, flags, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("locks: %w", err)
|
return fmt.Errorf("locks: %w", err)
|
||||||
}
|
}
|
||||||
|
defer cleanup()
|
||||||
writeLocks(out, cfg, locks)
|
writeLocks(out, cfg, locks)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -63,26 +64,31 @@ func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if strings.TrimSpace(flags.sessionID) == "" {
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
return fmt.Errorf("locks add: session_id is required")
|
return fmt.Errorf("locks add: session_id is required")
|
||||||
}
|
}
|
||||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
cfg, store, locks, _, cleanup, err := loadHelperContext(ctx, flags, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("locks add: %w", err)
|
return fmt.Errorf("locks add: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
|
defer cleanup()
|
||||||
|
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks add"); err != nil {
|
||||||
return fmt.Errorf("locks add: %w", err)
|
return fmt.Errorf("locks add: %w", err)
|
||||||
}
|
}
|
||||||
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
||||||
return fmt.Errorf("locks add: source %q is locked by pipeline config and cannot be modified remotely", source)
|
return fmt.Errorf("locks add: source %q is locked by pipeline config and cannot be modified remotely", source)
|
||||||
}
|
}
|
||||||
remoteSet := lockSourceSet(locks.Remote)
|
if err := mutateRemoteLockStore(ctx, cfg, store, func(lockStore *config.PublishLockStore) error {
|
||||||
if _, exists := remoteSet[source]; exists && !force {
|
remoteSet := lockSourceSet(lockStore.Locks)
|
||||||
return fmt.Errorf("locks add: remote lock for %q already exists; pass --force to update", source)
|
if _, exists := remoteSet[source]; exists && !force {
|
||||||
}
|
return fmt.Errorf("remote lock for %q already exists; pass --force to update", source)
|
||||||
remoteSet[source] = config.PublishLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
}
|
||||||
remoteLocks := lockMapValues(remoteSet)
|
remoteSet[source] = config.PublishLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
||||||
if _, err := config.ValidatePublishLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
|
lockStore.Locks = lockMapValues(remoteSet)
|
||||||
return fmt.Errorf("locks add: %w", err)
|
normalized, err := config.ValidatePublishLockRules(lockStore.Locks, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks")
|
||||||
}
|
if err != nil {
|
||||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
return err
|
||||||
|
}
|
||||||
|
lockStore.Locks = normalized
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
return fmt.Errorf("locks add: %w", err)
|
return fmt.Errorf("locks add: %w", err)
|
||||||
}
|
}
|
||||||
_, err = fmt.Fprintf(out, "narratio session locks add: locked %s\n", source)
|
_, err = fmt.Fprintf(out, "narratio session locks add: locked %s\n", source)
|
||||||
@@ -102,23 +108,26 @@ func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if strings.TrimSpace(flags.sessionID) == "" {
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
return fmt.Errorf("locks remove: session_id is required")
|
return fmt.Errorf("locks remove: session_id is required")
|
||||||
}
|
}
|
||||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
cfg, store, locks, _, cleanup, err := loadHelperContext(ctx, flags, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("locks remove: %w", err)
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
|
defer cleanup()
|
||||||
|
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks remove"); err != nil {
|
||||||
return fmt.Errorf("locks remove: %w", err)
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
}
|
}
|
||||||
remoteSet := lockSourceSet(locks.Remote)
|
if err := mutateRemoteLockStore(ctx, cfg, store, func(lockStore *config.PublishLockStore) error {
|
||||||
if _, ok := remoteSet[source]; !ok {
|
remoteSet := lockSourceSet(lockStore.Locks)
|
||||||
if _, static := lockSourceSet(locks.Static)[source]; static {
|
if _, ok := remoteSet[source]; !ok {
|
||||||
return fmt.Errorf("locks remove: source %q is locked by pipeline config and cannot be unlocked remotely", source)
|
if _, static := lockSourceSet(locks.Static)[source]; static {
|
||||||
|
return fmt.Errorf("source %q is locked by pipeline config and cannot be unlocked remotely", source)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("remote lock for %q does not exist", source)
|
||||||
}
|
}
|
||||||
return fmt.Errorf("locks remove: remote lock for %q does not exist", source)
|
delete(remoteSet, source)
|
||||||
}
|
lockStore.Locks = lockMapValues(remoteSet)
|
||||||
delete(remoteSet, source)
|
return nil
|
||||||
remoteLocks := lockMapValues(remoteSet)
|
}); err != nil {
|
||||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
|
||||||
return fmt.Errorf("locks remove: %w", err)
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
}
|
}
|
||||||
_, err = fmt.Fprintf(out, "narratio session locks remove: unlocked %s\n", source)
|
_, err = fmt.Fprintf(out, "narratio session locks remove: unlocked %s\n", source)
|
||||||
|
|||||||
52
internal/app/operator_session_init_test.go
Normal file
52
internal/app/operator_session_init_test.go
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSessionInitRejectsRenderedTemplateThatOmitsExpectedPreviousSession(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
campaignDir := filepath.Dir(campaignPath)
|
||||||
|
templatePath := filepath.Join(campaignDir, "session.template.yml")
|
||||||
|
template := `session_id: {{ session_id }}
|
||||||
|
campaign: sample-campaign
|
||||||
|
inputs:
|
||||||
|
audio_dir: ./audio
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(templatePath, []byte(template), 0o644); err != nil {
|
||||||
|
t.Fatalf("write session template: %v", err)
|
||||||
|
}
|
||||||
|
campaign := `campaign_id: sample-campaign
|
||||||
|
session_template_file: session.template.yml
|
||||||
|
inputs:
|
||||||
|
speakers_file: ./speakers.yml
|
||||||
|
autocorrect_file: ./autocorrect.yml
|
||||||
|
glossary_file: ./glossary.yml
|
||||||
|
players_file: ./players.yml
|
||||||
|
party_file: ./party.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(campaignPath, []byte(campaign), 0o644); err != nil {
|
||||||
|
t.Fatalf("write campaign config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := SessionInit(context.Background(), []string{
|
||||||
|
"2026-05-03",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign-file", campaignPath,
|
||||||
|
"--output", filepath.Join(t.TempDir(), "session.yml"),
|
||||||
|
"--previous-session-id", "2026-04-26",
|
||||||
|
}, &out)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "unused template variable value(s): previous_session_id") {
|
||||||
|
t.Fatalf("SessionInit() error = %q, want missing previous-session template error", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,11 +25,13 @@ func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
findings := []finding{}
|
findings := []finding{}
|
||||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
findings = append(findings, errorFinding("config", err.Error()))
|
findings = append(findings, errorFinding("config", err.Error()))
|
||||||
return renderFindings(out, "", "", findings)
|
return renderFindings(out, "", "", findings)
|
||||||
}
|
}
|
||||||
|
defer func() { _ = loaded.Close() }()
|
||||||
|
cfg := loaded.Config
|
||||||
if err := config.Validate(cfg); err != nil {
|
if err := config.Validate(cfg); err != nil {
|
||||||
findings = append(findings, errorFinding("config", err.Error()))
|
findings = append(findings, errorFinding("config", err.Error()))
|
||||||
} else {
|
} else {
|
||||||
@@ -53,16 +55,23 @@ func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg))
|
effective, effectiveErr := resolveEffectiveArtifacts(cfg, nil)
|
||||||
|
if effectiveErr != nil {
|
||||||
|
findings = append(findings, errorFinding("config", effectiveErr.Error()))
|
||||||
|
return renderFindings(out, cfg.Session.Campaign, cfg.Session.SessionID, findings)
|
||||||
|
}
|
||||||
|
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg), effective)
|
||||||
previous := inspectPreviousArtifactReadiness(ctx, cfg, store, requirements)
|
previous := inspectPreviousArtifactReadiness(ctx, cfg, store, requirements)
|
||||||
if len(previous.Requirements) == 0 {
|
if len(previous.Requirements) == 0 {
|
||||||
findings = append(findings, okFinding("previous", "no previous-session artifacts required"))
|
findings = append(findings, okFinding("previous", "no previous-session artifacts required"))
|
||||||
} else if previous.MissingID {
|
|
||||||
findings = append(findings, errorFinding("previous", "previous_session_id is required by configured previous-session artifacts"))
|
|
||||||
} else if previous.Err != nil {
|
} else if previous.Err != nil {
|
||||||
findings = append(findings, errorFinding("previous", previous.Err.Error()))
|
findings = append(findings, errorFinding("previous", previous.Err.Error()))
|
||||||
} else {
|
} else {
|
||||||
for _, req := range previous.Requirements {
|
for _, req := range previous.Requirements {
|
||||||
|
if !req.Required && previousRequirementSkipped(previous.SkippedMissing, req.Name) {
|
||||||
|
findings = append(findings, okFinding("previous", fmt.Sprintf("%s required=false unavailable", req.Name)))
|
||||||
|
continue
|
||||||
|
}
|
||||||
findings = append(findings, okFinding("previous", fmt.Sprintf("%s required=%t", req.Name, req.Required)))
|
findings = append(findings, okFinding("previous", fmt.Sprintf("%s required=%t", req.Name, req.Required)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -82,3 +91,12 @@ func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
return renderFindings(out, cfg.Session.Campaign, cfg.Session.SessionID, findings)
|
return renderFindings(out, cfg.Session.Campaign, cfg.Session.SessionID, findings)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func previousRequirementSkipped(values []string, name string) bool {
|
||||||
|
for _, value := range values {
|
||||||
|
if value == name {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Status reports effective local/remote session state.
|
// Status reports effective local/remote session state.
|
||||||
@@ -25,10 +26,12 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if strings.TrimSpace(flags.sessionID) == "" {
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
return fmt.Errorf("status: session_id is required")
|
return fmt.Errorf("status: session_id is required")
|
||||||
}
|
}
|
||||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("status: %w", err)
|
return fmt.Errorf("status: %w", err)
|
||||||
}
|
}
|
||||||
|
defer func() { _ = loaded.Close() }()
|
||||||
|
cfg := loaded.Config
|
||||||
if err := config.Validate(cfg); err != nil {
|
if err := config.Validate(cfg); err != nil {
|
||||||
return fmt.Errorf("status: %w", err)
|
return fmt.Errorf("status: %w", err)
|
||||||
}
|
}
|
||||||
@@ -40,16 +43,19 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
writeStatusStableInputs(out, inspectStableInputs(cfg))
|
writeStatusStableInputs(out, inspectStableInputs(cfg))
|
||||||
writeStatusLocalAudio(out, inspectLocalAudioPresence(cfg))
|
writeStatusLocalAudio(out, inspectLocalAudioPresence(cfg))
|
||||||
|
|
||||||
|
var localManifest *manifest.Manifest
|
||||||
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
|
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
|
||||||
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
|
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
|
||||||
} else if m == nil {
|
} else if m == nil {
|
||||||
fmt.Fprintln(out, "Local manifest: missing")
|
fmt.Fprintln(out, "Local manifest: missing")
|
||||||
} else {
|
} else {
|
||||||
|
localManifest = m
|
||||||
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
|
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
|
||||||
writeStageStatuses(out, m)
|
writeStageStatuses(out, m)
|
||||||
}
|
}
|
||||||
|
|
||||||
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||||
|
var remoteCurrent *RemoteCurrentState
|
||||||
if storeErr != nil {
|
if storeErr != nil {
|
||||||
fmt.Fprintf(out, "Remote publish: unavailable: %v\n", storeErr)
|
fmt.Fprintf(out, "Remote publish: unavailable: %v\n", storeErr)
|
||||||
} else if store != nil {
|
} else if store != nil {
|
||||||
@@ -57,22 +63,27 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if current.Err != nil {
|
if current.Err != nil {
|
||||||
fmt.Fprintf(out, "Remote publish: missing or unavailable: %v\n", current.Err)
|
fmt.Fprintf(out, "Remote publish: missing or unavailable: %v\n", current.Err)
|
||||||
} else {
|
} else {
|
||||||
|
remoteCurrent = current.State
|
||||||
fmt.Fprintf(out, "Remote publish: current run %s\n", current.State.RunID)
|
fmt.Fprintf(out, "Remote publish: current run %s\n", current.State.RunID)
|
||||||
fmt.Fprintf(out, "Remote manifest: %s\n", current.State.CurrentManifestKey)
|
fmt.Fprintf(out, "Remote manifest: %s\n", current.State.CurrentManifestKey)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
writeStatusRemoteAudio(ctx, out, cfg, store, storeErr)
|
writeStatusRemoteAudio(ctx, out, cfg, store, storeErr)
|
||||||
|
effective, effectiveErr := resolveEffectiveArtifacts(cfg, nil)
|
||||||
|
if effectiveErr != nil {
|
||||||
|
return fmt.Errorf("status: resolve effective artifacts: %w", effectiveErr)
|
||||||
|
}
|
||||||
writeStatusPreviousArtifacts(out, inspectPreviousArtifactReadiness(
|
writeStatusPreviousArtifacts(out, inspectPreviousArtifactReadiness(
|
||||||
ctx,
|
ctx,
|
||||||
cfg,
|
cfg,
|
||||||
store,
|
store,
|
||||||
artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)),
|
artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg), effective),
|
||||||
))
|
))
|
||||||
|
|
||||||
lockChecks := inspectEffectiveLocks(ctx, cfg, store)
|
lockChecks := inspectEffectiveLocks(ctx, cfg, store)
|
||||||
locks := lockChecks.Locks
|
locks := lockChecks.Locks
|
||||||
lockErr := lockChecks.Err
|
lockErr := lockChecks.Err
|
||||||
if catalog, catalogErr := buildHelperArtifactCatalog(cfg); catalogErr != nil {
|
if catalog, catalogErr := buildHelperArtifactCatalog(cfg, localManifest); catalogErr != nil {
|
||||||
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
|
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
|
||||||
} else if storeErr == nil {
|
} else if storeErr == nil {
|
||||||
catalogLocks := locks
|
catalogLocks := locks
|
||||||
@@ -84,7 +95,7 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
publishedRemoteState := map[string]string{}
|
publishedRemoteState := map[string]string{}
|
||||||
if store != nil {
|
if store != nil {
|
||||||
publishedRemoteState = remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
publishedRemoteState = remotePublishedOutputAvailabilityForCurrent(ctx, cfg, store, catalog, remoteCurrent)
|
||||||
}
|
}
|
||||||
fmt.Fprintln(out, "Remote outputs:")
|
fmt.Fprintln(out, "Remote outputs:")
|
||||||
writeArtifactList(out, cfg, catalog, catalogLocks, publishedRemoteState)
|
writeArtifactList(out, cfg, catalog, catalogLocks, publishedRemoteState)
|
||||||
@@ -149,10 +160,6 @@ func writeStatusPreviousArtifacts(out io.Writer, readiness previousArtifactReadi
|
|||||||
fmt.Fprintln(out, "Previous-session artifacts: not required")
|
fmt.Fprintln(out, "Previous-session artifacts: not required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if readiness.MissingID {
|
|
||||||
fmt.Fprintln(out, "Previous-session artifacts: unavailable: previous_session_id is required by configured previous-session artifacts")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if readiness.Err != nil {
|
if readiness.Err != nil {
|
||||||
fmt.Fprintf(out, "Previous-session artifacts: unavailable: %v\n", readiness.Err)
|
fmt.Fprintf(out, "Previous-session artifacts: unavailable: %v\n", readiness.Err)
|
||||||
return
|
return
|
||||||
@@ -162,5 +169,9 @@ func writeStatusPreviousArtifacts(out io.Writer, readiness previousArtifactReadi
|
|||||||
names = append(names, fmt.Sprintf("%s(required=%t)", req.Name, req.Required))
|
names = append(names, fmt.Sprintf("%s(required=%t)", req.Name, req.Required))
|
||||||
}
|
}
|
||||||
sort.Strings(names)
|
sort.Strings(names)
|
||||||
|
if len(readiness.SkippedMissing) > 0 {
|
||||||
|
fmt.Fprintf(out, "Previous-session artifacts: ready: %s; optional unavailable: %s\n", strings.Join(names, ", "), strings.Join(readiness.SkippedMissing, ", "))
|
||||||
|
return
|
||||||
|
}
|
||||||
fmt.Fprintf(out, "Previous-session artifacts: ready: %s\n", strings.Join(names, ", "))
|
fmt.Fprintf(out, "Previous-session artifacts: ready: %s\n", strings.Join(names, ", "))
|
||||||
}
|
}
|
||||||
|
|||||||
16
internal/app/paths_test_helpers_test.go
Normal file
16
internal/app/paths_test_helpers_test.go
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func mustPreviousArtifactPathForCampaign(t *testing.T, root, campaign, sessionID, relative string) string {
|
||||||
|
t.Helper()
|
||||||
|
path, err := artifacts.SessionPreviousArtifactPathForCampaign(root, campaign, sessionID, relative)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SessionPreviousArtifactPathForCampaign() error = %v", err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
@@ -22,7 +22,7 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
var flags commonConfigFlags
|
var flags commonConfigFlags
|
||||||
var force bool
|
var force bool
|
||||||
addCommonConfigFlags(fs, &flags)
|
addCommonConfigFlags(fs, &flags)
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
fs.BoolVar(&force, "force", false, "show all stages as scheduled to rerun")
|
||||||
|
|
||||||
if err := parseSessionAwareFlags("plan", fs, args, &flags.sessionID); err != nil {
|
if err := parseSessionAwareFlags("plan", fs, args, &flags.sessionID); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -30,10 +30,12 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if flags.sessionID == "" {
|
if flags.sessionID == "" {
|
||||||
return fmt.Errorf("plan: session_id is required")
|
return fmt.Errorf("plan: session_id is required")
|
||||||
}
|
}
|
||||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("plan: %w", err)
|
return fmt.Errorf("plan: %w", err)
|
||||||
}
|
}
|
||||||
|
defer func() { _ = loaded.Close() }()
|
||||||
|
cfg := loaded.Config
|
||||||
if err := config.Validate(cfg); err != nil {
|
if err := config.Validate(cfg); err != nil {
|
||||||
return fmt.Errorf("plan: %w", err)
|
return fmt.Errorf("plan: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,12 +27,12 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
|||||||
if !strings.Contains(got, "narratio session plan: workdir prepared at") {
|
if !strings.Contains(got, "narratio session plan: workdir prepared at") {
|
||||||
t.Fatalf("first output = %q, want workdir prepared", got)
|
t.Fatalf("first output = %q, want workdir prepared", got)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||||
if !strings.Contains(got, name+": run") {
|
if !strings.Contains(got, name+": run") {
|
||||||
t.Fatalf("first output = %q, missing stage %q", got, name)
|
t.Fatalf("first output = %q, missing stage %q", got, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !strings.Contains(got, "totals: run=10 skip=0") {
|
if !strings.Contains(got, "totals: run=11 skip=0") {
|
||||||
t.Fatalf("first output = %q, want totals", got)
|
t.Fatalf("first output = %q, want totals", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,8 +84,8 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
|||||||
if !strings.Contains(got, "trim: run") {
|
if !strings.Contains(got, "trim: run") {
|
||||||
t.Fatalf("output = %q, want trim run", got)
|
t.Fatalf("output = %q, want trim run", got)
|
||||||
}
|
}
|
||||||
if !strings.Contains(got, "totals: run=8 skip=2") {
|
if !strings.Contains(got, "totals: run=9 skip=2") {
|
||||||
t.Fatalf("output = %q, want totals run=8 skip=2", got)
|
t.Fatalf("output = %q, want totals run=9 skip=2", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -109,7 +109,7 @@ seriatim:
|
|||||||
audita:
|
audita:
|
||||||
binary: audita
|
binary: audita
|
||||||
notification:
|
notification:
|
||||||
timeout: 10s
|
mode: noop
|
||||||
`
|
`
|
||||||
sessionYAML := `session_id: 2026-05-03
|
sessionYAML := `session_id: 2026-05-03
|
||||||
campaign: sample-campaign
|
campaign: sample-campaign
|
||||||
@@ -133,8 +133,8 @@ inputs:
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error, got nil")
|
t.Fatal("expected error, got nil")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "read secrets env_dir") {
|
if !strings.Contains(err.Error(), "validate secrets env_dir") {
|
||||||
t.Fatalf("error = %q, want secrets read error context", err.Error())
|
t.Fatalf("error = %q, want secrets validation error context", err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import "testing"
|
|||||||
|
|
||||||
func TestBuildFullPlanOrder(t *testing.T) {
|
func TestBuildFullPlanOrder(t *testing.T) {
|
||||||
got := BuildFullPlan()
|
got := BuildFullPlan()
|
||||||
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"}
|
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"}
|
||||||
if len(got) != len(want) {
|
if len(got) != len(want) {
|
||||||
t.Fatalf("len(plan) = %d, want %d", len(got), len(want))
|
t.Fatalf("len(plan) = %d, want %d", len(got), len(want))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,113 +3,113 @@ package app
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var removeRunScopedDirFn = removeRunScopedDir
|
||||||
|
|
||||||
func runPostPublishCleanup(ctx context.Context, env *Env, manifestPath string, m *manifest.Manifest, executed []string) error {
|
func runPostPublishCleanup(ctx context.Context, env *Env, manifestPath string, m *manifest.Manifest, executed []string) error {
|
||||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil || m == nil {
|
if env == nil || env.Config == nil || env.Config.Pipeline == nil || m == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
spoolRequested := env.Config.Pipeline.Spool.DeleteAudioAfterPublish
|
cleanup := m.PostPublishCleanup
|
||||||
workRequested := env.Config.Pipeline.Workspace.CleanupAfterPublish
|
if cleanup == nil {
|
||||||
if !spoolRequested && !workRequested {
|
var err error
|
||||||
return nil
|
cleanup, err = createPostPublishCleanup(env.Config, m, executed)
|
||||||
}
|
if err != nil {
|
||||||
|
return err
|
||||||
sr := publishStageRecordForCleanup(m, executed)
|
|
||||||
if sr == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if sr.Metadata == nil {
|
|
||||||
sr.Metadata = map[string]any{}
|
|
||||||
}
|
|
||||||
sr.Metadata["spool_cleanup_requested"] = spoolRequested
|
|
||||||
sr.Metadata["workdir_cleanup_requested"] = workRequested
|
|
||||||
|
|
||||||
eligible, reason := publishCleanupEligible(env.Config, sr)
|
|
||||||
if !eligible {
|
|
||||||
sr.Metadata["cleanup_skipped"] = true
|
|
||||||
sr.Metadata["cleanup_skipped_reason"] = reason
|
|
||||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
|
||||||
return fmt.Errorf("save manifest cleanup skip metadata %q: %w", manifestPath, err)
|
|
||||||
}
|
}
|
||||||
return nil
|
if cleanup == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
m.PostPublishCleanup = cleanup
|
||||||
|
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||||
|
return fmt.Errorf("persist post-publish cleanup obligation %q: %w", manifestPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for index := range cleanup.Targets {
|
||||||
|
target := &cleanup.Targets[index]
|
||||||
|
if target.Completed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := removeRunScopedDirFn(target.Root, target.Path, target.Policy); err != nil {
|
||||||
|
return fmt.Errorf("complete post-publish cleanup for %q: %w", target.Path, err)
|
||||||
|
}
|
||||||
|
target.Completed = true
|
||||||
|
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||||
|
target.Completed = false
|
||||||
|
return fmt.Errorf("persist post-publish cleanup completion %q: %w", manifestPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func createPostPublishCleanup(cfg *config.Config, m *manifest.Manifest, executed []string) (*manifest.PostPublishCleanup, error) {
|
||||||
|
spoolRequested := cfg.Pipeline.Spool.DeleteAudioAfterPublish
|
||||||
|
workRequested := cfg.Pipeline.Workspace.CleanupAfterPublish
|
||||||
|
if !spoolRequested && !workRequested {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sr := publishStageRecordForCleanup(m)
|
||||||
|
publishedRunID, eligible, _ := publishCleanupEligible(cfg, m, sr, executed)
|
||||||
|
if !eligible {
|
||||||
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
spoolDir := strings.TrimSpace(m.LocalSpoolDir)
|
spoolDir := strings.TrimSpace(m.LocalSpoolDir)
|
||||||
if spoolDir == "" {
|
if spoolDir == "" {
|
||||||
spoolDir = artifacts.SessionSpoolAudioDir(
|
spoolDir = artifacts.SessionSpoolAudioDir(
|
||||||
env.Config.Pipeline.Spool.Root,
|
cfg.Pipeline.Spool.Root,
|
||||||
strings.TrimSpace(env.Config.Session.Campaign),
|
strings.TrimSpace(m.Campaign),
|
||||||
strings.TrimSpace(env.Config.Session.SessionID),
|
strings.TrimSpace(m.SessionID),
|
||||||
strings.TrimSpace(m.RunID),
|
publishedRunID,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
workDir := strings.TrimSpace(m.LocalWorkDir)
|
workDir := strings.TrimSpace(m.LocalWorkDir)
|
||||||
if workDir == "" {
|
if workDir == "" {
|
||||||
workDir = artifacts.SessionRunRootForCampaign(
|
workDir = artifacts.SessionRunRootForCampaign(
|
||||||
env.Config.Pipeline.Workspace.Root,
|
cfg.Pipeline.Workspace.Root,
|
||||||
strings.TrimSpace(env.Config.Session.Campaign),
|
strings.TrimSpace(m.Campaign),
|
||||||
strings.TrimSpace(env.Config.Session.SessionID),
|
strings.TrimSpace(m.SessionID),
|
||||||
strings.TrimSpace(m.RunID),
|
publishedRunID,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cleanup := &manifest.PostPublishCleanup{
|
||||||
|
CommittedRunID: publishedRunID,
|
||||||
|
RemoteCommitKey: strings.TrimSpace(asString(sr.Metadata["remote_commit_key"])),
|
||||||
|
CurrentCommitPointerKey: strings.TrimSpace(asString(sr.Metadata["current_commit_pointer_key"])),
|
||||||
|
}
|
||||||
if spoolRequested {
|
if spoolRequested {
|
||||||
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Spool.Root), spoolDir, "pipeline.spool.delete_audio_after_publish"); err != nil {
|
cleanup.Targets = append(cleanup.Targets, manifest.CleanupTarget{
|
||||||
sr.Metadata["cleanup_failed"] = true
|
Policy: "pipeline.spool.delete_audio_after_publish",
|
||||||
sr.Metadata["cleanup_failed_policy"] = "pipeline.spool.delete_audio_after_publish"
|
Root: strings.TrimSpace(cfg.Pipeline.Spool.Root),
|
||||||
sr.Metadata["cleanup_failed_path"] = spoolDir
|
Path: filepath.Clean(spoolDir),
|
||||||
_ = env.ManifestStore.Save(ctx, manifestPath, m)
|
})
|
||||||
return err
|
|
||||||
}
|
|
||||||
sr.Metadata["spool_cleanup_deleted"] = filepath.Clean(spoolDir)
|
|
||||||
}
|
}
|
||||||
|
if workRequested {
|
||||||
if !workRequested {
|
cleanup.Targets = append(cleanup.Targets, manifest.CleanupTarget{
|
||||||
sr.Metadata["cleanup_completed"] = true
|
Policy: "pipeline.workspace.cleanup_after_publish",
|
||||||
sr.Metadata["cleanup_skipped"] = false
|
Root: strings.TrimSpace(cfg.Pipeline.Workspace.Root),
|
||||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
Path: filepath.Clean(workDir),
|
||||||
return fmt.Errorf("save manifest cleanup metadata %q: %w", manifestPath, err)
|
})
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
return cleanup, nil
|
||||||
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Workspace.Root), workDir, "pipeline.workspace.cleanup_after_publish"); err != nil {
|
|
||||||
sr.Metadata["cleanup_failed"] = true
|
|
||||||
sr.Metadata["cleanup_failed_policy"] = "pipeline.workspace.cleanup_after_publish"
|
|
||||||
sr.Metadata["cleanup_failed_path"] = workDir
|
|
||||||
_ = env.ManifestStore.Save(ctx, manifestPath, m)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
sr.Metadata["workdir_cleanup_deleted"] = filepath.Clean(workDir)
|
|
||||||
sr.Metadata["cleanup_completed"] = true
|
|
||||||
sr.Metadata["cleanup_skipped"] = false
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func publishStageRecordForCleanup(m *manifest.Manifest, executed []string) *manifest.StageRecord {
|
func publishStageRecordForCleanup(m *manifest.Manifest) *manifest.StageRecord {
|
||||||
if m == nil {
|
if m == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
publishRan := false
|
|
||||||
for _, name := range executed {
|
|
||||||
if name == "publish" {
|
|
||||||
publishRan = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !publishRan {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
sr := m.Stages["publish"]
|
sr := m.Stages["publish"]
|
||||||
if sr == nil || sr.Status != manifest.StatusSucceeded {
|
if sr == nil || sr.Status != manifest.StatusSucceeded {
|
||||||
return nil
|
return nil
|
||||||
@@ -117,40 +117,56 @@ func publishStageRecordForCleanup(m *manifest.Manifest, executed []string) *mani
|
|||||||
return sr
|
return sr
|
||||||
}
|
}
|
||||||
|
|
||||||
func publishCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool, string) {
|
func publishCleanupEligible(cfg *config.Config, m *manifest.Manifest, sr *manifest.StageRecord, executed []string) (string, bool, string) {
|
||||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Publish == nil {
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Publish == nil {
|
||||||
return false, "publish configuration is missing"
|
return "", false, "publish configuration is missing"
|
||||||
}
|
}
|
||||||
enabled := true
|
enabled := true
|
||||||
if cfg.Pipeline.Publish.Enabled != nil {
|
if cfg.Pipeline.Publish.Enabled != nil {
|
||||||
enabled = *cfg.Pipeline.Publish.Enabled
|
enabled = *cfg.Pipeline.Publish.Enabled
|
||||||
}
|
}
|
||||||
if !enabled {
|
if !enabled {
|
||||||
return false, "publish.enabled is false"
|
return "", false, "publish.enabled is false"
|
||||||
}
|
}
|
||||||
uploadRun := true
|
uploadRun := true
|
||||||
if cfg.Pipeline.Publish.UploadRun != nil {
|
if cfg.Pipeline.Publish.UploadRun != nil {
|
||||||
uploadRun = *cfg.Pipeline.Publish.UploadRun
|
uploadRun = *cfg.Pipeline.Publish.UploadRun
|
||||||
}
|
}
|
||||||
if !uploadRun {
|
if !uploadRun {
|
||||||
return false, "publish.upload_run is false"
|
return "", false, "publish.upload_run is false"
|
||||||
}
|
}
|
||||||
if sr == nil || sr.Metadata == nil {
|
if sr == nil || sr.Metadata == nil {
|
||||||
return false, "publish metadata is missing"
|
return "", false, "publish metadata is missing"
|
||||||
}
|
}
|
||||||
if skipped, _ := sr.Metadata["skipped"].(bool); skipped {
|
if skipped, _ := sr.Metadata["skipped"].(bool); skipped {
|
||||||
return false, "publish stage was skipped"
|
return "", false, "publish stage was skipped"
|
||||||
}
|
}
|
||||||
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
|
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
|
||||||
return false, "publish did not upload run record"
|
return "", false, "publish did not upload run record"
|
||||||
}
|
}
|
||||||
if pointer, _ := sr.Metadata["current_pointer_written"].(bool); !pointer {
|
if strings.TrimSpace(asString(sr.Metadata["remote_commit_key"])) == "" {
|
||||||
return false, "publish did not write current pointer"
|
return "", false, "publish remote commit key is missing"
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(asString(sr.Metadata["current_run_id_key"])) == "" {
|
if strings.TrimSpace(asString(sr.Metadata["current_commit_pointer_key"])) == "" {
|
||||||
return false, "publish current run pointer key is missing"
|
return "", false, "publish current commit pointer key is missing"
|
||||||
}
|
}
|
||||||
return true, ""
|
publishedRunID := strings.TrimSpace(asString(sr.Metadata["published_run_id"]))
|
||||||
|
if publishedRunID == "" && containsStage(executed, "publish") && m != nil {
|
||||||
|
publishedRunID = strings.TrimSpace(m.RunID)
|
||||||
|
}
|
||||||
|
if publishedRunID == "" {
|
||||||
|
return "", false, "publish run id is missing"
|
||||||
|
}
|
||||||
|
return publishedRunID, true, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsStage(names []string, target string) bool {
|
||||||
|
for _, name := range names {
|
||||||
|
if name == target {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
type scopedDir struct {
|
type scopedDir struct {
|
||||||
@@ -167,7 +183,7 @@ func removeRunScopedDir(root, target, policy string) error {
|
|||||||
if !dir.Exists {
|
if !dir.Exists {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if err := os.RemoveAll(dir.TargetAbs); err != nil {
|
if err := fileops.RemoveAllUnderRoot(dir.RootAbs, dir.TargetAbs); err != nil {
|
||||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package app
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -18,16 +19,33 @@ import (
|
|||||||
|
|
||||||
type publishSuccessStage struct {
|
type publishSuccessStage struct {
|
||||||
metadata map[string]any
|
metadata map[string]any
|
||||||
|
targets *cleanupSeed
|
||||||
}
|
}
|
||||||
|
|
||||||
func (publishSuccessStage) Name() string { return "publish" }
|
func (publishSuccessStage) Name() string { return "publish" }
|
||||||
func (publishSuccessStage) Declares() stage.IODecl { return stage.IODecl{} }
|
func (s publishSuccessStage) Run(_ context.Context, _ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||||
func (s publishSuccessStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
if s.targets != nil {
|
||||||
|
s.targets.runWorkDir = m.LocalWorkDir
|
||||||
|
s.targets.spoolAudioDir = m.LocalSpoolDir
|
||||||
|
if err := os.MkdirAll(filepath.Join(m.LocalWorkDir, "logs"), 0o755); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(m.LocalWorkDir, "logs", "stage.log"), []byte("log\n"), 0o644); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(m.LocalSpoolDir, 0o755); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(m.LocalSpoolDir, "speaker.flac"), []byte("flac\n"), 0o644); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
md := map[string]any{
|
md := map[string]any{
|
||||||
"stage": "publish",
|
"stage": "publish",
|
||||||
"uploaded": true,
|
"uploaded": true,
|
||||||
"current_pointer_written": true,
|
"published_run_id": m.RunID,
|
||||||
"current_run_id_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/current/run_id.txt",
|
"remote_commit_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/20260519T010203Z-a1b2c3d4/commit.json",
|
||||||
|
"current_commit_pointer_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/current/commit-pointer.json",
|
||||||
}
|
}
|
||||||
for k, v := range s.metadata {
|
for k, v := range s.metadata {
|
||||||
md[k] = v
|
md[k] = v
|
||||||
@@ -37,8 +55,7 @@ func (s publishSuccessStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Ma
|
|||||||
|
|
||||||
type notifyFailStage struct{}
|
type notifyFailStage struct{}
|
||||||
|
|
||||||
func (notifyFailStage) Name() string { return "notify" }
|
func (notifyFailStage) Name() string { return "notify" }
|
||||||
func (notifyFailStage) Declares() stage.IODecl { return stage.IODecl{} }
|
|
||||||
func (notifyFailStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
func (notifyFailStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||||
return nil, errors.New("notify failed")
|
return nil, errors.New("notify failed")
|
||||||
}
|
}
|
||||||
@@ -48,7 +65,7 @@ func TestPostPublishCleanupDisabledKeepsLocalDirs(t *testing.T) {
|
|||||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = false
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = false
|
||||||
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,7 +79,7 @@ func TestPostPublishCleanupSpoolOnly(t *testing.T) {
|
|||||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,7 +93,7 @@ func TestPostPublishCleanupWorkdirOnly(t *testing.T) {
|
|||||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = false
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = false
|
||||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -92,7 +109,7 @@ func TestPostPublishCleanupBothPolicies(t *testing.T) {
|
|||||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,6 +119,115 @@ func TestPostPublishCleanupBothPolicies(t *testing.T) {
|
|||||||
assertExists(t, seed.previousCachePath)
|
assertExists(t, seed.previousCachePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPostPublishCleanupRetriesWhenInitialObligationSaveFails(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||||
|
|
||||||
|
failed := false
|
||||||
|
store := &cleanupFailingManifestStore{
|
||||||
|
delegate: &manifest.LocalStore{},
|
||||||
|
fail: func(m *manifest.Manifest) error {
|
||||||
|
if !failed && m.PostPublishCleanup != nil {
|
||||||
|
failed = true
|
||||||
|
return errors.New("injected obligation save failure")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{
|
||||||
|
Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "post-publish cleanup incomplete") {
|
||||||
|
t.Fatalf("executeStages() error = %v, want incomplete cleanup", err)
|
||||||
|
}
|
||||||
|
assertExists(t, seed.spoolAudioDir)
|
||||||
|
assertCleanupPending(t, cfg)
|
||||||
|
|
||||||
|
store.fail = nil
|
||||||
|
if _, err := executeStages(context.Background(), cfg, nil, RunOptions{Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
|
t.Fatalf("retry executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
assertMissing(t, seed.spoolAudioDir)
|
||||||
|
assertCleanupComplete(t, cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostPublishCleanupRetriesFailedDeletionWithoutTouchingOtherRuns(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
|
|
||||||
|
originalRemove := removeRunScopedDirFn
|
||||||
|
removeRunScopedDirFn = func(root, target, policy string) error {
|
||||||
|
if policy == "pipeline.workspace.cleanup_after_publish" {
|
||||||
|
return errors.New("injected deletion failure")
|
||||||
|
}
|
||||||
|
return originalRemove(root, target, policy)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { removeRunScopedDirFn = originalRemove })
|
||||||
|
|
||||||
|
_, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "post-publish cleanup incomplete") {
|
||||||
|
t.Fatalf("executeStages() error = %v, want incomplete cleanup", err)
|
||||||
|
}
|
||||||
|
assertMissing(t, seed.spoolAudioDir)
|
||||||
|
assertExists(t, seed.runWorkDir)
|
||||||
|
assertCleanupPending(t, cfg)
|
||||||
|
assertExists(t, seed.otherRunDir)
|
||||||
|
assertExists(t, seed.previousCachePath)
|
||||||
|
|
||||||
|
removeRunScopedDirFn = originalRemove
|
||||||
|
if _, err := executeStages(context.Background(), cfg, nil, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
|
t.Fatalf("retry executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
assertMissing(t, seed.runWorkDir)
|
||||||
|
assertExists(t, seed.otherRunDir)
|
||||||
|
assertExists(t, seed.previousCachePath)
|
||||||
|
assertCleanupComplete(t, cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostPublishCleanupRetriesWhenCompletionEvidenceSaveFails(t *testing.T) {
|
||||||
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
|
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||||
|
|
||||||
|
failed := false
|
||||||
|
store := &cleanupFailingManifestStore{
|
||||||
|
delegate: &manifest.LocalStore{},
|
||||||
|
fail: func(m *manifest.Manifest) error {
|
||||||
|
if m.PostPublishCleanup == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, target := range m.PostPublishCleanup.Targets {
|
||||||
|
if !failed && target.Completed {
|
||||||
|
failed = true
|
||||||
|
return errors.New("injected completion evidence failure")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{
|
||||||
|
Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}},
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "post-publish cleanup incomplete") {
|
||||||
|
t.Fatalf("executeStages() error = %v, want incomplete cleanup", err)
|
||||||
|
}
|
||||||
|
assertMissing(t, seed.spoolAudioDir)
|
||||||
|
assertCleanupPending(t, cfg)
|
||||||
|
|
||||||
|
store.fail = nil
|
||||||
|
if _, err := executeStages(context.Background(), cfg, nil, RunOptions{Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
|
t.Fatalf("retry executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
assertCleanupComplete(t, cfg)
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, nil, RunOptions{Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
|
t.Fatalf("idempotent retry executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
assertMissing(t, seed.spoolAudioDir)
|
||||||
|
}
|
||||||
|
|
||||||
func TestPostPublishCleanupNotRunWhenPublishFails(t *testing.T) {
|
func TestPostPublishCleanupNotRunWhenPublishFails(t *testing.T) {
|
||||||
cfg, seed := cleanupFixtureConfig(t)
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
@@ -121,7 +247,7 @@ func TestPostPublishCleanupNotRunWhenPublishSkipped(t *testing.T) {
|
|||||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{metadata: map[string]any{"skipped": true}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{metadata: map[string]any{"skipped": true}, targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -129,12 +255,12 @@ func TestPostPublishCleanupNotRunWhenPublishSkipped(t *testing.T) {
|
|||||||
assertExists(t, seed.runWorkDir)
|
assertExists(t, seed.runWorkDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPostPublishCleanupNotRunWhenCurrentPointerMissing(t *testing.T) {
|
func TestPostPublishCleanupNotRunWhenCommitPointerMissing(t *testing.T) {
|
||||||
cfg, seed := cleanupFixtureConfig(t)
|
cfg, seed := cleanupFixtureConfig(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{metadata: map[string]any{"current_pointer_written": false}}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{metadata: map[string]any{"current_commit_pointer_key": ""}, targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,7 +274,7 @@ func TestPostPublishCleanupNotRunWhenPublishUploadDisabled(t *testing.T) {
|
|||||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
cfg.Pipeline.Publish.UploadRun = boolPtr(false)
|
cfg.Pipeline.Publish.UploadRun = boolPtr(false)
|
||||||
|
|
||||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -181,12 +307,28 @@ func TestPostPublishCleanupFailsOnUnsafePath(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Load() error = %v", err)
|
t.Fatalf("Load() error = %v", err)
|
||||||
}
|
}
|
||||||
m.LocalSpoolDir = filepath.Join(filepath.Dir(cfg.Pipeline.Spool.Root), "outside-spool")
|
m.MarkStageSucceeded("publish", time.Now().UTC(), nil)
|
||||||
|
m.Stages["publish"].Metadata = map[string]any{
|
||||||
|
"uploaded": true,
|
||||||
|
"published_run_id": m.RunID,
|
||||||
|
"remote_commit_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/20260516T010203Z-1a2b3c4d/commit.json",
|
||||||
|
"current_commit_pointer_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/current/commit-pointer.json",
|
||||||
|
}
|
||||||
|
m.PostPublishCleanup = &manifest.PostPublishCleanup{
|
||||||
|
CommittedRunID: m.RunID,
|
||||||
|
RemoteCommitKey: m.Stages["publish"].Metadata["remote_commit_key"].(string),
|
||||||
|
CurrentCommitPointerKey: m.Stages["publish"].Metadata["current_commit_pointer_key"].(string),
|
||||||
|
Targets: []manifest.CleanupTarget{{
|
||||||
|
Policy: "pipeline.spool.delete_audio_after_publish",
|
||||||
|
Root: cfg.Pipeline.Spool.Root,
|
||||||
|
Path: filepath.Join(filepath.Dir(cfg.Pipeline.Spool.Root), "outside-spool"),
|
||||||
|
}},
|
||||||
|
}
|
||||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||||
t.Fatalf("Save() error = %v", err)
|
t.Fatalf("Save() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
_, err = executeStages(context.Background(), cfg, nil, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||||
if err == nil || !strings.Contains(err.Error(), "refusing to delete path outside root") {
|
if err == nil || !strings.Contains(err.Error(), "refusing to delete path outside root") {
|
||||||
t.Fatalf("executeStages() error = %v, want safe-path failure", err)
|
t.Fatalf("executeStages() error = %v, want safe-path failure", err)
|
||||||
}
|
}
|
||||||
@@ -215,32 +357,33 @@ func TestPostPublishCleanupNotRunWhenOutputIsMissing(t *testing.T) {
|
|||||||
assertExists(t, artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID))
|
assertExists(t, artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPostPublishCleanupNotRunWhenCurrentManifestUploadFails(t *testing.T) {
|
func TestPostPublishCleanupNotRunWhenCommittedManifestUploadFails(t *testing.T) {
|
||||||
cfg, seed, _ := publishStageCleanupFixture(t)
|
cfg, seed, _ := publishStageCleanupFixture(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
failKey := seed.sessionPrefix + "current/manifest.json"
|
|
||||||
|
|
||||||
publishStageImpl, err := stage.Select("publish")
|
publishStageImpl, err := stage.Select("publish")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Select(publish) error = %v", err)
|
t.Fatalf("Select(publish) error = %v", err)
|
||||||
}
|
}
|
||||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishStageImpl}, RunOptions{
|
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishStageImpl}, RunOptions{
|
||||||
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, fail: func(key string) bool {
|
||||||
|
return strings.HasSuffix(key, "/session-manifest.json")
|
||||||
|
}}},
|
||||||
})
|
})
|
||||||
if err == nil || !strings.Contains(err.Error(), "current manifest") {
|
if err == nil || !strings.Contains(err.Error(), "immutable object") {
|
||||||
t.Fatalf("executeStages() error = %v, want current-manifest failure", err)
|
t.Fatalf("executeStages() error = %v, want committed-manifest failure", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
assertExists(t, seed.spoolAudioDir)
|
assertExists(t, seed.spoolAudioDir)
|
||||||
assertExists(t, seed.runWorkDir)
|
assertExists(t, seed.runWorkDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPostPublishCleanupNotRunWhenCurrentPointerUploadFails(t *testing.T) {
|
func TestPostPublishCleanupNotRunWhenCommitPointerUploadFails(t *testing.T) {
|
||||||
cfg, seed, _ := publishStageCleanupFixture(t)
|
cfg, seed, _ := publishStageCleanupFixture(t)
|
||||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||||
failKey := seed.sessionPrefix + "current/run_id.txt"
|
failKey := artifacts.S3CurrentCommitPointerKey(seed.sessionPrefix)
|
||||||
|
|
||||||
publishStageImpl, err := stage.Select("publish")
|
publishStageImpl, err := stage.Select("publish")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -249,8 +392,8 @@ func TestPostPublishCleanupNotRunWhenCurrentPointerUploadFails(t *testing.T) {
|
|||||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishStageImpl}, RunOptions{
|
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishStageImpl}, RunOptions{
|
||||||
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
||||||
})
|
})
|
||||||
if err == nil || !strings.Contains(err.Error(), "current run pointer") {
|
if err == nil || !strings.Contains(err.Error(), "current commit pointer") {
|
||||||
t.Fatalf("executeStages() error = %v, want current-run-pointer failure", err)
|
t.Fatalf("executeStages() error = %v, want current-commit-pointer failure", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
assertExists(t, seed.spoolAudioDir)
|
assertExists(t, seed.spoolAudioDir)
|
||||||
@@ -258,6 +401,7 @@ func TestPostPublishCleanupNotRunWhenCurrentPointerUploadFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type cleanupSeed struct {
|
type cleanupSeed struct {
|
||||||
|
runID string
|
||||||
runWorkDir string
|
runWorkDir string
|
||||||
otherRunDir string
|
otherRunDir string
|
||||||
spoolAudioDir string
|
spoolAudioDir string
|
||||||
@@ -266,6 +410,28 @@ type cleanupSeed struct {
|
|||||||
sessionPrefix string
|
sessionPrefix string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type cleanupFailingManifestStore struct {
|
||||||
|
delegate manifest.Store
|
||||||
|
fail func(*manifest.Manifest) error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *cleanupFailingManifestStore) Create(ctx context.Context, sessionID string) (*manifest.Manifest, error) {
|
||||||
|
return s.delegate.Create(ctx, sessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *cleanupFailingManifestStore) Load(ctx context.Context, path string) (*manifest.Manifest, error) {
|
||||||
|
return s.delegate.Load(ctx, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *cleanupFailingManifestStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||||
|
if s.fail != nil {
|
||||||
|
if err := s.fail(m); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return s.delegate.Save(ctx, path, m)
|
||||||
|
}
|
||||||
|
|
||||||
func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
|
func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -277,7 +443,7 @@ func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
|
|||||||
runWorkDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
runWorkDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||||
otherRunDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, "20260516T010204Z-5e6f7a8b")
|
otherRunDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, "20260516T010204Z-5e6f7a8b")
|
||||||
spoolAudioDir := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
spoolAudioDir := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||||
previousCachePath := artifacts.SessionPreviousArtifactPathForCampaign(
|
previousCachePath := mustPreviousArtifactPathForCampaign(t,
|
||||||
cfg.Pipeline.Workspace.Root,
|
cfg.Pipeline.Workspace.Root,
|
||||||
cfg.Session.Campaign,
|
cfg.Session.Campaign,
|
||||||
cfg.Session.SessionID,
|
cfg.Session.SessionID,
|
||||||
@@ -311,6 +477,7 @@ func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return cfg, cleanupSeed{
|
return cfg, cleanupSeed{
|
||||||
|
runID: runID,
|
||||||
runWorkDir: runWorkDir,
|
runWorkDir: runWorkDir,
|
||||||
otherRunDir: otherRunDir,
|
otherRunDir: otherRunDir,
|
||||||
spoolAudioDir: spoolAudioDir,
|
spoolAudioDir: spoolAudioDir,
|
||||||
@@ -355,7 +522,7 @@ func publishStageCleanupFixture(t *testing.T) (*config.Config, cleanupSeed, stri
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Load() error = %v", err)
|
t.Fatalf("Load() error = %v", err)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||||
seedManifest.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
seedManifest.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||||
}
|
}
|
||||||
seedManifest.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
|
seedManifest.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
@@ -385,23 +552,46 @@ func writePublishFixtureRunFiles(t *testing.T, runWorkDir, sessionRoot string) {
|
|||||||
type failKeyStore struct {
|
type failKeyStore struct {
|
||||||
delegate *storage.FakeBackend
|
delegate *storage.FakeBackend
|
||||||
failKey string
|
failKey string
|
||||||
|
fail func(string) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *failKeyStore) fails(key string) bool {
|
||||||
|
return strings.TrimSpace(key) == strings.TrimSpace(s.failKey) || (s.fail != nil && s.fail(key))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *failKeyStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
|
func (s *failKeyStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
|
||||||
return s.delegate.List(ctx, prefix)
|
return s.delegate.List(ctx, prefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *failKeyStore) Read(ctx context.Context, key string) (storage.ObjectInfo, io.ReadCloser, error) {
|
||||||
|
return s.delegate.Read(ctx, key)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *failKeyStore) Download(ctx context.Context, key, localPath string) error {
|
func (s *failKeyStore) Download(ctx context.Context, key, localPath string) error {
|
||||||
return s.delegate.Download(ctx, key, localPath)
|
return s.delegate.Download(ctx, key, localPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *failKeyStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
|
func (s *failKeyStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
|
||||||
if strings.TrimSpace(key) == strings.TrimSpace(s.failKey) {
|
if s.fails(key) {
|
||||||
return storage.ObjectInfo{}, errors.New("forced upload failure")
|
return storage.ObjectInfo{}, errors.New("forced upload failure")
|
||||||
}
|
}
|
||||||
return s.delegate.Upload(ctx, localPath, key, opts)
|
return s.delegate.Upload(ctx, localPath, key, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *failKeyStore) UploadReader(ctx context.Context, source io.Reader, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
|
||||||
|
if s.fails(key) {
|
||||||
|
return storage.ObjectInfo{}, errors.New("forced upload failure")
|
||||||
|
}
|
||||||
|
return s.delegate.UploadReader(ctx, source, key, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *failKeyStore) UploadConditional(ctx context.Context, source io.Reader, key string, opts storage.UploadOptions, condition storage.WriteCondition) (storage.ObjectInfo, error) {
|
||||||
|
if s.fails(key) {
|
||||||
|
return storage.ObjectInfo{}, errors.New("forced upload failure")
|
||||||
|
}
|
||||||
|
return s.delegate.UploadConditional(ctx, source, key, opts, condition)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *failKeyStore) Exists(ctx context.Context, key string) (bool, error) {
|
func (s *failKeyStore) Exists(ctx context.Context, key string) (bool, error) {
|
||||||
return s.delegate.Exists(ctx, key)
|
return s.delegate.Exists(ctx, key)
|
||||||
}
|
}
|
||||||
@@ -419,3 +609,36 @@ func assertMissing(t *testing.T, path string) {
|
|||||||
t.Fatalf("expected path to be removed %q, stat err=%v", path, err)
|
t.Fatalf("expected path to be removed %q, stat err=%v", path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assertCleanupPending(t *testing.T, cfg *config.Config) {
|
||||||
|
t.Helper()
|
||||||
|
m, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if m.PostPublishCleanup == nil {
|
||||||
|
t.Fatal("expected a persisted cleanup obligation")
|
||||||
|
}
|
||||||
|
for _, target := range m.PostPublishCleanup.Targets {
|
||||||
|
if !target.Completed {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("expected at least one cleanup target to remain incomplete")
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertCleanupComplete(t *testing.T, cfg *config.Config) {
|
||||||
|
t.Helper()
|
||||||
|
m, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if m.PostPublishCleanup == nil {
|
||||||
|
t.Fatal("expected a persisted cleanup obligation")
|
||||||
|
}
|
||||||
|
for _, target := range m.PostPublishCleanup.Targets {
|
||||||
|
if !target.Completed {
|
||||||
|
t.Fatalf("cleanup target remains incomplete: %#v", target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -10,6 +12,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
)
|
)
|
||||||
|
|
||||||
type effectiveLocks struct {
|
type effectiveLocks struct {
|
||||||
@@ -19,6 +22,12 @@ type effectiveLocks struct {
|
|||||||
Key string
|
Key string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
remoteLockMutationAttempts = 4
|
||||||
|
// MaxRemoteLockStoreBytes bounds the mutable remote publish-lock document.
|
||||||
|
MaxRemoteLockStoreBytes int64 = 1 << 20
|
||||||
|
)
|
||||||
|
|
||||||
func remoteLocksKey(cfg *config.Config) (string, error) {
|
func remoteLocksKey(cfg *config.Config) (string, error) {
|
||||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||||
return "", fmt.Errorf("resolved config is required")
|
return "", fmt.Errorf("resolved config is required")
|
||||||
@@ -34,32 +43,33 @@ func remoteLocksKey(cfg *config.Config) (string, error) {
|
|||||||
return artifacts.S3SessionLocksKey(sessionPrefix), nil
|
return artifacts.S3SessionLocksKey(sessionPrefix), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.PublishLockStore, string, error) {
|
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.PublishLockStore, string, string, error) {
|
||||||
key, err := remoteLocksKey(cfg)
|
key, err := remoteLocksKey(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", err
|
return nil, "", "", err
|
||||||
}
|
}
|
||||||
exists, err := store.Exists(ctx, key)
|
if store == nil {
|
||||||
|
return nil, key, "", fmt.Errorf("remote lock store is required")
|
||||||
|
}
|
||||||
|
info, data, err := storage.ReadObjectBounded(ctx, store, key, MaxRemoteLockStoreBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, key, fmt.Errorf("check remote locks %q: %w", key, err)
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return &config.PublishLockStore{}, key, "", nil
|
||||||
|
}
|
||||||
|
var limitErr *storage.ReadLimitError
|
||||||
|
if errors.As(err, &limitErr) {
|
||||||
|
return nil, key, "", fmt.Errorf("read remote lock control object %q with %d-byte limit: %w", key, MaxRemoteLockStoreBytes, err)
|
||||||
|
}
|
||||||
|
return nil, key, "", fmt.Errorf("read remote locks %q: %w", key, err)
|
||||||
}
|
}
|
||||||
if !exists {
|
if strings.TrimSpace(info.ETag) == "" {
|
||||||
return &config.PublishLockStore{}, key, nil
|
return nil, key, "", fmt.Errorf("read remote locks %q: object has no generation", key)
|
||||||
}
|
}
|
||||||
tmp, err := storage.DownloadObjectToTemp(ctx, store, key, "narratio-locks-*.yml")
|
lockStore, err := config.LoadPublishLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, key, fmt.Errorf("download remote locks %q: %w", key, err)
|
return nil, key, "", err
|
||||||
}
|
}
|
||||||
defer func() { _ = os.Remove(tmp) }()
|
return lockStore, key, info.ETag, nil
|
||||||
data, err := os.ReadFile(tmp)
|
|
||||||
if err != nil {
|
|
||||||
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
|
|
||||||
}
|
|
||||||
lockStore, err := config.LoadPublishLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium)
|
|
||||||
if err != nil {
|
|
||||||
return nil, key, err
|
|
||||||
}
|
|
||||||
return lockStore, key, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*effectiveLocks, error) {
|
func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*effectiveLocks, error) {
|
||||||
@@ -70,7 +80,7 @@ func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.O
|
|||||||
All: append([]config.PublishLockRule(nil), staticLocks...),
|
All: append([]config.PublishLockRule(nil), staticLocks...),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
lockStore, key, err := loadRemoteLockStore(ctx, cfg, store)
|
lockStore, key, _, err := loadRemoteLockStore(ctx, cfg, store)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -100,28 +110,38 @@ func applyEffectiveLocks(cfg *config.Config, locks []config.PublishLockRule) {
|
|||||||
cfg.Pipeline.Publish.Locks = append([]config.PublishLockRule(nil), locks...)
|
cfg.Pipeline.Publish.Locks = append([]config.PublishLockRule(nil), locks...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func uploadRemoteLockStore(ctx context.Context, store storage.ObjectStore, key string, lockStore *config.PublishLockStore) error {
|
func mutateRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore, mutate func(*config.PublishLockStore) error) error {
|
||||||
data, err := config.MarshalPublishLockStore(lockStore)
|
for attempt := 0; attempt < remoteLockMutationAttempts; attempt++ {
|
||||||
if err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
lockStore, key, generation, err := loadRemoteLockStore(ctx, cfg, store)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := mutate(lockStore); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err := config.MarshalPublishLockStore(lockStore)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
condition := storage.WriteCondition{MatchETag: generation}
|
||||||
|
if generation == "" {
|
||||||
|
condition = storage.WriteCondition{RequireAbsent: true}
|
||||||
|
}
|
||||||
|
_, err = store.UploadConditional(ctx, bytes.NewReader(data), key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}, condition)
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !errors.Is(err, storage.ErrConditionNotMet) {
|
||||||
|
return fmt.Errorf("upload remote locks %q: %w", key, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
tmp, err := os.CreateTemp("", "narratio-locks-upload-*.yml")
|
return fmt.Errorf("update remote locks: concurrent updates prevented a conditional write after %d attempts", remoteLockMutationAttempts)
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("create lock store temp file: %w", err)
|
|
||||||
}
|
|
||||||
tmpPath := tmp.Name()
|
|
||||||
defer func() { _ = os.Remove(tmpPath) }()
|
|
||||||
if _, err := tmp.Write(data); err != nil {
|
|
||||||
_ = tmp.Close()
|
|
||||||
return fmt.Errorf("write lock store temp file: %w", err)
|
|
||||||
}
|
|
||||||
if err := tmp.Close(); err != nil {
|
|
||||||
return fmt.Errorf("close lock store temp file: %w", err)
|
|
||||||
}
|
|
||||||
if _, err := store.Upload(ctx, tmpPath, key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}); err != nil {
|
|
||||||
return fmt.Errorf("upload remote locks %q: %w", key, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func lockSourceSet(locks []config.PublishLockRule) map[string]config.PublishLockRule {
|
func lockSourceSet(locks []config.PublishLockRule) map[string]config.PublishLockRule {
|
||||||
@@ -150,8 +170,11 @@ func writeLocalFile(path string, data []byte, force bool) error {
|
|||||||
return fmt.Errorf("check output file %q: %w", cleaned, err)
|
return fmt.Errorf("check output file %q: %w", cleaned, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := os.MkdirAll(filepath.Dir(cleaned), 0o755); err != nil {
|
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(cleaned)); err != nil {
|
||||||
return fmt.Errorf("create output directory: %w", err)
|
return fmt.Errorf("create output directory: %w", err)
|
||||||
}
|
}
|
||||||
return os.WriteFile(cleaned, data, 0o644)
|
if err := os.WriteFile(cleaned, data, fileops.WorkspaceFileMode); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Chmod(cleaned, fileops.WorkspaceFileMode)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestExecuteRemoteSessionFallbackLoadsFromObjectStore(t *testing.T) {
|
func TestExecuteRemoteSessionFallbackLoadsFromObjectStore(t *testing.T) {
|
||||||
@@ -44,6 +45,146 @@ inputs:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRemoteSessionConfigIsRemovedAfterEveryCommandExit(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
sessionYAML string
|
||||||
|
command []string
|
||||||
|
configureRun func()
|
||||||
|
wantSuccessful bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "success",
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`,
|
||||||
|
command: []string{"session", "plan", "2026-05-03"},
|
||||||
|
wantSuccessful: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation failure",
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
`,
|
||||||
|
command: []string{"session", "plan", "2026-05-03"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "load failure",
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
unknown: true
|
||||||
|
`,
|
||||||
|
command: []string{"session", "plan", "2026-05-03"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "adapter failure",
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`,
|
||||||
|
command: []string{"run", "2026-05-03"},
|
||||||
|
configureRun: func() {
|
||||||
|
executeStagesFn = func(context.Context, *config.Config, []stage.Stage, RunOptions) (*RunSummary, error) {
|
||||||
|
return nil, errors.New("adapter failed")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cancellation",
|
||||||
|
sessionYAML: `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`,
|
||||||
|
command: []string{"run", "2026-05-03"},
|
||||||
|
configureRun: func() {
|
||||||
|
executeStagesFn = func(context.Context, *config.Config, []stage.Stage, RunOptions) (*RunSummary, error) {
|
||||||
|
return nil, context.Canceled
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
seedRemoteSessionConfig(t, fake, "2026-05-03", tt.sessionYAML)
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
|
|
||||||
|
var downloadedPath string
|
||||||
|
captureRemoteSessionTempPath(t, &downloadedPath)
|
||||||
|
if tt.configureRun != nil {
|
||||||
|
origExecuteStagesFn := executeStagesFn
|
||||||
|
t.Cleanup(func() { executeStagesFn = origExecuteStagesFn })
|
||||||
|
tt.configureRun()
|
||||||
|
}
|
||||||
|
|
||||||
|
args := append(append([]string(nil), tt.command...), "--config", pipelinePath, "--campaign-file", campaignPath)
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute(args, &stdout, &stderr)
|
||||||
|
if tt.wantSuccessful && code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !tt.wantSuccessful && code == 0 {
|
||||||
|
t.Fatal("exit code = 0, want non-zero")
|
||||||
|
}
|
||||||
|
if downloadedPath == "" {
|
||||||
|
t.Fatal("remote session download path was not captured")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(downloadedPath); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("downloaded remote session path still exists or could not be inspected: %q, err=%v", downloadedPath, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRemoteSessionConfigCloseIsIdempotent(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
seedRemoteSessionConfig(t, fake, "2026-05-03", `session_id: 2026-05-03
|
||||||
|
inputs:
|
||||||
|
audio_s3:
|
||||||
|
prefix: audio/
|
||||||
|
`)
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||||
|
|
||||||
|
var downloadedPath string
|
||||||
|
captureRemoteSessionTempPath(t, &downloadedPath)
|
||||||
|
loaded, err := loadCommandConfig(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := loaded.Close(); err != nil {
|
||||||
|
t.Fatalf("first Close() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := loaded.Close(); err != nil {
|
||||||
|
t.Fatalf("second Close() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(downloadedPath); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("downloaded remote session path still exists or could not be inspected: %q, err=%v", downloadedPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func captureRemoteSessionTempPath(t *testing.T, destination *string) {
|
||||||
|
t.Helper()
|
||||||
|
original := downloadObjectToTempFn
|
||||||
|
downloadObjectToTempFn = func(ctx context.Context, store storage.ObjectStore, key, pattern string) (string, error) {
|
||||||
|
path, err := original(ctx, store, key, pattern)
|
||||||
|
if err == nil {
|
||||||
|
*destination = path
|
||||||
|
}
|
||||||
|
return path, err
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { downloadObjectToTempFn = original })
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteRemoteSessionFallbackLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
func TestExecuteRemoteSessionFallbackLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||||
@@ -51,8 +192,8 @@ func TestExecuteRemoteSessionFallbackLoadsSecretsBeforeObjectStoreInit(t *testin
|
|||||||
secretKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_SECRET"
|
secretKeyEnv := "NARRATIO_TEST_REMOTE_SESSION_SECRET"
|
||||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||||
secretsDir := t.TempDir()
|
secretsDir := t.TempDir()
|
||||||
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "remote-session-key-id\n")
|
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "remote-session-key-id\n")
|
||||||
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "remote-session-secret\n")
|
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "remote-session-secret\n")
|
||||||
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
||||||
|
|
||||||
fake := &storage.FakeBackend{}
|
fake := &storage.FakeBackend{}
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ var buildRestorePlanFn = buildRestorePlan
|
|||||||
var executeRestorePlanFn = executeRestorePlan
|
var executeRestorePlanFn = executeRestorePlan
|
||||||
|
|
||||||
// Restore validates restore CLI/config inputs and storage preflight for future restore phases.
|
// Restore validates restore CLI/config inputs and storage preflight for future restore phases.
|
||||||
func Restore(ctx context.Context, args []string, out io.Writer) error {
|
func Restore(ctx context.Context, args []string, out io.Writer) (resultErr error) {
|
||||||
positionalSessionID, args := pullLeadingSessionID(args)
|
positionalSessionID, args := pullLeadingSessionID(args)
|
||||||
fs := flag.NewFlagSet("restore", flag.ContinueOnError)
|
fs := flag.NewFlagSet("restore", flag.ContinueOnError)
|
||||||
fs.SetOutput(out)
|
fs.SetOutput(out)
|
||||||
@@ -54,10 +54,12 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if strings.TrimSpace(flags.sessionID) == "" {
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
return fmt.Errorf("restore: session_id is required")
|
return fmt.Errorf("restore: session_id is required")
|
||||||
}
|
}
|
||||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("restore: %w", err)
|
return fmt.Errorf("restore: %w", err)
|
||||||
}
|
}
|
||||||
|
defer func() { _ = loaded.Close() }()
|
||||||
|
cfg := loaded.Config
|
||||||
if err := config.Validate(cfg); err != nil {
|
if err := config.Validate(cfg); err != nil {
|
||||||
return fmt.Errorf("restore: %w", err)
|
return fmt.Errorf("restore: %w", err)
|
||||||
}
|
}
|
||||||
@@ -69,23 +71,23 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("restore: %w", err)
|
return fmt.Errorf("restore: %w", err)
|
||||||
}
|
}
|
||||||
plan, err := buildRestorePlanFn(ctx, cfg, current, objectStore, RestorePlanOptions{
|
|
||||||
IncludeAudio: includeAudio,
|
|
||||||
Force: force,
|
|
||||||
DryRun: dryRun,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("restore: %w", err)
|
|
||||||
}
|
|
||||||
report, err := newRestoreReport(current, plan, RestorePlanOptions{
|
|
||||||
IncludeAudio: includeAudio,
|
|
||||||
Force: force,
|
|
||||||
DryRun: dryRun,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("restore: %w", err)
|
|
||||||
}
|
|
||||||
if dryRun {
|
if dryRun {
|
||||||
|
plan, err := buildRestorePlanFn(ctx, cfg, current, objectStore, RestorePlanOptions{
|
||||||
|
IncludeAudio: includeAudio,
|
||||||
|
Force: force,
|
||||||
|
DryRun: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("restore: %w", err)
|
||||||
|
}
|
||||||
|
report, err := newRestoreReport(current, plan, RestorePlanOptions{
|
||||||
|
IncludeAudio: includeAudio,
|
||||||
|
Force: force,
|
||||||
|
DryRun: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("restore: %w", err)
|
||||||
|
}
|
||||||
if err := writeRestoreDryRunSummary(out, report); err != nil {
|
if err := writeRestoreDryRunSummary(out, report); err != nil {
|
||||||
return fmt.Errorf("restore: write plan output: %w", err)
|
return fmt.Errorf("restore: write plan output: %w", err)
|
||||||
}
|
}
|
||||||
@@ -96,27 +98,54 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if _, err := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID); err != nil {
|
if _, err := artifactStore.EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID); err != nil {
|
||||||
return fmt.Errorf("restore: prepare workdir: %w", err)
|
return fmt.Errorf("restore: prepare workdir: %w", err)
|
||||||
}
|
}
|
||||||
lock, err := artifactStore.AcquireSessionLockFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
lock, err := artifactStore.AcquireSessionLockForContext(ctx, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("restore: acquire session lock: %w", err)
|
return fmt.Errorf("restore: acquire session lock: %w", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
_ = artifactStore.ReleaseSessionLock(lock)
|
if releaseErr := artifactStore.ReleaseSessionLock(lock); releaseErr != nil {
|
||||||
|
if resultErr == nil {
|
||||||
|
resultErr = fmt.Errorf("restore: release session lock: %w", releaseErr)
|
||||||
|
} else {
|
||||||
|
resultErr = errors.Join(resultErr, fmt.Errorf("restore: release session lock: %w", releaseErr))
|
||||||
|
}
|
||||||
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
if plan.ConflictCount > 0 && !force {
|
// Classification and installation share the same transition lock as a
|
||||||
|
// runner. This prevents a runner from making reuse decisions against state
|
||||||
|
// that restore is about to replace.
|
||||||
|
plan, err := buildRestorePlanFn(ctx, cfg, current, objectStore, RestorePlanOptions{
|
||||||
|
IncludeAudio: includeAudio,
|
||||||
|
Force: force,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("restore: %w", err)
|
||||||
|
}
|
||||||
|
report, err := newRestoreReport(current, plan, RestorePlanOptions{
|
||||||
|
IncludeAudio: includeAudio,
|
||||||
|
Force: force,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("restore: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if plan.ConflictCount > 0 {
|
||||||
report.setFailed(fmt.Errorf("conflict: %d conflicting path(s)", plan.ConflictCount))
|
report.setFailed(fmt.Errorf("conflict: %d conflicting path(s)", plan.ConflictCount))
|
||||||
if _, reportErr := persistRestoreReport(artifactStore, cfg, report); reportErr != nil {
|
if _, reportErr := persistRestoreReport(artifactStore, cfg, report); reportErr != nil {
|
||||||
return fmt.Errorf("restore: report failure: %w", reportErr)
|
return fmt.Errorf("restore: report failure: %w", reportErr)
|
||||||
}
|
}
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"restore conflict: %d conflicting path(s); rerun with --force to overwrite (download=%d skip_same=%d conflicts=%d)",
|
"restore conflict: %d conflicting path(s); --force can replace eligible regular files but not unresolved conflicts (download=%d skip_same=%d conflicts=%d)",
|
||||||
plan.ConflictCount,
|
plan.ConflictCount,
|
||||||
plan.DownloadCount,
|
plan.DownloadCount,
|
||||||
plan.SkipSameCount,
|
plan.SkipSameCount,
|
||||||
plan.ConflictCount,
|
plan.ConflictCount,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
if err := writeRestoreMarker(artifactStore.SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)); err != nil {
|
||||||
|
return fmt.Errorf("restore: mark incomplete restore: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
result, err := executeRestorePlanFn(ctx, cfg, current, plan, report, objectStore)
|
result, err := executeRestorePlanFn(ctx, cfg, current, plan, report, objectStore)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -128,6 +157,9 @@ func Restore(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
report.Execution.Downloaded = result.DownloadedCount
|
report.Execution.Downloaded = result.DownloadedCount
|
||||||
report.setSucceeded()
|
report.setSucceeded()
|
||||||
|
if err := clearRestoreMarker(artifactStore.SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)); err != nil {
|
||||||
|
return fmt.Errorf("restore: clear incomplete restore marker: %w", err)
|
||||||
|
}
|
||||||
if _, err := persistRestoreReport(artifactStore, cfg, report); err != nil {
|
if _, err := persistRestoreReport(artifactStore, cfg, report); err != nil {
|
||||||
return fmt.Errorf("restore: write report: %w", err)
|
return fmt.Errorf("restore: write report: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
308
internal/app/restore_commit_test.go
Normal file
308
internal/app/restore_commit_test.go
Normal file
@@ -0,0 +1,308 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCommittedRestorePlanUsesOnlyDeclaredObjects(t *testing.T) {
|
||||||
|
cfg := restorePlanConfig(t)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
current := seedCommittedRestoreSnapshot(t, cfg, fake, "20260519T010203Z-a1b2c3d4", map[string][]byte{
|
||||||
|
"transcripts/full.json": []byte(`{"segments":[1]}`),
|
||||||
|
})
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: current.SessionPrefix + "artifacts/stale.md", Data: []byte("stale")})
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(current.SessionPrefix, "20260519T010204Z-e5f6a7b8"), "transcripts/other.json"), Data: []byte("other")})
|
||||||
|
|
||||||
|
plan, err := buildRestorePlan(context.Background(), cfg, current, fake, RestorePlanOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
got := actionRelPaths(plan.Actions)
|
||||||
|
want := []string{"manifest.json", "transcripts/full.json"}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("action paths = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
for index := range want {
|
||||||
|
if got[index] != want[index] {
|
||||||
|
t.Fatalf("action paths = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommittedRestoreReusesVerifiedManifestCandidate(t *testing.T) {
|
||||||
|
cfg := restorePlanConfig(t)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
current := seedCommittedRestoreSnapshot(t, cfg, fake, "20260519T010203Z-a1b2c3d4", nil)
|
||||||
|
|
||||||
|
plan, err := buildRestorePlan(context.Background(), cfg, current, fake, RestorePlanOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := executeRestorePlan(context.Background(), cfg, current, plan, nil, fake); err != nil {
|
||||||
|
t.Fatalf("executeRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := fakeReadCount(fake, current.CurrentManifestKey); got != 1 {
|
||||||
|
t.Fatalf("manifest reads = %d, want one discovery read reused by restore", got)
|
||||||
|
}
|
||||||
|
if got := fakeDownloadCount(fake, current.CurrentManifestKey); got != 0 {
|
||||||
|
t.Fatalf("manifest downloads = %d, want no temporary download", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommittedRestoreRejectsChangedGenerationForVerifiedManifestCandidate(t *testing.T) {
|
||||||
|
cfg := restorePlanConfig(t)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
current := seedCommittedRestoreSnapshot(t, cfg, fake, "20260519T010203Z-a1b2c3d4", nil)
|
||||||
|
plan, err := buildRestorePlan(context.Background(), cfg, current, fake, RestorePlanOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: current.CurrentManifestKey, Data: []byte("changed manifest"), ETag: "changed-generation"})
|
||||||
|
|
||||||
|
_, err = executeRestorePlan(context.Background(), cfg, current, plan, nil, fake)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "generation mismatch") {
|
||||||
|
t.Fatalf("executeRestorePlan() error = %v, want generation mismatch", err)
|
||||||
|
}
|
||||||
|
if got := fakeReadCount(fake, current.CurrentManifestKey); got != 1 {
|
||||||
|
t.Fatalf("manifest reads = %d, want no second transfer for rejected candidate", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommittedStatusReportsOnlyDeclaredPublishedOutputs(t *testing.T) {
|
||||||
|
cfg := restorePlanConfig(t)
|
||||||
|
cfg.Pipeline.Publish = &config.PublishConfig{Outputs: []config.PublishOutputRule{{
|
||||||
|
Source: "narratio.transcript.final_trimmed",
|
||||||
|
Dest: "transcripts/full.json",
|
||||||
|
}}}
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
current := seedCommittedRestoreSnapshot(t, cfg, fake, "20260519T010203Z-a1b2c3d4", map[string][]byte{
|
||||||
|
"transcripts/full.json": []byte("declared\n"),
|
||||||
|
})
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: current.SessionPrefix + "transcripts/full.json", Data: []byte("mutable stale copy\n")})
|
||||||
|
|
||||||
|
availability := remotePublishedOutputAvailabilityForCurrent(context.Background(), cfg, fake, nil, current)
|
||||||
|
key := publishedOutputRemoteStateKey("narratio.transcript.final_trimmed", "transcripts/full.json")
|
||||||
|
if availability[key] != "remote=published" {
|
||||||
|
t.Fatalf("availability = %#v, want committed published output", availability)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommittedRestoreKeepsSelectedSnapshotWhenPointerChanges(t *testing.T) {
|
||||||
|
cfg := restorePlanConfig(t)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
first := seedCommittedRestoreSnapshot(t, cfg, fake, "20260519T010203Z-a1b2c3d4", map[string][]byte{
|
||||||
|
"transcripts/full.json": []byte("from first commit\n"),
|
||||||
|
})
|
||||||
|
plan, err := buildRestorePlan(context.Background(), cfg, first, fake, RestorePlanOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
_ = seedCommittedRestoreSnapshot(t, cfg, fake, "20260519T010204Z-e5f6a7b8", map[string][]byte{
|
||||||
|
"transcripts/full.json": []byte("from second commit\n"),
|
||||||
|
})
|
||||||
|
|
||||||
|
if _, err := executeRestorePlan(context.Background(), cfg, first, plan, nil, fake); err != nil {
|
||||||
|
t.Fatalf("executeRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
root := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
mustReadEquals(t, filepath.Join(root, "transcripts", "full.json"), "from first commit\n")
|
||||||
|
restored, err := (&manifest.LocalStore{}).Load(context.Background(), filepath.Join(root, "manifest.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load restored manifest: %v", err)
|
||||||
|
}
|
||||||
|
if restored.RunID != first.RunID {
|
||||||
|
t.Fatalf("restored run id = %q, want %q", restored.RunID, first.RunID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommittedRestoreRejectsChangedDeclaredObjectBeforeManifestInstall(t *testing.T) {
|
||||||
|
cfg := restorePlanConfig(t)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
current := seedCommittedRestoreSnapshot(t, cfg, fake, "20260519T010203Z-a1b2c3d4", map[string][]byte{
|
||||||
|
"transcripts/full.json": []byte("committed bytes\n"),
|
||||||
|
})
|
||||||
|
plan, err := buildRestorePlan(context.Background(), cfg, current, fake, RestorePlanOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
key := artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(current.SessionPrefix, current.RunID), "transcripts/full.json")
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("changed bytes\n")})
|
||||||
|
|
||||||
|
if _, err := executeRestorePlan(context.Background(), cfg, current, plan, nil, fake); err == nil {
|
||||||
|
t.Fatal("executeRestorePlan() error = nil, want committed-object verification failure")
|
||||||
|
}
|
||||||
|
root := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
if _, err := os.Stat(filepath.Join(root, "manifest.json")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("manifest should not be installed after failed restore; stat err=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommittedRestoreRejectsChangedDeclaredObjectGeneration(t *testing.T) {
|
||||||
|
cfg := restorePlanConfig(t)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
current := seedCommittedRestoreSnapshot(t, cfg, fake, "20260519T010203Z-a1b2c3d4", map[string][]byte{
|
||||||
|
"transcripts/full.json": []byte("committed bytes\n"),
|
||||||
|
})
|
||||||
|
plan, err := buildRestorePlan(context.Background(), cfg, current, fake, RestorePlanOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
key := artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(current.SessionPrefix, current.RunID), "transcripts/full.json")
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("committed bytes\n"), ETag: "replacement-generation"})
|
||||||
|
|
||||||
|
if _, err := executeRestorePlan(context.Background(), cfg, current, plan, nil, fake); err == nil {
|
||||||
|
t.Fatal("executeRestorePlan() error = nil, want generation verification failure")
|
||||||
|
}
|
||||||
|
root := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
if _, err := os.Stat(filepath.Join(root, "manifest.json")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("manifest should not be installed after failed restore; stat err=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommittedRestoreRejectsMissingDeclaredObjectBeforeManifestInstall(t *testing.T) {
|
||||||
|
cfg := restorePlanConfig(t)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
current := seedCommittedRestoreSnapshot(t, cfg, fake, "20260519T010203Z-a1b2c3d4", map[string][]byte{
|
||||||
|
"transcripts/full.json": []byte("committed bytes\n"),
|
||||||
|
})
|
||||||
|
plan, err := buildRestorePlan(context.Background(), cfg, current, fake, RestorePlanOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
missingKey := artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(current.SessionPrefix, current.RunID), "transcripts/full.json")
|
||||||
|
fake.DownloadHook = func(call storage.FakeDownloadCall) error {
|
||||||
|
if call.Key == missingKey {
|
||||||
|
return os.ErrNotExist
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := executeRestorePlan(context.Background(), cfg, current, plan, nil, fake); err == nil {
|
||||||
|
t.Fatal("executeRestorePlan() error = nil, want missing-object failure")
|
||||||
|
}
|
||||||
|
root := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
if _, err := os.Stat(filepath.Join(root, "manifest.json")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("manifest should not be installed after failed restore; stat err=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommittedRestoreForceRetainsDirectoryConflict(t *testing.T) {
|
||||||
|
cfg := restorePlanConfig(t)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
current := seedCommittedRestoreSnapshot(t, cfg, fake, "20260519T010203Z-a1b2c3d4", map[string][]byte{
|
||||||
|
"transcripts/full.json": []byte("committed bytes\n"),
|
||||||
|
})
|
||||||
|
root := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
if err := os.MkdirAll(filepath.Join(root, "transcripts", "full.json"), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
plan, err := buildRestorePlan(context.Background(), cfg, current, fake, RestorePlanOptions{Force: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildRestorePlan() error = %v", err)
|
||||||
|
}
|
||||||
|
if plan.ConflictCount != 1 {
|
||||||
|
t.Fatalf("ConflictCount = %d, want 1", plan.ConflictCount)
|
||||||
|
}
|
||||||
|
for _, action := range plan.Actions {
|
||||||
|
if action.LocalRelativePath == "transcripts/full.json" && action.ConflictKind != RestoreConflictDirectory {
|
||||||
|
t.Fatalf("ConflictKind = %q, want %q", action.ConflictKind, RestoreConflictDirectory)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func seedCommittedRestoreSnapshot(t *testing.T, cfg *config.Config, fake *storage.FakeBackend, runID string, outputs map[string][]byte) *RemoteCurrentState {
|
||||||
|
t.Helper()
|
||||||
|
sessionPrefix := artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
remoteManifest := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
|
remoteManifest.Campaign = cfg.Session.Campaign
|
||||||
|
remoteManifest.RunID = runID
|
||||||
|
manifestData, err := json.Marshal(remoteManifest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal manifest: %v", err)
|
||||||
|
}
|
||||||
|
manifestData = append(manifestData, '\n')
|
||||||
|
manifestKey := artifacts.S3RunSessionManifestKey(sessionPrefix, runID)
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: manifestKey, Data: manifestData})
|
||||||
|
artifactsByKey := []artifacts.RemoteArtifact{remoteRestoreArtifact(fake, artifacts.RemoteArtifactTypeSessionManifest, "session.manifest", manifestKey)}
|
||||||
|
|
||||||
|
paths := make([]string, 0, len(outputs))
|
||||||
|
for relative := range outputs {
|
||||||
|
paths = append(paths, relative)
|
||||||
|
}
|
||||||
|
sort.Strings(paths)
|
||||||
|
for _, relative := range paths {
|
||||||
|
key := artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(sessionPrefix, runID), relative)
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: key, Data: outputs[relative]})
|
||||||
|
artifactsByKey = append(artifactsByKey, remoteRestoreArtifact(fake, artifacts.RemoteArtifactTypePublishedOutput, "narratio.test", key))
|
||||||
|
}
|
||||||
|
commit := artifacts.RemoteCommitManifest{
|
||||||
|
FormatVersion: artifacts.RemoteCommitFormatVersion,
|
||||||
|
Campaign: cfg.Session.Campaign,
|
||||||
|
SessionID: cfg.Session.SessionID,
|
||||||
|
RunID: runID,
|
||||||
|
Artifacts: artifactsByKey,
|
||||||
|
}
|
||||||
|
commitData, err := artifacts.EncodeRemoteCommitManifest(commit)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encode remote commit: %v", err)
|
||||||
|
}
|
||||||
|
commitKey := artifacts.S3RunCommitKey(sessionPrefix, runID)
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: commitKey, Data: commitData})
|
||||||
|
commitObject := fake.Objects[commitKey]
|
||||||
|
pointerData, err := artifacts.EncodeCurrentCommitPointer(artifacts.CurrentCommitPointer{
|
||||||
|
FormatVersion: artifacts.RemoteCommitFormatVersion,
|
||||||
|
Campaign: cfg.Session.Campaign,
|
||||||
|
SessionID: cfg.Session.SessionID,
|
||||||
|
RunID: runID,
|
||||||
|
CommitKey: commitKey,
|
||||||
|
CommitSHA256: restoreCommitSHA256(commitData),
|
||||||
|
CommitSize: int64(len(commitData)),
|
||||||
|
CommitGeneration: commitObject.ETag,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encode current pointer: %v", err)
|
||||||
|
}
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: artifacts.S3CurrentCommitPointerKey(sessionPrefix), Data: pointerData})
|
||||||
|
current, err := discoverRemoteCurrentState(context.Background(), cfg, fake)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("discoverRemoteCurrentState() error = %v", err)
|
||||||
|
}
|
||||||
|
return current
|
||||||
|
}
|
||||||
|
|
||||||
|
func remoteRestoreArtifact(fake *storage.FakeBackend, artifactType artifacts.RemoteArtifactType, source, key string) artifacts.RemoteArtifact {
|
||||||
|
object := fake.Objects[key]
|
||||||
|
return artifacts.RemoteArtifact{
|
||||||
|
Type: artifactType, Source: source, DestinationKey: key, SHA256: restoreCommitSHA256(object.Data), Size: int64(len(object.Data)), Generation: object.ETag,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fakeReadCount(fake *storage.FakeBackend, key string) int {
|
||||||
|
count := 0
|
||||||
|
for _, call := range fake.Reads {
|
||||||
|
if call.Key == key {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
func restoreCommitSHA256(data []byte) string {
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user