Compare commits
78 Commits
a90231ce0c
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| e433c86203 | |||
| a2a144dffa | |||
| 8ef6e99d69 | |||
| 2545faef6c | |||
| 80be8be4d6 | |||
| 801adb385d | |||
| feba7b9d74 | |||
| b89224bbde | |||
| 131ffd9887 | |||
| af492c9e97 | |||
| f39fc94610 | |||
| 4e4eff6ba7 | |||
| 8ff1b4fa66 | |||
| 702f622e18 | |||
| 9da2c1e144 | |||
| 32653f54f9 | |||
| 72a200968a | |||
| b39b68add7 | |||
| d9fa1d9328 | |||
| 8375ad83f3 | |||
| 4158394dcf | |||
| eac7e155a5 | |||
| 0cf2cbfeb3 | |||
| 361dbb4ca8 | |||
| d6deccf3e8 | |||
| ee747243fe | |||
| a1ceb457e9 | |||
| 9900211fa4 | |||
| 60cebf0e4b | |||
| 7bd575187e | |||
| ab5a7e8e3d | |||
| 99b2e1cd81 | |||
| 363313d99c | |||
| 18ddf00d3d | |||
| 59f3fe3d1d | |||
| 1dccf5f140 | |||
| 0b40cf8026 | |||
| a7ec195587 | |||
| 13de820931 | |||
| 14ef59aaed | |||
| f387222fce | |||
| 9cb9008dfc | |||
| 083decc5b4 | |||
| 57cac5d3f7 | |||
| 0a772e03b4 | |||
| 0920062a38 | |||
| 39afe644eb | |||
| e3ee3de10a | |||
| 9c72db56e9 | |||
| bb2d606dbb | |||
| 9850767a8a | |||
| 74e2d21de5 | |||
| 7cb18a1a40 | |||
| b556fc2f4f | |||
| b99bd38eb4 | |||
| 701b6726d7 | |||
| 665039f4dc | |||
| ef8dae776e | |||
| d01775b68a | |||
| 0d6f2dd0ce | |||
| df40cbec6e | |||
| 0341e0c7c0 | |||
| 39af7d4f3c | |||
| bba582b4ca | |||
| 1f16a85330 | |||
| f9482639d4 | |||
| dce721cdbd | |||
| 98734644d6 | |||
| 951383226c | |||
| df58595d1e | |||
| c3c14e7468 | |||
| e7319ea016 | |||
| bd2d5e2496 | |||
| 115a44f629 | |||
| 18411dc5b5 | |||
| e23dc1ab6e | |||
| e1359ea227 | |||
| 7fdd99ec27 |
@@ -2,8 +2,33 @@ when:
|
||||
- event: tag
|
||||
|
||||
steps:
|
||||
- name: build-release-assets
|
||||
validate:
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- go test ./...
|
||||
- go test -race ./...
|
||||
- go vet ./...
|
||||
- go build ./...
|
||||
- go test ./internal/doccheck
|
||||
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||
|
||||
cross-build:
|
||||
image: golang:1.25
|
||||
depends_on: validate
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
output_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "$output_dir"' EXIT
|
||||
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
|
||||
goos="${target%/*}"
|
||||
goarch="${target#*/}"
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
|
||||
done
|
||||
|
||||
build-release-assets:
|
||||
image: golang:1.25
|
||||
depends_on: [validate, cross-build]
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
@@ -33,7 +58,7 @@ steps:
|
||||
build_binary windows amd64 ".exe"
|
||||
build_binary windows arm64 ".exe"
|
||||
|
||||
- name: publish-release
|
||||
publish-release:
|
||||
image: woodpeckerci/plugin-release
|
||||
depends_on:
|
||||
- build-release-assets
|
||||
|
||||
8
.woodpecker/shuffle.yml
Normal file
8
.woodpecker/shuffle.yml
Normal file
@@ -0,0 +1,8 @@
|
||||
when:
|
||||
- event: cron
|
||||
|
||||
steps:
|
||||
shuffled-race-tests:
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- go test -race -shuffle=on -count=3 ./...
|
||||
47
.woodpecker/verify.yml
Normal file
47
.woodpecker/verify.yml
Normal file
@@ -0,0 +1,47 @@
|
||||
when:
|
||||
- event: [push, pull_request]
|
||||
|
||||
steps:
|
||||
tests:
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- go test ./...
|
||||
|
||||
race-tests:
|
||||
image: golang:1.25
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go test -race ./...
|
||||
|
||||
static-analysis:
|
||||
image: golang:1.25
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go vet ./...
|
||||
|
||||
build:
|
||||
image: golang:1.25
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go build ./...
|
||||
|
||||
documentation-and-examples:
|
||||
image: golang:1.25
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go test ./internal/doccheck
|
||||
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||
|
||||
cross-build:
|
||||
image: golang:1.25
|
||||
depends_on: [race-tests, static-analysis, build, documentation-and-examples]
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
output_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "$output_dir"' EXIT
|
||||
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64; do
|
||||
goos="${target%/*}"
|
||||
goarch="${target#*/}"
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -o "$output_dir/narratio-$goos-$goarch" ./cmd/narratio
|
||||
done
|
||||
44
README.md
44
README.md
@@ -1,22 +1,42 @@
|
||||
# narratio
|
||||
|
||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into polished transcripts and generated artifacts.
|
||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
||||
polished transcripts, validated Notarius extraction lanes, and generated
|
||||
artifacts.
|
||||
|
||||
It runs a deterministic workflow across `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `render`, `analyze`, and `publish`, with manifest-driven continuation and restore support.
|
||||
It runs a deterministic workflow with manifest-driven continuation, remote
|
||||
publish, and restore support.
|
||||
|
||||
```bash
|
||||
```sh
|
||||
narratio run 2026-04-04
|
||||
```
|
||||
|
||||
This requires resolvable `pipeline.yml`, `campaign.yml`, and concrete `session.yml` (or explicit config flags).
|
||||
This requires resolvable `pipeline.yml`, `campaign.yml`, and concrete
|
||||
`session.yml` files or their explicit command-line alternatives.
|
||||
|
||||
## Documentation
|
||||
|
||||
- [CLI Reference](docs/cli.md)
|
||||
- [Configuration](docs/config.md)
|
||||
- [Operations](docs/operations.md)
|
||||
- [Troubleshooting](docs/troubleshooting.md)
|
||||
- [Internal Component Contracts](docs/internal/README.md)
|
||||
- [Development Guide](docs/policy/development.md)
|
||||
- [Architecture Principles](docs/policy/architecture.md)
|
||||
- [Maintained Examples](examples/)
|
||||
- [CLI reference](docs/cli.md) — commands, arguments, flags, and invocation
|
||||
behavior.
|
||||
- [Configuration](docs/config.md) — discovery, fields, defaults, and
|
||||
validation.
|
||||
- [Operations](docs/operations.md) — runtime workflow, state, publishing,
|
||||
recovery, and cleanup.
|
||||
- [Troubleshooting](docs/troubleshooting.md) — symptom-driven diagnosis and
|
||||
safe remedies.
|
||||
- [Integration contracts](docs/integrations/) — external tools, formats, and
|
||||
compatibility expectations.
|
||||
- [Maintained examples](examples/README.md) — complete copyable configuration
|
||||
and input files.
|
||||
|
||||
## Maintainer Documentation
|
||||
|
||||
- [Development guide](docs/development.md) — first-read orientation and
|
||||
task-specific reading routes.
|
||||
- [Internal overview](docs/internal/overview.md) — implemented component map.
|
||||
- [Architecture](docs/policy/architecture.md) — normative boundaries and
|
||||
invariants.
|
||||
- [Documentation policy](docs/policy/documentation.md) — canonical ownership
|
||||
and maintenance rules.
|
||||
- [Testing policy](docs/policy/testing.md) — test value, boundaries, and
|
||||
sufficiency.
|
||||
|
||||
68
docs/cli.md
68
docs/cli.md
@@ -47,7 +47,11 @@ Rules:
|
||||
- `--campaign` and `--campaign-file` are mutually exclusive.
|
||||
- `--session` is not used by `session init`.
|
||||
- if both positional `<session_id>` and `--session-id` are provided, values must match.
|
||||
- `--previous-session-id` is a strict expectation: the selected session file
|
||||
must contain the same `previous_session_id`.
|
||||
- `clean --all` cannot be combined with campaign/session selectors.
|
||||
- notification delivery is currently limited to the configured `noop` mode; see
|
||||
the [configuration reference](./config.md#notifications).
|
||||
|
||||
## Session ID Input Rules
|
||||
|
||||
@@ -75,7 +79,10 @@ narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common co
|
||||
Behavior:
|
||||
|
||||
- evaluates full stage order;
|
||||
- skips already-succeeded stages unless `--force` is set;
|
||||
- runs `extract` between `trim` and `render`; an omitted or disabled Notarius
|
||||
configuration records an explicit `notarius_disabled` self-skip;
|
||||
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
||||
resume check finds its durable result obsolete;
|
||||
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- writes session and run manifests.
|
||||
|
||||
@@ -93,6 +100,7 @@ Valid stage names:
|
||||
- `polish`
|
||||
- `normalize`
|
||||
- `trim`
|
||||
- `extract`
|
||||
- `render`
|
||||
- `analyze`
|
||||
- `publish`
|
||||
@@ -135,14 +143,13 @@ narratio clean --all [--dry-run] [--clear-cache] [--config <pipeline.yml>]
|
||||
|
||||
Behavior:
|
||||
|
||||
- session mode removes:
|
||||
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||
- `{spool.root}/{campaign}/{session_id}`
|
||||
- `--all` removes:
|
||||
- `{workspace.root}/work/*`
|
||||
- direct children under `{spool.root}`
|
||||
- session mode removes the selected session's local work and spool state;
|
||||
- `--all` removes all local session work and spool state;
|
||||
- cache remains unless `--clear-cache` is provided.
|
||||
|
||||
See [Operations: Cleanup](./operations.md#cleanup) for deletion scope and
|
||||
post-publish cleanup behavior.
|
||||
|
||||
### `session plan`
|
||||
|
||||
```bash
|
||||
@@ -165,7 +172,8 @@ Read-only preflight checks for config validity, required inputs, audio mode, pre
|
||||
narratio session status <session_id> [...common config flags]
|
||||
```
|
||||
|
||||
Prints local manifest state and, when storage is available, remote current-state and published-output status.
|
||||
Prints local manifest state and, when storage is available, status for the
|
||||
pointer-selected remote commit and its declared published outputs.
|
||||
|
||||
### `session init`
|
||||
|
||||
@@ -207,17 +215,12 @@ Behavior:
|
||||
|
||||
- discovers committed remote current state;
|
||||
- plans local restores;
|
||||
- writes `reports/restore-latest.json` on execution;
|
||||
- blocks conflicting overwrites unless `--force` is set.
|
||||
- writes an execution report;
|
||||
- blocks unresolved conflicts. `--force` permits replacement only of eligible
|
||||
regular files.
|
||||
|
||||
Default restore scope:
|
||||
|
||||
- `manifest.json`
|
||||
- `transcripts/**`
|
||||
- `artifacts/**`
|
||||
- `previous/**` when required by configured previous-session inputs
|
||||
|
||||
`audio/**` is included only with `--include-audio`.
|
||||
See [Operations: Restore Workflow](./operations.md#restore-workflow) for the
|
||||
default restore scope, report location, and conflict-handling workflow.
|
||||
|
||||
### `session artifacts`
|
||||
|
||||
@@ -225,7 +228,10 @@ Default restore scope:
|
||||
narratio session artifacts <session_id> [--remote] [...common config flags]
|
||||
```
|
||||
|
||||
Lists effective built-in and configured artifact sources, publish rules, lock state, and optional remote published-state availability.
|
||||
Lists effective built-in, configured Scriptorium, and configured extraction
|
||||
sources; reports planned, available, unavailable, and published state without
|
||||
reading payload bodies; and includes publish rules, lock state, and optional
|
||||
remote published-state availability.
|
||||
|
||||
### `session locks`
|
||||
|
||||
@@ -237,10 +243,13 @@ narratio session locks remove <session_id> <source> [...common config flags]
|
||||
|
||||
Behavior:
|
||||
|
||||
- list mode merges static `pipeline.publish.locks` with remote `{session_prefix}/locks.yml`;
|
||||
- list mode reports the effective merge of static and remote locks;
|
||||
- add/remove mutate only remote locks;
|
||||
- static locks from pipeline config cannot be removed by CLI commands.
|
||||
|
||||
See [Operations: Publish Locks](./operations.md#publish-locks) for lock storage
|
||||
and precedence.
|
||||
|
||||
## `--artifacts` Selection Rules
|
||||
|
||||
- accepted on `run`, `run-stage`, `analyze`, and `publish`;
|
||||
@@ -252,7 +261,9 @@ Effects:
|
||||
|
||||
- filters analyze execution to selected configured artifacts;
|
||||
- filters publish rules that source `narratio.artifact.<name>`;
|
||||
- does not filter built-in transcript/bounds publish sources.
|
||||
- does not filter built-in transcript/bounds or explicitly configured
|
||||
`narratio.extraction.<name>` publish sources; and
|
||||
- does not select or filter Notarius lanes.
|
||||
|
||||
## Common Workflows
|
||||
|
||||
@@ -279,3 +290,18 @@ Force publish only:
|
||||
```bash
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
## Output And Exit Behavior
|
||||
|
||||
- Successful commands write their result or summary to standard output and
|
||||
exit with status `0`.
|
||||
- Command failures and invalid invocations write an error to standard error and
|
||||
exit with status `1`.
|
||||
- An unknown top-level command also prints the top-level usage summary to
|
||||
standard error.
|
||||
- `session restore --help` prints its command-specific usage and exits with
|
||||
status `0`.
|
||||
|
||||
Output is intended for operator inspection. Narratio does not currently offer
|
||||
a machine-readable CLI output mode; durable machine-readable state is recorded
|
||||
in manifests and reports described in [Operations](./operations.md).
|
||||
|
||||
123
docs/config.md
123
docs/config.md
@@ -38,9 +38,25 @@ If local session discovery fails and a `session_id` is known, Narratio attempts
|
||||
|
||||
using configured object storage.
|
||||
|
||||
The downloaded remote session file is command-scoped: Narratio removes it after
|
||||
the command finishes and records only the remote object provenance alongside
|
||||
the durable copied session input.
|
||||
|
||||
### Identity segments
|
||||
|
||||
Campaign IDs (`campaign_id` and `default_campaign_id`), session IDs, previous
|
||||
session IDs, and Narratio run IDs are opaque portable segments. They must use
|
||||
only ASCII letters, digits, `.`, `_`, and `-`; empty values, `.`/`..`, path
|
||||
separators, drive forms, whitespace, control characters, and non-ASCII text are
|
||||
rejected. Narratio does not trim or rewrite these values. Existing manifests or
|
||||
remote state with an unsafe legacy identity must be migrated before use.
|
||||
|
||||
## Validation and Merge Rules
|
||||
|
||||
- YAML decode is strict (`KnownFields(true)`): unknown fields fail load.
|
||||
- YAML decode is strict (`KnownFields(true)`) and accepts exactly one document:
|
||||
unknown fields or trailing documents fail load.
|
||||
- Configured timeout and retry-delay durations must be positive. An omitted
|
||||
artifact timeout continues to inherit its configured Scriptorium timeout.
|
||||
- Session files must be concrete; unresolved `{{ ... }}` placeholders fail load.
|
||||
- Pipeline defaults are applied before validation.
|
||||
- Campaign and session identities must agree.
|
||||
@@ -85,7 +101,13 @@ inputs:
|
||||
|
||||
- Do not place raw secrets in YAML.
|
||||
- Use env var names in config (for example `pipeline.audita.llm_api_key_env`).
|
||||
- Optionally load env files from `pipeline.secrets.env_dir`.
|
||||
- Optionally load credential files from `pipeline.secrets.env_dir`. Each valid
|
||||
environment-variable filename supplies one value; trailing CR/LF is removed.
|
||||
- An existing process environment value takes precedence over a credential file.
|
||||
- Credential directories and files must not be symlinks and must be regular,
|
||||
bounded files (at most 8 KiB per value). On POSIX, provision the directory
|
||||
with no group/other access (normally `0700`) and files with no group/other
|
||||
access (normally `0600`).
|
||||
- Commands that need storage/auth load filesystem secrets before constructing adapters.
|
||||
|
||||
## Publish Configuration Summary
|
||||
@@ -118,6 +140,9 @@ Rules:
|
||||
|
||||
- `outputs[].source` is required.
|
||||
- `outputs[].dest` may be omitted when derivable from source.
|
||||
- extraction sources require an explicit `outputs[].dest` and publish only when
|
||||
a rule names that source; the Notarius index and complete bundle are not
|
||||
publish sources.
|
||||
- `outputs[].required` defaults to `true`.
|
||||
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
||||
|
||||
@@ -132,8 +157,8 @@ Rules:
|
||||
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||
| `pipeline.secrets.env_dir` | string | No | empty |
|
||||
| `pipeline.storage.backend` | string | No | empty |
|
||||
| `pipeline.storage.s3.bucket` | string | Conditional | required for S3 session-audio and for publish upload when backend is `s3` |
|
||||
| `pipeline.storage.backend` | string | No | `local`; supported values are `local` and `s3` (case-insensitive) |
|
||||
| `pipeline.storage.s3.bucket` | string | Conditional | required when backend is `s3` and S3 session-audio or publish upload is enabled |
|
||||
| `pipeline.storage.s3.root_prefix` | string | No | `dnd` |
|
||||
| `pipeline.storage.s3.region` | string | No | empty |
|
||||
| `pipeline.storage.s3.endpoint` | string | No | empty |
|
||||
@@ -153,7 +178,7 @@ Rules:
|
||||
| `pipeline.publish.locks[]` | list | No | empty |
|
||||
| `pipeline.publish.locks[].source` | string | Yes (per lock) | must reference supported publish source |
|
||||
| `pipeline.publish.locks[].reason` | string | No | empty |
|
||||
| `pipeline.whisperx.transcribe_url` | string | Yes | valid URL |
|
||||
| `pipeline.whisperx.transcribe_url` | string | Yes | absolute `http` or `https` URL |
|
||||
| `pipeline.whisperx.language` | string | No | `en` |
|
||||
| `pipeline.whisperx.timeout` | duration | No | `30m` |
|
||||
| `pipeline.whisperx.retries` | int | No | `3` |
|
||||
@@ -196,6 +221,13 @@ Rules:
|
||||
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
||||
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
||||
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||
| `pipeline.notarius.enabled` | bool | No | `false` |
|
||||
| `pipeline.notarius.binary` | string | No | `notarius` |
|
||||
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
||||
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
||||
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
||||
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
||||
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
||||
| `pipeline.render.enabled` | bool | No | `true` |
|
||||
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
|
||||
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
|
||||
@@ -207,9 +239,27 @@ Rules:
|
||||
| `pipeline.scriptorium.timeout` | duration | No | `10m` |
|
||||
| `pipeline.scriptorium.render_debug` | bool | No | `false` |
|
||||
| `pipeline.scriptorium.artifacts` | map | No | empty |
|
||||
| `pipeline.notification.backend` | string | No | empty |
|
||||
| `pipeline.notification.recipient` | string | No | empty |
|
||||
| `pipeline.notification.timeout` | duration | No | `30s` |
|
||||
| `pipeline.notification.mode` | string | No | `noop`; the only supported notification mode until a provider is implemented |
|
||||
|
||||
### Notarius Output Entries
|
||||
|
||||
For each `pipeline.notarius.outputs.<name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `lane_id` | string | Yes | unique Notarius lane ID |
|
||||
| `media_type` | string | Yes | exact accepted descriptor media type |
|
||||
| `schema_id` | string | Yes | exact accepted descriptor schema ID |
|
||||
| `schema_version` | string | Yes | exact accepted descriptor schema version |
|
||||
| `module_key` | string | No | exact accepted module key when set |
|
||||
|
||||
Output names must match `^[a-z][a-z0-9_]*$` and become selectable sources named
|
||||
`narratio.extraction.<name>`. Lane IDs must be unique. Every declared output is
|
||||
required from a successful Notarius result; a missing, rejected, duplicate, or
|
||||
contract-incompatible lane fails extraction. See the
|
||||
[complete maintained example](../examples/pipeline.full.annotated.yml) for the
|
||||
current ten-lane D&D mapping and the [Notarius contract](./integrations/notarius.md)
|
||||
for compatibility ownership.
|
||||
|
||||
### Scriptorium Artifact Entries
|
||||
|
||||
@@ -229,20 +279,39 @@ For each `pipeline.scriptorium.artifacts.<name>`:
|
||||
|
||||
Narratio adds `session_id=narratio-session-<session_id>` to every Scriptorium request for sticky upstream LLM routing. If an artifact config sets `vars.session_id`, Narratio replaces that value before invoking Scriptorium. Use a different variable name if a prompt needs the raw Narratio session ID as content.
|
||||
|
||||
Without `--artifacts`, analyze executes enabled configured artifacts. With an
|
||||
explicit `--artifacts` list, the exact named configured artifacts are the
|
||||
one-invocation execution set even if their `enabled` values are false; the list
|
||||
does not automatically include dependencies. Named artifacts must therefore be
|
||||
configured with valid executable fields, and their configured dependencies must
|
||||
already be available to analyze. This override affects analyze planning only;
|
||||
publish uses the list only to filter configured
|
||||
`narratio.artifact.<name>` output rules.
|
||||
|
||||
For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
||||
| `artifact` | string | No | optional passthrough adapter field |
|
||||
| `path` | string | No | optional passthrough adapter field |
|
||||
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.extraction.<name>`, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
||||
| `required` | bool | No | optional input requirement |
|
||||
|
||||
`artifact` and `path` are obsolete and rejected by strict configuration
|
||||
loading. Use the canonical `source` identifier to select the input; Narratio
|
||||
does not provide adapter-specific input passthrough fields.
|
||||
|
||||
### Notifications
|
||||
|
||||
Narratio currently supports only `notification.mode: noop`, which is also the
|
||||
default when the section is omitted. The notify stage performs no delivery in
|
||||
this mode. Backend, recipient, timeout, and other provider settings are
|
||||
rejected by strict configuration loading until Narratio has a provider
|
||||
integration.
|
||||
|
||||
### Campaign
|
||||
|
||||
| Field | Type | Required | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `campaign_id` | string | Yes | canonical campaign identity |
|
||||
| `campaign_id` | string | Yes | canonical opaque campaign identity |
|
||||
| `session_template_file` | string | No | used by `session init` when set |
|
||||
| `inputs.speakers_file` | string | Yes | stable input default |
|
||||
| `inputs.autocorrect_file` | string | Yes | stable input default |
|
||||
@@ -254,9 +323,9 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
|
||||
|
||||
| Field | Type | Required in session file | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `session_id` | string | Yes | must match CLI session target when provided |
|
||||
| `previous_session_id` | string | No | must not equal `session_id` |
|
||||
| `campaign` | string | No | filled from `campaign_id` during resolve if omitted |
|
||||
| `session_id` | string | Yes | opaque identity; must match CLI session target when provided |
|
||||
| `previous_session_id` | string | No | opaque identity; must not equal `session_id` |
|
||||
| `campaign` | string | No | opaque identity; filled from `campaign_id` during resolve if omitted |
|
||||
| `date` | string | No | metadata |
|
||||
| `title` | string | No | metadata |
|
||||
| `inputs.speakers_file` | string | No | overrides campaign stable input |
|
||||
@@ -271,13 +340,23 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
|
||||
Audio rules:
|
||||
|
||||
- configure local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
||||
- `audio_s3` requires `pipeline.storage.backend: s3` and a configured S3 bucket.
|
||||
|
||||
### Storage backend selection
|
||||
|
||||
`local` is the default and disables remote object-store operations. Configure
|
||||
`s3` explicitly before supplying `storage.s3`; a populated S3 block does not
|
||||
select a backend on its own. Unknown backend names and an S3 block paired with
|
||||
`local` are rejected during configuration validation.
|
||||
|
||||
### Previous-session expectation
|
||||
|
||||
`previous_session_id` is optional in a session file. When a command supplies
|
||||
`--previous-session-id`, however, the session file must contain the same value;
|
||||
an omitted or different value is rejected before the command performs work.
|
||||
|
||||
## Maintained Examples
|
||||
|
||||
- `examples/pipeline.minimal.yml`
|
||||
- `examples/pipeline.production.yml`
|
||||
- `examples/pipeline.full.annotated.yml`
|
||||
- `examples/campaigns/sample-campaign/campaign.yml`
|
||||
- `examples/session.local-audio.yml`
|
||||
- `examples/session.s3-audio.yml`
|
||||
- `examples/session.template.yml`
|
||||
See the [maintained examples index](../examples/README.md) for complete pipeline,
|
||||
campaign, session, template, and input fixtures. Keep complete copyable files
|
||||
there rather than duplicating them in this reference.
|
||||
|
||||
58
docs/development.md
Normal file
58
docs/development.md
Normal file
@@ -0,0 +1,58 @@
|
||||
# Development
|
||||
|
||||
This is the first-read landing page for people and LLM coding agents working on
|
||||
Narratio. It provides a concise repository orientation and routes each kind of
|
||||
change to its canonical documentation.
|
||||
|
||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
||||
polished transcripts and generated artifacts. Start with the
|
||||
[README](../README.md) for product context,
|
||||
[Architecture](policy/architecture.md) for normative system boundaries, and the
|
||||
[Internal Overview](internal/overview.md) for implemented component ownership.
|
||||
|
||||
## What To Read
|
||||
|
||||
| When working on | Read | Why |
|
||||
| --- | --- | --- |
|
||||
| Finding the package or component that owns current behavior | [Internal Overview](internal/overview.md) | It is the implemented component inventory and routes to focused internal documents. |
|
||||
| Application shape, boundaries, dependency direction, runtime invariants, safety properties, or dependencies | [Architecture](policy/architecture.md) | It defines the intended system shape, ownership, and non-goals. |
|
||||
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical owners, audiences, current-behavior rules, and maintenance requirements. |
|
||||
| Adding, changing, reviewing, rewriting, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and test lifecycle decisions. |
|
||||
| CLI composition or command behavior | [Internal Overview](internal/overview.md) and [CLI Reference](cli.md) | The overview routes to command ownership; the reference owns public syntax and invocation behavior. |
|
||||
| Configuration loading, resolution, or user-visible configuration | [Internal Overview](internal/overview.md) and [Configuration](config.md) | The overview routes to implementation ownership; the reference owns fields, defaults, discovery, and validation. |
|
||||
| Session workflow, status, restore, cleanup, or object storage | [Restore Internals](internal/command-restore.md), [Workspace Internals](internal/workspace.md), [Storage Internals](internal/storage.md), [Operations](operations.md), and [Troubleshooting](troubleshooting.md) | These separate implementation mechanics, operator procedures, and symptom-driven recovery. |
|
||||
| Pipeline sequencing or the behavior of a stage | [Internal Overview](internal/overview.md) and its focused stage documents | The overview owns the implemented stage inventory and routes to each stage contract. |
|
||||
| Adapters or external tool contracts | [Adapter Internals](internal/adapters.md) and [Integration Contracts](integrations/README.md) | The internal guide owns adapter composition and mechanics; integration documents own external formats and protocols. |
|
||||
| Manifests, artifacts, workspace paths, or publish behavior | [Manifest Internals](internal/manifest.md), [Artifact Internals](internal/artifacts.md), [Workspace Internals](internal/workspace.md), [Publish Internals](internal/stage-publish.md), and [Operations](operations.md) | These separate implementation state and resolution from operator-visible layout and lifecycle. |
|
||||
| Maintained configuration or input examples | [Configuration](config.md) and [Examples](../examples/README.md) | The reference owns field meanings; the examples directory owns complete copyable files. |
|
||||
| Proposed or unimplemented behavior | `docs/roadmap/` | Future work belongs only in roadmap documentation until implemented. |
|
||||
|
||||
For an existing subsystem, also inspect its focused tests and package-level
|
||||
contracts before changing behavior.
|
||||
|
||||
## Validation
|
||||
|
||||
Use focused package tests while iterating. Every pull request and push runs the
|
||||
following repository-wide checks before it can be accepted:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
go test ./internal/doccheck
|
||||
go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||
```
|
||||
|
||||
The documentation check verifies local Markdown links and the dependency graph
|
||||
of the Woodpecker workflows. The configuration check loads every maintained
|
||||
pipeline and session example. Release automation repeats these checks and
|
||||
cross-compiles the CLI before it builds release assets; publishing depends on
|
||||
that validation path, so a failure cannot publish a release.
|
||||
|
||||
Woodpecker also runs `go test -race -shuffle=on -count=3 ./...` on its scheduled
|
||||
job to expose ordering and repeatability defects. Current runners cross-compile
|
||||
for macOS and Windows, but do not provide native macOS or Windows execution.
|
||||
Those cross-builds establish compilation only, not platform-equivalent runtime
|
||||
evidence. Add native checks only when official runner labels and successful
|
||||
native-run evidence are available.
|
||||
@@ -1,19 +1,35 @@
|
||||
# Integrations Index
|
||||
|
||||
## Audience
|
||||
Developers and coding agents changing Narratio's external integration boundaries.
|
||||
|
||||
Operators, developers, and coding agents who need to understand Narratio's
|
||||
externally observable integration boundaries.
|
||||
|
||||
## Scope
|
||||
`docs/integrations/` is the implementation-level reference for downstream tool adapter contracts.
|
||||
|
||||
These docs cover what Narratio expects from external tools and what each adapter guarantees back to stage code.
|
||||
`docs/integrations/` is the canonical reference for protocols, invocation and
|
||||
data contracts, logical outputs, and compatibility behavior at external tool
|
||||
boundaries.
|
||||
|
||||
These documents describe what Narratio sends or invokes, what it accepts in
|
||||
return, and how failures are surfaced. Internal composition and stage mechanics
|
||||
belong in [the adapter implementation guide](../internal/adapters.md) and the
|
||||
focused stage documents.
|
||||
|
||||
## Integration Contracts
|
||||
- `audita.md`: transcript polishing adapter (`audita process`).
|
||||
- `seriatim.md`: merge/normalize/trim/render adapter (`seriatim`).
|
||||
- `scriptorium.md`: artifact run/render adapter (`scriptorium run|render`).
|
||||
|
||||
- [Audita](./audita.md): transcript polishing (`audita process`).
|
||||
- [Notarius](./notarius.md): complete pipeline execution and safe JSON bundle
|
||||
discovery (`notarius run`).
|
||||
- [Seriatim](./seriatim.md): merge, normalize, trim, and render operations.
|
||||
- [Scriptorium](./scriptorium.md): artifact generation and debug rendering
|
||||
(`scriptorium run|render`).
|
||||
- [WhisperX](./whisperx.md): speaker-audio transcription over HTTP.
|
||||
|
||||
## Related Canonical Docs
|
||||
- `docs/config.md`: operator-facing configuration reference.
|
||||
- `docs/internal/adapters.md`: shared adapter boundary and runner wiring.
|
||||
- `docs/internal/stage-*.md`: stage-specific integration usage.
|
||||
|
||||
- [Configuration](../config.md): operator-facing configuration reference.
|
||||
- [Adapter implementation](../internal/adapters.md): shared adapter boundary and
|
||||
runner wiring.
|
||||
- [Internal documentation](../internal/overview.md): stage-specific integration
|
||||
usage and component ownership.
|
||||
|
||||
@@ -3,23 +3,24 @@
|
||||
## Purpose
|
||||
Define the Audita adapter contract used by the `polish` stage.
|
||||
|
||||
## Adapter Boundary
|
||||
Interface:
|
||||
- `audita.Runner`
|
||||
- method: `Run(ctx, PolishRequest) (PolishResult, error)`
|
||||
## External Boundary
|
||||
|
||||
Primary implementation:
|
||||
- `internal/adapters/audita/SubprocessRunner`
|
||||
|
||||
Execution mode:
|
||||
- subprocess invocation of `audita process`
|
||||
Narratio invokes `audita process` as a subprocess for each polish operation.
|
||||
The configured timeout and parent cancellation bound the invocation. Internal
|
||||
runner composition is documented in
|
||||
[the adapter implementation guide](../internal/adapters.md).
|
||||
|
||||
## Request Contract
|
||||
`PolishRequest` carries:
|
||||
- required transcript/glossary/output/work-dir paths;
|
||||
- optional report path (required when report mode is enabled);
|
||||
- generated config and stdout/stderr log paths;
|
||||
- optional module/model/base-url/config/output-schema/concurrency settings.
|
||||
- optional per-invocation module override.
|
||||
|
||||
The constructed runner owns static Audita settings: binary, timeout,
|
||||
credentials, default modules, model and endpoint settings, validation and output
|
||||
settings, report mode, and concurrency. The `polish` stage supplies only
|
||||
invocation-specific paths and may override modules for that invocation.
|
||||
|
||||
## Result Contract
|
||||
`PolishResult` returns:
|
||||
@@ -45,6 +46,9 @@ Run fails for:
|
||||
- invalid processed transcript JSON (`segments` array required);
|
||||
- invalid report JSON when reporting is enabled.
|
||||
|
||||
Processed transcript JSON is limited to 64 MiB and optional report JSON to 16
|
||||
MiB. Both must be regular files without symlinked path components.
|
||||
|
||||
Failure results still include output/log/config/exit metadata for diagnostics.
|
||||
|
||||
## Deterministic Behavior
|
||||
@@ -52,9 +56,12 @@ Failure results still include output/log/config/exit metadata for diagnostics.
|
||||
- Generated invocation YAML (`audita.generated.v1`) is emitted when requested.
|
||||
- Manifest writes are stage-owned; adapter itself is stateless.
|
||||
|
||||
## Config Mapping
|
||||
Config fields consumed through runner/stage wiring are under `pipeline.audita.*`.
|
||||
## Configuration
|
||||
|
||||
Operator-selected values are defined under `pipeline.audita.*` in the
|
||||
[configuration reference](../config.md#pipeline).
|
||||
|
||||
Maintained example with Audita config:
|
||||
- `examples/pipeline.full.annotated.yml`
|
||||
- `examples/pipeline.production.yml`
|
||||
|
||||
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
|
||||
- [Production-shaped pipeline](../../examples/pipeline.production.yml)
|
||||
|
||||
83
docs/integrations/notarius.md
Normal file
83
docs/integrations/notarius.md
Normal file
@@ -0,0 +1,83 @@
|
||||
# Notarius Integration Contract
|
||||
|
||||
## Boundary
|
||||
|
||||
Narratio uses Notarius as a subprocess to extract configured structured JSON
|
||||
lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
|
||||
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
||||
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
||||
|
||||
Canonical Notarius references:
|
||||
|
||||
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/subprocess.md)
|
||||
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/dnd-pipeline.md)
|
||||
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/run-result.md)
|
||||
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/json-output.md)
|
||||
|
||||
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
||||
records the exact current constraints for all ten D&D lanes. Treat the linked
|
||||
Notarius documents as canonical when changing those values; Narratio does not
|
||||
duplicate the complete schemas.
|
||||
|
||||
## Invocation
|
||||
|
||||
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
||||
configuration path, input path, output directory, and working directory to
|
||||
absolute paths and invokes:
|
||||
|
||||
```text
|
||||
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> --json
|
||||
```
|
||||
|
||||
Standard output is reserved for the JSON receipt. Standard error is captured
|
||||
separately as diagnostic output. Narratio applies the configured timeout and
|
||||
does not interpret stdout as a receipt unless the subprocess exits successfully.
|
||||
It does not pass a Narratio session ID or run `notarius config validate`
|
||||
automatically; the configured working directory and Narratio's minimal child
|
||||
environment apply to the subprocess.
|
||||
|
||||
## Accepted Result
|
||||
|
||||
Narratio currently accepts receipt schema `notarius.run-result.v1`. The receipt
|
||||
must identify the configured pipeline, and its `index_file` must be exactly
|
||||
`index.json` beneath the reported bundle root. The production index must name
|
||||
the management files exactly as `manifest.json`, `rejected.json`, and
|
||||
`warnings.json`. All receipt, index, and lane paths must stay inside that
|
||||
bundle; symlinks and non-regular lane payloads are rejected.
|
||||
|
||||
Supported receipt and index shapes tolerate unknown fields for forward
|
||||
compatibility, while required identity, validation, count, manifest,
|
||||
rejection, warning, and lane-list fields remain mandatory. Narratio applies
|
||||
bounded reads to the receipt, index, rejection, and warning documents. Optional
|
||||
chunk-map and evidence-context descriptors must carry their complete generic
|
||||
contract metadata when present.
|
||||
|
||||
For every entry in `pipeline.notarius.outputs`, Narratio requires exactly one
|
||||
index descriptor with the configured lane ID, media type, schema ID, schema
|
||||
version, and, when configured, module key. Missing, duplicate, rejected, or
|
||||
incompatible required lanes fail extraction even if Notarius exited zero.
|
||||
Unconfigured lanes may remain in the preserved bundle but do not become
|
||||
selectable Narratio sources.
|
||||
|
||||
Each accepted configured lane is registered as
|
||||
`narratio.extraction.<output_key>`. The bundle index is retained for audit and
|
||||
resume validation but is not selectable. Scriptorium and publish rules consume
|
||||
only explicitly named lane sources; `--artifacts` never selects Notarius lanes.
|
||||
|
||||
## Failure And Compatibility Behavior
|
||||
|
||||
- Startup and nonzero-exit errors fail extraction and retain captured diagnostics.
|
||||
- Invalid receipt JSON or an unsupported receipt schema fails before bundle use.
|
||||
- Unsafe or incompatible index data and required-lane rejection fail before the
|
||||
staged bundle is promoted to durable storage.
|
||||
- Contract and external provenance metadata are preserved on lane artifact
|
||||
records and through explicit publication.
|
||||
|
||||
Rejection and warning summaries retain structured stage, scope, lane, and
|
||||
reason-code fields for diagnostics without exposing free-form external messages
|
||||
or reading lane payload bodies.
|
||||
|
||||
Configuration fields and defaults are in [Configuration](../config.md).
|
||||
Operator paths, rerun procedures, and bundle retention are in
|
||||
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
||||
for failure recovery.
|
||||
@@ -3,20 +3,17 @@
|
||||
## Purpose
|
||||
Define the Scriptorium adapter contract used by `analyze` and trim-bounds generation in `trim`.
|
||||
|
||||
## Adapter Boundary
|
||||
Interface:
|
||||
- `scriptorium.Runner`
|
||||
- methods:
|
||||
- `RunArtifact(ctx, RunArtifactRequest)`
|
||||
- `RenderArtifact(ctx, RenderArtifactRequest)`
|
||||
## External Boundary
|
||||
|
||||
Primary implementation:
|
||||
- `internal/adapters/scriptorium/SubprocessRunner`
|
||||
Narratio invokes Scriptorium as a subprocess in these modes:
|
||||
|
||||
Execution modes:
|
||||
- `scriptorium run`
|
||||
- `scriptorium render`
|
||||
|
||||
The request timeout and parent cancellation bound each invocation. Internal
|
||||
runner composition is documented in
|
||||
[the adapter implementation guide](../internal/adapters.md).
|
||||
|
||||
## Request Contract
|
||||
Both request types carry:
|
||||
- binary/config/prompt/profile IDs;
|
||||
@@ -49,6 +46,9 @@ Run behavior:
|
||||
- `run` exit code `2` is mapped to `ValidationFailed=true`;
|
||||
- successful subprocess still fails if output file is missing or empty.
|
||||
|
||||
Each artifact result is limited to 64 MiB and must be a regular file without
|
||||
symlinked path components.
|
||||
|
||||
Render behavior:
|
||||
- subprocess errors propagate;
|
||||
- output file must exist and be non-empty.
|
||||
@@ -59,9 +59,13 @@ Render behavior:
|
||||
- generated invocation YAML (`scriptorium.generated.v1`) is emitted when requested.
|
||||
- adapter is stateless and does not own artifact-selection policy.
|
||||
|
||||
## Config Mapping
|
||||
Config fields consumed through runner/stage wiring are under `pipeline.scriptorium.*` plus per-artifact settings under `pipeline.scriptorium.artifacts.*`.
|
||||
## Configuration
|
||||
|
||||
Operator-selected values are defined under `pipeline.scriptorium.*`, including
|
||||
per-artifact settings under `pipeline.scriptorium.artifacts.*`, in the
|
||||
[configuration reference](../config.md#pipeline).
|
||||
|
||||
Maintained examples with Scriptorium config:
|
||||
- `examples/pipeline.full.annotated.yml`
|
||||
- `examples/pipeline.production.yml`
|
||||
|
||||
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
|
||||
- [Production-shaped pipeline](../../examples/pipeline.production.yml)
|
||||
|
||||
@@ -3,24 +3,19 @@
|
||||
## Purpose
|
||||
Define the Seriatim adapter contract used by `merge`, `normalize`, `trim`, and `render`.
|
||||
|
||||
## Adapter Boundary
|
||||
Interface:
|
||||
- `seriatim.Runner`
|
||||
- methods:
|
||||
- `Run(ctx, MergeRequest)`
|
||||
- `Normalize(ctx, NormalizeRequest)`
|
||||
- `Trim(ctx, TrimRequest)`
|
||||
- `Render(ctx, RenderRequest)`
|
||||
## External Boundary
|
||||
|
||||
Primary implementation:
|
||||
- `internal/adapters/seriatim/SubprocessRunner`
|
||||
Narratio invokes Seriatim as a subprocess in these modes:
|
||||
|
||||
Execution modes:
|
||||
- `seriatim merge`
|
||||
- `seriatim normalize`
|
||||
- `seriatim trim`
|
||||
- `seriatim render`
|
||||
|
||||
The configured timeout and parent cancellation bound each invocation. Internal
|
||||
runner composition is documented in
|
||||
[the adapter implementation guide](../internal/adapters.md).
|
||||
|
||||
## Request/Result Contracts
|
||||
- `MergeRequest`/`MergeResult`: multi-input merge to base transcript, optional report.
|
||||
- `NormalizeRequest`/`NormalizeResult`: transcript normalization with explicit schema.
|
||||
@@ -46,6 +41,8 @@ Invocation fails on:
|
||||
- empty render output files.
|
||||
|
||||
When report paths are provided/enabled, report files must parse as JSON.
|
||||
Each Seriatim JSON or rendered-text result is limited to 64 MiB and must be a
|
||||
regular file without symlinked path components.
|
||||
|
||||
## Deterministic Behavior
|
||||
- argument ordering is deterministic per command construction.
|
||||
@@ -53,9 +50,13 @@ When report paths are provided/enabled, report files must parse as JSON.
|
||||
- generated invocation YAML (`seriatim.generated.v1`) is emitted when requested.
|
||||
- adapter does not write manifests or choose stage inputs.
|
||||
|
||||
## Config Mapping
|
||||
Config fields consumed through runner/stage wiring are under `pipeline.seriatim.*` and `pipeline.render.*`.
|
||||
## Configuration
|
||||
|
||||
Operator-selected values are defined under `pipeline.seriatim.*` and
|
||||
`pipeline.render.*` in the
|
||||
[configuration reference](../config.md#pipeline).
|
||||
|
||||
Maintained examples with Seriatim config:
|
||||
- `examples/pipeline.full.annotated.yml`
|
||||
- `examples/pipeline.production.yml`
|
||||
|
||||
- [Full annotated pipeline](../../examples/pipeline.full.annotated.yml)
|
||||
- [Production-shaped pipeline](../../examples/pipeline.production.yml)
|
||||
|
||||
71
docs/integrations/whisperx.md
Normal file
71
docs/integrations/whisperx.md
Normal file
@@ -0,0 +1,71 @@
|
||||
# Integration: WhisperX
|
||||
|
||||
## Purpose
|
||||
|
||||
WhisperX transcribes each prepared speaker audio file for Narratio's
|
||||
`transcribe` stage. Narratio uses an HTTP boundary and installs each successful
|
||||
response as that speaker's raw transcript JSON.
|
||||
|
||||
## HTTP Boundary
|
||||
|
||||
Narratio sends an HTTP `POST` to the configured transcription URL using
|
||||
`multipart/form-data` with:
|
||||
|
||||
- `file`: the audio file, retaining its base filename; and
|
||||
- `language`: the configured language string.
|
||||
|
||||
The server must return a `2xx` response whose body is valid JSON. Narratio does
|
||||
not currently require a more specific response schema at this boundary.
|
||||
|
||||
The transcription URL must be an absolute `http` or `https` URL. The audio body
|
||||
is streamed through a fresh multipart writer for every attempt, so its memory
|
||||
use is bounded by the transport buffer rather than by the complete audio file.
|
||||
WhisperX response acquisition is capped at 10 MiB.
|
||||
|
||||
## Request And Result Contract
|
||||
|
||||
Each adapter request identifies a speaker, a readable audio file, and the
|
||||
destination for the raw transcript. The HTTP request carries the audio and
|
||||
language; the speaker identifier remains Narratio orchestration metadata.
|
||||
|
||||
On success, Narratio atomically writes the response body to the requested
|
||||
destination. The adapter result reports that logical output together with the
|
||||
attempt count, final HTTP status when available, elapsed duration, and adapter
|
||||
identity metadata. A failed or invalid response is not installed as the
|
||||
transcript output.
|
||||
|
||||
## Retry, Timeout, And Cancellation
|
||||
|
||||
- The configured timeout applies independently to each HTTP attempt.
|
||||
- `retries` means additional attempts after the first.
|
||||
- HTTP `429`, HTTP `5xx`, attempt timeouts, and network errors are retryable.
|
||||
- Other HTTP `4xx` responses and explicit cancellation are not retryable.
|
||||
- Narratio waits the configured retry delay between attempts and aborts that
|
||||
wait when the parent context is canceled.
|
||||
|
||||
## Validation And Failure Semantics
|
||||
|
||||
Client construction rejects a missing or non-HTTP(S) absolute transcription URL,
|
||||
a missing language, a non-positive timeout, negative retries, or a negative
|
||||
retry delay. A request fails before transmission when its audio or output path
|
||||
is missing.
|
||||
|
||||
Non-`2xx` status, transport failure, response-size overflow, invalid JSON, or
|
||||
failure to install the output causes the transcription to fail. Errors include
|
||||
attempt context, and the result retains attempts, final status when available,
|
||||
and elapsed duration for diagnostics.
|
||||
|
||||
## Determinism And Concurrency
|
||||
|
||||
Each audio request has stable multipart field names, and successful bytes are
|
||||
installed atomically. The transcribe stage may process speaker files in
|
||||
parallel, bounded by the configured concurrency. It records results in stable
|
||||
speaker order after all work completes; any speaker failure fails the stage.
|
||||
|
||||
## Related Canonical Docs
|
||||
|
||||
- [Configuration](../config.md#pipeline) defines the operator-selected
|
||||
WhisperX URL, language, timeouts, retry policy, and concurrency.
|
||||
- [Adapter implementation](../internal/adapters.md) describes internal wiring.
|
||||
- [Transcribe stage](../internal/stage-transcribe.md) describes stage mechanics,
|
||||
durable artifacts, and manifests.
|
||||
@@ -1,45 +0,0 @@
|
||||
# Internal Documentation Index
|
||||
|
||||
## Audience
|
||||
Developers and coding agents changing Narratio internals.
|
||||
|
||||
## Scope
|
||||
`docs/internal/` documents implemented internal contracts: stage boundaries, manifest/state behavior, artifact resolution, restore behavior, storage boundaries, and workspace invariants.
|
||||
|
||||
User and operator behavior belongs in:
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
- `docs/operations.md`
|
||||
- `docs/troubleshooting.md`
|
||||
|
||||
## Pipeline Stage Set
|
||||
Canonical stage order from `internal/stage.All()`:
|
||||
1. `prepare`
|
||||
2. `transcribe`
|
||||
3. `merge`
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `render`
|
||||
8. `analyze`
|
||||
9. `publish`
|
||||
10. `notify` (placeholder)
|
||||
|
||||
`notify` is currently a placeholder stage with optional notifier call behavior; it has no persisted pipeline outputs.
|
||||
|
||||
## Internal Component Docs
|
||||
- `adapters.md`: external adapter boundaries and default runtime wiring.
|
||||
- `artifacts.md`: canonical source IDs, runtime catalog behavior, and resolution rules.
|
||||
- `manifest.md`: session and run manifest contracts.
|
||||
- `storage.md`: object-store interface and S3 implementation behavior.
|
||||
- `workspace.md`: local session layout, run-local layout, and cleanup guardrails.
|
||||
- `command-restore.md`: restore discovery, planning, execution, and reporting.
|
||||
- `stage-prepare.md`
|
||||
- `stage-transcribe.md`
|
||||
- `stage-merge.md`
|
||||
- `stage-polish.md`
|
||||
- `stage-normalize.md`
|
||||
- `stage-trim.md`
|
||||
- `stage-render.md`
|
||||
- `stage-analyze.md`
|
||||
- `stage-publish.md`
|
||||
@@ -1,49 +1,90 @@
|
||||
# Internal: Adapters
|
||||
|
||||
## Purpose
|
||||
Define external integration boundaries and default adapter wiring used by app/stage orchestration.
|
||||
|
||||
Explain the adapter interfaces and production composition used by application
|
||||
and stage orchestration. Externally observable protocols and formats belong in
|
||||
the [integration contracts](../integrations/).
|
||||
|
||||
## Adapter Boundaries
|
||||
|
||||
Narratio stage logic depends on adapter interfaces, not transport-specific details.
|
||||
|
||||
Primary adapters:
|
||||
|
||||
- `whisperx.Client`
|
||||
- `seriatim.Runner`
|
||||
- `audita.Runner`
|
||||
- `scriptorium.Runner`
|
||||
- `notarius.Runner`
|
||||
- `storage.ObjectStore`
|
||||
- `notify.Sender`
|
||||
|
||||
## Ownership
|
||||
|
||||
Adapters own:
|
||||
|
||||
- HTTP/subprocess/SDK argument and transport details.
|
||||
- Backend-specific request/response mapping.
|
||||
|
||||
Adapters do not own:
|
||||
|
||||
- stage ordering/skip/force logic;
|
||||
- manifest transitions;
|
||||
- canonical path policy.
|
||||
|
||||
## Default Wiring
|
||||
|
||||
`internal/app/runner.go` initializes default adapters when not injected:
|
||||
|
||||
- WhisperX HTTP client from pipeline config.
|
||||
- Seriatim subprocess runner.
|
||||
- Audita subprocess runner.
|
||||
- Scriptorium subprocess runner.
|
||||
- Notarius subprocess runner when extraction is enabled.
|
||||
- Noop notifier (`notify.NoopSender`).
|
||||
- Object store only when required by selected stages/config.
|
||||
|
||||
Object-store construction goes through `newCommandObjectStore`, which loads configured filesystem secrets before adapter initialization.
|
||||
Notarius is composed only when extraction is enabled; the extract stage owns
|
||||
receipt, bundle, and configured-lane policy rather than the adapter.
|
||||
|
||||
Object-store construction goes through `newCommandObjectStore`, which loads
|
||||
configured filesystem secrets before adapter initialization.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
- Constructor errors fail stage execution setup early.
|
||||
- Runtime adapter errors propagate to stage code and then manifest failure handling.
|
||||
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
||||
- Shared subprocess execution starts an owned process group on Linux/macOS or a
|
||||
kill-on-close job object on Windows. Every terminal path disposes of that
|
||||
owned tree before returning. After a natural leader exit, Unix checks for
|
||||
remaining group members and uses bounded graceful then forceful termination;
|
||||
Windows closes the job so kill-on-close applies. Cancellation, deadlines, and
|
||||
diagnostic limits use the same terminal disposal path without losing their
|
||||
original result classification. Child environments contain only the execution
|
||||
baseline and adapter-specified values; configured credentials are explicit
|
||||
sensitive values. Stdout and stderr are redacted while streaming into separate
|
||||
8 MiB diagnostic captures; a bounded wait closes a stream retained by a
|
||||
departed leader's descendant. Unsupported platforms reject owned command
|
||||
execution.
|
||||
|
||||
## Test Surfaces
|
||||
## Implementation And Tests
|
||||
|
||||
- Composition: `internal/app/runner.go`, `internal/app/object_store.go`
|
||||
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
||||
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
|
||||
- `internal/adapters/whisperx/http_test.go`
|
||||
- `internal/adapters/seriatim/subprocess_test.go`
|
||||
- `internal/adapters/audita/subprocess_test.go`
|
||||
- `internal/adapters/scriptorium/subprocess_test.go`
|
||||
- `internal/adapters/notarius/subprocess_test.go`
|
||||
- `internal/adapters/storage/*_test.go`
|
||||
- `internal/app/runner_test.go`
|
||||
|
||||
See the [WhisperX](../integrations/whisperx.md),
|
||||
[Seriatim](../integrations/seriatim.md), [Audita](../integrations/audita.md),
|
||||
[Scriptorium](../integrations/scriptorium.md), and
|
||||
[Notarius](../integrations/notarius.md) contracts before changing an
|
||||
externally visible boundary. Operator-selected values belong in
|
||||
[Configuration](../config.md).
|
||||
|
||||
@@ -1,34 +1,59 @@
|
||||
# Internal: Artifacts
|
||||
|
||||
## Purpose
|
||||
Define canonical artifact IDs, runtime catalog behavior, source resolution rules, and shared current-state mechanics used by app and previous-cache code.
|
||||
|
||||
Explain the artifact registry, runtime catalog, resolver, previous-input
|
||||
requirements, and shared remote current-state mechanics implemented by
|
||||
`internal/artifacts`. Configuration fields that accept source IDs belong in
|
||||
[Configuration](../config.md); physical placement belongs in
|
||||
[Operations](../operations.md).
|
||||
|
||||
## Built-in Source IDs
|
||||
|
||||
- `narratio.transcript.base` -> `transcripts/base.json` (`merge`)
|
||||
- `narratio.transcript.polished` -> `transcripts/polished.json` (`polish`)
|
||||
- `narratio.transcript.final` -> `transcripts/final.json` (`normalize`)
|
||||
- `narratio.transcript.final_trimmed` -> `transcripts/final.trimmed.json` (`trim`)
|
||||
- `narratio.transcript.final_markdown` -> `transcripts/final.md` (`render`)
|
||||
- `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md` (`render`)
|
||||
- `narratio.bounds.session` -> `artifacts/session_bounds.json` (`trim`)
|
||||
The internal registry recognizes these stable built-in source IDs:
|
||||
|
||||
## Configured and Previous-Session Sources
|
||||
- `narratio.transcript.base`
|
||||
- `narratio.transcript.polished`
|
||||
- `narratio.transcript.final`
|
||||
- `narratio.transcript.final_trimmed`
|
||||
- `narratio.transcript.final_markdown`
|
||||
- `narratio.transcript.final_trimmed_markdown`
|
||||
- `narratio.bounds.session`
|
||||
|
||||
Registry entries bind each ID to its producer, output kind, canonical fallback,
|
||||
and content validator. The focused stage documents own their input/output flow;
|
||||
[Configuration](../config.md) owns where operators may select these IDs.
|
||||
|
||||
## Configured, Extraction, And Previous-Session Sources
|
||||
|
||||
- configured source ID format: `narratio.artifact.<artifact_key>`
|
||||
- extraction source ID format: `narratio.extraction.<output_key>`
|
||||
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
||||
|
||||
Both formats are validated by strict source-policy rules.
|
||||
All formats are validated by strict source-policy rules. Configured artifact and
|
||||
extraction keys use `^[a-z][a-z0-9_]*$`; source parsers never normalize an
|
||||
unrecognized token into a valid source. Extraction sources are registered only
|
||||
from `pipeline.notarius.outputs`; the Notarius index has no selectable source
|
||||
ID.
|
||||
|
||||
## Runtime Catalog
|
||||
|
||||
`ArtifactCatalog` tracks:
|
||||
|
||||
- `planned`: source registered for run context;
|
||||
- `executable`: selected and enabled for analyze execution;
|
||||
- `executable`: included in the effective analyze artifact set;
|
||||
- `available`: local file exists and validates;
|
||||
- `provenance`: availability source.
|
||||
|
||||
Configured definitions are always registered. Without an explicit selection,
|
||||
the effective analyze set contains enabled definitions. With `--artifacts`, the
|
||||
exact named configured definitions become the effective set for that invocation,
|
||||
regardless of their `enabled` value; dependencies are not added implicitly.
|
||||
Availability is separate from executability: a non-executable configured output
|
||||
may be reused from a canonical non-empty file, while an executable definition
|
||||
is generated by analyze. Extraction entries are registered from configuration
|
||||
and become available only after compatible extraction evidence is hydrated.
|
||||
|
||||
Current provenance values:
|
||||
|
||||
- `generated.current_analyze_run`
|
||||
@@ -47,12 +72,28 @@ Configured sources (`narratio.artifact.*`):
|
||||
|
||||
- resolve only through runtime catalog availability.
|
||||
|
||||
Extraction sources (`narratio.extraction.*`):
|
||||
|
||||
- use the shared typed bundle evidence inspection in `extraction_evidence.go`;
|
||||
- require a current successful extract record with the exact configured source,
|
||||
compatible contract and Notarius provenance, a confined regular durable
|
||||
payload, matching checksum, and the current resolved trimmed-transcript
|
||||
identity;
|
||||
- remain unavailable unless catalog hydration receives valid evidence. Resume
|
||||
treats absent or obsolete evidence as a rerun decision and unsafe evidence as
|
||||
an error; and
|
||||
- are never inferred by scanning the Notarius bundle directory.
|
||||
|
||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||
|
||||
- resolve only from local `previous/` cache state;
|
||||
- prefer manifest-backed previous-input paths;
|
||||
- fallback to existing previous-cache filesystem paths.
|
||||
|
||||
Source absence is evaluated by the consuming artifact input. An optional input
|
||||
is omitted from that invocation; a required input fails resolution. This is
|
||||
separate from a stage's lifecycle outcome.
|
||||
|
||||
Validation by content type:
|
||||
|
||||
- transcript JSON built-ins: JSON with top-level `segments` array;
|
||||
@@ -64,7 +105,7 @@ Validation by content type:
|
||||
|
||||
`CollectPreviousArtifactRequirements`:
|
||||
|
||||
- scans enabled configured artifacts only;
|
||||
- scans the effective configured artifact set;
|
||||
- extracts only canonical previous-session sources;
|
||||
- deduplicates by artifact key;
|
||||
- merges required and optional references (required wins);
|
||||
@@ -72,14 +113,34 @@ Validation by content type:
|
||||
|
||||
## Current-State Helpers
|
||||
|
||||
Artifacts package owns shared remote current-state loading mechanics used by restore, status/validate checks, and previous-cache planning.
|
||||
Artifacts package owns shared remote current-state loading mechanics used by
|
||||
restore, status and validation checks, and previous-cache planning.
|
||||
|
||||
For a new-protocol current state, the pointer-selected immutable commit is the
|
||||
complete restore authority. Callers receive its declared object identities and
|
||||
must not supplement them by listing mutable session prefixes. The legacy reader
|
||||
is intentionally separate and remains migration-only support.
|
||||
|
||||
The reader opens each small control object directly and enforces owner-specific
|
||||
limits before decoding: 64 KiB for the mutable commit pointer, 4 MiB for the
|
||||
immutable commit manifest, and 8 MiB for the selected session manifest. Legacy
|
||||
compatibility applies a 4 KiB limit to `current/run_id.txt` and the same 8 MiB
|
||||
manifest limit to `current/manifest.json`. These are exposed as
|
||||
`MaxCurrentCommitPointerBytes`, `MaxRemoteCommitManifestBytes`,
|
||||
`MaxRemoteSessionManifestBytes`, `MaxLegacyCurrentRunPointerBytes`, and
|
||||
`MaxLegacyCurrentManifestBytes`.
|
||||
|
||||
Each read uses the generation and size metadata returned with its opened body.
|
||||
Actual bytes remain subject to a limit-plus-one read even if size metadata is
|
||||
absent or inaccurate. Immutable selections then retain their declared-size,
|
||||
checksum, generation, and identity checks. No current-state control object is
|
||||
downloaded through a temporary file.
|
||||
|
||||
Core helpers:
|
||||
|
||||
- `LoadCurrentRunPointer`
|
||||
- `LoadCurrentManifest`
|
||||
- `LoadCurrentState`
|
||||
- `ValidateCurrentStateIdentity`
|
||||
- `RemoteCommitManifest` and `CurrentCommitPointer`
|
||||
|
||||
Typed missing-state errors:
|
||||
|
||||
@@ -107,9 +168,47 @@ Caller policy is intentionally outside artifacts helpers:
|
||||
- spool/cache paths;
|
||||
- S3 session/run/current-state key layout.
|
||||
|
||||
New publication creates run-scoped immutable objects, including
|
||||
`runs/{run_id}/commit.json` and `runs/{run_id}/session-manifest.json`. The sole
|
||||
mutable selector is `current/commit-pointer.json`; readers verify its selected
|
||||
commit and declared object generations/checksums. Legacy current-pair loading
|
||||
is confined to `current_state_legacy.go` for migration only.
|
||||
|
||||
Campaign, session, and Narratio run IDs are validated as portable opaque
|
||||
segments at configuration and artifact boundaries before they can be used in a
|
||||
workspace or S3 namespace. Previous-artifact destinations remain typed,
|
||||
multi-segment relative paths and are confined beneath `previous/artifacts`; they
|
||||
are not treated as opaque identifiers.
|
||||
|
||||
See [Workspace Internals](workspace.md) for how callers consume local helpers
|
||||
and [Operations](../operations.md#local-state-layout) for the authoritative
|
||||
physical layout.
|
||||
|
||||
## Invariants
|
||||
|
||||
- source ID formats are stable contracts;
|
||||
- artifact resolution is deterministic and manifest-aware;
|
||||
- extraction sources are available only from a compatible successful manifest
|
||||
record;
|
||||
- previous-session source resolution in `analyze` is local-only;
|
||||
- remote current-state key construction remains centralized in artifacts helpers.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
||||
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`,
|
||||
`internal/artifacts/extraction_catalog.go`,
|
||||
`internal/artifacts/extraction_evidence.go`,
|
||||
`internal/artifacts/extraction_input.go`
|
||||
- Current state: `internal/artifacts/current_state.go`,
|
||||
`internal/artifacts/current_state_commit.go`,
|
||||
`internal/artifacts/current_state_legacy.go`
|
||||
- Paths and keys: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/s3_keys.go`
|
||||
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
||||
- Tests: `internal/artifacts/artifact_resolver_test.go`,
|
||||
`internal/artifacts/catalog_test.go`,
|
||||
`internal/artifacts/extraction_catalog_test.go`,
|
||||
`internal/artifacts/current_state_test.go`,
|
||||
`internal/artifacts/paths_model_test.go`,
|
||||
`internal/artifacts/previous_requirements_test.go`
|
||||
|
||||
@@ -1,24 +1,24 @@
|
||||
# Internal: Command Restore
|
||||
|
||||
## Purpose
|
||||
Define the implemented `narratio session restore` command contract:
|
||||
|
||||
- committed remote current-state discovery;
|
||||
- deterministic restore planning;
|
||||
- safe local install semantics;
|
||||
- durable restore reporting.
|
||||
Explain the implemented restore discovery, planning, installation, and
|
||||
reporting flow in `internal/app`. User invocation belongs in
|
||||
[CLI](../cli.md#session-restore), and the operator recovery procedure and
|
||||
physical restore scope belong in
|
||||
[Operations](../operations.md#restore-workflow).
|
||||
|
||||
Restore separates remote authority, local conflict policy, and filesystem
|
||||
mutation so each remains testable independently.
|
||||
|
||||
## Discovery Contract
|
||||
|
||||
Restore resolves remote committed state from the session publish current pointers:
|
||||
|
||||
- `current/run_id.txt` (required, non-empty);
|
||||
- `current/manifest.json` (required, decodable).
|
||||
|
||||
Current-state discovery uses shared artifacts-level mechanics and validates identity against the resolved request config:
|
||||
Discovery delegates current-state pointer and manifest loading to
|
||||
`internal/artifacts`, then validates the result against the resolved request:
|
||||
|
||||
- campaign must match;
|
||||
- session ID must match.
|
||||
- run ID must match the pointer-selected committed run.
|
||||
|
||||
Restore treats any missing or invalid remote current state as a command error.
|
||||
|
||||
@@ -32,28 +32,20 @@ Restore planner action kinds:
|
||||
|
||||
Planner behavior:
|
||||
|
||||
- remote list scope is the resolved session prefix;
|
||||
- a new-protocol restore uses only the selected commit's declared artifact set;
|
||||
each action carries that artifact's immutable key, checksum, size, and
|
||||
generation. Coherent legacy state remains on the isolated compatibility path;
|
||||
- remote-to-local mapping is traversal-safe;
|
||||
- actions are sorted deterministically by local relative path.
|
||||
- actions are sorted by local relative path and then remote key;
|
||||
- force converts differing eligible regular files from conflicts to downloads;
|
||||
directories and other non-regular targets remain conflicts.
|
||||
|
||||
Restore scope from current remote state:
|
||||
For a non-dry-run restore, planning/classification happens only after acquiring
|
||||
the session lock. Runner manifest/reuse checks acquire that same lock first.
|
||||
|
||||
- include `manifest.json`;
|
||||
- include `transcripts/**`;
|
||||
- include `artifacts/**`;
|
||||
- include `audio/**` only with `--include-audio`.
|
||||
|
||||
Explicit exclusions from current remote state mapping:
|
||||
|
||||
- `current/**`;
|
||||
- `runs/**`;
|
||||
- `logs/**`;
|
||||
- `reports/**`;
|
||||
- `config/**`;
|
||||
- `inputs/**`;
|
||||
- `previous/**`.
|
||||
|
||||
Previous-cache files are planned separately through `previouscache.BuildPlan` when configured previous-session requirements exist.
|
||||
Previous-cache readiness is resolved through `previouscache.Resolve` for restore,
|
||||
prepare, status, and validation. A committed source is selected only by its
|
||||
exact source identity; legacy fallback remains isolated and rejects ambiguity.
|
||||
|
||||
## Execution Contract
|
||||
|
||||
@@ -63,22 +55,56 @@ Execution order and safety:
|
||||
- `manifest.json` installs last;
|
||||
- downloads use sibling temp files plus atomic rename;
|
||||
- manifest replacement is validated before rename;
|
||||
- each committed object is verified against its declared checksum, size, and
|
||||
generation before installation;
|
||||
- a committed manifest already verified during discovery is retained for the
|
||||
matching restore action and revalidated before installation, avoiding a
|
||||
second body transfer;
|
||||
- failed installs do not roll back files already written in the same execution.
|
||||
- a durable `.restore-incomplete.json` marker is written before installation.
|
||||
It blocks runners until a restore retry completes all verified installs and
|
||||
the local manifest replacement, at which point it is removed.
|
||||
- restored manifest local references are rebased beneath the selected local
|
||||
session root. Unsafe relative references and producer-machine absolute paths
|
||||
outside the manifest's producer session root are rejected; producer-local
|
||||
spool/cache and cleanup locations are not restored as authority.
|
||||
|
||||
Audio restore path:
|
||||
|
||||
- uses `audio.MaterializeS3Audio`;
|
||||
- integrates spool and S3 audio cache paths;
|
||||
- supports cache-hit reuse without object redownload.
|
||||
- reuses cached audio only when its no-follow regular file, content digest, and
|
||||
identity sidecar all match the selected remote object version; otherwise it
|
||||
refreshes through the durable download path.
|
||||
|
||||
## Reporting Contract
|
||||
|
||||
- `--dry-run`: prints summary only; no local writes.
|
||||
- non-dry-run: writes `reports/restore-latest.json`.
|
||||
- dry-run mode prints a summary, performs no durable session writes, and may
|
||||
read remote current-state or object-identity data to produce that summary;
|
||||
- execution mode persists the canonical restore report described in
|
||||
[Operations](../operations.md#restore-workflow);
|
||||
- report includes plan counts, per-action status, and execution failures.
|
||||
|
||||
## Invariants
|
||||
|
||||
- restore uses committed remote current state as authority;
|
||||
- `current/run_id.txt` is the remote publish commit marker;
|
||||
- one restore or status inspection observes the single pointer-selected commit
|
||||
loaded at discovery; later pointer changes cannot add objects or substitute a
|
||||
different run into its plan;
|
||||
- a verified `current/commit-pointer.json` and its selected immutable commit
|
||||
establish new-protocol remote commitment; coherent legacy
|
||||
`current/run_id.txt` plus `current/manifest.json` remains read-only migration
|
||||
support;
|
||||
- restore does not execute pipeline stages.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Discovery: `internal/app/restore_discovery.go`
|
||||
- Planning: `internal/app/restore_plan.go`, `internal/previouscache`
|
||||
- Execution: `internal/app/restore_execute.go`
|
||||
- Reporting and command coordination: `internal/app/restore_report.go`,
|
||||
`internal/app/restore.go`
|
||||
- Tests: `internal/app/restore_discovery_test.go`,
|
||||
`internal/app/restore_plan_test.go`,
|
||||
`internal/app/restore_execution_test.go`,
|
||||
`internal/app/restore_workflow_test.go`
|
||||
|
||||
71
docs/internal/fileops.md
Normal file
71
docs/internal/fileops.md
Normal file
@@ -0,0 +1,71 @@
|
||||
# Internal: File Operations
|
||||
|
||||
`internal/fileops` owns the narrow mechanics for durable replacement of one
|
||||
byte file. Callers keep ownership of serialization, validation, cancellation,
|
||||
and destination-directory policy.
|
||||
|
||||
## Destination Confinement
|
||||
|
||||
Before it creates, replaces, or installs a destination file, `fileops` opens
|
||||
each ancestor from the filesystem root and rejects symbolic links or components
|
||||
that change during traversal. The resulting parent-directory handle is retained
|
||||
for sibling temporary-file creation and rename, so a later pathname swap cannot
|
||||
redirect the replacement. Existing destination symlinks are replaced as leaf
|
||||
entries; their targets are never followed.
|
||||
|
||||
Remote object acquisition uses a writer supplied by the storage owner. The
|
||||
writer receives a `fileops`-owned, already-open sibling temporary file rather
|
||||
than a mutable destination path. Callers still own remote object selection,
|
||||
validation, conflict handling, and final mode.
|
||||
|
||||
Directory promotion keeps the verified destination parent open while it creates
|
||||
the temporary tree, copies regular source entries, and performs the platform
|
||||
no-replace rename. Platforms without a verified handle-relative atomic
|
||||
no-replace primitive reject promotion before writing a temporary tree.
|
||||
|
||||
## Cleanup Contract
|
||||
|
||||
`RemoveAllUnderRoot` accepts an explicit root and a proper descendant. It opens
|
||||
the root and each target ancestor without following symlinks, then removes the
|
||||
tree through those directory handles. It rejects root deletion and any symlink
|
||||
encountered in the target path or tree; repeated removal of a missing target is
|
||||
successful. Command and post-publish policy remains owned by `internal/app`.
|
||||
|
||||
## Confined Reads
|
||||
|
||||
`ReadRegularFileUnderRoot` is the no-follow, bounded read primitive for a
|
||||
caller-selected root and relative file path; `ReadRegularFile` is its
|
||||
path-based convenience wrapper. They verify every ancestor through directory
|
||||
handles and admit only a stable regular-file handle. Callers enforce their own
|
||||
byte limits and access policy. Credential mode policy and environment
|
||||
precedence remain owned by `internal/app`.
|
||||
|
||||
## Replacement Contract
|
||||
|
||||
`ReplaceFileAtomic` requires an existing destination directory. It creates a
|
||||
sibling temporary file, writes the complete byte sequence, applies the
|
||||
caller-supplied mode, syncs and closes the file, runs an optional pre-rename
|
||||
check, replaces the destination with a rename, then syncs the containing
|
||||
directory.
|
||||
|
||||
The pre-rename check is the last point at which a caller can cancel without
|
||||
installing a new destination. A failure before the rename leaves the old
|
||||
destination unchanged and removes the temporary file; any cleanup failure is
|
||||
returned alongside the primary failure. A failure after the rename may leave
|
||||
the new file visible, but it is not reported as crash-durable.
|
||||
|
||||
Replacement follows the operating system's same-filesystem rename semantics.
|
||||
If a platform cannot replace an existing destination, the operation returns an
|
||||
error and never removes the old file as an emulation step.
|
||||
|
||||
## Directory-Sync Support
|
||||
|
||||
Linux and macOS attempt to sync the destination directory. Windows opens the
|
||||
directory with backup semantics and flushes its buffers. If either operation
|
||||
is unavailable for the platform, directory handle, or filesystem,
|
||||
`ErrDirectorySyncUnsupported` is returned. Narratio does not treat that result
|
||||
as successful crash-durable replacement.
|
||||
|
||||
`WriteFileAtomic`, copy helpers, and downloaded temporary-file installation
|
||||
retain their compatibility behavior of creating the destination parent with
|
||||
the repository's workspace permissions before using this contract.
|
||||
@@ -1,21 +1,32 @@
|
||||
# Internal: Manifest
|
||||
|
||||
## Purpose
|
||||
Define durable session state (`manifest.json`) and invocation state (`runs/{run_id}/manifest.json`) contracts.
|
||||
|
||||
Explain the session-progress and invocation-audit models implemented by
|
||||
`internal/manifest`. Physical manifest placement belongs in
|
||||
[Operations](../operations.md#local-state-layout).
|
||||
|
||||
## Session Manifest
|
||||
Path:
|
||||
- `{workspace.root}/work/{campaign}/{session_id}/manifest.json`
|
||||
|
||||
Primary model (`manifest.Manifest`):
|
||||
`manifest.Manifest` records:
|
||||
|
||||
- identity (`session_id`, `campaign`, `run_id`)
|
||||
- local path metadata (`local_workdir`, `local_spool_dir`)
|
||||
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
|
||||
- `inputs` records
|
||||
- durable `artifacts` records
|
||||
- per-stage `stages` map
|
||||
- an optional `post_publish_cleanup` obligation, which binds a committed run,
|
||||
remote commit identity, and each exact root-confined local target to its
|
||||
completion evidence
|
||||
|
||||
Session, campaign, and run identities in local and downloaded manifests must be
|
||||
portable opaque segments. Unsafe legacy identities are rejected with migration
|
||||
guidance rather than being normalized into a different workspace or remote
|
||||
namespace.
|
||||
|
||||
The model admits these stage states:
|
||||
|
||||
Stage status enum:
|
||||
- `pending`
|
||||
- `running`
|
||||
- `succeeded`
|
||||
@@ -25,33 +36,122 @@ Stage status enum:
|
||||
- `interrupted`
|
||||
|
||||
## Run Manifest
|
||||
Path:
|
||||
- `{workspace.root}/work/{campaign}/{session_id}/runs/{run_id}/manifest.json`
|
||||
|
||||
Run model (`manifest.RunManifest`):
|
||||
`manifest.RunManifest` is created for each invocation and records:
|
||||
|
||||
- invocation identity and `force` flag
|
||||
- requested stages
|
||||
- per-stage action (`run` or `skip`)
|
||||
- per-stage status
|
||||
- overall run status (`running`, `succeeded`, `failed`)
|
||||
|
||||
## Remote Commit Manifest
|
||||
|
||||
`artifacts.RemoteCommitManifest` is a separate, versioned remote snapshot
|
||||
contract. It is not a serialized session manifest and contains no local
|
||||
post-publication assertion such as `current_pointer_written`. A remote commit
|
||||
identifies one campaign, session, and run and declares its immutable artifact
|
||||
set. Each artifact has a typed source, immutable destination key, SHA-256
|
||||
checksum, size, and storage generation.
|
||||
|
||||
`current/commit-pointer.json` is the sole mutable selector for the new
|
||||
contract. It identifies exactly one run-scoped `runs/{run_id}/commit.json` and
|
||||
binds that object by checksum, size, and generation. Readers strictly reject
|
||||
unknown fields, version mismatches, pointer/commit identity mismatches, and
|
||||
objects that do not match their declaration.
|
||||
|
||||
The reader retains a temporary, clearly isolated compatibility path for a
|
||||
coherent legacy `current/manifest.json` plus `current/run_id.txt` pair. That
|
||||
path is removable after migration and is never used to write new state.
|
||||
|
||||
## Persistence Semantics
|
||||
|
||||
`manifest.LocalStore`:
|
||||
|
||||
- validates loaded documents;
|
||||
- normalizes missing maps/stage records;
|
||||
- writes atomically via temp file + rename;
|
||||
- writes through a sibling temporary file, syncing the completed file and
|
||||
destination directory after atomic replacement;
|
||||
- updates `updated_at` on save.
|
||||
|
||||
If the operating system or filesystem cannot sync a directory, save returns an
|
||||
explicit error instead of claiming crash-durable replacement. A returned error
|
||||
after the rename can therefore leave the new manifest visible but not confirmed
|
||||
durable; callers must reload it before retrying.
|
||||
|
||||
## Execution Semantics
|
||||
Runner updates both manifests per stage transition:
|
||||
- mark running
|
||||
- mark succeeded/failed/skipped
|
||||
- persist logs/generated config refs and metadata
|
||||
|
||||
The application runner marks an executing stage running and then succeeded or
|
||||
failed in both manifests, persisting each transition. On success it records
|
||||
outputs, logs, generated configuration references, and metadata. Artifact
|
||||
records may include optional contract and external provenance objects; old
|
||||
manifests remain compatible when those fields are absent. A successful forced
|
||||
rerun marks only succeeded downstream session-stage records stale.
|
||||
|
||||
Starting an execution clears the current session-stage record's prior outputs,
|
||||
logs, generated configuration references, and metadata. Failed and skipped
|
||||
transitions enforce the same clearing rule directly, while success repopulates
|
||||
only fields returned by the new result. Marking a record stale does not clear
|
||||
those details because resume validation and diagnosis may still require them
|
||||
before execution begins. Invocation run manifests remain immutable audit
|
||||
records of their own outcomes.
|
||||
|
||||
A stage may explicitly return a skipped disposition and stable reason. The
|
||||
runner persists that outcome in both manifests, clears older outputs for the
|
||||
session-stage record along with older logs, generated configuration references,
|
||||
and metadata, then applies any bounded details from the current skip and
|
||||
continues. This self-skip is distinct from deciding not to execute an
|
||||
already-succeeded stage and is reconsidered on later runs. Skipped results
|
||||
cannot contain outputs.
|
||||
|
||||
When an already-succeeded stage is skipped, the invocation run manifest records
|
||||
the `skip` action and reason. The session manifest deliberately retains its
|
||||
existing succeeded record because it remains the cross-invocation progress
|
||||
authority. Stages with a resume validator, currently extraction, may reject an
|
||||
otherwise eligible skip when the recorded durable result is obsolete; the
|
||||
runner marks it stale and executes it.
|
||||
|
||||
Session manifest is the authoritative stage-progress ledger across invocations.
|
||||
Run manifest is invocation-scoped audit state.
|
||||
|
||||
After a publish commits remotely, any configured local cleanup is first recorded
|
||||
as a session-manifest obligation before deletion begins. Each target becomes
|
||||
complete only after its confined deletion (or safe absence check) and a
|
||||
successful manifest save. An incomplete obligation is retried on later
|
||||
invocations independently of their selected stages and retains the committed
|
||||
run and remote identity that authorized it.
|
||||
|
||||
Each invocation derives campaign, session, run, local-path, and remote-prefix
|
||||
metadata from the validated resolved configuration as one projection. A persisted
|
||||
session manifest must agree on campaign and session identity before execution;
|
||||
the current projection is refreshed for every invocation while stage progress,
|
||||
inputs, and durable artifacts remain session history.
|
||||
|
||||
For handled failures after an invocation record is created, the runner records
|
||||
the failure on the session ledger and persists it before persisting the failed
|
||||
run audit record. This preserves the resume authority while making a partial
|
||||
persistence disagreement visible. Abrupt process death remains an accepted case
|
||||
where a durable running record can require operator interpretation.
|
||||
|
||||
## Invariants
|
||||
|
||||
- stage resume/skip decisions are session-manifest driven.
|
||||
- running, failed, and self-skipped stages do not retain result payloads from
|
||||
an earlier success.
|
||||
- stale stages retain prior details until replacement execution starts.
|
||||
- force reruns stale downstream succeeded stages.
|
||||
- run manifest does not replace session manifest as progress authority.
|
||||
- remote commitment is established by a verified current pointer and remote
|
||||
commit relationship, never by a mutable session-manifest boolean.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Models and transitions: `internal/manifest/manifest.go`,
|
||||
`internal/manifest/run_manifest.go`
|
||||
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
|
||||
`internal/artifacts/current_state_commit.go`,
|
||||
`internal/artifacts/current_state_legacy.go`
|
||||
- Persistence and validation: `internal/manifest/store.go`
|
||||
- Package tests: `internal/manifest/*_test.go`
|
||||
- Assembled execution behavior: `internal/app/runner_test.go`,
|
||||
`internal/app/run_stage_test.go`
|
||||
|
||||
98
docs/internal/overview.md
Normal file
98
docs/internal/overview.md
Normal file
@@ -0,0 +1,98 @@
|
||||
# Internal Overview
|
||||
|
||||
This document is the implemented component map for Narratio. Normative system
|
||||
boundaries and dependency direction belong in
|
||||
[Architecture](../policy/architecture.md). User and operator contracts belong
|
||||
in the [CLI](../cli.md), [Configuration](../config.md),
|
||||
[Operations](../operations.md), and [Troubleshooting](../troubleshooting.md).
|
||||
Externally observable tool and format contracts belong under
|
||||
[Integrations](../integrations/).
|
||||
|
||||
## Execution Path
|
||||
|
||||
```text
|
||||
cmd/narratio -> internal/app -> configuration and production composition
|
||||
-> internal/stage -> adapters and external systems
|
||||
-> manifests and artifact resolution -> durable local/remote output
|
||||
```
|
||||
|
||||
The executable delegates process behavior to the application boundary. The
|
||||
application resolves configuration, composes concrete collaborators, acquires
|
||||
session safety controls, and runs commands. Pipeline commands execute the
|
||||
canonical stage sequence through adapter interfaces, while manifests record
|
||||
progress and artifact services resolve durable inputs and outputs.
|
||||
|
||||
## Components
|
||||
|
||||
| Area | Implemented owners | Responsibility |
|
||||
| --- | --- | --- |
|
||||
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
|
||||
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, cleanup gates, and user-facing reporting. |
|
||||
| Configuration | `internal/config` | Strict YAML loading, discovery, defaults, normalization, session templating, and validation. |
|
||||
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
|
||||
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
|
||||
| Manifests | `internal/manifest` | Durable session progress, invocation audit state, stage transitions, validation, and atomic persistence. |
|
||||
| Artifacts and paths | `internal/artifacts`, `internal/pathsafe` | Artifact identities and resolution, local and remote path/key models, current-state discovery, and confined relative destinations. |
|
||||
| Previous-session cache | `internal/previouscache` | Deterministic planning and materialization requirements for configured previous-session inputs. |
|
||||
| Artifact policy | `internal/artifactpolicy` | Source and destination policy, configured artifact identity validation, and publish destination safety. |
|
||||
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, [`internal/fileops`](fileops.md) | Transcript and artifact data contracts plus durable single-file replacement helpers; unsupported directory syncing is reported explicitly. |
|
||||
| Logging | `internal/logging` | Application logger construction and shared structured logging behavior. |
|
||||
|
||||
The application boundary composes concrete implementations. Stages depend on
|
||||
Narratio-level contracts; external transport and SDK details remain in
|
||||
adapters. The normative rules for these relationships remain in
|
||||
[Architecture](../policy/architecture.md).
|
||||
|
||||
## Pipeline Stage Set
|
||||
|
||||
The implemented canonical order is:
|
||||
|
||||
1. [`prepare`](stage-prepare.md)
|
||||
2. [`transcribe`](stage-transcribe.md)
|
||||
3. [`merge`](stage-merge.md)
|
||||
4. [`polish`](stage-polish.md)
|
||||
5. [`normalize`](stage-normalize.md)
|
||||
6. [`trim`](stage-trim.md)
|
||||
7. [`extract`](stage-extract.md)
|
||||
8. [`render`](stage-render.md)
|
||||
9. [`analyze`](stage-analyze.md)
|
||||
10. [`publish`](stage-publish.md)
|
||||
11. `notify` (no-op)
|
||||
|
||||
`notify` currently has no persisted pipeline outputs and uses the explicit
|
||||
`noop` notification mode. The focused stage documents own implementation
|
||||
mechanics. The
|
||||
[CLI](../cli.md) and [Operations](../operations.md) own user-visible invocation
|
||||
and execution semantics.
|
||||
|
||||
## Focused Documentation
|
||||
|
||||
- [Adapter Internals](adapters.md): external adapter boundaries, composition,
|
||||
failure behavior, and test surfaces.
|
||||
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
|
||||
resolution, previous requirements, and current-state helpers.
|
||||
- [Manifest Internals](manifest.md): session and run records, persistence, and
|
||||
execution transitions.
|
||||
- [Storage Internals](storage.md): object-store interface and S3 behavior.
|
||||
- [Workspace Internals](workspace.md): local layout, locking, and cleanup
|
||||
guardrails.
|
||||
- [Restore Internals](command-restore.md): discovery, planning, execution, and
|
||||
reporting.
|
||||
- [`prepare`](stage-prepare.md)
|
||||
- [`transcribe`](stage-transcribe.md)
|
||||
- [`merge`](stage-merge.md)
|
||||
- [`polish`](stage-polish.md)
|
||||
- [`normalize`](stage-normalize.md)
|
||||
- [`trim`](stage-trim.md)
|
||||
- [`extract`](stage-extract.md)
|
||||
- [`render`](stage-render.md)
|
||||
- [`analyze`](stage-analyze.md)
|
||||
- [`publish`](stage-publish.md)
|
||||
|
||||
Use this map to find an owner, then read its focused documentation and tests
|
||||
before changing behavior.
|
||||
|
||||
The stage registry is implemented in `internal/stage/placeholders.go` and its
|
||||
ordering is protected by `internal/app/planner_test.go`. Cross-invocation skip,
|
||||
force, failure, and invalidation behavior is exercised in
|
||||
`internal/app/runner_test.go` and `internal/app/run_stage_test.go`.
|
||||
@@ -1,42 +1,73 @@
|
||||
# Stage: analyze
|
||||
|
||||
## Purpose
|
||||
|
||||
Execute selected configured Scriptorium artifacts in dependency order and materialize outputs.
|
||||
|
||||
## Inputs
|
||||
|
||||
- configured artifacts from `pipeline.scriptorium.artifacts`
|
||||
- optional selected artifact filter (`--artifacts`)
|
||||
- built-in/configured/previous-session source references in artifact inputs
|
||||
- optional selected artifact keys supplied through the stage environment
|
||||
- built-in, configured, extraction, and previous-session source references in
|
||||
artifact inputs
|
||||
|
||||
Supported source families:
|
||||
- built-ins: `narratio.transcript.*`, `narratio.bounds.session`
|
||||
- prepared stable inputs: `narratio.input.players`, `narratio.input.party`, `narratio.input.glossary`
|
||||
- prepared stable inputs: `narratio.input.players`, `narratio.input.party`,
|
||||
`narratio.input.glossary`
|
||||
- configured artifacts: `narratio.artifact.<key>`
|
||||
- extraction lanes: `narratio.extraction.<key>`
|
||||
- previous-session cache: `narratio.previous_session.artifact.<key>`
|
||||
|
||||
## Outputs
|
||||
|
||||
- one materialized output per executed configured artifact (`output_path`)
|
||||
- stage metadata describing selected/generated/reused artifacts
|
||||
|
||||
## Key Behavior
|
||||
- skips with metadata when Scriptorium config is missing or no executable artifacts remain.
|
||||
- builds runtime artifact catalog (built-ins + configured artifacts).
|
||||
|
||||
- when Scriptorium is absent or no configured artifact is executable, completes
|
||||
successfully with no outputs and records explanatory metadata. This is not an
|
||||
explicit self-skip: both manifests record success, satisfy publish's
|
||||
prerequisite, and an ordinary later run reuses the result until forced.
|
||||
- builds a runtime artifact catalog containing built-ins, configured artifacts,
|
||||
and configured extraction lanes. Extraction availability is hydrated only
|
||||
from compatible successful extraction evidence.
|
||||
- uses enabled configured artifacts by default. An explicit `--artifacts`
|
||||
selection is a one-invocation override: it makes exactly the named configured
|
||||
artifacts executable even when disabled, and does not automatically include
|
||||
dependencies. A selected artifact's dependencies must instead already be
|
||||
available to the catalog.
|
||||
- marks non-executable configured artifacts as reusable when output files already exist.
|
||||
- validates selected artifact dependency order (cycle-safe topo ordering).
|
||||
- resolves required/optional inputs per artifact source definition.
|
||||
- omits an unavailable optional input; an unavailable required input fails.
|
||||
- resolves prepared stable input sources from `inputs/*.yml` materialized by `prepare`.
|
||||
- resolves previous-session sources from local `previous/` cache only.
|
||||
- runs optional render-debug, then artifact execution.
|
||||
- validates non-empty output files and materializes canonical outputs.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
- required missing configured/previous-session inputs fail.
|
||||
- missing required prepared stable input source includes prepare rerun guidance.
|
||||
- missing required previous-session source includes prepare rerun guidance.
|
||||
- missing required `narratio.transcript.final_markdown` or `narratio.transcript.final_trimmed_markdown` inputs includes render rerun guidance.
|
||||
- missing required `narratio.transcript.final_markdown` or
|
||||
`narratio.transcript.final_trimmed_markdown` inputs includes render rerun
|
||||
guidance.
|
||||
- dependency cycles or unavailable required dependencies fail.
|
||||
- adapter validation failures fail stage.
|
||||
|
||||
## Invariants
|
||||
|
||||
- `analyze` performs no remote storage calls for previous-session source resolution.
|
||||
- output provenance and metadata are deterministic per execution.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Configuration](../config.md#scriptorium-artifact-entries) owns artifact
|
||||
fields and source-selection rules.
|
||||
- [CLI](../cli.md) owns user-visible artifact selection.
|
||||
- [Scriptorium](../integrations/scriptorium.md) owns the subprocess contract.
|
||||
- Implementation and tests: `internal/stage/analyze.go`,
|
||||
`internal/stage/analyze_test.go`
|
||||
|
||||
90
docs/internal/stage-extract.md
Normal file
90
docs/internal/stage-extract.md
Normal file
@@ -0,0 +1,90 @@
|
||||
# Internal: Extract Stage
|
||||
|
||||
## Responsibility
|
||||
|
||||
`extract` runs after `trim` and before `render`. It converts the canonical
|
||||
`narratio.transcript.final_trimmed` JSON into configured Notarius lane artifacts.
|
||||
An omitted or disabled Notarius section makes the stage explicitly self-skip
|
||||
with reason `notarius_disabled`, no outputs, and no Notarius runner.
|
||||
|
||||
The external protocol is documented in the
|
||||
[Notarius integration contract](../integrations/notarius.md). Configuration
|
||||
fields belong in [Configuration](../config.md), and physical paths and force
|
||||
procedures belong in [Operations](../operations.md).
|
||||
|
||||
## Lifecycle
|
||||
|
||||
`internal/stage/extract.go`:
|
||||
|
||||
1. resolves the final trimmed transcript from the shared artifact catalog;
|
||||
2. resolves and fingerprints the Notarius invocation contract;
|
||||
3. creates a run-local staging directory and invokes the injected
|
||||
`notarius.Runner`;
|
||||
4. validates the successful receipt, confined index, configured required lane
|
||||
descriptors, and regular payload files;
|
||||
5. atomically promotes the complete bundle to its immutable durable location;
|
||||
6. records one non-selectable `notarius_index` output and one selectable
|
||||
`notarius_lane` output per configured lane; and
|
||||
7. registers each lane as `narratio.extraction.<output_key>` for downstream
|
||||
Scriptorium and publish resolution.
|
||||
|
||||
Lane records retain checksum, contract, producer run ID, and Notarius system,
|
||||
run, pipeline, and lane provenance. Stage metadata retains the durable bundle
|
||||
root, receipt, diagnostic paths, rejection/warning summaries, producing
|
||||
Narratio run ID, the resolved trimmed-input identity, and invocation
|
||||
fingerprint. The input identity binds the exact transcript bytes, canonical
|
||||
source ID, producer stage/output/run identity, and resolution provenance.
|
||||
Validation completes before
|
||||
promotion, so a rejected result cannot expose a partial durable bundle.
|
||||
|
||||
Any executed extraction outcome that replaces a different effective outcome
|
||||
marks succeeded downstream stages stale. Repeating the same disabled self-skip
|
||||
with no outputs is stable and does not repeatedly invalidate downstream stages.
|
||||
|
||||
## Resume Validation
|
||||
|
||||
`internal/stage/extract_resume.go` permits a skip only when the existing stage
|
||||
record succeeded and still matches the current invocation fingerprint. The
|
||||
fingerprint covers the resolved executable and config paths, pipeline ID,
|
||||
timeout, working directory, sorted configured output contracts, and the current
|
||||
direct trimmed-transcript identity. The same identity is resolved again for
|
||||
artifact evidence, so changing the current transcript bytes or producer
|
||||
identity makes the prior extraction obsolete.
|
||||
|
||||
The validator then checks the producing run identity, canonical immutable
|
||||
bundle root, path confinement and absence of symlink components, receipt
|
||||
identity, exactly one canonical index, the exact configured source set,
|
||||
contracts and provenance, regular-file status, and stored checksums. Missing or
|
||||
obsolete results are non-resumable and run again; unsafe filesystem conditions
|
||||
return an error rather than silently accepting or replacing data.
|
||||
|
||||
The fingerprint cannot observe files imported by Notarius configuration,
|
||||
profile contents, prompt/module definitions, or other transitive inputs.
|
||||
Operators must force extraction after changing any such input.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
|
||||
index compatibility, required-lane rejection, payload inspection, checksum, or
|
||||
promotion errors fail the stage through ordinary manifest transition handling.
|
||||
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
|
||||
not given selectable extraction sources unless the complete configured result
|
||||
has passed validation and promotion.
|
||||
|
||||
When a replacement attempt begins, the current session-stage record no longer
|
||||
advertises payload from the previous success. A failed replacement therefore
|
||||
has no current outputs, logs, generated configuration references, or metadata,
|
||||
while the earlier invocation manifest and immutable promoted bundle remain
|
||||
available for audit and recovery.
|
||||
|
||||
## Implementation And Focused Tests
|
||||
|
||||
- Stage execution, selection, and resume validation: `internal/stage/extract.go`,
|
||||
`internal/stage/extract_resume.go`,
|
||||
`internal/stage/extract_test.go`
|
||||
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
||||
`internal/adapters/notarius/subprocess_test.go`
|
||||
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
||||
`internal/artifacts/extraction_catalog_test.go`
|
||||
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
||||
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
||||
@@ -1,18 +1,22 @@
|
||||
# Stage: merge
|
||||
|
||||
## Purpose
|
||||
|
||||
Normalize raw transcript inputs and merge into base transcript via Seriatim.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/raw/*.json`
|
||||
- `inputs/speakers.yml`
|
||||
- `inputs/autocorrect.yml`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/base.json`
|
||||
- optional `artifacts/seriatim.report.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- discovers and validates raw transcript inputs.
|
||||
- normalizes each raw transcript (`seriatim.Normalize`) into run-local scratch output.
|
||||
- merges normalized inputs (`seriatim.Run`) into base transcript.
|
||||
@@ -20,6 +24,14 @@ Normalize raw transcript inputs and merge into base transcript via Seriatim.
|
||||
- materializes canonical outputs and records stage logs/generated configs.
|
||||
|
||||
## Invariants
|
||||
|
||||
- merge always consumes normalized forms of raw inputs.
|
||||
- base transcript must validate before stage success.
|
||||
- report output is config-gated.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Seriatim](../integrations/seriatim.md) owns subprocess and output semantics.
|
||||
- [Configuration](../config.md#pipeline) owns operator-selected Seriatim values.
|
||||
- Implementation and tests: `internal/stage/merge.go`,
|
||||
`internal/stage/merge_test.go`
|
||||
|
||||
@@ -1,16 +1,20 @@
|
||||
# Stage: normalize
|
||||
|
||||
## Purpose
|
||||
|
||||
Normalize polished transcript into final transcript using Seriatim.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/polished.json`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/final.json` (or configured normalize output path)
|
||||
- optional `artifacts/seriatim.normalize.report.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- resolves polished transcript from manifest outputs/canonical fallback.
|
||||
- applies `pipeline.normalize` config or default normalize config.
|
||||
- runs Seriatim normalize with configured timeout/binary.
|
||||
@@ -18,5 +22,13 @@ Normalize polished transcript into final transcript using Seriatim.
|
||||
- materializes canonical outputs and records logs/generated configs.
|
||||
|
||||
## Invariants
|
||||
|
||||
- final transcript must validate as processed transcript JSON (`segments` array).
|
||||
- normalize defaults are applied when `pipeline.normalize` is unset.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Seriatim](../integrations/seriatim.md) owns subprocess and output semantics.
|
||||
- [Configuration](../config.md#pipeline) owns normalize fields and defaults.
|
||||
- Implementation and tests: `internal/stage/normalize.go`,
|
||||
`internal/stage/normalize_test.go`
|
||||
|
||||
@@ -1,23 +1,37 @@
|
||||
# Stage: polish
|
||||
|
||||
## Purpose
|
||||
|
||||
Run Audita polishing on base transcript and produce polished transcript.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/base.json`
|
||||
- `inputs/glossary.yml`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/polished.json`
|
||||
- optional `artifacts/audita.report.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- resolves base transcript from merge outputs/canonical fallback.
|
||||
- invokes Audita with configured model/module/runtime options.
|
||||
- invokes an Audita runner configured with static model/runtime options; the
|
||||
invocation supplies paths and modules.
|
||||
- validates processed transcript structure (`segments` array required).
|
||||
- validates optional report JSON.
|
||||
- materializes canonical outputs; records logs/generated config and adapter metadata.
|
||||
|
||||
## Invariants
|
||||
|
||||
- polished transcript schema validation is mandatory.
|
||||
- report output is config-gated.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Audita](../integrations/audita.md) owns subprocess, validation, and failure
|
||||
semantics.
|
||||
- [Configuration](../config.md#pipeline) owns operator-selected Audita values.
|
||||
- Implementation and tests: `internal/stage/polish.go`,
|
||||
`internal/stage/polish_test.go`
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
# Stage: prepare
|
||||
|
||||
## Purpose
|
||||
|
||||
Materialize canonical current-session inputs before processing stages.
|
||||
|
||||
## Inputs
|
||||
- resolved `campaign.yml`, `session.yml`, and pipeline config
|
||||
|
||||
- resolved campaign, session, and pipeline configuration
|
||||
- stable input files (`speakers`, `autocorrect`, `glossary`, `players`, `party`)
|
||||
- audio source:
|
||||
- local `audio_dir`/`audio_files`, or
|
||||
- S3 `audio_s3.prefix`
|
||||
- one resolved local or S3 audio source
|
||||
- enabled configured artifact input requirements for previous-session sources
|
||||
|
||||
## Outputs
|
||||
|
||||
- `inputs/campaign.yml`
|
||||
- `inputs/session.yml`
|
||||
- `inputs/pipeline.resolved.yml`
|
||||
@@ -26,19 +27,39 @@ Materialize canonical current-session inputs before processing stages.
|
||||
- deterministic `manifest.inputs` entries (checksums + provenance)
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- validates required config/store state.
|
||||
- enforces local audio vs S3 audio mutual exclusivity.
|
||||
- rejects duplicate explicit local audio sources after resolution.
|
||||
- gives distinct local source paths with the same basename deterministic unique
|
||||
prepared filenames so neither source is overwritten.
|
||||
- materializes S3 audio through spool/cache-aware logic.
|
||||
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
|
||||
- when previous requirements exist:
|
||||
- clears managed `previous/` state;
|
||||
- builds previous-cache remote plan;
|
||||
- clears managed `previous/` state on every invocation, then, when requirements exist:
|
||||
- resolves the pointer-selected previous source through the shared resolver;
|
||||
- downloads previous manifest/artifacts;
|
||||
- records previous inputs in `manifest.inputs`.
|
||||
|
||||
Required previous-session inputs fail when unavailable; optional missing inputs are skipped.
|
||||
Required previous-session inputs fail when unavailable; optional missing inputs
|
||||
are typed skipped results. Committed sources use their exact source-to-destination
|
||||
mapping, while the isolated legacy reader rejects ambiguous fallback matches.
|
||||
|
||||
## Invariants
|
||||
|
||||
- only `prepare` hydrates canonical `previous/` cache state.
|
||||
- managed previous artifacts are stored under `previous/artifacts/**` without duplicate `artifacts/artifacts/` nesting.
|
||||
- managed previous artifacts are stored under `previous/artifacts/**` without
|
||||
duplicate `artifacts/artifacts/` nesting.
|
||||
- managed `previous/` state represents only the current requirement set.
|
||||
- `manifest.inputs` ordering is deterministic (`kind`, `path`).
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Configuration](../config.md) owns audio selection, stable input fields, and
|
||||
previous-session settings.
|
||||
- [Operations](../operations.md) owns physical input, audio, spool, cache, and
|
||||
previous-state layout.
|
||||
- [Storage Internals](storage.md) and [Artifact Internals](artifacts.md) explain
|
||||
the internal collaborators.
|
||||
- Implementation and tests: `internal/stage/prepare.go`,
|
||||
`internal/stage/prepare_test.go`, `internal/audio/s3_audio_test.go`,
|
||||
`internal/previouscache/*_test.go`
|
||||
|
||||
@@ -1,33 +1,63 @@
|
||||
# Stage: publish
|
||||
|
||||
## Purpose
|
||||
|
||||
Upload run/session outputs to object storage and atomically advance remote current state.
|
||||
|
||||
## Inputs
|
||||
- successful prerequisite stages: `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `render`, `analyze`
|
||||
- run root `runs/{run_id}/**`
|
||||
- publish output rules (`pipeline.publish.outputs`)
|
||||
- effective publish locks (static + remote merged lock set)
|
||||
- local `previous/**` files when present
|
||||
|
||||
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
|
||||
- invocation-scoped run files
|
||||
- resolved publish output rules
|
||||
- effective publish locks (static + remote merged lock set), revalidated at the
|
||||
remote commit boundary
|
||||
- durable previous-session cache files when present
|
||||
|
||||
## Outputs
|
||||
- uploaded run files under remote `runs/{run_id}/...` (excluding `audio/**`)
|
||||
- uploaded selected publish outputs under session prefix
|
||||
- uploaded `previous/**` files under session prefix when present
|
||||
- uploaded `current/manifest.json`
|
||||
- uploaded `current/run_id.txt` written last
|
||||
|
||||
- uploaded invocation record and selected publish outputs;
|
||||
- uploaded durable previous-session cache files when present;
|
||||
- immutable run-scoped commit manifest; and
|
||||
- current commit pointer, written last.
|
||||
|
||||
Exact remote placement and the operator workflow belong in
|
||||
[Operations](../operations.md#publish-workflow).
|
||||
|
||||
## Key Behavior
|
||||
- stage can self-skip when publish disabled or run upload disabled.
|
||||
|
||||
- when publishing or run upload is disabled, completes successfully with no
|
||||
outputs and records explanatory metadata. This is not an explicit self-skip:
|
||||
both manifests record success, and an ordinary later run reuses that result
|
||||
until publish is forced.
|
||||
- validates prerequisite stage success and object-store availability.
|
||||
- collects deterministic run file list plus run `manifest.json`.
|
||||
- derives a deterministic run-archive allowlist from the validated run
|
||||
`manifest.json`: declared run-local outputs, logs, generated configs, and the
|
||||
manifest itself. Unlisted workspace files are not archive candidates.
|
||||
- opens each archive candidate beneath its archive root without following
|
||||
symlinked ancestors or leaf entries, verifies that it is a regular file and
|
||||
checks a declared checksum when present, then streams the opened descriptor.
|
||||
- derives the durable previous-cache archive from its validated manifest using
|
||||
the same confinement and regular-file checks.
|
||||
- resolves publish output sources through runtime artifact catalog and manifest-aware resolution.
|
||||
- publishes extraction lanes only through explicit configured output rules;
|
||||
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
|
||||
- selected artifact filter applies to configured artifact sources only.
|
||||
- locked outputs are skipped intentionally (including required ones).
|
||||
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
||||
- writes remote current manifest before current run pointer.
|
||||
- creates one complete immutable source-to-destination mapping before upload;
|
||||
- uploads and verifies every declared immutable object and the commit manifest;
|
||||
- updates `current/commit-pointer.json` exactly once, last; and
|
||||
- does not write the legacy `current/manifest.json` or `current/run_id.txt` pair.
|
||||
- rechecks remote lock state immediately before the pointer update. A newly
|
||||
committed lock aborts selection, leaving any uploaded immutable attempt
|
||||
unselected.
|
||||
- reads the mutable remote lock document through a direct limit-plus-one read
|
||||
capped by `MaxRemoteLockStoreBytes` (1 MiB), retaining the generation returned
|
||||
with the opened body for conditional updates. Oversized lock documents fail
|
||||
before YAML decoding; published artifact payloads do not use this limit.
|
||||
|
||||
## Metadata Signals
|
||||
|
||||
Includes counts/lists for:
|
||||
- run uploads
|
||||
- published output uploads
|
||||
@@ -35,10 +65,35 @@ Includes counts/lists for:
|
||||
- skipped optional outputs
|
||||
- skipped unselected outputs
|
||||
- locked outputs
|
||||
- current-state key paths
|
||||
- `current_pointer_written`
|
||||
- remote commit and current-pointer key paths
|
||||
- the run identifier selected by the commit
|
||||
|
||||
## Invariants
|
||||
- `current/run_id.txt` is the remote commit marker and is written last.
|
||||
- run upload excludes `audio/**`.
|
||||
- publish locks are not overridden by `--force`.
|
||||
|
||||
- `current/commit-pointer.json` is the remote commit marker and is written last.
|
||||
- run files, selected outputs, previous-cache files, and the committed session
|
||||
manifest are all declared by an immutable commit under the run prefix.
|
||||
- run and previous uploads contain only manifest-declared regular files opened
|
||||
from verified descriptors; symlinks, special files, replacement races, and
|
||||
undeclared entries are rejected or ignored before uploads begin.
|
||||
- run-local diagnostics, including Notarius receipt and stderr files, are
|
||||
archived only when recorded by the run manifest.
|
||||
- publish locks are not overridden by `--force`; remote locks are revalidated
|
||||
immediately before current-state selection.
|
||||
- post-commit local cleanup is authorized by the committed publish metadata and
|
||||
is durably recorded by the application lifecycle before any local deletion.
|
||||
|
||||
The commit boundary and cleanup gate are normative architecture invariants; see
|
||||
[Architecture](../policy/architecture.md#publish-commit-boundary).
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Configuration](../config.md#publish-configuration-summary) owns output and
|
||||
static-lock fields.
|
||||
- [Operations](../operations.md#publish-locks) owns remote lock lifecycle and
|
||||
physical remote state.
|
||||
- [Artifact Internals](artifacts.md) explains source resolution and current-state
|
||||
helpers.
|
||||
- Implementation and tests: `internal/stage/publish.go`,
|
||||
`internal/stage/publish_test.go`, `internal/app/operator_helpers_test.go`,
|
||||
`internal/app/post_publish_cleanup_test.go`
|
||||
|
||||
@@ -1,29 +1,44 @@
|
||||
# Stage: render
|
||||
|
||||
## Purpose
|
||||
|
||||
Render Markdown transcript artifacts from normalized JSON transcripts via Seriatim.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `narratio.transcript.final` (`transcripts/final.json`)
|
||||
- `narratio.transcript.final_trimmed` (`transcripts/final.trimmed.json`)
|
||||
|
||||
## Outputs
|
||||
|
||||
- `narratio.transcript.final_markdown` -> `transcripts/final.md`
|
||||
- `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- uses `pipeline.render` settings (enabled/format/title/booleans).
|
||||
- resolves inputs manifest-first, then canonical fallback.
|
||||
- writes run-local outputs first, then materializes canonical session outputs.
|
||||
- records input provenance, output paths, adapter metadata, logs, and generated config refs.
|
||||
- skips with stage metadata when `pipeline.render.enabled=false`.
|
||||
- when `pipeline.render.enabled=false`, completes successfully with no outputs
|
||||
and records explanatory metadata. This is not an explicit self-skip: both
|
||||
manifests record success, and enabling render later requires a forced run.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
- missing normalized input fails with normalize rerun guidance.
|
||||
- missing trimmed input fails with trim rerun guidance.
|
||||
- adapter/subprocess failure fails stage.
|
||||
- empty render output files fail validation.
|
||||
|
||||
## Invariants
|
||||
|
||||
- only `format: markdown` is supported.
|
||||
- render stage owns production of built-in Markdown transcript sources.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Seriatim](../integrations/seriatim.md) owns render subprocess behavior.
|
||||
- [Configuration](../config.md#pipeline) owns render fields and defaults.
|
||||
- Implementation and tests: `internal/stage/render.go`,
|
||||
`internal/stage/render_test.go`
|
||||
|
||||
@@ -1,22 +1,41 @@
|
||||
# Stage: transcribe
|
||||
|
||||
## Purpose
|
||||
|
||||
Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `audio/*.flac` from `prepare`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/raw/<speaker>.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- discovers prepared audio from manifest inputs or canonical audio directory.
|
||||
- derives speaker ID from `.flac` basename.
|
||||
- runs WhisperX with configured concurrency/retry settings.
|
||||
- derives the transcript identity from the prepared `.flac` filename.
|
||||
- dispatches WhisperX requests through a bounded worker pool.
|
||||
- validates each output as JSON.
|
||||
- writes run-local outputs then materializes canonical transcript outputs.
|
||||
- writes run-local outputs then materializes canonical transcript outputs only
|
||||
after every planned request succeeds.
|
||||
|
||||
## Invariants
|
||||
- speaker basenames must be unique.
|
||||
|
||||
- prepared audio identities must be unique; prepare disambiguates distinct
|
||||
source paths that share a basename.
|
||||
- output path returned by adapter must match requested output path.
|
||||
- each successful output is validated before stage success.
|
||||
- an empty adapter result path means the requested path; adapters cannot select
|
||||
an alternate destination.
|
||||
- each successful output is validated before stage success, and cancellation or
|
||||
incomplete dispatch cannot be reported as a successful result.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [WhisperX](../integrations/whisperx.md) owns HTTP, retry, timeout, and
|
||||
cancellation semantics.
|
||||
- [Configuration](../config.md#pipeline) owns concurrency and other
|
||||
operator-selected values.
|
||||
- Implementation and tests: `internal/stage/transcribe.go`,
|
||||
`internal/stage/transcribe_test.go`
|
||||
|
||||
@@ -1,16 +1,21 @@
|
||||
# Stage: trim
|
||||
|
||||
## Purpose
|
||||
Produce a final-trimmed transcript. By default, the stage generates bounds and applies a bounds-driven trim.
|
||||
|
||||
Produce a final-trimmed transcript. By default, the stage generates bounds and
|
||||
applies a bounds-driven trim.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `transcripts/final.json`
|
||||
|
||||
## Outputs
|
||||
|
||||
- `transcripts/final.trimmed.json` (or configured trim output path)
|
||||
- when trim enabled: `artifacts/session_bounds.json`
|
||||
|
||||
## Key Behavior
|
||||
|
||||
When `trim.enabled=true`:
|
||||
- runs Scriptorium bounds artifact generation;
|
||||
- optionally runs render-debug output generation;
|
||||
@@ -23,6 +28,16 @@ When `trim.enabled=false`:
|
||||
- copies normalized transcript to trimmed output.
|
||||
|
||||
## Invariants
|
||||
|
||||
- normalized transcript is required input.
|
||||
- bounds output exists only in enabled trim path.
|
||||
- render-debug output is diagnostic and not a declared stage output.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Scriptorium](../integrations/scriptorium.md) owns bounds generation and
|
||||
debug-render subprocess behavior.
|
||||
- [Seriatim](../integrations/seriatim.md) owns transcript trimming behavior.
|
||||
- [Configuration](../config.md#pipeline) owns trim fields and defaults.
|
||||
- Implementation and tests: `internal/stage/trim.go`,
|
||||
`internal/stage/trim_test.go`
|
||||
|
||||
@@ -1,34 +1,64 @@
|
||||
# Internal: Storage
|
||||
|
||||
## Purpose
|
||||
Document remote object-store contracts and S3 implementation behavior.
|
||||
|
||||
Explain the object-store interface and S3 implementation used by Narratio.
|
||||
Remote key layout and lifecycle belong in [Operations](../operations.md), while
|
||||
operator-selected storage fields and credential mechanisms belong in
|
||||
[Configuration](../config.md).
|
||||
|
||||
## Primary Contract
|
||||
|
||||
`storage.ObjectStore` interface:
|
||||
|
||||
- `List(ctx, prefix)`
|
||||
- `Read(ctx, key)` returns an object body and the generation observed with it
|
||||
- `Download(ctx, key, localPath)`
|
||||
- `Upload(ctx, localPath, key, opts)`
|
||||
- `UploadConditional(ctx, source, key, opts, condition)`
|
||||
- `Exists(ctx, key)`
|
||||
|
||||
Key invariant:
|
||||
- callers pass full bucket-relative keys;
|
||||
- storage implementations do not infer campaign/session/run prefixes.
|
||||
|
||||
## Configuration
|
||||
`NewObjectStoreFromConfig` currently supports S3-backed stores from `pipeline.storage.*` config.
|
||||
`ReadObjectBounded` is the shared mechanism for small control objects. It opens
|
||||
one object version, returns the metadata observed with that body, rejects an
|
||||
oversized known size before transfer, and still performs a context-aware
|
||||
limit-plus-one read. It closes the body on every exit. Callers own the policy
|
||||
limit and add the control-object category to errors; this helper is not used for
|
||||
large artifact payloads.
|
||||
|
||||
S3 constructor behavior:
|
||||
- requires configured bucket;
|
||||
- uses region/endpoint/path-style options when set;
|
||||
- resolves credentials from configured env var names (with defaults).
|
||||
## Composition
|
||||
|
||||
`NewObjectStoreFromConfig` constructs the S3-backed implementation from
|
||||
resolved configuration. The application loads configured filesystem secrets
|
||||
before calling it. The storage adapter consumes already-resolved values; it does
|
||||
not own discovery, defaults, or configuration validation.
|
||||
|
||||
## S3 Backend Behavior
|
||||
|
||||
- normalizes object keys.
|
||||
- `List` paginates and returns normalized `ObjectInfo`.
|
||||
- A truncated S3 listing must supply a new, non-empty continuation token;
|
||||
otherwise listing fails with bucket and prefix context instead of looping.
|
||||
- `Download` writes local files with parent directory creation.
|
||||
- `Upload` streams local file and returns remote metadata.
|
||||
- `Read` binds a returned body to its S3 ETag. `UploadConditional` maps an ETag
|
||||
match or absence precondition directly to the provider request and reports a
|
||||
failed precondition without performing a local check-then-write replacement.
|
||||
- `Exists` maps not-found responses to `false`.
|
||||
|
||||
## Invariants
|
||||
|
||||
- storage layer is stateless regarding manifest/stage progression.
|
||||
- bounded reads never retain more than the caller's limit plus one byte and do
|
||||
not replace owner-specific size policy.
|
||||
- publish ordering semantics are owned by stage/app code, not storage adapters.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Contract and S3 adapter: `internal/adapters/storage`
|
||||
- Composition: `internal/app/object_store.go`
|
||||
- Tests: `internal/adapters/storage/*_test.go`,
|
||||
`internal/app/object_store_test.go`
|
||||
|
||||
@@ -1,57 +1,94 @@
|
||||
# Internal: Workspace
|
||||
|
||||
## Purpose
|
||||
Define local session layout, run-local stage layout, and cleanup guardrails.
|
||||
|
||||
## Canonical Session Layout
|
||||
Session root:
|
||||
- `{workspace.root}/work/{campaign}/{session_id}`
|
||||
Explain the helpers that construct local session and run paths, coordinate
|
||||
single-writer access, and confine cleanup. The authoritative physical layout and
|
||||
retention workflow belong in [Operations](../operations.md#local-state-layout).
|
||||
|
||||
Core directories/files:
|
||||
- `inputs/`
|
||||
- `audio/`
|
||||
- `transcripts/`
|
||||
- `artifacts/`
|
||||
- `reports/`
|
||||
- `logs/`
|
||||
- `config/`
|
||||
- `current/`
|
||||
- `runs/`
|
||||
- `previous/`
|
||||
- `manifest.json`
|
||||
- `.lock`
|
||||
## Path Ownership
|
||||
|
||||
`previous/` reserved files:
|
||||
- `previous/manifest.json`
|
||||
- `previous/artifacts/**`
|
||||
`internal/artifacts` owns canonical session, run, spool, cache, and
|
||||
previous-cache path construction. `SessionPathsFor` provides the session-scoped
|
||||
path model, and layout creation goes through `EnsureLayoutFor`. Callers should
|
||||
consume those helpers instead of rebuilding relative paths.
|
||||
|
||||
`internal/pathsafe` validates relative destinations. `internal/fileops` opens
|
||||
cleanup roots and their descendants through no-follow directory handles before
|
||||
removing them.
|
||||
|
||||
`internal/fileops` owns the ordinary workspace mode contract. On POSIX,
|
||||
`WorkspaceDirectoryMode` is setgid `02775` and `WorkspaceFileMode` is `0664`.
|
||||
`EnsureWorkspaceDirectory` reapplies the directory mode after creation so a
|
||||
restrictive umask cannot remove group access, while retaining existing ownership
|
||||
and group. Credential paths are outside this contract; the platform-specific
|
||||
operational requirements are in [Operations](../operations.md#workspace-permissions).
|
||||
|
||||
## Run-Local Stage Layout
|
||||
When run context is available, stages use:
|
||||
- `runs/{run_id}/{stage}/outputs/`
|
||||
- `runs/{run_id}/{stage}/logs/`
|
||||
- `runs/{run_id}/{stage}/reports/`
|
||||
- `runs/{run_id}/{stage}/config/`
|
||||
- `runs/{run_id}/{stage}/scratch/`
|
||||
|
||||
Run-local outputs are materialized back into canonical session paths before stage success.
|
||||
`previous/**` writes are never redirected to run-local output paths.
|
||||
`internal/stage/run_local.go` maps stage outputs and diagnostics into an
|
||||
invocation-scoped layout. Successful outputs are validated and atomically
|
||||
materialized into canonical session paths before stage success. Managed
|
||||
previous-session cache paths remain session-durable and are never redirected
|
||||
into run-local output space.
|
||||
|
||||
Extraction uses run-local receipt, stderr, and output-root helpers, then
|
||||
promotes the validated external bundle to the unique immutable Notarius bundle
|
||||
path supplied by `internal/artifacts`. `internal/fileops.PromoteDirectory`
|
||||
copies only regular files and directories to a same-filesystem temporary
|
||||
sibling. Source traversal uses confined directory handles and identity checks
|
||||
so replacing an inspected root, directory, or file is rejected rather than
|
||||
followed. The completed tree is atomically renamed without replacing an
|
||||
existing destination. Exact physical paths belong in
|
||||
[Operations](../operations.md#extraction-workflow).
|
||||
|
||||
## Locking
|
||||
`artifacts.LocalStore` enforces single-writer session lock via `.lock` file (`ErrLockConflict` on contention).
|
||||
|
||||
`artifacts.LocalStore` enforces the single-writer session lock via an
|
||||
operating-system lock held on `.lock` (`ErrLockConflict` on contention). The
|
||||
file retains owner metadata after release or process death; its existence is
|
||||
not evidence that a lock is active. Command and restore flows wait for this
|
||||
lock only while their context remains active, and report a release failure.
|
||||
|
||||
## Cleanup Semantics
|
||||
Automatic post-publish cleanup:
|
||||
- only runs when publish actually executed and succeeded;
|
||||
- requires `uploaded=true` and `current_pointer_written=true` metadata;
|
||||
- respects `pipeline.spool.delete_audio_after_publish` and `pipeline.workspace.cleanup_after_publish`;
|
||||
- refuses unsafe deletes (root delete, out-of-root delete, symlink paths).
|
||||
|
||||
Manual clean command:
|
||||
- `clean <session_id>` removes session work and spool subtree.
|
||||
- `clean --all` removes all workspace work and spool children.
|
||||
- durable cache is preserved unless `--clear-cache` is requested.
|
||||
Automatic post-publish cleanup:
|
||||
|
||||
- is created only after a successful publish commit with complete publish
|
||||
metadata, then is persisted before any deletion;
|
||||
- requires `uploaded=true`, a remote commit key, and a current commit-pointer
|
||||
key in publish metadata;
|
||||
- consumes the resolved cleanup policy described in
|
||||
[Configuration](../config.md);
|
||||
- refuses unsafe deletes (root delete, out-of-root delete, and symlinked
|
||||
ancestors or entries);
|
||||
- retries any recorded incomplete target on later invocations even when no
|
||||
publish work is selected. Missing targets are a successful, idempotent
|
||||
cleanup result only after the completion evidence is saved.
|
||||
|
||||
Manual cleanup uses the same root-confined deletion mechanism. Invocation
|
||||
syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
||||
[Operations](../operations.md#cleanup).
|
||||
|
||||
## Invariants
|
||||
|
||||
- campaign-aware session root is mandatory.
|
||||
- manifest-driven stage state is durable across runs.
|
||||
- cleanup guardrails prevent destructive root/out-of-scope deletion.
|
||||
- ordinary workspace paths retain group-writable directory and file modes across
|
||||
nested creation, replacement, and Notarius promotion.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Path model and local store: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/local.go`
|
||||
- Run-local materialization: `internal/stage/run_local.go`
|
||||
- Immutable bundle promotion: `internal/fileops/directory.go`
|
||||
- Workspace modes: `internal/fileops/modes.go`
|
||||
- Cleanup confinement: `internal/fileops/cleanup.go`,
|
||||
`internal/app/cleanup_targets.go`, `internal/app/post_publish_cleanup.go`
|
||||
- Tests: `internal/artifacts/paths_model_test.go`,
|
||||
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
||||
`internal/fileops/directory_test.go`, `internal/fileops/modes_posix_test.go`,
|
||||
`internal/fileops/cleanup_test.go`, `internal/app/cleanup_targets_test.go`,
|
||||
`internal/app/post_publish_cleanup_test.go`
|
||||
|
||||
@@ -75,16 +75,30 @@ Canonical stage order:
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `render`
|
||||
8. `analyze`
|
||||
9. `publish`
|
||||
10. `notify`
|
||||
7. `extract`
|
||||
8. `render`
|
||||
9. `analyze`
|
||||
10. `publish`
|
||||
11. `notify`
|
||||
|
||||
Execution rules:
|
||||
|
||||
- succeeded stages are skipped unless `--force` is set;
|
||||
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- force rerunning a succeeded upstream stage marks succeeded downstream stages as `stale`.
|
||||
- forcing an upstream stage marks succeeded downstream stages as `stale` before
|
||||
the replacement runs; and
|
||||
- an executed failure, changed self-skip, or success that replaces a different
|
||||
effective upstream outcome also marks succeeded downstream stages stale. A
|
||||
repeated self-skip with the same reason and no outputs is stable and does not
|
||||
perpetually rerun downstream work.
|
||||
|
||||
An explicit self-skip is a durable `skipped` stage outcome that later runs
|
||||
reconsider. It differs from successful no-output execution: disabled `render`
|
||||
and `publish`, and absent or no-executable `analyze`, record `succeeded` with
|
||||
metadata and no outputs. Ordinary later runs reuse those successful results;
|
||||
force the affected stage after enabling or configuring it. Optional artifact
|
||||
inputs are omitted only from the consuming artifact invocation and do not make
|
||||
the stage self-skip.
|
||||
|
||||
Single-stage execution:
|
||||
|
||||
@@ -101,7 +115,80 @@ Selection behavior:
|
||||
- validates names against `pipeline.scriptorium.artifacts`;
|
||||
- filters analyze execution to selected configured artifacts;
|
||||
- filters publish rules for `narratio.artifact.<name>` sources only;
|
||||
- does not suppress built-in transcript or bounds publish sources.
|
||||
- does not suppress built-in transcript, bounds, or explicitly configured
|
||||
`narratio.extraction.<name>` publish sources; and
|
||||
- never partially selects Notarius lanes.
|
||||
|
||||
## Extraction Workflow
|
||||
|
||||
When Notarius is omitted or disabled, `extract` records an explicit skipped
|
||||
outcome with reason `notarius_disabled` and no outputs. A later invocation
|
||||
reconsiders the skipped stage, so enabling Notarius does not require force.
|
||||
|
||||
When Notarius extraction is enabled, the stage consumes the final trimmed JSON
|
||||
and preserves the complete validated Notarius bundle at:
|
||||
|
||||
- `artifacts/notarius/{narratio_run_id}/`
|
||||
|
||||
The directory is immutable once promoted. Configured lanes become
|
||||
`narratio.extraction.<name>` sources for Scriptorium and explicit publish rules;
|
||||
the bundle and `index.json` are retained for audit and resume validation but
|
||||
are not selectable or published implicitly.
|
||||
|
||||
Starting a replacement clears the previous extraction payload from the current
|
||||
session-stage record. If that replacement fails or self-skips, the current
|
||||
record does not fall back to the earlier outputs. The earlier run manifest and
|
||||
immutable bundle remain available for inspection, but downstream resolution
|
||||
requires a new current successful extraction record.
|
||||
|
||||
Atomic Notarius bundle promotion is supported on Linux and macOS. On Windows
|
||||
and other operating systems, extraction fails before copying the bundle into a
|
||||
temporary promotion tree because Narratio has no verified atomic no-replace
|
||||
directory primitive there. This is an extraction limitation, not a broader
|
||||
platform-support guarantee for every Narratio workflow.
|
||||
|
||||
## External Command Lifecycle
|
||||
|
||||
When an external command is cancelled or times out, Narratio terminates its
|
||||
owned descendants as well as the command itself. Cancellation first requests
|
||||
termination where the platform supports it, then force terminates after a
|
||||
bounded wait. A command is not considered finished until its leader has been
|
||||
reaped, and descendants that keep standard output or error open cannot keep
|
||||
the invocation blocked. Other operating systems fail closed rather than launch
|
||||
a command without tree ownership.
|
||||
|
||||
Subprocess stdout and stderr diagnostics are separately redacted and capped at
|
||||
8 MiB per invocation. Narratio does not retain configured credential values in
|
||||
these logs or their error tails; reaching a capture limit terminates the command
|
||||
tree and reports which stream exceeded the limit.
|
||||
|
||||
Run-local diagnostics are:
|
||||
|
||||
- `runs/{run_id}/extract/notarius.receipt.json`
|
||||
- `runs/{run_id}/extract/notarius.stderr.log`
|
||||
- `runs/{run_id}/extract/notarius-output/` before durable promotion
|
||||
|
||||
The run-record upload is an allowlist derived from the validated run manifest,
|
||||
not a workspace scan. Each declared source is opened without following
|
||||
symlinked ancestors or the leaf, verified as a regular file, and streamed from
|
||||
that verified descriptor. Unlisted files and unsafe entries are never uploaded.
|
||||
The durable bundle is never scanned for implicit publication; only lanes named
|
||||
by explicit `pipeline.publish.outputs` rules are uploaded.
|
||||
|
||||
To intentionally replace the current extraction result, run:
|
||||
|
||||
```bash
|
||||
narratio run-stage extract 2026-04-04 --force
|
||||
```
|
||||
|
||||
Narratio automatically reruns extraction when its recorded invocation contract
|
||||
or durable output validation changes. It cannot fingerprint configuration
|
||||
files, profiles, prompts, modules, or references loaded transitively by
|
||||
Notarius. Force extraction after changing any of those inputs, even when the
|
||||
top-level Narratio and Notarius config paths remain the same. A forced extract
|
||||
marks successful downstream stages stale. Ordinary extraction failures or
|
||||
outcome changes also stale affected downstream stages, while an identical
|
||||
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
||||
|
||||
## Publish Workflow
|
||||
|
||||
@@ -119,13 +206,31 @@ narratio run-stage publish 2026-04-04 --force
|
||||
|
||||
Publish commit model:
|
||||
|
||||
- uploads run files under `{session_prefix}/runs/{run_id}/`;
|
||||
- uploads configured published outputs;
|
||||
- uploads `previous/**` cache files when present;
|
||||
- writes `current/manifest.json`;
|
||||
- writes `current/run_id.txt` last.
|
||||
- uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
|
||||
audio and the run-local Notarius staging bundle;
|
||||
- uploads configured published outputs and `previous/**` cache files into the
|
||||
same immutable run scope, including only explicitly configured extraction
|
||||
lanes;
|
||||
- writes `{session_prefix}/runs/{run_id}/commit.json` after all declared
|
||||
immutable objects are uploaded and verified; and
|
||||
- writes `{session_prefix}/current/commit-pointer.json` once, last.
|
||||
|
||||
`current/run_id.txt` is the remote current-state commit marker.
|
||||
`current/commit-pointer.json` is the remote current-state commit marker. It
|
||||
selects exactly one immutable commit, which declares the complete object set.
|
||||
|
||||
## Remote Commit Migration
|
||||
|
||||
The immutable remote commit contract uses
|
||||
`runs/{run_id}/commit.json` to declare a run's complete object set and a small
|
||||
`current/commit-pointer.json` to select it. The pointer binds the selected
|
||||
commit by version, checksum, size, and storage generation; committed artifacts
|
||||
are also checksum- and generation-bound. Readers accept this contract now and
|
||||
strictly reject mismatched or unknown data.
|
||||
|
||||
Legacy reads are limited to a coherent `current/manifest.json` and
|
||||
`current/run_id.txt` pair; a torn pair is rejected. New publication does not
|
||||
write that pair and remote commit state does not carry local
|
||||
`current_pointer_written` metadata.
|
||||
|
||||
## Publish Locks
|
||||
|
||||
@@ -139,7 +244,14 @@ Effective lock rules:
|
||||
- static and remote locks are merged;
|
||||
- static locks win on source collisions;
|
||||
- locked outputs are intentional skips;
|
||||
- lock add/remove commands mutate only remote lock state.
|
||||
- lock add/remove commands mutate only remote lock state through generation-bound
|
||||
conditional writes. A command retries a bounded number of concurrent
|
||||
conflicts while its invocation context remains active, so it never replaces a
|
||||
different lock-document generation; and
|
||||
- a publish re-reads remote locks immediately before it writes the current
|
||||
commit pointer. A lock committed before that recheck prevents selecting the
|
||||
new snapshot, even though its already-uploaded immutable objects may remain
|
||||
available for a later retry.
|
||||
|
||||
Examples:
|
||||
|
||||
@@ -165,20 +277,30 @@ Apply:
|
||||
narratio session restore 2026-04-04
|
||||
```
|
||||
|
||||
`--dry-run` does not write durable session files. It still reads the selected
|
||||
remote current state and may read object identity/content needed to classify the
|
||||
plan, so it is not a network-free operation.
|
||||
|
||||
Default restore scope:
|
||||
|
||||
- `manifest.json`
|
||||
- `transcripts/**`
|
||||
- `artifacts/**`
|
||||
- the committed session manifest and the committed transcript/artifact objects
|
||||
declared by the selected remote commit
|
||||
- `previous/**` when needed by configured previous-session artifact inputs
|
||||
|
||||
Optional:
|
||||
|
||||
- `--include-audio` to include `audio/**`
|
||||
- `--force` to overwrite local conflicts
|
||||
- `--force` to overwrite eligible conflicting regular files; it never replaces
|
||||
directories or other non-regular local targets
|
||||
|
||||
Restore writes an execution report at `reports/restore-latest.json`.
|
||||
|
||||
If restore fails after beginning installation, it leaves a durable
|
||||
`.restore-incomplete.json` marker in the session root. Pipeline runs will stop
|
||||
until you rerun the same restore command and it completes. Restore intentionally
|
||||
does not try to roll back files already installed; retrying the selected remote
|
||||
snapshot is the recovery procedure.
|
||||
|
||||
## Local State Layout
|
||||
|
||||
Session root:
|
||||
@@ -198,6 +320,10 @@ Durable session paths:
|
||||
- `config/**`
|
||||
- `runs/**`
|
||||
|
||||
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
||||
stderr, and pre-promotion output remain in the producing run's `extract`
|
||||
directory as described in [Extraction Workflow](#extraction-workflow).
|
||||
|
||||
Run-local layout:
|
||||
|
||||
- `runs/{run_id}/{stage}/outputs`
|
||||
@@ -215,6 +341,38 @@ Cache layout (durable S3 audio cache):
|
||||
|
||||
- `{cache.root}/s3/{bucket}/...`
|
||||
|
||||
Each cached audio file has an adjacent managed identity record. It binds the
|
||||
file to its remote object version and verified digest; deleting or altering the
|
||||
record simply causes Narratio to download and verify the object again.
|
||||
|
||||
### Workspace Permissions
|
||||
|
||||
Ordinary Narratio workspace content is intentionally shareable with the
|
||||
workspace group. On POSIX systems, Narratio-created workspace, spool, and cache
|
||||
directories converge on setgid `02775`; ordinary files, including manifests,
|
||||
transcripts, generated configuration, logs, reports, and Notarius artifacts,
|
||||
converge on `0664`. Narratio explicitly applies these modes so a restrictive
|
||||
caller umask does not remove group write or setgid. It does not change file or
|
||||
directory ownership: the configured workspace's existing group is inherited.
|
||||
|
||||
Windows does not implement POSIX mode bits or setgid semantics. Configure the
|
||||
workspace, spool, and cache locations with an ACL that grants the collaborating
|
||||
group read/write access, and configure credential locations with an ACL limited
|
||||
to the intended credential owner. Do not use POSIX mode displays as evidence of
|
||||
Windows access control.
|
||||
|
||||
API keys are credentials, not ordinary workspace data. Store them outside the
|
||||
shared workspace or in a separately restricted credential location; ordinary
|
||||
workspace group access must never be treated as authorization to read keys.
|
||||
On POSIX, provision a credential directory as `0700` and credential files as
|
||||
`0600`; Narratio rejects group- or other-readable configured credential paths.
|
||||
On Windows, restrict the directory and files with ACLs to the credential owner.
|
||||
|
||||
External adapter results are individually bounded before Narratio validates or
|
||||
materializes them. These per-file limits do not reserve disk space: prevent hard
|
||||
disk exhaustion with filesystem, service, container, or volume quotas sized for
|
||||
the session workload.
|
||||
|
||||
## Cleanup
|
||||
|
||||
Session-scoped cleanup:
|
||||
@@ -240,9 +398,14 @@ Rules:
|
||||
|
||||
- `clean` deletes work/spool session state;
|
||||
- cache is preserved unless `--clear-cache` is set;
|
||||
- each deletion is confined beneath its configured workspace, spool, or cache
|
||||
root and refuses symlinked paths;
|
||||
- automatic post-publish cleanup is gated by successful publish commit plus:
|
||||
- `pipeline.spool.delete_audio_after_publish=true`
|
||||
- `pipeline.workspace.cleanup_after_publish=true`
|
||||
- Narratio first records the exact run-scoped cleanup obligation. If cleanup
|
||||
reports incomplete, the remote committed snapshot remains current; rerun
|
||||
Narratio to retry only the outstanding confined local cleanup.
|
||||
|
||||
## Operational Caveats
|
||||
|
||||
|
||||
@@ -1,202 +1,272 @@
|
||||
# Narratio Architecture
|
||||
# Architecture
|
||||
|
||||
## Purpose
|
||||
This document defines Narratio's intended high-level architecture and the
|
||||
invariants that changes must preserve. Implemented component details belong in
|
||||
the [Internal Overview](../internal/overview.md) and its linked documents.
|
||||
Significant architectural decision history belongs under `docs/adr/` when such
|
||||
records exist.
|
||||
|
||||
`narratio` is a Go orchestration application for processing D&D session audio into polished transcripts and generated session artifacts.
|
||||
## System Shape
|
||||
|
||||
This document defines the development principles for the project. It is inward-facing: its audience is developers and LLM coding agents. It should guide future changes, not serve as a complete implementation reference.
|
||||
Narratio is a small Go application that turns D&D session audio into polished
|
||||
transcripts and generated session artifacts. It is an explicit, stage-driven
|
||||
orchestrator, not a general workflow engine.
|
||||
|
||||
Implemented component details belong under `docs/internal/`.
|
||||
Narratio coordinates specialized external systems rather than reimplementing
|
||||
their domains:
|
||||
|
||||
## Project Shape
|
||||
- WhisperX performs transcription;
|
||||
- Seriatim performs deterministic transcript processing and rendering;
|
||||
- Audita performs transcript correction and polishing;
|
||||
- Notarius extracts validated structured artifact bundles; and
|
||||
- Scriptorium executes prompts and produces configured artifacts.
|
||||
|
||||
Narratio is a modular, stage-driven orchestrator.
|
||||
Narratio owns orchestration, configuration resolution, session and run state,
|
||||
artifact and path modeling, manifest persistence, stage sequencing, resume,
|
||||
restore, cleanup gates, and publish semantics. External contracts are defined
|
||||
in the [integration documentation](../integrations/).
|
||||
|
||||
It coordinates specialized downstream systems rather than reimplementing their domains:
|
||||
The pipeline has one canonical ordered stage set. Configuration may enable,
|
||||
disable, or parameterize supported behavior, but it must not turn that sequence
|
||||
into an arbitrary DAG or hide orchestration in generic workflow abstractions.
|
||||
The implemented stage inventory belongs in the
|
||||
[Internal Overview](../internal/overview.md).
|
||||
|
||||
- WhisperX handles transcription.
|
||||
- Seriatim handles deterministic transcript merge/normalization/trim behavior.
|
||||
- Audita handles transcript correction and polishing.
|
||||
- Scriptorium handles prompt execution and generated artifacts.
|
||||
Narratio is contract-first without being abstraction-heavy. Interfaces and
|
||||
extension points should protect demonstrated boundaries. New abstraction is not
|
||||
itself an architectural goal.
|
||||
|
||||
Narratio owns orchestration, configuration loading, session/run state, local and remote path modeling, manifest persistence, stage sequencing, resume behavior, and publish semantics.
|
||||
## Ownership And Dependency Direction
|
||||
|
||||
Narratio should remain explicit and comprehensible. It is not intended to become a generic workflow engine.
|
||||
The application boundary owns command dispatch, configuration selection,
|
||||
production composition, session locking, and top-level lifecycle. It may depend
|
||||
on concrete implementations to assemble a run.
|
||||
|
||||
## Core Principles
|
||||
Stage orchestration expresses intent in Narratio-level data and interfaces.
|
||||
Stages may depend on configuration, manifest, artifact, path, and adapter
|
||||
contracts, but they must not depend on transport-specific request types,
|
||||
subprocess argument construction, cloud SDK types, or downstream tool internals.
|
||||
|
||||
### Modular and composable
|
||||
Adapters translate between Narratio contracts and external systems. They own
|
||||
HTTP, subprocess, notification, and object-storage mechanics, including command
|
||||
construction, transport behavior, provider response handling, and external
|
||||
error adaptation. External dependency types must remain inside the adapter that
|
||||
owns them unless that dependency is the adapter's explicit public contract.
|
||||
WhisperX HTTP behavior, Seriatim, Audita, Notarius, and Scriptorium command
|
||||
construction, notification transport, and object-storage SDK details remain
|
||||
behind these boundaries.
|
||||
|
||||
Code should be organized around clear responsibilities. Stages, adapters, config loading, manifest persistence, path construction, and storage behavior should remain separable and independently testable.
|
||||
State and path services must not infer stage policy. Storage implementations
|
||||
receive explicit bucket-relative keys and do not infer campaign, session, run,
|
||||
or root-prefix semantics. Manifest persistence records transitions but does not
|
||||
choose orchestration policy. Artifact resolution identifies and validates
|
||||
artifacts but does not execute producers.
|
||||
|
||||
### Hexagonal boundaries
|
||||
Dependencies should remain narrow and point toward Narratio-owned contracts.
|
||||
Prefer the Go standard library. Add an external dependency only when it provides
|
||||
a clear correctness, security, interoperability, or complexity benefit, and
|
||||
confine it to the boundary that needs it.
|
||||
|
||||
External systems should be isolated behind narrow adapters. Stage logic should depend on Narratio-level interfaces and data structures, not on external SDK types, subprocess argument construction, or transport-specific details.
|
||||
## Stage Boundaries
|
||||
|
||||
### Standard library preference
|
||||
Each stage has one explicit responsibility and declares:
|
||||
|
||||
Prefer the Go standard library. Add dependencies only when they provide substantial value, are necessary for an external integration, or are a widely used de facto standard.
|
||||
|
||||
Accepted examples include a YAML library for configuration and the AWS SDK for S3-compatible storage.
|
||||
|
||||
### Explicit orchestration
|
||||
|
||||
The pipeline should remain stage-driven and explicit. New behavior should be added through clear stage, adapter, config, or manifest contracts rather than implicit side effects or generic workflow abstraction.
|
||||
|
||||
## Stage Design
|
||||
|
||||
Each stage should have a clear scope of responsibility.
|
||||
|
||||
A stage should define:
|
||||
|
||||
- its purpose;
|
||||
- required input state;
|
||||
- produced output state;
|
||||
- config fields it consumes;
|
||||
- configuration it consumes;
|
||||
- external adapters it uses;
|
||||
- manifest refs it reads or writes;
|
||||
- skip, force, and resume behavior;
|
||||
- failure behavior;
|
||||
- tests that protect its contract.
|
||||
- manifest references and metadata it reads or writes;
|
||||
- skip, force, invalidation, and resume behavior; and
|
||||
- failure behavior.
|
||||
|
||||
Stages should avoid reaching across boundaries. If shared behavior is needed, prefer a helper or service with a narrow interface over duplicating ad hoc logic between stages.
|
||||
Stages write and validate run-local results before materializing canonical
|
||||
outputs where that distinction applies. A stage is complete only after its
|
||||
required outputs have been written, validated, and recorded in durable manifest
|
||||
state. Later stages depend on recorded success and artifact resolution, not
|
||||
merely on incidental files existing on disk.
|
||||
|
||||
## Transactionality and Resume
|
||||
A failed or interrupted stage must not be presented as successful. Failure
|
||||
should preserve enough local state and diagnostics for inspection, recovery,
|
||||
and resume. Forcing an upstream stage invalidates succeeded downstream work
|
||||
according to the canonical stage order.
|
||||
|
||||
A stage should behave transactionally.
|
||||
A stage may explicitly self-skip with a stable reason and no outputs. That
|
||||
outcome is persisted, clears older outputs owned by the stage, and is
|
||||
reconsidered on a later invocation. A stage may also validate whether an
|
||||
otherwise successful recorded result is still resumable; an obsolete result
|
||||
is staled and rerun, while an unsafe condition that prevents a sound decision
|
||||
stops execution.
|
||||
|
||||
A stage is complete only when its outputs have been written, validated, and recorded in the manifest. If a stage fails, Narratio should preserve enough local state for inspection, recovery, and resume.
|
||||
Shared behavior should live behind a narrow service or helper with one clear
|
||||
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
|
||||
artifact, or path policy ad hoc.
|
||||
|
||||
A failed or incomplete run must not be treated as successful. Later stages should depend on manifest-recorded success, not merely on incidental files existing on disk.
|
||||
## Manifest, Resume, And Restore
|
||||
|
||||
## Manifest Model
|
||||
The session manifest is the durable ledger for progress across invocations. It
|
||||
records session and run identity, stage state, input and output references,
|
||||
diagnostic references, checksums or provenance where useful, and non-secret
|
||||
adapter and publish metadata.
|
||||
|
||||
The manifest is the durable local ledger for a run.
|
||||
Resume and skip decisions are manifest-driven. Filesystem state may be
|
||||
inspected and validated, but file presence alone does not replace recorded
|
||||
stage state. Invocation-scoped run records provide an audit of one execution;
|
||||
they do not replace the session manifest as progress authority.
|
||||
|
||||
It should record:
|
||||
Restore treats committed remote current state as its authority. It must plan
|
||||
deterministically, confine remote-to-local paths, protect local conflicts, and
|
||||
install the validated session manifest after other restored durable files. The
|
||||
physical workflow and recovery procedures belong in
|
||||
[Operations](../operations.md).
|
||||
|
||||
- run identity;
|
||||
- stage status;
|
||||
- input and output refs;
|
||||
- logs and generated config refs;
|
||||
- checksums or provenance where useful;
|
||||
- non-secret adapter and publish metadata.
|
||||
Restore and runner transitions for one session use the same local lock. A
|
||||
durable incomplete-restore marker blocks runner reuse after a partial restore;
|
||||
safe retry, rather than rollback of arbitrary local effects, is the recovery
|
||||
mechanism. Restored manifest-local references must be confined to the selected
|
||||
local session root, never trusted as producer-machine absolute paths.
|
||||
|
||||
Resume behavior should be manifest-driven. Filesystem state may be inspected and validated, but it should not replace manifest stage state as the source of run progress.
|
||||
For the immutable remote-commit protocol, a restore or status operation binds
|
||||
to one pointer-selected commit and only its declared object identities. A force
|
||||
flag may replace an eligible regular managed file, but never turns a directory
|
||||
or other non-regular conflict into a successful restore.
|
||||
|
||||
## Adapter Boundaries
|
||||
## Configuration
|
||||
|
||||
Adapters own external integration details.
|
||||
Configuration is strict, explicit, centralized, and operator-oriented.
|
||||
|
||||
Expected boundaries:
|
||||
- YAML decoding rejects unknown fields.
|
||||
- Defaults are centralized and testable.
|
||||
- Empty configured values do not silently replace meaningful defaults.
|
||||
- Validation rejects invalid composition before stage execution where
|
||||
practical.
|
||||
- Session templating remains narrow and deterministic rather than becoming a
|
||||
general configuration language.
|
||||
- Secret values are supplied indirectly and are not persisted in ordinary
|
||||
configuration.
|
||||
|
||||
- WhisperX HTTP details stay in the WhisperX adapter.
|
||||
- Seriatim CLI construction stays in the Seriatim adapter.
|
||||
- Audita CLI construction stays in the Audita adapter.
|
||||
- Scriptorium CLI construction stays in the Scriptorium adapter.
|
||||
- Object-storage details stay behind the storage adapter interface.
|
||||
- AWS SDK types stay inside the S3 storage implementation.
|
||||
Narratio must not become a second configuration system for downstream tools.
|
||||
External systems own their runtime defaults wherever practical; Narratio passes
|
||||
the paths required by its stage contracts and explicit operator overrides. The
|
||||
field-level contract and credential-supply mechanisms belong in
|
||||
[Configuration](../config.md).
|
||||
|
||||
Stage code should express intent in Narratio terms and call adapters through narrow contracts.
|
||||
## Artifacts, Paths, And Storage
|
||||
|
||||
## Configuration Philosophy
|
||||
Artifact identities and local and remote paths are application contracts.
|
||||
Canonical helpers own workspace, spool, cache, session, run, input, transcript,
|
||||
artifact, log, report, configuration, and publish-current paths. Callers must
|
||||
not reconstruct canonical paths through scattered string concatenation.
|
||||
|
||||
Configuration should be strict, explicit, and operator-friendly.
|
||||
Reusable audio cache entries require a typed record that binds a confined,
|
||||
no-follow regular file and its digest to the selected remote object identity.
|
||||
Size alone and unqualified multipart ETags are not content-integrity evidence.
|
||||
|
||||
Principles:
|
||||
Artifact resolution is deterministic and manifest-aware. Producers materialize
|
||||
canonical outputs before reporting success, and consumers resolve declared
|
||||
artifact identities rather than infer files from unrelated directory contents.
|
||||
External artifact bundles become current only through validated immutable
|
||||
promotion and manifest records; directory presence alone never establishes
|
||||
availability.
|
||||
|
||||
- YAML decoding should reject unknown fields.
|
||||
- Defaults should be centralized and testable.
|
||||
- Empty configured values should not silently override meaningful defaults.
|
||||
- Session templating should remain narrow and deterministic.
|
||||
- Template support should serve operator convenience, not become a general configuration language.
|
||||
Writes, moves, replacements, and deletions must use narrow, explicit,
|
||||
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
||||
relative destinations must not expand the scope of an operation. Cleanup is
|
||||
permitted only through explicit operator action or configured post-publish
|
||||
gates, and it must preserve durable cache unless cache removal is explicitly
|
||||
requested.
|
||||
|
||||
Narratio should not become a secondary configuration system for downstream tools. Seriatim, Audita, and Scriptorium should own their runtime defaults wherever practical. Narratio should pass required stage-contract paths and explicit operator overrides.
|
||||
Physical layout, retention, and operational lifecycle belong in
|
||||
[Operations](../operations.md). Logical external formats and durable integration
|
||||
contracts belong under [Integrations](../integrations/).
|
||||
|
||||
## Path and Storage Discipline
|
||||
## Publish Commit Boundary
|
||||
|
||||
Local and remote paths are part of Narratio’s application contract.
|
||||
Publish has one explicit remote commit boundary. A remote run becomes current
|
||||
only after Narratio has successfully uploaded its immutable run-scoped objects,
|
||||
the immutable commit manifest, and finally the current commit pointer.
|
||||
|
||||
Code should use centralized path helpers for workspace, spool, session, run, artifact, log, config, and publish/current paths. Stages should avoid reconstructing canonical paths through scattered string concatenation.
|
||||
`current/commit-pointer.json` is the sole mutable selector and must be written
|
||||
exactly once, last. Failed, incomplete, skipped, or uncommitted publish attempts
|
||||
must not be presented as current remote state. Publish locks remain authoritative
|
||||
and are not bypassed by a forced run. Mutable remote locks use provider-enforced
|
||||
generation preconditions and are revalidated immediately before pointer
|
||||
selection; loss of that check leaves the prior committed snapshot current.
|
||||
|
||||
Storage backends should receive explicit bucket-relative keys. Storage implementations should not infer campaign, session, run, or root-prefix semantics.
|
||||
Automatic local cleanup is permitted only after a successful publish commit,
|
||||
only when explicitly configured, and only through the path-safety guardrails.
|
||||
It is a durable local obligation bound to that committed run and its exact
|
||||
targets, not an inferred side effect of the current stage list. A cleanup
|
||||
failure makes the invocation incomplete while leaving the committed remote
|
||||
snapshot authoritative; later invocations resume the recorded obligation.
|
||||
|
||||
## Publish Invariants
|
||||
## Security, Privacy, And Diagnostics
|
||||
|
||||
Publish behavior must preserve a clear commit boundary.
|
||||
Narratio distinguishes ordinary workspace data from credentials. Campaign and
|
||||
session material—including manifests, transcripts, prompts, generated
|
||||
configuration, logs, reports, diagnostics, and Notarius artifacts—is
|
||||
intentionally shareable with the configured workspace group. API-key material
|
||||
is sensitive and is not covered by the ordinary workspace-sharing policy.
|
||||
|
||||
A remote run is current only after the publish stage has successfully uploaded the run record, required published outputs, `current/manifest.json`, and finally `current/run_id.txt`.
|
||||
On POSIX systems, Narratio-created ordinary workspace directories converge on
|
||||
setgid `02775` and ordinary workspace files on `0664`, even when the caller's
|
||||
umask is restrictive. This preserves the existing workspace group for nested
|
||||
creation and atomic replacements without changing ownership. API-key storage
|
||||
uses a separate restrictive contract. On Windows, POSIX mode bits and setgid
|
||||
are not authoritative; operators must provide the equivalent shared-group and
|
||||
credential-restricted ACLs described in [Operations](../operations.md#workspace-permissions).
|
||||
|
||||
`current/run_id.txt` is the final remote commit marker and must be written last.
|
||||
Raw secrets must not be stored in pipeline, campaign, or session YAML or written
|
||||
to manifests, logs, generated configuration, reports, publish metadata,
|
||||
documentation, or examples. Secrets enter through configured environment
|
||||
variable names or secret-file references. Diagnostics should avoid transcript
|
||||
and prompt content unless a deliberate, bounded inspection mechanism requires
|
||||
it.
|
||||
|
||||
Failed, incomplete, skipped, or uncommitted publish attempts must not be presented as current remote state. Local cleanup is permitted only after successful publish commit and only when explicitly configured.
|
||||
Logs, reports, generated invocation files, generated configuration, and render
|
||||
debug files are diagnostics, not canonical pipeline products. They should be
|
||||
durable and discoverable where configured, and manifest references must preserve
|
||||
the distinction between diagnostics and artifacts.
|
||||
|
||||
## Security and Privacy
|
||||
Documentation security rules belong in the
|
||||
[Documentation Policy](documentation.md). Credential supply belongs in
|
||||
[Configuration](../config.md), while permissions, sensitive runtime-artifact
|
||||
handling, and recovery belong in [Operations](../operations.md).
|
||||
|
||||
Narratio handles private campaign material.
|
||||
## Determinism And Testability
|
||||
|
||||
Rules:
|
||||
Narratio prefers deterministic behavior where practical, including stable local
|
||||
and remote layouts, sorted operation order, predictable generated
|
||||
configuration, repeatable command construction, deterministic artifact
|
||||
resolution, and reproducible planning.
|
||||
|
||||
- Do not store raw secrets in pipeline or session YAML.
|
||||
- Use environment variable names or secret-file references for secret handling.
|
||||
- Do not write raw secret values to manifests, logs, generated configs, or publish metadata.
|
||||
- Treat transcripts, generated artifacts, prompts, reports, and logs as potentially sensitive.
|
||||
- Avoid logging transcript or prompt content unless there is a deliberate diagnostic reason.
|
||||
Run IDs and timestamps may be intentionally variable, but surrounding behavior
|
||||
must remain controllable in tests. Core behavior should be testable without live
|
||||
external services; expensive, nondeterministic, destructive, or external
|
||||
boundaries should be replaceable with focused test doubles. General testing
|
||||
philosophy and sufficiency rules belong in the [Testing Policy](testing.md).
|
||||
|
||||
## Diagnostics
|
||||
## Documentation And Decision Records
|
||||
|
||||
Diagnostics should be durable and discoverable, but distinct from canonical outputs.
|
||||
Documentation follows the [Documentation Policy](documentation.md). Current
|
||||
behavior belongs in its canonical user, operator, integration, architecture, or
|
||||
internal owner. Proposed behavior and implementation status belong under
|
||||
`docs/roadmap/`.
|
||||
|
||||
Logs, reports, generated invocation/config files, and render-debug files support debugging. Transcript tiers and configured artifacts are pipeline products.
|
||||
Significant architectural decisions may be recorded under `docs/adr/` using the
|
||||
format and lifecycle defined by the documentation policy. ADR acceptance does
|
||||
not establish that a decision has been implemented.
|
||||
|
||||
Manifest refs should preserve that distinction.
|
||||
## Architectural Non-Goals
|
||||
|
||||
## Determinism
|
||||
|
||||
Where practical, Narratio should prefer deterministic behavior:
|
||||
|
||||
- stable local path layout;
|
||||
- stable remote key layout;
|
||||
- sorted upload order;
|
||||
- predictable generated config files;
|
||||
- repeatable command construction;
|
||||
- tests that do not depend on live external services.
|
||||
|
||||
Run IDs and timestamps may be intentionally variable, but surrounding behavior should remain testable.
|
||||
|
||||
## Testing Expectations
|
||||
|
||||
Core behavior should be testable without live external services.
|
||||
|
||||
Tests should cover:
|
||||
|
||||
- config loading, defaults, and validation;
|
||||
- CLI parsing and command construction;
|
||||
- path helpers;
|
||||
- manifest transitions;
|
||||
- stage success, failure, skip, and resume behavior;
|
||||
- adapter command construction;
|
||||
- fake storage behavior;
|
||||
- publish commit ordering;
|
||||
- example config validity where practical.
|
||||
|
||||
Live S3, WhisperX, LLM, or subprocess integration tests should be explicit integration tests, not required for ordinary unit test runs.
|
||||
|
||||
## Documentation Expectations
|
||||
|
||||
Documentation must follow `docs/documentation/policy.md`.
|
||||
|
||||
Current behavior belongs in user-facing docs and `docs/internal/`. Future, planned, aspirational, experimental, or unimplemented work belongs only under `docs/roadmap/`.
|
||||
|
||||
`docs/architecture.md` should remain concise and principle-focused. It should not duplicate the full config reference, CLI reference, operations guide, or internal stage documentation.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
Narratio is not:
|
||||
Narratio does not aim to provide:
|
||||
|
||||
- a generic DAG or workflow engine;
|
||||
- a replacement configuration layer for Seriatim, Audita, or Scriptorium;
|
||||
- a storage backend abstraction beyond the needs of this pipeline;
|
||||
- a place to embed raw secrets;
|
||||
- a place for stage logic to depend directly on AWS SDK types or downstream tool internals;
|
||||
- a replacement configuration layer for WhisperX, Seriatim, Audita,
|
||||
Scriptorium, or other downstream tools;
|
||||
- a storage abstraction broader than the needs of this pipeline;
|
||||
- stage logic coupled directly to cloud SDKs, transports, subprocess details,
|
||||
or downstream implementation internals;
|
||||
- raw-secret persistence;
|
||||
- implicit cross-stage behavior that bypasses manifest and artifact contracts;
|
||||
or
|
||||
- a prompt-authoring system.
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
# Development Guide
|
||||
|
||||
## Purpose
|
||||
Canonical contributor workflow and engineering conventions for implemented Narratio behavior.
|
||||
|
||||
## Repository layout
|
||||
|
||||
- `cmd/narratio/`: CLI entrypoint.
|
||||
- `internal/app/`: command handlers, run/stage orchestration, cleanup gates, secrets loading.
|
||||
- `internal/config/`: strict YAML loading, defaults, and validation.
|
||||
- `internal/stage/`: stage implementations and stage registry/order.
|
||||
- `internal/adapters/`: external boundary adapters (WhisperX, Seriatim, Audita, Scriptorium, storage, notify).
|
||||
- `internal/manifest/`: session/run manifest types and persistence.
|
||||
- `internal/artifacts/`: canonical local/remote path helpers and local artifact store.
|
||||
- `docs/`: canonical documentation set.
|
||||
- `examples/`: maintained config examples used by tests.
|
||||
|
||||
## Build and test commands
|
||||
|
||||
- Run focused CLI behavior checks:
|
||||
|
||||
```bash
|
||||
go test ./internal/app -run TestExecute -v
|
||||
```
|
||||
|
||||
- Run config example load/validate checks:
|
||||
|
||||
```bash
|
||||
go test ./internal/config -run TestExamplesLoadAndValidate -v
|
||||
```
|
||||
|
||||
- Run full test suite:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
```
|
||||
|
||||
## Coding conventions
|
||||
|
||||
- Keep orchestration explicit and stage-driven; do not introduce generic workflow/DAG abstractions.
|
||||
- Keep external-system details inside adapter packages; stages should consume Narratio-level contracts only.
|
||||
- Use centralized path helpers from `internal/artifacts` rather than ad hoc path concatenation.
|
||||
- Preserve manifest-driven state transitions (`running`, `succeeded`, `failed`, `skipped`, `stale`) as the source of run progress.
|
||||
- Keep user/operator docs implementation-accurate; planned work belongs only under `docs/roadmap/`.
|
||||
|
||||
For design principles and invariants, see [docs/architecture.md](./architecture.md). For stage/adapter contracts, see [docs/internal/README.md](./internal/README.md).
|
||||
|
||||
## Dependency policy
|
||||
|
||||
- Prefer Go standard library where practical.
|
||||
- Add third-party dependencies only when they provide clear value for required behavior.
|
||||
- Keep dependency additions narrow to the boundary package that needs them.
|
||||
|
||||
## Change playbooks
|
||||
|
||||
### Add config fields
|
||||
|
||||
1. Add fields to config structs in `internal/config`.
|
||||
2. Set defaults in `internal/config/defaults.go` when appropriate.
|
||||
3. Add validation rules in `internal/config/validate.go`.
|
||||
4. Add or update load/validate tests in `internal/config/*_test.go`.
|
||||
5. Update canonical config docs and examples:
|
||||
- [docs/config.md](./config.md)
|
||||
- relevant files under `examples/`
|
||||
|
||||
### Add CLI flags or commands
|
||||
|
||||
1. Update command parsing and behavior in `internal/app`.
|
||||
2. Add or update command tests (`TestExecute` and command-specific tests).
|
||||
3. Update [docs/cli.md](./cli.md) and, if operator workflow changes, [docs/operations.md](./operations.md).
|
||||
|
||||
Remote-storage commands must obtain object storage through the app-level command object-store helper. Do not call `storage.NewObjectStoreFromConfig` directly from command handlers; the helper loads configured filesystem secrets before constructing the storage adapter.
|
||||
|
||||
### Add or modify stages/adapters
|
||||
|
||||
1. Implement stage behavior in `internal/stage` with clear input/output boundaries.
|
||||
2. Keep external transport/subprocess details in `internal/adapters`.
|
||||
3. Preserve manifest and publish-output semantics expected by runner and publish logic.
|
||||
4. Add/update stage and adapter tests.
|
||||
5. Update internal component contracts in `docs/internal/`.
|
||||
|
||||
### Update examples
|
||||
|
||||
1. Keep canonical examples only in `examples/`.
|
||||
2. Ensure examples load and validate through runtime config paths.
|
||||
3. Update `internal/config/load_validate_test.go` as needed.
|
||||
4. Update links in `docs/config.md` if example filenames change.
|
||||
|
||||
### Update docs and roadmap
|
||||
|
||||
1. Keep implemented behavior in canonical docs (`README`, `docs/*.md`, `docs/internal/`).
|
||||
2. Keep planned/unimplemented behavior only in `docs/roadmap/`.
|
||||
3. After completing roadmap items, remove or mark them complete in `docs/roadmap/documentation.md`.
|
||||
4. Run a link/path sweep before finalizing changes.
|
||||
@@ -1,356 +1,148 @@
|
||||
# Go Project Documentation Policy
|
||||
# Documentation Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
Project documentation must help four audiences:
|
||||
|
||||
1. users who need to run the application;
|
||||
2. administrators/operators who need to configure and operate it;
|
||||
3. developers who need to understand and change it safely;
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
||||
|
||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||
to keep Narratio documentation accurate, concise, discoverable, and resistant
|
||||
to drift for users, operators, developers, integrators, and LLM coding agents.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### 1. Keep docs concise
|
||||
### One Canonical Owner
|
||||
|
||||
Each authoritative fact belongs in one document. A non-owning document may give
|
||||
a short, stable summary for orientation, but it must link to the canonical owner
|
||||
instead of repeating volatile details.
|
||||
|
||||
Volatile details include commands, flags, configuration fields and defaults,
|
||||
stage or integration keys, schemas, file names, paths, status codes, retry
|
||||
behavior, and runtime guarantees. If readers could reasonably treat a statement
|
||||
as a contract, maintain it only in the owning document.
|
||||
|
||||
Each document should cover a defined scope and only the essentials for that scope.
|
||||
### Current And Future Behavior
|
||||
|
||||
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||
Partial features may be described only to their implemented boundary.
|
||||
|
||||
ADRs are the narrow exception: an ADR may record an accepted architectural
|
||||
decision before implementation, but acceptance must not be presented as proof
|
||||
that the behavior exists. The roadmap owns implementation status and sequencing
|
||||
until the decision is implemented. Current architecture, user, operator,
|
||||
integration, and internal documentation are updated when the behavior lands.
|
||||
|
||||
### Audience And Detail
|
||||
|
||||
Write for the document's stated audience and include only the detail needed for
|
||||
its owned topic. User and operator docs should not expose implementation detail.
|
||||
Developer docs should link to user-facing and external contracts rather than
|
||||
restate them.
|
||||
|
||||
### Examples
|
||||
|
||||
Avoid:
|
||||
- long background explanations;
|
||||
- repeated reference material;
|
||||
- implementation detail in user-facing docs;
|
||||
- aspirational language outside roadmap docs;
|
||||
- verbose examples where one minimal example is clearer.
|
||||
|
||||
### 2. Document only implemented behavior outside roadmap files
|
||||
|
||||
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
|
||||
|
||||
- `docs/roadmap/`
|
||||
|
||||
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
|
||||
|
||||
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
|
||||
|
||||
### 3. Use canonical homes
|
||||
|
||||
Each type of information should have one canonical location.
|
||||
|
||||
Canonical homes:
|
||||
|
||||
- project purpose and quickstart: `README.md`
|
||||
- development principles: `docs/architecture.md`
|
||||
- configuration reference: `docs/config.md`
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
- troubleshooting: `docs/troubleshooting.md`
|
||||
- implemented internals: `docs/internal/`
|
||||
- future work: `docs/roadmap/`
|
||||
- contributor workflow: `docs/development.md`
|
||||
- copyable examples: `examples/`
|
||||
|
||||
Other files should summarize briefly and link to the canonical source.
|
||||
|
||||
### 4. Keep examples real
|
||||
|
||||
Examples should be valid, maintained, and free of secrets.
|
||||
|
||||
Where practical:
|
||||
- example configs should load successfully;
|
||||
- example commands should match real CLI syntax;
|
||||
- important examples should be covered by tests.
|
||||
|
||||
## Documentation Profiles
|
||||
|
||||
All projects require:
|
||||
|
||||
- `README.md`
|
||||
- `docs/architecture.md`
|
||||
|
||||
Additional docs depend on the project.
|
||||
|
||||
### Small library
|
||||
|
||||
Recommended:
|
||||
- `docs/development.md`, if contributor conventions are non-obvious
|
||||
|
||||
### Simple CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/development.md`
|
||||
|
||||
### Config-driven CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
|
||||
Recommended:
|
||||
- `examples/`
|
||||
- `docs/development.md`
|
||||
|
||||
### Stateful or operator-facing application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `examples/`
|
||||
- `docs/development.md`
|
||||
|
||||
### Modular, staged, service-oriented, or orchestration application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
## Required Documents
|
||||
|
||||
### README.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
The README is the outward-facing project orientation page.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. concise description;
|
||||
2. elevator pitch;
|
||||
3. shortest useful command or usage example;
|
||||
4. links to targeted docs.
|
||||
|
||||
The README should be short. It is not a manual.
|
||||
|
||||
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.)
|
||||
|
||||
### docs/architecture.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
`docs/architecture.md` is required for every project.
|
||||
|
||||
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
|
||||
|
||||
It should include:
|
||||
|
||||
- project shape;
|
||||
- core design principles;
|
||||
- package and boundary philosophy;
|
||||
- state/persistence philosophy, if applicable;
|
||||
- external integration philosophy, if applicable;
|
||||
- error-handling and logging principles;
|
||||
- testing expectations;
|
||||
- documentation expectations;
|
||||
- architectural invariants;
|
||||
- explicit non-goals, if useful.
|
||||
|
||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||
|
||||
### docs/config.md
|
||||
|
||||
**Audience:** administrators, operators, advanced users
|
||||
|
||||
Required for applications with configuration files.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. config file locations and discovery precedence;
|
||||
2. minimal working config;
|
||||
3. production-oriented config;
|
||||
4. full configuration reference;
|
||||
5. secrets handling, if applicable;
|
||||
6. links to maintained examples.
|
||||
|
||||
The full configuration reference should be canonical.
|
||||
|
||||
### docs/cli.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
Required for CLI applications.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. shortest useful command;
|
||||
2. command overview;
|
||||
3. complete flag reference;
|
||||
4. common workflows;
|
||||
5. diagnostic or recovery commands, if applicable.
|
||||
|
||||
Explain when commands are useful, not just their syntax.
|
||||
|
||||
### docs/operations.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
|
||||
It should cover:
|
||||
|
||||
- normal workflow;
|
||||
- filesystem layout;
|
||||
- remote storage layout, if applicable;
|
||||
- logs and manifests;
|
||||
- resume/retry behavior;
|
||||
- cleanup behavior;
|
||||
- archive/backup behavior;
|
||||
- safe recovery procedures;
|
||||
- operational caveats.
|
||||
|
||||
### docs/troubleshooting.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Recommended once recurring failure modes exist.
|
||||
|
||||
Each entry should include:
|
||||
|
||||
- symptom;
|
||||
- likely cause;
|
||||
- diagnostic command or inspection step;
|
||||
- safe fix;
|
||||
- relevant links.
|
||||
|
||||
### docs/development.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects maintained by humans and LLM coding agents.
|
||||
|
||||
It should include:
|
||||
|
||||
- repository layout;
|
||||
- build/test commands;
|
||||
- coding conventions;
|
||||
- dependency policy;
|
||||
- how to add config fields;
|
||||
- how to add CLI flags;
|
||||
- how to add stages/modules/adapters, if applicable;
|
||||
- how to update examples;
|
||||
- documentation update expectations.
|
||||
|
||||
### docs/internal/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for modular, staged, service-oriented, or orchestration projects.
|
||||
|
||||
This directory describes implemented internal components. It is not the roadmap.
|
||||
|
||||
Use one file per major component where useful.
|
||||
|
||||
Each component doc should include:
|
||||
|
||||
1. purpose;
|
||||
2. inputs and outputs;
|
||||
3. boundaries;
|
||||
4. config fields used;
|
||||
5. external adapters used;
|
||||
6. state or manifest behavior, if applicable;
|
||||
7. skip/resume behavior, if applicable;
|
||||
8. failure behavior;
|
||||
9. tests to inspect before changing;
|
||||
10. architectural invariants.
|
||||
|
||||
### docs/roadmap/
|
||||
|
||||
**Audience:** maintainers, developers, LLM coding agents
|
||||
|
||||
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
|
||||
|
||||
Roadmap docs should clearly distinguish:
|
||||
|
||||
- proposed work;
|
||||
- accepted plans;
|
||||
- deferred ideas;
|
||||
- rejected ideas;
|
||||
- implementation prompts or task breakdowns, if useful.
|
||||
|
||||
Roadmap docs should not be confused with current behavior.
|
||||
|
||||
### docs/integrations/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
## Examples Directory
|
||||
|
||||
Projects with non-trivial configuration or workflows should include `examples/`.
|
||||
|
||||
Useful examples include:
|
||||
|
||||
- minimal working config;
|
||||
- production-oriented config;
|
||||
- full annotated config;
|
||||
- local development config;
|
||||
- remote/object-storage config;
|
||||
- minimal session/input file.
|
||||
|
||||
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
|
||||
|
||||
## Security and Privacy
|
||||
|
||||
Docs and examples must not include:
|
||||
|
||||
- real API keys;
|
||||
- tokens;
|
||||
- passwords;
|
||||
- private keys;
|
||||
- private environment dumps;
|
||||
- sensitive user data;
|
||||
- raw private transcripts;
|
||||
- private infrastructure details unless intentionally public.
|
||||
|
||||
Document secret-handling mechanisms, not actual secret values.
|
||||
|
||||
## Maintenance Rules
|
||||
|
||||
When docs change, verify the affected behavior.
|
||||
|
||||
Where practical:
|
||||
|
||||
- load example config files in tests;
|
||||
- test CLI examples or command parser behavior;
|
||||
- validate documented flags against real flags;
|
||||
- remove stale references;
|
||||
- update links after renames;
|
||||
- keep roadmap content out of non-roadmap docs.
|
||||
|
||||
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
|
||||
|
||||
Documentation is complete only when it matches the current code.
|
||||
|
||||
## Documentation Change Checklist
|
||||
|
||||
Before merging documentation changes, verify:
|
||||
|
||||
- README is concise and orientation-focused.
|
||||
- `docs/architecture.md` describes development principles.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
- Defaults appear in the canonical config reference.
|
||||
- No secrets or private data are included.
|
||||
- Links are accurate.
|
||||
Complete copyable files belong in `examples/`. Documentation may use the
|
||||
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||
to maintained examples instead of embedding a second complete copy.
|
||||
|
||||
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||
configuration used in documentation should match the application.
|
||||
|
||||
### Security And Privacy
|
||||
|
||||
Documentation and examples must not contain real credentials, private keys,
|
||||
private environment dumps, sensitive source material, or private infrastructure
|
||||
details unless intentionally public. Document secret-handling mechanisms, not
|
||||
secret values.
|
||||
|
||||
## Canonical Ownership
|
||||
|
||||
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||
| --- | --- | --- | --- |
|
||||
| Product orientation and minimal end-to-end quickstart | `README.md` | What Narratio is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
|
||||
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, output conventions, and exit behavior. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, stage implementation details. |
|
||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schemas, fields, defaults, environment overrides, validation rules, and user-selectable stage or integration settings. | Complete example files, CLI syntax, runtime state lifecycle, implementation details. |
|
||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and remote-state layout, output and diagnostic handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical artifact schemas, implementation mechanics. |
|
||||
| Troubleshooting | `docs/troubleshooting.md` | Symptom-driven diagnosis, likely causes, safe inspection steps and remedies, and links to relevant contracts. | CLI syntax, configuration definitions, operational procedures, integration contracts, implementation mechanics. |
|
||||
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical artifact paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||
| Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
|
||||
|
||||
Documents that do not exist are required only when the corresponding interface
|
||||
or responsibility exists. Do not create placeholder API, consumer, integration,
|
||||
or operations documents for behavior the application does not have.
|
||||
|
||||
## Boundary Rules
|
||||
|
||||
### Orientation
|
||||
|
||||
The README owns product orientation. The developer guide routes contributors.
|
||||
Architecture owns normative structure. Internal overview owns the current
|
||||
concrete component map. These documents may link to one another but should not
|
||||
maintain parallel package or behavior descriptions.
|
||||
|
||||
### Commands, Configuration, Operations, And Troubleshooting
|
||||
|
||||
CLI documentation answers how to invoke the application. Configuration
|
||||
documentation answers what settings mean. Operations answers what happens to
|
||||
runtime state and how to operate or recover the application. Troubleshooting
|
||||
starts from observable symptoms and links readers to the owning command,
|
||||
configuration, operational, or integration contract. When a workflow crosses
|
||||
these topics, choose the document that owns the task and link to the other
|
||||
contracts.
|
||||
|
||||
### Contracts And Implementation
|
||||
|
||||
Integration and API documents define externally observable shapes and
|
||||
semantics. Internal documents explain how Narratio implements or consumes those
|
||||
contracts. Internal docs may name a field, file, or protocol to identify a
|
||||
dependency, but must link to its canonical contract for the definition.
|
||||
|
||||
### Security Topics
|
||||
|
||||
This policy owns what documentation and examples may contain. Architecture owns
|
||||
application security invariants. Configuration owns credential-supply
|
||||
mechanisms. Operations owns permissions and handling of sensitive runtime
|
||||
artifacts. Troubleshooting owns safe diagnostic and remediation guidance.
|
||||
Internal docs own implementation mechanisms only.
|
||||
|
||||
## Architecture Decision Records
|
||||
|
||||
Use sequentially numbered ADR filenames such as
|
||||
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
|
||||
|
||||
1. title;
|
||||
2. status;
|
||||
3. date;
|
||||
4. context;
|
||||
5. decision;
|
||||
6. alternatives considered;
|
||||
7. consequences.
|
||||
|
||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
||||
belong in the roadmap.
|
||||
|
||||
## Maintenance
|
||||
|
||||
When behavior changes, update its canonical owner in the same change. If
|
||||
ownership moves, remove the old definition and replace it with a link where
|
||||
navigation remains useful.
|
||||
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||
implementation;
|
||||
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||
- remove stale references and validate links;
|
||||
- confirm that non-owning documents summarize and link rather than redefine;
|
||||
- confirm that no secrets or sensitive private data were added.
|
||||
|
||||
296
docs/policy/testing.md
Normal file
296
docs/policy/testing.md
Normal file
@@ -0,0 +1,296 @@
|
||||
# Testing Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||
|
||||
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||
|
||||
## Every test has a cost
|
||||
|
||||
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||
|
||||
A test must be:
|
||||
|
||||
- written and reviewed;
|
||||
- understood by future maintainers and coding agents;
|
||||
- executed in local and CI workflows;
|
||||
- diagnosed when it fails;
|
||||
- updated when legitimate behavior changes;
|
||||
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||
|
||||
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||
|
||||
A test is warranted only when the confidence it provides justifies these costs.
|
||||
|
||||
Apply this cost-benefit analysis at two levels:
|
||||
|
||||
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||
|
||||
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||
|
||||
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||
|
||||
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||
|
||||
## Default testing style
|
||||
|
||||
Use a **classical/Detroit-style** approach:
|
||||
|
||||
- Test observable behavior, resulting state, contracts, and invariants.
|
||||
- Use real internal collaborators when they are fast and deterministic.
|
||||
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
|
||||
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
|
||||
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
|
||||
|
||||
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||
|
||||
## Test execution requirements
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
|
||||
|
||||
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
|
||||
|
||||
## What deserves tests
|
||||
|
||||
Prioritize tests for:
|
||||
|
||||
1. Public and package-level contracts.
|
||||
2. Domain rules and important invariants.
|
||||
3. Boundary conditions and malformed input.
|
||||
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||
5. Serialization, schemas, compatibility, and round trips.
|
||||
6. Previously observed or plausible regressions.
|
||||
7. Representative integration and end-to-end workflows.
|
||||
|
||||
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||
|
||||
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||
|
||||
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||
|
||||
## Choose the right test boundary
|
||||
|
||||
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||
|
||||
This is often the package API, but it may instead be:
|
||||
|
||||
- a smaller pure function when dense domain logic is most clearly isolated there;
|
||||
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||
|
||||
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
|
||||
|
||||
## Test behavior, not implementation
|
||||
|
||||
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
|
||||
|
||||
Before adding or retaining a test, ask:
|
||||
|
||||
> What realistic defect would this test catch?
|
||||
|
||||
A test is suspect when its main purpose is to detect that someone:
|
||||
|
||||
- changed an internal constant;
|
||||
- renamed or split a private helper;
|
||||
- reordered equivalent internal operations;
|
||||
- changed incidental formatting;
|
||||
- replaced one correct algorithm with another; or
|
||||
- refactored internal object structure without changing behavior.
|
||||
|
||||
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||
|
||||
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
|
||||
|
||||
## Expected effects of different changes
|
||||
|
||||
Use the following expectations when evaluating test failures and test maintenance:
|
||||
|
||||
| Change | Expected effect on tests |
|
||||
|---|---|
|
||||
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
|
||||
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
|
||||
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
|
||||
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
|
||||
|
||||
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
|
||||
|
||||
## Separate mechanism from policy
|
||||
|
||||
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||
|
||||
For example, do not encode an internal concurrency limit indirectly:
|
||||
|
||||
```go
|
||||
// Production policy:
|
||||
const maxConcurrency = 4
|
||||
|
||||
// Brittle test:
|
||||
err := startProcesses(5)
|
||||
require.Error(t, err)
|
||||
```
|
||||
|
||||
Instead, test the mechanism relationally:
|
||||
|
||||
```go
|
||||
const limit = 2
|
||||
runner := NewRunner(limit)
|
||||
|
||||
require.NoError(t, runner.Start(limit))
|
||||
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||
```
|
||||
|
||||
The test should prove:
|
||||
|
||||
- the configured limit is accepted; and
|
||||
- one beyond the configured limit is rejected.
|
||||
|
||||
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||
|
||||
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||
|
||||
For concurrency limits, test both kinds of behavior when relevant:
|
||||
|
||||
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||
|
||||
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||
|
||||
## Avoid semantic duplication across layers
|
||||
|
||||
Each behavior should have a clear test owner.
|
||||
|
||||
- Parser tests own parsing cases.
|
||||
- Validator tests own validation rules.
|
||||
- Domain tests own transformations and invariants.
|
||||
- Adapter tests own external integration behavior.
|
||||
- Orchestrator tests own coordination and failure propagation.
|
||||
- CLI tests own argument and configuration mapping.
|
||||
- End-to-end tests prove that representative assembled workflows work.
|
||||
|
||||
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||
|
||||
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||
|
||||
## Use test doubles deliberately
|
||||
|
||||
Choose the least elaborate test double that provides the required control or observation.
|
||||
|
||||
As a default:
|
||||
|
||||
1. Prefer real collaborators when they are fast and deterministic.
|
||||
2. Use small in-memory fakes when realistic stateful behavior is helpful.
|
||||
3. Use stubs when a dependency only needs to provide controlled responses.
|
||||
4. Use mocks when the interaction itself is contractual.
|
||||
|
||||
Mocks are appropriate when the contract includes facts such as:
|
||||
|
||||
- a notification is sent exactly once;
|
||||
- a transaction is committed only after successful writes;
|
||||
- cancellation reaches a subprocess;
|
||||
- an expensive API is called no more than once; or
|
||||
- a security audit event is emitted.
|
||||
|
||||
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||
|
||||
## Go-specific guidance
|
||||
|
||||
Use:
|
||||
|
||||
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||
- `t.TempDir()` for real filesystem behavior;
|
||||
- `httptest.Server` for realistic HTTP interactions;
|
||||
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||
- golden files only when the complete output is intentionally stable;
|
||||
- integration tests where correctness depends on component interaction; and
|
||||
- a small number of representative end-to-end tests.
|
||||
|
||||
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||
|
||||
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
|
||||
|
||||
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||
|
||||
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||
|
||||
## Coverage
|
||||
|
||||
Coverage is a diagnostic, not a target.
|
||||
|
||||
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||
|
||||
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||
|
||||
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||
|
||||
## Regression tests
|
||||
|
||||
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||
|
||||
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||
|
||||
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||
|
||||
## Deleting or rewriting tests
|
||||
|
||||
Tests are maintained code, not permanent historical artifacts.
|
||||
|
||||
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||
|
||||
Strong candidates include tests that:
|
||||
|
||||
- require updates after harmless internal changes;
|
||||
- directly assert private constants without protecting a real contract;
|
||||
- duplicate the same policy across several layers;
|
||||
- verify mock choreography rather than outcomes;
|
||||
- snapshot large amounts of incidental output;
|
||||
- test trivial private helpers already exercised through stable package behavior;
|
||||
- protect risks already covered more effectively elsewhere;
|
||||
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||
- no longer correspond to a plausible failure mode.
|
||||
|
||||
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||
|
||||
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||
|
||||
## Reviewing a proposed test
|
||||
|
||||
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||
|
||||
1. What realistic defect would it catch?
|
||||
2. How likely is that defect?
|
||||
3. How consequential would it be?
|
||||
4. Is the behavior already protected elsewhere?
|
||||
5. At which layer should this behavior be owned?
|
||||
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||
8. What should cause this test to fail?
|
||||
9. What legitimate changes should not cause this test to fail?
|
||||
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||
11. Is there a smaller or more direct test that protects the same risk?
|
||||
|
||||
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||
|
||||
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||
|
||||
## Definition of sufficient
|
||||
|
||||
A test suite is sufficient when:
|
||||
|
||||
- important contracts and invariants are protected;
|
||||
- meaningful boundaries and failure modes are exercised;
|
||||
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||
- important external boundaries have realistic integration coverage;
|
||||
- representative complete workflows are tested;
|
||||
- failures provide useful signal rather than redundant noise;
|
||||
- legitimate internal changes usually do not require test edits; and
|
||||
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||
|
||||
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
|
||||
|
||||
The governing rule is:
|
||||
|
||||
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||
@@ -24,6 +24,8 @@ Safe fix:
|
||||
|
||||
- pass explicit `--config`, `--campaign` or `--campaign-file`, and `--session`.
|
||||
|
||||
Relevant reference: [Configuration discovery](./config.md#discovery-and-selection).
|
||||
|
||||
## Session template placeholders rejected
|
||||
|
||||
Symptom:
|
||||
@@ -44,6 +46,8 @@ Safe fix:
|
||||
|
||||
- generate concrete session YAML with `narratio session init`.
|
||||
|
||||
Relevant reference: [Operations: Session Initialization](./operations.md#session-initialization).
|
||||
|
||||
## Strict decode or schema validation failure
|
||||
|
||||
Symptom:
|
||||
@@ -62,7 +66,10 @@ narratio session plan 2026-04-04 --config /path/pipeline.yml --campaign-file /pa
|
||||
|
||||
Safe fix:
|
||||
|
||||
- align config with [docs/config.md](./config.md) and maintained files under `examples/`.
|
||||
- align config with [Configuration](./config.md) and the
|
||||
[maintained examples](../examples/README.md).
|
||||
|
||||
Relevant reference: [Configuration](./config.md).
|
||||
|
||||
## Audio mode conflict
|
||||
|
||||
@@ -74,10 +81,18 @@ Likely cause:
|
||||
|
||||
- configured both local and S3 session audio inputs.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session validate 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- use local mode (`audio_dir` or `audio_files`) or S3 mode (`audio_s3.prefix`), not both.
|
||||
|
||||
Relevant reference: [Session configuration](./config.md#session).
|
||||
|
||||
## `--artifacts` selection error
|
||||
|
||||
Symptom:
|
||||
@@ -90,10 +105,159 @@ Likely causes:
|
||||
- empty list entry (for example trailing comma);
|
||||
- `run-stage` used with non-`analyze`/`publish` target.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session artifacts 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
|
||||
- provide only configured keys and use `--artifacts` with supported commands/stages.
|
||||
|
||||
Relevant reference: [CLI artifact selection](./cli.md).
|
||||
|
||||
## Notarius executable missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails while resolving or starting the Notarius executable.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- `pipeline.notarius.binary` is not installed, executable, or on `PATH`;
|
||||
- a configured executable path is wrong.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- install a compatible Notarius release or correct the binary setting, then
|
||||
rerun extraction.
|
||||
|
||||
Relevant references: [Notarius configuration](./config.md#notarius-output-entries)
|
||||
and [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Notarius exits nonzero
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction reports a Notarius exit error instead of a receipt.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
- inspect `runs/{run_id}/extract/notarius.stderr.log`; stdout is reserved for
|
||||
the receipt and is not merged with diagnostics.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct the reported Notarius pipeline, input, provider, or configuration
|
||||
failure and rerun extraction. Do not edit a staged output bundle into place.
|
||||
|
||||
After a failed replacement, an older immutable bundle may still exist even
|
||||
though the current session manifest has no successful extraction payload. This
|
||||
is expected audit state, not a signal to relink the old bundle manually.
|
||||
|
||||
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Atomic Notarius promotion unsupported
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails with `atomic no-replace directory promotion is unsupported`
|
||||
before a durable bundle or temporary promotion tree is created.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Narratio is running on an operating system other than Linux, macOS, or
|
||||
Windows, where the required atomic no-replace directory primitive has not
|
||||
been implemented and verified.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- run extraction on Linux, macOS, or Windows. Do not replace the atomic commit
|
||||
with a manual copy or move; the session manifest must never observe a partial
|
||||
or overwritten bundle.
|
||||
|
||||
This is an extraction-specific platform boundary, not a support statement for
|
||||
unrelated Narratio workflows. See
|
||||
[Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Notarius receipt or index incompatible
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction rejects the receipt schema, pipeline identity, bundle/index path,
|
||||
lane descriptor, or payload path even though Notarius exited successfully.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- Narratio and Notarius versions disagree on their consumer contract;
|
||||
- the configured pipeline or lane constraints are stale;
|
||||
- output paths escape the bundle or traverse symlinks.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- compare installed Notarius output with the canonical Notarius contracts,
|
||||
including receipt `index_file: index.json` and index management names
|
||||
`manifest.json`, `rejected.json`, and `warnings.json`; align
|
||||
`pipeline.notarius` constraints and rerun. Do not bypass confinement or schema
|
||||
checks.
|
||||
|
||||
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Required Notarius lane rejected or missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails because a configured lane is rejected, missing, duplicated,
|
||||
or incompatible, including after a zero exit.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- inspect the Notarius diagnostic log and bundle rejection/warning information;
|
||||
- correct the Notarius module or the exact declared lane contract;
|
||||
- remove an output declaration only if downstream consumers genuinely no longer
|
||||
require that source, then rerun extraction.
|
||||
|
||||
Every configured output is required. Narratio does not promote a partial result.
|
||||
|
||||
## Extraction resume invalidated
|
||||
|
||||
Symptom:
|
||||
|
||||
- a previously successful extraction runs again during ordinary continuation.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the executable/config path, pipeline ID, timeout, working directory, or
|
||||
configured output contracts changed;
|
||||
- the durable bundle, index, lane set, provenance, regular-file status, or
|
||||
checksum no longer validates.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- allow the automatic rerun after verifying the current configuration. Treat
|
||||
an unsafe path or symlink error as filesystem corruption or tampering and
|
||||
investigate it rather than replacing files manually.
|
||||
|
||||
## Notarius transitive configuration changed
|
||||
|
||||
Symptom:
|
||||
|
||||
- Notarius profiles, prompts, modules, imported files, or references changed,
|
||||
but Narratio still considers the previous extraction resumable.
|
||||
|
||||
Safe fix:
|
||||
|
||||
```bash
|
||||
narratio run-stage extract 2026-04-04 --force
|
||||
```
|
||||
|
||||
Narratio fingerprints its invocation contract, not the contents of transitive
|
||||
Notarius inputs. Always force extraction after changing them; downstream
|
||||
successful stages are then marked stale normally.
|
||||
|
||||
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Previous-session artifact input missing
|
||||
|
||||
Symptom:
|
||||
@@ -124,6 +288,8 @@ or rerun prepare after correcting session config:
|
||||
narratio run-stage prepare 2026-04-04 --force
|
||||
```
|
||||
|
||||
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
|
||||
|
||||
## Session lock conflict (`.lock`)
|
||||
|
||||
Symptom:
|
||||
@@ -133,7 +299,7 @@ Symptom:
|
||||
Likely causes:
|
||||
|
||||
- another process is running for the same session;
|
||||
- stale lock left by interrupted process.
|
||||
- a process still holds the operating-system lock while it is shutting down.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
@@ -145,7 +311,10 @@ ps aux | grep narratio
|
||||
Safe fix:
|
||||
|
||||
- wait for active process completion;
|
||||
- remove stale lock only after confirming no live process owns it.
|
||||
- retry after an interrupted holder has exited; the kernel releases its lock
|
||||
even though the `.lock` metadata file remains for inspection.
|
||||
|
||||
Relevant reference: [Operations: Local State Layout](./operations.md#local-state-layout).
|
||||
|
||||
## Restore conflict without `--force`
|
||||
|
||||
@@ -168,6 +337,8 @@ Safe fix:
|
||||
- review conflicts;
|
||||
- rerun with `--force` only when remote state should overwrite local.
|
||||
|
||||
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
|
||||
|
||||
## Restore current-state discovery failure
|
||||
|
||||
Symptom:
|
||||
@@ -191,6 +362,8 @@ Safe fix:
|
||||
- resolve storage/auth issue;
|
||||
- republish from healthy local state if current pointer is missing.
|
||||
|
||||
Relevant reference: [Operations: Publish Workflow](./operations.md#publish-workflow).
|
||||
|
||||
## Publish output failure
|
||||
|
||||
Symptom:
|
||||
@@ -217,6 +390,8 @@ Safe fix:
|
||||
- correct publish source/destination rules;
|
||||
- retry after storage failure is resolved.
|
||||
|
||||
Relevant reference: [Publish configuration](./config.md#publish-configuration-summary).
|
||||
|
||||
## Render markdown source missing
|
||||
|
||||
Symptom:
|
||||
@@ -232,7 +407,6 @@ Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio session status 2026-04-04
|
||||
narratio run-stage render 2026-04-04 --force
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
@@ -244,6 +418,8 @@ narratio run-stage render 2026-04-04 --force
|
||||
narratio run-stage analyze 2026-04-04 --force
|
||||
```
|
||||
|
||||
Relevant reference: [Operations: Stage Execution](./operations.md#stage-execution-and-continuation-behavior).
|
||||
|
||||
## Secrets or storage credential failure
|
||||
|
||||
Symptom:
|
||||
@@ -260,7 +436,7 @@ Diagnostics:
|
||||
|
||||
```bash
|
||||
ls -la /path/to/secrets_dir
|
||||
env | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
|
||||
env | sed 's/=.*//' | grep -E 'OBJECT_STORAGE|AWS|AUDITA|SCRIPTORIUM'
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
@@ -269,6 +445,8 @@ Safe fix:
|
||||
- provide required env vars;
|
||||
- keep secret values out of YAML.
|
||||
|
||||
Relevant reference: [Secrets](./config.md#secrets-handling).
|
||||
|
||||
## S3 audio prepare failure
|
||||
|
||||
Symptom:
|
||||
@@ -284,7 +462,7 @@ Likely causes:
|
||||
Diagnostics:
|
||||
|
||||
```bash
|
||||
narratio run-stage prepare 2026-04-04 --force
|
||||
narratio session validate 2026-04-04
|
||||
```
|
||||
|
||||
Safe fix:
|
||||
@@ -292,6 +470,8 @@ Safe fix:
|
||||
- verify prefix contents and storage access;
|
||||
- keep session audio mode consistent.
|
||||
|
||||
Relevant reference: [Operations](./operations.md).
|
||||
|
||||
## References
|
||||
|
||||
- [docs/cli.md](./cli.md)
|
||||
|
||||
48
examples/README.md
Normal file
48
examples/README.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# Maintained Examples
|
||||
|
||||
These files are safe, copyable starting points for Narratio configuration and
|
||||
input structure. Replace placeholder identifiers, storage names, integration
|
||||
URLs, and paths for the target environment. Field meanings and defaults belong
|
||||
in the [configuration reference](../docs/config.md).
|
||||
|
||||
## Pipeline Configuration
|
||||
|
||||
- [Minimal pipeline](pipeline.minimal.yml): campaign discovery plus the required
|
||||
WhisperX URL.
|
||||
- [Production-shaped pipeline](pipeline.production.yml): S3 storage, publish,
|
||||
external tools, and configured Scriptorium artifacts.
|
||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||
pipeline section with explanatory comments.
|
||||
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||
Scriptorium artifact consuming only three declared Notarius lanes.
|
||||
|
||||
The existing `internal/config` example test loads and validates each pipeline
|
||||
with the sample campaign and a compatible local- or S3-audio session.
|
||||
|
||||
## Campaign And Session Configuration
|
||||
|
||||
- [Sample campaign](campaigns/sample-campaign/campaign.yml), its
|
||||
[session template](campaigns/sample-campaign/session.template.yml), and its
|
||||
adjacent stable inputs provide a complete campaign directory shape.
|
||||
- [Local-audio session](session.local-audio.yml) and
|
||||
[S3-audio session](session.s3-audio.yml) are concrete session files.
|
||||
- [Session template](session.template.yml) and the campaign-local equivalent
|
||||
demonstrate the narrow placeholder syntax consumed by `session init`; they
|
||||
are templates, not runtime session files.
|
||||
|
||||
## Input Fixtures
|
||||
|
||||
- [Speakers](speakers.yml), [autocorrect](autocorrect.yml), and
|
||||
[glossary](glossary.yml) show the standalone input shapes.
|
||||
- The sample campaign references its local
|
||||
[speakers](campaigns/sample-campaign/speakers.yml),
|
||||
[autocorrect](campaigns/sample-campaign/autocorrect.yml),
|
||||
[glossary](campaigns/sample-campaign/glossary.yml),
|
||||
[players](campaigns/sample-campaign/players.yml), and
|
||||
[party](campaigns/sample-campaign/party.yml) fixtures.
|
||||
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
|
||||
reserves the expected filename and directory shape. Replace it with a real
|
||||
FLAC file before running transcription.
|
||||
|
||||
The examples contain environment-variable names but no credential values. They
|
||||
use fictional campaign content and reserved example domains.
|
||||
@@ -1,5 +1,5 @@
|
||||
match:
|
||||
- speaker: "Eric Rakestraw"
|
||||
- speaker: "Example Speaker"
|
||||
match:
|
||||
- "Eric_Rakestraw"
|
||||
- "Eric"
|
||||
- "Example_Speaker"
|
||||
- "Example"
|
||||
|
||||
55
examples/pipeline.extraction-subset.yml
Normal file
55
examples/pipeline.extraction-subset.yml
Normal file
@@ -0,0 +1,55 @@
|
||||
# Purpose-specific extraction example: a Scriptorium session brief consumes
|
||||
# only the three Notarius lanes it needs.
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
|
||||
notarius:
|
||||
enabled: true
|
||||
binary: notarius
|
||||
config_path: /usr/local/etc/notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
timeout: 3h
|
||||
outputs:
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-registry
|
||||
location_registry:
|
||||
lane_id: location-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/location-registry
|
||||
scene_descriptions:
|
||||
lane_id: scene-descriptions
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.scene_descriptions
|
||||
schema_version: v1
|
||||
module_key: dnd/scene-descriptions
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
artifacts:
|
||||
session_brief:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_brief
|
||||
output_path: artifacts/session_brief.md
|
||||
inputs:
|
||||
npcs:
|
||||
source: narratio.extraction.npc_registry
|
||||
required: true
|
||||
locations:
|
||||
source: narratio.extraction.location_registry
|
||||
required: true
|
||||
scenes:
|
||||
source: narratio.extraction.scene_descriptions
|
||||
required: true
|
||||
|
||||
@@ -12,7 +12,7 @@ workspace:
|
||||
# env_dir: ./secrets
|
||||
|
||||
storage:
|
||||
# Optional storage backend selector; use "s3" for publish + S3 audio workflows.
|
||||
# Defaults to "local". Use "s3" explicitly for publish + S3 audio workflows.
|
||||
backend: s3
|
||||
s3:
|
||||
# Required when using S3 audio or S3 publish uploads.
|
||||
@@ -60,6 +60,11 @@ publish:
|
||||
- source: narratio.artifact.player_handout
|
||||
dest: artifacts/player_handout.md
|
||||
required: false
|
||||
# Extraction lanes publish only when named explicitly; the bundle and index
|
||||
# are never implicit publish sources.
|
||||
- source: narratio.extraction.npc_registry
|
||||
dest: artifacts/extraction/npc-registry.json
|
||||
required: true
|
||||
|
||||
whisperx:
|
||||
# Required.
|
||||
@@ -123,6 +128,78 @@ trim:
|
||||
seriatim:
|
||||
report: false
|
||||
|
||||
notarius:
|
||||
# Optional structured extraction between trim and render.
|
||||
enabled: true
|
||||
binary: notarius
|
||||
config_path: /usr/local/etc/notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
timeout: 3h
|
||||
working_directory: /usr/local/etc/notarius
|
||||
# Each key creates source narratio.extraction.<key>. These constraints match
|
||||
# the current Notarius D&D lane contracts; update them with Notarius.
|
||||
outputs:
|
||||
item_registry:
|
||||
lane_id: item-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.item_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/item-registry
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-registry
|
||||
location_registry:
|
||||
lane_id: location-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/location-registry
|
||||
scene_descriptions:
|
||||
lane_id: scene-descriptions
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.scene_descriptions
|
||||
schema_version: v1
|
||||
module_key: dnd/scene-descriptions
|
||||
item_occurrences:
|
||||
lane_id: item-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.item_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/item-occurrences
|
||||
spells:
|
||||
lane_id: spells
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.spells
|
||||
schema_version: v1
|
||||
module_key: dnd/spells
|
||||
combat_turns:
|
||||
lane_id: combat-turns
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.combat_turns
|
||||
schema_version: v1
|
||||
module_key: dnd/combat-turns
|
||||
npc_occurrences:
|
||||
lane_id: npc-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-occurrences
|
||||
location_occurrences:
|
||||
lane_id: location-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/location-occurrences
|
||||
enemy_events:
|
||||
lane_id: enemy-events
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.enemy_events
|
||||
schema_version: v1
|
||||
module_key: dnd/enemy-events
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
@@ -183,7 +260,5 @@ scriptorium:
|
||||
output_kind: player_handout
|
||||
|
||||
notification:
|
||||
# Optional notification settings.
|
||||
backend: ""
|
||||
recipient: ""
|
||||
timeout: 30s
|
||||
# No delivery provider is currently implemented.
|
||||
mode: noop
|
||||
|
||||
@@ -125,4 +125,4 @@ scriptorium:
|
||||
output_kind: player_handout
|
||||
|
||||
notification:
|
||||
timeout: 30s
|
||||
mode: noop
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
match:
|
||||
- speaker: "Eric Rakestraw"
|
||||
- speaker: "Example Speaker"
|
||||
match:
|
||||
- "Eric_Rakestraw"
|
||||
- "Eric"
|
||||
- "Example_Speaker"
|
||||
- "Example"
|
||||
|
||||
1
go.mod
1
go.mod
@@ -7,6 +7,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
golang.org/x/sys v0.47.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
|
||||
2
go.sum
2
go.sum
@@ -34,6 +34,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -4,10 +4,10 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
// NoopRunner is a deterministic no-op audita adapter.
|
||||
@@ -84,17 +84,17 @@ func materializePlaceholders(req PolishRequest) error {
|
||||
"merged_transcript_path": req.MergedTranscriptPath,
|
||||
"output_path": req.OutputProcessedPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
@@ -117,14 +117,14 @@ func writeJSONIfRequested(path string, payload any) error {
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
|
||||
return fmt.Errorf("create parent directory %q: %w", filepath.Dir(path), err)
|
||||
}
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal placeholder json for %q: %w", path, err)
|
||||
}
|
||||
if err := subprocess.WriteFileAtomic(path, data, 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write placeholder json %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// TODO: implement a real Audita subprocess/service adapter.
|
||||
|
||||
// Runner is the adapter boundary for audita polish invocations.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, req PolishRequest) (PolishResult, error)
|
||||
@@ -15,25 +13,15 @@ type Runner interface {
|
||||
|
||||
// PolishRequest describes an audita invocation.
|
||||
type PolishRequest struct {
|
||||
GeneratedConfigPath string
|
||||
MergedTranscriptPath string
|
||||
OutputProcessedPath string
|
||||
GlossaryPath string
|
||||
ReportPath string
|
||||
WorkDir string
|
||||
Modules []string
|
||||
BaseURL string
|
||||
Model string
|
||||
TranscriptDescription string
|
||||
ConfigPath string
|
||||
OutputSchema string
|
||||
WorkDirRetention string
|
||||
TotalLLMConcurrency *int
|
||||
ProposalLLMConcurrency *int
|
||||
ValidationModel string
|
||||
ValidationLLMConcurrency *int
|
||||
StdoutLogPath string
|
||||
StderrLogPath string
|
||||
GeneratedConfigPath string
|
||||
MergedTranscriptPath string
|
||||
OutputProcessedPath string
|
||||
GlossaryPath string
|
||||
ReportPath string
|
||||
WorkDir string
|
||||
Modules []string
|
||||
StdoutLogPath string
|
||||
StderrLogPath string
|
||||
}
|
||||
|
||||
// PolishResult describes a polish output.
|
||||
|
||||
@@ -11,8 +11,15 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
// MaxProcessedOutputBytes bounds Audita's processed-transcript JSON result.
|
||||
const MaxProcessedOutputBytes int64 = 64 * 1024 * 1024
|
||||
|
||||
// MaxReportOutputBytes bounds Audita's optional report JSON result.
|
||||
const MaxReportOutputBytes int64 = 16 * 1024 * 1024
|
||||
|
||||
// SubprocessRunnerConfig defines deterministic settings for Audita CLI execution.
|
||||
type SubprocessRunnerConfig struct {
|
||||
Binary string
|
||||
@@ -207,12 +214,13 @@ func (r *SubprocessRunner) Run(ctx context.Context, req PolishRequest) (PolishRe
|
||||
}
|
||||
|
||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||
Executable: r.binary,
|
||||
Args: args,
|
||||
Timeout: r.timeout,
|
||||
EnvOverrides: env,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
Executable: r.binary,
|
||||
Args: args,
|
||||
Timeout: r.timeout,
|
||||
EnvOverrides: env,
|
||||
DiagnosticOwner: "audita",
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
})
|
||||
if err != nil {
|
||||
wrappedMessage := fmt.Sprintf(
|
||||
@@ -370,13 +378,13 @@ func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []strin
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_present": credentialPresent,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
}
|
||||
|
||||
func validateProcessedOutput(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
data, err := readAuditaResult(path, MaxProcessedOutputBytes, "processed transcript")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read file: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
var payload map[string]any
|
||||
@@ -405,9 +413,9 @@ func addSubprocessStreamHint(message string, runErr error) string {
|
||||
}
|
||||
|
||||
func validateJSONFile(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
data, err := readAuditaResult(path, MaxReportOutputBytes, "report")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read file: %w", err)
|
||||
return err
|
||||
}
|
||||
var v any
|
||||
if err := json.Unmarshal(data, &v); err != nil {
|
||||
@@ -415,3 +423,11 @@ func validateJSONFile(path string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readAuditaResult(path string, limit int64, category string) ([]byte, error) {
|
||||
data, err := fileops.ReadRegularFile(path, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("audita %s result exceeds or cannot be read within %d-byte limit: %w", category, limit, err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
@@ -189,7 +189,7 @@ func TestSubprocessRunnerUnconfiguredCredentialEnvOmitsCredential(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
|
||||
func TestSubprocessRunnerOmitsUnspecifiedParentEnvironment(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("helper wrapper script uses /bin/sh")
|
||||
}
|
||||
@@ -214,8 +214,8 @@ func TestSubprocessRunnerInheritsParentEnvironment(t *testing.T) {
|
||||
}
|
||||
|
||||
rec := readAuditaHelperRecord(t, recordPath)
|
||||
if rec.Env["AUDITA_INHERITED_MARKER"] != "inherited-from-parent" {
|
||||
t.Fatalf("AUDITA_INHERITED_MARKER = %q, want inherited-from-parent", rec.Env["AUDITA_INHERITED_MARKER"])
|
||||
if rec.Env["AUDITA_INHERITED_MARKER"] != "" {
|
||||
t.Fatalf("AUDITA_INHERITED_MARKER = %q, want omitted from the child environment", rec.Env["AUDITA_INHERITED_MARKER"])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
22
internal/adapters/notarius/fake.go
Normal file
22
internal/adapters/notarius/fake.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package notarius
|
||||
|
||||
import "context"
|
||||
|
||||
// FakeRunner is a configurable in-memory runner for stage tests.
|
||||
type FakeRunner struct {
|
||||
Requests []RunRequest
|
||||
Result RunResult
|
||||
Err error
|
||||
}
|
||||
|
||||
// Run records the request and returns the configured result or error.
|
||||
func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return RunResult{}, err
|
||||
}
|
||||
f.Requests = append(f.Requests, req)
|
||||
if f.Err != nil {
|
||||
return RunResult{}, f.Err
|
||||
}
|
||||
return f.Result, nil
|
||||
}
|
||||
108
internal/adapters/notarius/runner.go
Normal file
108
internal/adapters/notarius/runner.go
Normal file
@@ -0,0 +1,108 @@
|
||||
// Package notarius declares the adapter contract for Notarius CLI invocations.
|
||||
package notarius
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
const ReceiptSchemaVersion = "notarius.run-result.v1"
|
||||
|
||||
// Runner is the adapter boundary for a complete Notarius pipeline invocation.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, req RunRequest) (RunResult, error)
|
||||
}
|
||||
|
||||
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
|
||||
type RunRequest struct {
|
||||
Binary string
|
||||
ConfigPath string
|
||||
PipelineID string
|
||||
InputPath string
|
||||
OutputRoot string
|
||||
WorkingDirectory string
|
||||
ReceiptPath string
|
||||
LogPath string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Receipt is the transport-neutral successful run receipt.
|
||||
type Receipt struct {
|
||||
SchemaVersion string
|
||||
RunID string
|
||||
PipelineID string
|
||||
OutputDirectory string
|
||||
IndexFile string
|
||||
NormalizedOutputCount int
|
||||
RejectedOutputCount int
|
||||
WarningCount int
|
||||
ValidationStatus string
|
||||
DebugDirectory string
|
||||
}
|
||||
|
||||
// LaneDescriptor identifies one normalized lane payload discovered through the index.
|
||||
type LaneDescriptor struct {
|
||||
LaneID string
|
||||
File string
|
||||
Path string
|
||||
MediaType string
|
||||
ModuleKey string
|
||||
SchemaID string
|
||||
SchemaName string
|
||||
SchemaVersion string
|
||||
}
|
||||
|
||||
// PipelineDescriptor identifies a pipeline-wide artifact discovered through the index.
|
||||
type PipelineDescriptor struct {
|
||||
ArtifactKind string
|
||||
File string
|
||||
Path string
|
||||
MediaType string
|
||||
SchemaID string
|
||||
SchemaName string
|
||||
SchemaVersion string
|
||||
}
|
||||
|
||||
// Index describes the validated bundle-management and artifact paths.
|
||||
type Index struct {
|
||||
Path string
|
||||
ManifestFile string
|
||||
ManifestPath string
|
||||
RejectedFile string
|
||||
RejectedPath string
|
||||
WarningsFile string
|
||||
WarningsPath string
|
||||
Lanes []LaneDescriptor
|
||||
ChunkMap *PipelineDescriptor
|
||||
EvidenceContext *PipelineDescriptor
|
||||
}
|
||||
|
||||
// RejectionSummary retains structured rejection identity without free-form messages.
|
||||
type RejectionSummary struct {
|
||||
Stage string
|
||||
StepID string
|
||||
LaneID string
|
||||
ModuleKey string
|
||||
ChunkID string
|
||||
ValidatorName string
|
||||
ReasonCode string
|
||||
}
|
||||
|
||||
// WarningSummary retains structured warning identity without free-form messages.
|
||||
type WarningSummary struct {
|
||||
Scope string
|
||||
ReasonCode string
|
||||
}
|
||||
|
||||
// RunResult describes a successfully decoded and validated Notarius bundle.
|
||||
type RunResult struct {
|
||||
Receipt Receipt
|
||||
Index Index
|
||||
BundleRoot string
|
||||
ReceiptPath string
|
||||
LogPath string
|
||||
ExitCode int
|
||||
Duration time.Duration
|
||||
Rejections []RejectionSummary
|
||||
Warnings []WarningSummary
|
||||
}
|
||||
502
internal/adapters/notarius/subprocess.go
Normal file
502
internal/adapters/notarius/subprocess.go
Normal file
@@ -0,0 +1,502 @@
|
||||
package notarius
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
maxReceiptBytes = 1 << 20
|
||||
maxIndexBytes = 4 << 20
|
||||
maxSummaryBytes = 4 << 20
|
||||
canonicalIndexFile = "index.json"
|
||||
canonicalManifestFile = "manifest.json"
|
||||
canonicalRejectedFile = "rejected.json"
|
||||
canonicalWarningsFile = "warnings.json"
|
||||
)
|
||||
|
||||
type subprocessRun func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error)
|
||||
|
||||
// SubprocessRunner invokes Notarius through its public CLI.
|
||||
type SubprocessRunner struct {
|
||||
run subprocessRun
|
||||
}
|
||||
|
||||
// NewSubprocessRunner constructs a production Notarius subprocess runner.
|
||||
func NewSubprocessRunner() *SubprocessRunner {
|
||||
return &SubprocessRunner{run: subprocess.Run}
|
||||
}
|
||||
|
||||
// Run executes a complete Notarius pipeline and discovers its published bundle.
|
||||
func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
if r == nil || r.run == nil {
|
||||
return RunResult{}, fmt.Errorf("notarius subprocess runner is nil")
|
||||
}
|
||||
if err := validateRunRequest(req); err != nil {
|
||||
return RunResult{}, err
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"run", req.PipelineID,
|
||||
"--config", req.ConfigPath,
|
||||
"--input", req.InputPath,
|
||||
"--output-dir", req.OutputRoot,
|
||||
"--json",
|
||||
}
|
||||
processResult, err := r.run(ctx, subprocess.RunRequest{
|
||||
Executable: req.Binary,
|
||||
Args: args,
|
||||
WorkingDir: req.WorkingDirectory,
|
||||
Timeout: req.Timeout,
|
||||
DiagnosticOwner: "notarius",
|
||||
StdoutLogPath: req.ReceiptPath,
|
||||
StderrLogPath: req.LogPath,
|
||||
})
|
||||
baseResult := RunResult{
|
||||
ReceiptPath: req.ReceiptPath,
|
||||
LogPath: req.LogPath,
|
||||
ExitCode: processResult.ExitCode,
|
||||
Duration: processResult.Duration,
|
||||
}
|
||||
if err != nil {
|
||||
return baseResult, fmt.Errorf("run notarius pipeline %q: %w", req.PipelineID, err)
|
||||
}
|
||||
|
||||
receipt, err := loadReceipt(req.ReceiptPath, req.PipelineID)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
bundleRoot, err := validateBundleRoot(req.OutputRoot, receipt.OutputDirectory)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
indexPath, err := resolveRegularFile(bundleRoot, receipt.IndexFile)
|
||||
if err != nil {
|
||||
return baseResult, fmt.Errorf("resolve receipt index file: %w", err)
|
||||
}
|
||||
index, err := loadIndex(bundleRoot, indexPath)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
rejections, err := loadRejections(index.RejectedPath)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
warnings, err := loadWarnings(index.WarningsPath)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
|
||||
baseResult.Receipt = receipt
|
||||
baseResult.Index = index
|
||||
baseResult.BundleRoot = bundleRoot
|
||||
baseResult.Rejections = rejections
|
||||
baseResult.Warnings = warnings
|
||||
return baseResult, nil
|
||||
}
|
||||
|
||||
func validateRunRequest(req RunRequest) error {
|
||||
if strings.TrimSpace(req.Binary) == "" {
|
||||
return fmt.Errorf("notarius binary is required")
|
||||
}
|
||||
if strings.TrimSpace(req.PipelineID) == "" {
|
||||
return fmt.Errorf("notarius pipeline id is required")
|
||||
}
|
||||
if req.Timeout <= 0 {
|
||||
return fmt.Errorf("notarius timeout must be positive")
|
||||
}
|
||||
for label, path := range map[string]string{
|
||||
"config": req.ConfigPath,
|
||||
"input": req.InputPath,
|
||||
"output root": req.OutputRoot,
|
||||
"working directory": req.WorkingDirectory,
|
||||
"receipt": req.ReceiptPath,
|
||||
"log": req.LogPath,
|
||||
} {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fmt.Errorf("notarius %s path is required", label)
|
||||
}
|
||||
if !filepath.IsAbs(path) {
|
||||
return fmt.Errorf("notarius %s path must be absolute", label)
|
||||
}
|
||||
}
|
||||
if filepath.Clean(req.ReceiptPath) == filepath.Clean(req.LogPath) {
|
||||
return fmt.Errorf("notarius receipt and log paths must be different")
|
||||
}
|
||||
if err := requireRegularFile(req.ConfigPath); err != nil {
|
||||
return fmt.Errorf("validate notarius config path: %w", err)
|
||||
}
|
||||
if err := requireRegularFile(req.InputPath); err != nil {
|
||||
return fmt.Errorf("validate notarius input path: %w", err)
|
||||
}
|
||||
if err := requireDirectory(req.OutputRoot); err != nil {
|
||||
return fmt.Errorf("validate notarius output root: %w", err)
|
||||
}
|
||||
if err := requireDirectory(req.WorkingDirectory); err != nil {
|
||||
return fmt.Errorf("validate notarius working directory: %w", err)
|
||||
}
|
||||
if err := validateLogDestination(req.ReceiptPath); err != nil {
|
||||
return fmt.Errorf("validate notarius receipt path: %w", err)
|
||||
}
|
||||
if err := validateLogDestination(req.LogPath); err != nil {
|
||||
return fmt.Errorf("validate notarius log path: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type receiptDocument struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
RunID string `json:"run_id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
OutputDirectory string `json:"output_directory"`
|
||||
IndexFile string `json:"index_file"`
|
||||
NormalizedOutputCount *int `json:"normalized_output_count"`
|
||||
RejectedOutputCount *int `json:"rejected_output_count"`
|
||||
WarningCount *int `json:"warning_count"`
|
||||
ValidationStatus string `json:"validation_status"`
|
||||
DebugDirectory string `json:"debug_directory"`
|
||||
}
|
||||
|
||||
func loadReceipt(path, pipelineID string) (Receipt, error) {
|
||||
var document receiptDocument
|
||||
if err := decodeBoundedJSON(path, maxReceiptBytes, &document); err != nil {
|
||||
return Receipt{}, fmt.Errorf("decode notarius receipt: %w", err)
|
||||
}
|
||||
if document.SchemaVersion != ReceiptSchemaVersion {
|
||||
return Receipt{}, fmt.Errorf("unsupported notarius receipt schema version %q", document.SchemaVersion)
|
||||
}
|
||||
if strings.TrimSpace(document.RunID) == "" || strings.TrimSpace(document.PipelineID) == "" ||
|
||||
strings.TrimSpace(document.OutputDirectory) == "" || strings.TrimSpace(document.ValidationStatus) == "" ||
|
||||
document.NormalizedOutputCount == nil ||
|
||||
document.RejectedOutputCount == nil || document.WarningCount == nil {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt is missing required fields")
|
||||
}
|
||||
if document.IndexFile != canonicalIndexFile {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt index_file %q is incompatible; want %q", document.IndexFile, canonicalIndexFile)
|
||||
}
|
||||
if document.PipelineID != pipelineID {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt pipeline id %q does not match requested pipeline %q", document.PipelineID, pipelineID)
|
||||
}
|
||||
if *document.NormalizedOutputCount < 0 || *document.RejectedOutputCount < 0 || *document.WarningCount < 0 {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt counts must be non-negative")
|
||||
}
|
||||
if !filepath.IsAbs(document.OutputDirectory) {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt output directory must be absolute")
|
||||
}
|
||||
if document.DebugDirectory != "" && !filepath.IsAbs(document.DebugDirectory) {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt debug directory must be absolute when present")
|
||||
}
|
||||
return Receipt{
|
||||
SchemaVersion: document.SchemaVersion,
|
||||
RunID: document.RunID,
|
||||
PipelineID: document.PipelineID,
|
||||
OutputDirectory: filepath.Clean(document.OutputDirectory),
|
||||
IndexFile: document.IndexFile,
|
||||
NormalizedOutputCount: *document.NormalizedOutputCount,
|
||||
RejectedOutputCount: *document.RejectedOutputCount,
|
||||
WarningCount: *document.WarningCount,
|
||||
ValidationStatus: document.ValidationStatus,
|
||||
DebugDirectory: document.DebugDirectory,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type indexDocument struct {
|
||||
ManifestFile string `json:"manifest_file"`
|
||||
OutputFiles *[]laneDocument `json:"output_files"`
|
||||
RejectedFile string `json:"rejected_file"`
|
||||
WarningsFile string `json:"warnings_file"`
|
||||
ChunkMap *pipelineDocument `json:"chunk_map"`
|
||||
EvidenceContext *pipelineDocument `json:"evidence_context"`
|
||||
}
|
||||
|
||||
type laneDocument struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
File string `json:"file"`
|
||||
MediaType string `json:"media_type"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
SchemaID string `json:"schema_id"`
|
||||
SchemaName string `json:"schema_name"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
}
|
||||
|
||||
type pipelineDocument struct {
|
||||
ArtifactKind string `json:"artifact_kind"`
|
||||
File string `json:"file"`
|
||||
MediaType string `json:"media_type"`
|
||||
SchemaID string `json:"schema_id"`
|
||||
SchemaName string `json:"schema_name"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
}
|
||||
|
||||
func loadIndex(bundleRoot, indexPath string) (Index, error) {
|
||||
var document indexDocument
|
||||
if err := decodeBoundedJSON(indexPath, maxIndexBytes, &document); err != nil {
|
||||
return Index{}, fmt.Errorf("decode notarius index: %w", err)
|
||||
}
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
got string
|
||||
want string
|
||||
}{
|
||||
{name: "manifest_file", got: document.ManifestFile, want: canonicalManifestFile},
|
||||
{name: "rejected_file", got: document.RejectedFile, want: canonicalRejectedFile},
|
||||
{name: "warnings_file", got: document.WarningsFile, want: canonicalWarningsFile},
|
||||
} {
|
||||
if field.got != field.want {
|
||||
return Index{}, fmt.Errorf("notarius index %s %q is incompatible; want %q", field.name, field.got, field.want)
|
||||
}
|
||||
}
|
||||
if document.OutputFiles == nil {
|
||||
return Index{}, fmt.Errorf("notarius index is missing required output_files")
|
||||
}
|
||||
|
||||
index := Index{
|
||||
Path: indexPath,
|
||||
ManifestFile: document.ManifestFile,
|
||||
RejectedFile: document.RejectedFile,
|
||||
WarningsFile: document.WarningsFile,
|
||||
}
|
||||
var err error
|
||||
if index.ManifestPath, err = resolveRegularFile(bundleRoot, index.ManifestFile); err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius manifest file: %w", err)
|
||||
}
|
||||
if index.RejectedPath, err = resolveRegularFile(bundleRoot, index.RejectedFile); err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius rejection file: %w", err)
|
||||
}
|
||||
if index.WarningsPath, err = resolveRegularFile(bundleRoot, index.WarningsFile); err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius warning file: %w", err)
|
||||
}
|
||||
|
||||
seenLanes := make(map[string]struct{}, len(*document.OutputFiles))
|
||||
for _, lane := range *document.OutputFiles {
|
||||
if strings.TrimSpace(lane.LaneID) == "" || strings.TrimSpace(lane.File) == "" {
|
||||
return Index{}, fmt.Errorf("notarius lane descriptors require lane_id and file")
|
||||
}
|
||||
if _, exists := seenLanes[lane.LaneID]; exists {
|
||||
return Index{}, fmt.Errorf("notarius index contains duplicate lane id %q", lane.LaneID)
|
||||
}
|
||||
seenLanes[lane.LaneID] = struct{}{}
|
||||
path, err := resolveRegularFile(bundleRoot, lane.File)
|
||||
if err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius lane %q file: %w", lane.LaneID, err)
|
||||
}
|
||||
index.Lanes = append(index.Lanes, LaneDescriptor{
|
||||
LaneID: lane.LaneID, File: lane.File, Path: path, MediaType: lane.MediaType,
|
||||
ModuleKey: lane.ModuleKey, SchemaID: lane.SchemaID, SchemaName: lane.SchemaName,
|
||||
SchemaVersion: lane.SchemaVersion,
|
||||
})
|
||||
}
|
||||
if document.ChunkMap != nil {
|
||||
index.ChunkMap, err = resolvePipelineDescriptor(bundleRoot, "chunk_map", *document.ChunkMap)
|
||||
if err != nil {
|
||||
return Index{}, err
|
||||
}
|
||||
}
|
||||
if document.EvidenceContext != nil {
|
||||
index.EvidenceContext, err = resolvePipelineDescriptor(bundleRoot, "evidence_context", *document.EvidenceContext)
|
||||
if err != nil {
|
||||
return Index{}, err
|
||||
}
|
||||
}
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func resolvePipelineDescriptor(bundleRoot, label string, document pipelineDocument) (*PipelineDescriptor, error) {
|
||||
if strings.TrimSpace(document.ArtifactKind) == "" || strings.TrimSpace(document.File) == "" ||
|
||||
strings.TrimSpace(document.MediaType) == "" || strings.TrimSpace(document.SchemaID) == "" ||
|
||||
strings.TrimSpace(document.SchemaName) == "" || strings.TrimSpace(document.SchemaVersion) == "" {
|
||||
return nil, fmt.Errorf("notarius %s descriptor is missing required fields", label)
|
||||
}
|
||||
path, err := resolveRegularFile(bundleRoot, document.File)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve notarius %s file: %w", label, err)
|
||||
}
|
||||
return &PipelineDescriptor{
|
||||
ArtifactKind: document.ArtifactKind, File: document.File, Path: path,
|
||||
MediaType: document.MediaType, SchemaID: document.SchemaID,
|
||||
SchemaName: document.SchemaName, SchemaVersion: document.SchemaVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type rejectionDocument struct {
|
||||
Rejected *[]struct {
|
||||
Stage string `json:"stage"`
|
||||
StepID string `json:"step_id"`
|
||||
LaneID string `json:"lane_id"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"rejected"`
|
||||
}
|
||||
|
||||
func loadRejections(path string) ([]RejectionSummary, error) {
|
||||
var document rejectionDocument
|
||||
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
||||
return nil, fmt.Errorf("decode notarius rejections: %w", err)
|
||||
}
|
||||
if document.Rejected == nil {
|
||||
return nil, fmt.Errorf("notarius rejection document is missing rejected array")
|
||||
}
|
||||
summaries := make([]RejectionSummary, 0, len(*document.Rejected))
|
||||
for _, item := range *document.Rejected {
|
||||
if strings.TrimSpace(item.Stage) == "" || strings.TrimSpace(item.Message) == "" {
|
||||
return nil, fmt.Errorf("notarius rejection entries require stage and message")
|
||||
}
|
||||
summaries = append(summaries, RejectionSummary{
|
||||
Stage: item.Stage, StepID: item.StepID, LaneID: item.LaneID,
|
||||
ModuleKey: item.ModuleKey, ChunkID: item.ChunkID,
|
||||
ValidatorName: item.ValidatorName, ReasonCode: item.ReasonCode,
|
||||
})
|
||||
}
|
||||
return summaries, nil
|
||||
}
|
||||
|
||||
type warningDocument struct {
|
||||
Warnings *[]struct {
|
||||
Scope string `json:"scope"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"warnings"`
|
||||
}
|
||||
|
||||
func loadWarnings(path string) ([]WarningSummary, error) {
|
||||
var document warningDocument
|
||||
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
||||
return nil, fmt.Errorf("decode notarius warnings: %w", err)
|
||||
}
|
||||
if document.Warnings == nil {
|
||||
return nil, fmt.Errorf("notarius warning document is missing warnings array")
|
||||
}
|
||||
summaries := make([]WarningSummary, 0, len(*document.Warnings))
|
||||
for _, item := range *document.Warnings {
|
||||
if strings.TrimSpace(item.ReasonCode) == "" || strings.TrimSpace(item.Message) == "" {
|
||||
return nil, fmt.Errorf("notarius warning entries require reason_code and message")
|
||||
}
|
||||
summaries = append(summaries, WarningSummary{Scope: item.Scope, ReasonCode: item.ReasonCode})
|
||||
}
|
||||
return summaries, nil
|
||||
}
|
||||
|
||||
func decodeBoundedJSON(path string, limit int64, destination any) error {
|
||||
data, err := fileops.ReadRegularFile(path, limit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("notarius JSON result exceeds or cannot be read within %d-byte limit: %w", limit, err)
|
||||
}
|
||||
if err := json.Unmarshal(data, destination); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBundleRoot(outputRoot, bundleRoot string) (string, error) {
|
||||
root := filepath.Clean(outputRoot)
|
||||
bundle := filepath.Clean(bundleRoot)
|
||||
relative, err := filepath.Rel(root, bundle)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("compare notarius output paths: %w", err)
|
||||
}
|
||||
if relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("notarius output directory %q is not beneath output root %q", bundleRoot, outputRoot)
|
||||
}
|
||||
if err := requireDirectoryTree(root, relative); err != nil {
|
||||
return "", fmt.Errorf("validate notarius output directory: %w", err)
|
||||
}
|
||||
return bundle, nil
|
||||
}
|
||||
|
||||
func resolveRegularFile(root, logicalPath string) (string, error) {
|
||||
resolved, err := pathsafe.JoinSlashRelativeUnderRoot(root, logicalPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
relative, err := filepath.Rel(root, resolved)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := requireRegularFileTree(root, relative); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func requireDirectoryTree(root, relative string) error {
|
||||
if err := requireDirectory(root); err != nil {
|
||||
return err
|
||||
}
|
||||
current := root
|
||||
for _, component := range strings.Split(relative, string(filepath.Separator)) {
|
||||
current = filepath.Join(current, component)
|
||||
if err := requireDirectory(current); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireRegularFileTree(root, relative string) error {
|
||||
components := strings.Split(relative, string(filepath.Separator))
|
||||
if len(components) == 0 {
|
||||
return fmt.Errorf("regular file path is required")
|
||||
}
|
||||
if err := requireDirectory(root); err != nil {
|
||||
return err
|
||||
}
|
||||
current := root
|
||||
for _, component := range components[:len(components)-1] {
|
||||
current = filepath.Join(current, component)
|
||||
if err := requireDirectory(current); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return requireRegularFile(filepath.Join(current, components[len(components)-1]))
|
||||
}
|
||||
|
||||
func requireDirectory(path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return fmt.Errorf("path %q must be a directory without symlinks", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireRegularFile(path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("path %q must be a regular file without symlinks", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateLogDestination(path string) error {
|
||||
if err := requireDirectory(filepath.Dir(path)); err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("path %q must be absent or a regular file without symlinks", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
572
internal/adapters/notarius/subprocess_test.go
Normal file
572
internal/adapters/notarius/subprocess_test.go
Normal file
@@ -0,0 +1,572 @@
|
||||
package notarius
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
)
|
||||
|
||||
func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
var captured sharedsubprocess.RunRequest
|
||||
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||
captured = processReq
|
||||
writeValidBundleAndReceipt(t, req, true)
|
||||
return sharedsubprocess.RunResult{ExitCode: 0, Duration: 2 * time.Second}, nil
|
||||
}}
|
||||
|
||||
result, err := runner.Run(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
wantArgs := []string{
|
||||
"run", "dnd-session", "--config", req.ConfigPath, "--input", req.InputPath,
|
||||
"--output-dir", req.OutputRoot, "--json",
|
||||
}
|
||||
if !reflect.DeepEqual(captured.Args, wantArgs) {
|
||||
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
|
||||
}
|
||||
if captured.Executable != req.Binary || captured.WorkingDir != req.WorkingDirectory || captured.Timeout != req.Timeout {
|
||||
t.Fatalf("subprocess request = %#v", captured)
|
||||
}
|
||||
if captured.StdoutLogPath != req.ReceiptPath || captured.StderrLogPath != req.LogPath {
|
||||
t.Fatalf("stream paths = stdout %q stderr %q", captured.StdoutLogPath, captured.StderrLogPath)
|
||||
}
|
||||
if captured.EnvOverrides != nil {
|
||||
t.Fatalf("environment overrides = %#v, want inherited environment only", captured.EnvOverrides)
|
||||
}
|
||||
for _, arg := range captured.Args {
|
||||
if arg == "--session-id" {
|
||||
t.Fatal("subprocess args unexpectedly contain --session-id")
|
||||
}
|
||||
}
|
||||
|
||||
if result.Receipt.SchemaVersion != ReceiptSchemaVersion || result.Receipt.RunID != "notarius-run-1" {
|
||||
t.Fatalf("receipt = %#v", result.Receipt)
|
||||
}
|
||||
if len(result.Index.Lanes) != 1 || result.Index.Lanes[0].LaneID != "npc-registry" {
|
||||
t.Fatalf("lanes = %#v", result.Index.Lanes)
|
||||
}
|
||||
if result.Index.ChunkMap == nil || result.Index.ChunkMap.ArtifactKind != "chunk_map" {
|
||||
t.Fatalf("chunk map = %#v", result.Index.ChunkMap)
|
||||
}
|
||||
if result.Index.EvidenceContext == nil || result.Index.EvidenceContext.ArtifactKind != "evidence_context" {
|
||||
t.Fatalf("evidence context = %#v", result.Index.EvidenceContext)
|
||||
}
|
||||
if len(result.Rejections) != 1 || result.Rejections[0].LaneID != "spells" || result.Rejections[0].ReasonCode != "invalid_spell" {
|
||||
t.Fatalf("rejections = %#v", result.Rejections)
|
||||
}
|
||||
if len(result.Warnings) != 1 || result.Warnings[0].Scope != "lane:npc-registry" || result.Warnings[0].ReasonCode != "normalized_name" {
|
||||
t.Fatalf("warnings = %#v", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
writeValidBundleAndReceipt(t, req, false)
|
||||
receiptFixture := req.ReceiptPath + ".fixture"
|
||||
data, err := os.ReadFile(req.ReceiptPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(receiptFixture, data, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
|
||||
}
|
||||
if err := os.Remove(req.ReceiptPath); err != nil {
|
||||
t.Fatalf("Remove(receipt) error = %v", err)
|
||||
}
|
||||
|
||||
captureDir := filepath.Join(filepath.Dir(req.ReceiptPath), "capture")
|
||||
if err := os.Mkdir(captureDir, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(capture) error = %v", err)
|
||||
}
|
||||
script := writeShellScript(t, `#!/bin/sh
|
||||
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
|
||||
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
|
||||
printf 'diagnostic stream\n' >&2
|
||||
cat "$NOTARIUS_RECEIPT_FIXTURE"
|
||||
`)
|
||||
req.Binary = script
|
||||
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
|
||||
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
|
||||
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
||||
|
||||
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
|
||||
assertTextFile(t, filepath.Join(captureDir, "environment"), "")
|
||||
assertTextFile(t, req.LogPath, "diagnostic stream\n")
|
||||
receiptBytes, err := os.ReadFile(req.ReceiptPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||
}
|
||||
if strings.Contains(string(receiptBytes), "diagnostic stream") {
|
||||
t.Fatal("receipt contains stderr output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
scriptBody string
|
||||
timeout time.Duration
|
||||
cancel bool
|
||||
want string
|
||||
}{
|
||||
{name: "nonzero", scriptBody: "printf '{malformed receipt'; printf 'failed\\n' >&2; exit 7\n", timeout: time.Second, want: "exit code 7"},
|
||||
{name: "timeout", scriptBody: "sleep 5\n", timeout: 20 * time.Millisecond, want: "timed out"},
|
||||
{name: "cancellation", scriptBody: "sleep 5\n", timeout: time.Second, cancel: true, want: "canceled"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
|
||||
req.Timeout = test.timeout
|
||||
ctx := context.Background()
|
||||
if test.cancel {
|
||||
cancelCtx, cancel := context.WithCancel(ctx)
|
||||
ctx = cancelCtx
|
||||
time.AfterFunc(20*time.Millisecond, cancel)
|
||||
}
|
||||
_, err := NewSubprocessRunner().Run(ctx, req)
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Run() error = %v, want fragment %q", err, test.want)
|
||||
}
|
||||
if strings.Contains(err.Error(), "decode notarius receipt") {
|
||||
t.Fatalf("Run() parsed stdout after process failure: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerReturnsSharedSubprocessErrorWithoutReadingReceipt(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
if err := os.WriteFile(req.ReceiptPath, []byte("not json"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(receipt) error = %v", err)
|
||||
}
|
||||
wantErr := errors.New("process failed")
|
||||
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||
return sharedsubprocess.RunResult{ExitCode: 9}, wantErr
|
||||
}}
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Run() error = %v, want wrapped process error", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "decode") {
|
||||
t.Fatalf("Run() parsed receipt after failure: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadReceiptValidation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
valid := map[string]any{
|
||||
"schema_version": ReceiptSchemaVersion, "run_id": "run-1", "pipeline_id": "pipeline-1",
|
||||
"output_directory": filepath.Join(root, "outputs", "run-1"), "index_file": "index.json",
|
||||
"normalized_output_count": 1, "rejected_output_count": 0, "warning_count": 0,
|
||||
"validation_status": "approved", "future_field": true,
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(map[string]any)
|
||||
raw []byte
|
||||
wantOK bool
|
||||
wantError string
|
||||
}{
|
||||
{name: "unknown fields tolerated", wantOK: true},
|
||||
{name: "malformed", raw: []byte("{")},
|
||||
{name: "unsupported version", mutate: func(v map[string]any) { v["schema_version"] = "notarius.run-result.v2" }},
|
||||
{name: "missing field", mutate: func(v map[string]any) { delete(v, "run_id") }},
|
||||
{name: "pipeline mismatch", mutate: func(v map[string]any) { v["pipeline_id"] = "other" }},
|
||||
{name: "relative output", mutate: func(v map[string]any) { v["output_directory"] = "run-1" }},
|
||||
{name: "negative count", mutate: func(v map[string]any) { v["warning_count"] = -1 }},
|
||||
{
|
||||
name: "nested index", mutate: func(v map[string]any) { v["index_file"] = "nested/index.json" },
|
||||
wantError: `index_file "nested/index.json"`,
|
||||
},
|
||||
{
|
||||
name: "cleanable index", mutate: func(v map[string]any) { v["index_file"] = "./index.json" },
|
||||
wantError: `index_file "./index.json"`,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
path := filepath.Join(root, strings.ReplaceAll(test.name, " ", "-")+".json")
|
||||
values := cloneMap(valid)
|
||||
if test.mutate != nil {
|
||||
test.mutate(values)
|
||||
}
|
||||
if test.raw != nil {
|
||||
if err := os.WriteFile(path, test.raw, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
} else {
|
||||
writeJSONFile(t, path, values)
|
||||
}
|
||||
_, err := loadReceipt(path, "pipeline-1")
|
||||
if test.wantOK && err != nil {
|
||||
t.Fatalf("loadReceipt() error = %v", err)
|
||||
}
|
||||
if !test.wantOK && err == nil {
|
||||
t.Fatal("loadReceipt() error = nil, want validation failure")
|
||||
}
|
||||
if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
|
||||
t.Fatalf("loadReceipt() error = %v, want fragment %q", err, test.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
oversized := filepath.Join(root, "oversized.json")
|
||||
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxReceiptBytes+1)), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(oversized) error = %v", err)
|
||||
}
|
||||
if _, err := loadReceipt(oversized, "pipeline-1"); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadReceipt(oversized) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateBundleRootRejectsEscapesAndSymlinks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outputRoot := filepath.Join(root, "output")
|
||||
if err := os.Mkdir(outputRoot, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(output root) error = %v", err)
|
||||
}
|
||||
validBundle := filepath.Join(outputRoot, "run-1")
|
||||
if err := os.Mkdir(validBundle, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(bundle) error = %v", err)
|
||||
}
|
||||
if _, err := validateBundleRoot(outputRoot, validBundle); err != nil {
|
||||
t.Fatalf("validateBundleRoot(valid) error = %v", err)
|
||||
}
|
||||
|
||||
outside := filepath.Join(root, "output-other")
|
||||
if err := os.Mkdir(outside, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(outside) error = %v", err)
|
||||
}
|
||||
for name, candidate := range map[string]string{"equal root": outputRoot, "escape": root, "prefix confusion": outside} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := validateBundleRoot(outputRoot, candidate); err == nil {
|
||||
t.Fatalf("validateBundleRoot(%q) error = nil", candidate)
|
||||
}
|
||||
})
|
||||
}
|
||||
symlink := filepath.Join(outputRoot, "linked")
|
||||
if err := os.Symlink(outside, symlink); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
if _, err := validateBundleRoot(outputRoot, symlink); err == nil {
|
||||
t.Fatal("validateBundleRoot(symlink) error = nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadIndexRejectsMalformedUnsafeAndUnsupportedDocuments(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
indexValue any
|
||||
prepare func(*testing.T, string)
|
||||
wantError string
|
||||
}{
|
||||
{name: "malformed", indexValue: json.RawMessage(`{"manifest_file":`)},
|
||||
{name: "unsupported output shape", indexValue: map[string]any{"manifest_file": "manifest.json", "output_files": map[string]any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
|
||||
{name: "missing management path", indexValue: map[string]any{"output_files": []any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
|
||||
{name: "renamed manifest", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["manifest_file"] = "metadata.json"
|
||||
return value
|
||||
}(), wantError: `manifest_file "metadata.json"`},
|
||||
{name: "cleanable manifest", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["manifest_file"] = "./manifest.json"
|
||||
return value
|
||||
}(), wantError: `manifest_file "./manifest.json"`},
|
||||
{name: "renamed rejections", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["rejected_file"] = "rejections.json"
|
||||
return value
|
||||
}(), wantError: `rejected_file "rejections.json"`},
|
||||
{name: "renamed warnings", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["warnings_file"] = "diagnostics/warnings.json"
|
||||
return value
|
||||
}(), wantError: `warnings_file "diagnostics/warnings.json"`},
|
||||
{name: "duplicate lane", indexValue: validIndexValue([]any{
|
||||
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
|
||||
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
|
||||
})},
|
||||
{name: "absolute logical path", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "/tmp/npc.json"}})},
|
||||
{name: "lexical traversal", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../outside.json"}})},
|
||||
{name: "root prefix confusion", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../bundle-other/npc.json"}})},
|
||||
{name: "file symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "lanes/npc.json"}}), prepare: func(t *testing.T, bundle string) {
|
||||
if err := os.Symlink(filepath.Join(bundle, "manifest.json"), filepath.Join(bundle, "lanes", "npc.json")); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "directory symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "linked/npc.json"}}), prepare: func(t *testing.T, bundle string) {
|
||||
if err := os.Symlink(filepath.Join(bundle, "lanes"), filepath.Join(bundle, "linked")); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "missing management file", indexValue: validIndexValue([]any{}), prepare: func(t *testing.T, bundle string) {
|
||||
if err := os.Remove(filepath.Join(bundle, "manifest.json")); err != nil {
|
||||
t.Fatalf("Remove(manifest) error = %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "incomplete pipeline descriptor", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["chunk_map"] = map[string]any{"artifact_kind": "chunk_map", "file": "chunk-map.json"}
|
||||
return value
|
||||
}()},
|
||||
{name: "pipeline descriptor escape", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["evidence_context"] = map[string]any{
|
||||
"artifact_kind": "evidence_context", "file": "../evidence.json", "media_type": "application/json",
|
||||
"schema_id": "evidence", "schema_name": "Evidence", "schema_version": "v1",
|
||||
}
|
||||
return value
|
||||
}()},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
bundle := createBundleSkeleton(t)
|
||||
indexPath := filepath.Join(bundle, "index.json")
|
||||
if raw, ok := test.indexValue.(json.RawMessage); ok {
|
||||
if err := os.WriteFile(indexPath, raw, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(index) error = %v", err)
|
||||
}
|
||||
} else {
|
||||
writeJSONFile(t, indexPath, test.indexValue)
|
||||
}
|
||||
if test.prepare != nil {
|
||||
test.prepare(t, bundle)
|
||||
}
|
||||
if _, err := loadIndex(bundle, indexPath); err == nil {
|
||||
t.Fatal("loadIndex() error = nil, want failure")
|
||||
} else if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
|
||||
t.Fatalf("loadIndex() error = %v, want fragment %q", err, test.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
bundle := createBundleSkeleton(t)
|
||||
oversizedIndex := filepath.Join(bundle, "index.json")
|
||||
if err := os.WriteFile(oversizedIndex, []byte(strings.Repeat("x", maxIndexBytes+1)), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(oversized index) error = %v", err)
|
||||
}
|
||||
if _, err := loadIndex(bundle, oversizedIndex); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadIndex(oversized) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
rejectedPath := filepath.Join(root, "rejected.json")
|
||||
warningsPath := filepath.Join(root, "warnings.json")
|
||||
writeJSONFile(t, rejectedPath, map[string]any{"rejected": []any{map[string]any{
|
||||
"stage": "validate", "lane_id": "spells", "reason_code": "invalid", "message": "do not retain this", "future": true,
|
||||
}}, "future": true})
|
||||
writeJSONFile(t, warningsPath, map[string]any{"warnings": []any{map[string]any{
|
||||
"scope": "lane:spells", "reason_code": "bounded", "message": "do not retain this", "future": true,
|
||||
}}, "future": true})
|
||||
rejections, err := loadRejections(rejectedPath)
|
||||
if err != nil || len(rejections) != 1 || rejections[0].ReasonCode != "invalid" {
|
||||
t.Fatalf("loadRejections() = %#v, %v", rejections, err)
|
||||
}
|
||||
warnings, err := loadWarnings(warningsPath)
|
||||
if err != nil || len(warnings) != 1 || warnings[0].Scope != "lane:spells" {
|
||||
t.Fatalf("loadWarnings() = %#v, %v", warnings, err)
|
||||
}
|
||||
|
||||
for name, path := range map[string]string{"rejections": rejectedPath, "warnings": warningsPath} {
|
||||
t.Run("malformed "+name, func(t *testing.T) {
|
||||
if err := os.WriteFile(path, []byte("{"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
var err error
|
||||
if name == "rejections" {
|
||||
_, err = loadRejections(path)
|
||||
} else {
|
||||
_, err = loadWarnings(path)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("summary decoder error = nil")
|
||||
}
|
||||
})
|
||||
}
|
||||
oversized := filepath.Join(root, "oversized.json")
|
||||
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxSummaryBytes+1)), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(oversized) error = %v", err)
|
||||
}
|
||||
if _, err := loadWarnings(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadWarnings(oversized) error = %v", err)
|
||||
}
|
||||
if _, err := loadRejections(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadRejections(oversized) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
|
||||
req := RunRequest{PipelineID: "pipeline"}
|
||||
want := RunResult{BundleRoot: "/bundle"}
|
||||
fake := &FakeRunner{Result: want}
|
||||
got, err := fake.Run(context.Background(), req)
|
||||
if err != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(fake.Requests, []RunRequest{req}) {
|
||||
t.Fatalf("Run() = %#v, %v; requests = %#v", got, err, fake.Requests)
|
||||
}
|
||||
|
||||
wantErr := errors.New("configured failure")
|
||||
fake.Err = wantErr
|
||||
if _, err := fake.Run(context.Background(), req); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Run(configured error) = %v", err)
|
||||
}
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
before := len(fake.Requests)
|
||||
if _, err := fake.Run(canceled, req); !errors.Is(err, context.Canceled) || len(fake.Requests) != before {
|
||||
t.Fatalf("Run(canceled) error = %v; requests = %d", err, len(fake.Requests))
|
||||
}
|
||||
}
|
||||
|
||||
func validRunRequest(t *testing.T) RunRequest {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
configPath := filepath.Join(root, "notarius.yml")
|
||||
inputPath := filepath.Join(root, "input.json")
|
||||
outputRoot := filepath.Join(root, "outputs")
|
||||
workingDirectory := filepath.Join(root, "work")
|
||||
diagnostics := filepath.Join(root, "diagnostics")
|
||||
for _, directory := range []string{outputRoot, workingDirectory, diagnostics} {
|
||||
if err := os.Mkdir(directory, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(%q) error = %v", directory, err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(config) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(inputPath, []byte("{}\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(input) error = %v", err)
|
||||
}
|
||||
return RunRequest{
|
||||
Binary: "notarius", ConfigPath: configPath, PipelineID: "dnd-session", InputPath: inputPath,
|
||||
OutputRoot: outputRoot, WorkingDirectory: workingDirectory,
|
||||
ReceiptPath: filepath.Join(diagnostics, "receipt.json"), LogPath: filepath.Join(diagnostics, "stderr.log"),
|
||||
Timeout: time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
|
||||
t.Helper()
|
||||
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
|
||||
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(bundle) error = %v", err)
|
||||
}
|
||||
for path, data := range map[string]string{
|
||||
"manifest.json": `{}`,
|
||||
"lanes/npc.json": `{}`,
|
||||
"chunk-map.json": `{}`,
|
||||
"evidence-context.json": `{}`,
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(path)), []byte(data), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||
}
|
||||
}
|
||||
rejection := map[string]any{"stage": "validate", "lane_id": "spells", "reason_code": "invalid_spell", "message": strings.Repeat("external detail", 20)}
|
||||
warning := map[string]any{"scope": "lane:npc-registry", "reason_code": "normalized_name", "message": strings.Repeat("external warning", 20)}
|
||||
if includeUnknown {
|
||||
rejection["future"] = true
|
||||
warning["future"] = true
|
||||
}
|
||||
writeJSONFile(t, filepath.Join(bundle, "rejected.json"), map[string]any{"rejected": []any{rejection}, "future": true})
|
||||
writeJSONFile(t, filepath.Join(bundle, "warnings.json"), map[string]any{"warnings": []any{warning}, "future": true})
|
||||
index := validIndexValue([]any{map[string]any{
|
||||
"lane_id": "npc-registry", "file": "lanes/npc.json", "media_type": "application/json",
|
||||
"module_key": "dnd/npc-registry", "schema_id": "notarius.dnd.npc_registry",
|
||||
"schema_name": "NPCRegistry", "schema_version": "v1", "future": true,
|
||||
}})
|
||||
index["chunk_map"] = map[string]any{
|
||||
"artifact_kind": "chunk_map", "file": "chunk-map.json", "media_type": "application/json",
|
||||
"schema_id": "notarius.chunk_map", "schema_name": "ChunkMap", "schema_version": "v1", "future": true,
|
||||
}
|
||||
index["evidence_context"] = map[string]any{
|
||||
"artifact_kind": "evidence_context", "file": "evidence-context.json", "media_type": "application/json",
|
||||
"schema_id": "notarius.evidence_context", "schema_name": "EvidenceContext", "schema_version": "v1", "future": true,
|
||||
}
|
||||
index["future"] = true
|
||||
writeJSONFile(t, filepath.Join(bundle, "index.json"), index)
|
||||
receipt := map[string]any{
|
||||
"schema_version": ReceiptSchemaVersion, "run_id": "notarius-run-1", "pipeline_id": req.PipelineID,
|
||||
"output_directory": bundle, "index_file": "index.json", "normalized_output_count": 1,
|
||||
"rejected_output_count": 1, "warning_count": 1, "validation_status": "rejected",
|
||||
}
|
||||
if includeUnknown {
|
||||
receipt["future"] = true
|
||||
}
|
||||
writeJSONFile(t, req.ReceiptPath, receipt)
|
||||
}
|
||||
|
||||
func createBundleSkeleton(t *testing.T) string {
|
||||
t.Helper()
|
||||
bundle := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(bundle) error = %v", err)
|
||||
}
|
||||
for _, name := range []string{"manifest.json", "rejected.json", "warnings.json", "lanes/npc.json", "chunk-map.json"} {
|
||||
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(name)), []byte("{}"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", name, err)
|
||||
}
|
||||
}
|
||||
return bundle
|
||||
}
|
||||
|
||||
func validIndexValue(lanes []any) map[string]any {
|
||||
return map[string]any{
|
||||
"manifest_file": "manifest.json", "output_files": lanes,
|
||||
"rejected_file": "rejected.json", "warnings_file": "warnings.json",
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSONFile(t *testing.T, path string, value any) {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeShellScript(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "notarius-helper")
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(script) error = %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func assertTextFile(t *testing.T, path, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q) error = %v", path, err)
|
||||
}
|
||||
if string(data) != want {
|
||||
t.Fatalf("ReadFile(%q) = %q, want %q", path, string(data), want)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneMap(source map[string]any) map[string]any {
|
||||
result := make(map[string]any, len(source))
|
||||
for key, value := range source {
|
||||
result[key] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
// NoopRunner is a deterministic no-op scriptorium adapter.
|
||||
@@ -142,7 +143,7 @@ func (f *FakeRunner) RenderArtifact(ctx context.Context, req RenderArtifactReque
|
||||
|
||||
func materializeRunPlaceholders(req RunArtifactRequest) error {
|
||||
if req.OutputPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write run output %q: %w", req.OutputPath, err)
|
||||
}
|
||||
}
|
||||
@@ -154,17 +155,17 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
|
||||
"prompt_id": req.PromptID,
|
||||
"output_path": req.OutputPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
@@ -173,7 +174,7 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
|
||||
|
||||
func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
||||
if req.OutputPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write render output %q: %w", req.OutputPath, err)
|
||||
}
|
||||
}
|
||||
@@ -185,17 +186,17 @@ func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
||||
"prompt_id": req.PromptID,
|
||||
"output_path": req.OutputPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,8 +9,12 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
// MaxOutputFileBytes bounds one Scriptorium artifact result.
|
||||
const MaxOutputFileBytes int64 = 64 * 1024 * 1024
|
||||
|
||||
// SubprocessRunner invokes Scriptorium through its public CLI.
|
||||
type SubprocessRunner struct{}
|
||||
|
||||
@@ -52,13 +56,17 @@ func (r *SubprocessRunner) RunArtifact(ctx context.Context, req RunArtifactReque
|
||||
}
|
||||
}
|
||||
|
||||
envOverrides, sensitiveNames := credentialEnvironment(req.APIKeyEnv)
|
||||
runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{
|
||||
Executable: req.Binary,
|
||||
Args: args,
|
||||
WorkingDir: req.WorkingDir,
|
||||
Timeout: req.Timeout,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
Executable: req.Binary,
|
||||
Args: args,
|
||||
WorkingDir: req.WorkingDir,
|
||||
Timeout: req.Timeout,
|
||||
EnvOverrides: envOverrides,
|
||||
SensitiveEnvNames: sensitiveNames,
|
||||
DiagnosticOwner: "scriptorium",
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
})
|
||||
|
||||
result := ArtifactResult{
|
||||
@@ -134,13 +142,17 @@ func (r *SubprocessRunner) RenderArtifact(ctx context.Context, req RenderArtifac
|
||||
}
|
||||
}
|
||||
|
||||
envOverrides, sensitiveNames := credentialEnvironment(req.APIKeyEnv)
|
||||
runRes, runErr := subprocess.Run(ctx, subprocess.RunRequest{
|
||||
Executable: req.Binary,
|
||||
Args: args,
|
||||
WorkingDir: req.WorkingDir,
|
||||
Timeout: req.Timeout,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
Executable: req.Binary,
|
||||
Args: args,
|
||||
WorkingDir: req.WorkingDir,
|
||||
Timeout: req.Timeout,
|
||||
EnvOverrides: envOverrides,
|
||||
SensitiveEnvNames: sensitiveNames,
|
||||
DiagnosticOwner: "scriptorium",
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
})
|
||||
|
||||
result := ArtifactResult{
|
||||
@@ -223,6 +235,15 @@ func validateCommonRunRequest(
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func credentialEnvironment(apiKeyEnv string) (map[string]string, []string) {
|
||||
name := strings.TrimSpace(apiKeyEnv)
|
||||
if name == "" {
|
||||
return nil, nil
|
||||
}
|
||||
value, _ := os.LookupEnv(name)
|
||||
return map[string]string{name: value}, []string{name}
|
||||
}
|
||||
|
||||
func buildRunArgs(req RunArtifactRequest) []string {
|
||||
args := []string{"run", "--prompt", strings.TrimSpace(req.PromptID)}
|
||||
if cfgPath := strings.TrimSpace(req.ConfigPath); cfgPath != "" {
|
||||
@@ -321,18 +342,15 @@ func writeInvocationConfig(path string, payload invocationPayload) error {
|
||||
"render_format": payload.RenderFormat,
|
||||
"render_prompt_logged": payload.RenderPromptStore,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(path, data, 0o644)
|
||||
return subprocess.WriteYAMLAtomic(path, data, fileops.WorkspaceFileMode)
|
||||
}
|
||||
|
||||
func validateNonEmptyOutput(path string) error {
|
||||
info, err := os.Stat(path)
|
||||
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat file: %w", err)
|
||||
return fmt.Errorf("scriptorium artifact output exceeds or cannot be read within %d-byte limit: %w", MaxOutputFileBytes, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return fmt.Errorf("path is a directory")
|
||||
}
|
||||
if info.Size() <= 0 {
|
||||
if len(data) == 0 {
|
||||
return fmt.Errorf("file is empty")
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
// NoopRunner is a deterministic no-op seriatim adapter.
|
||||
@@ -260,7 +261,7 @@ func (f *FakeRunner) Render(ctx context.Context, req RenderRequest) (RenderResul
|
||||
|
||||
func materializePlaceholders(req MergeRequest) error {
|
||||
if req.OutputMergedTranscriptPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write merged transcript %q: %w", req.OutputMergedTranscriptPath, err)
|
||||
}
|
||||
}
|
||||
@@ -271,22 +272,22 @@ func materializePlaceholders(req MergeRequest) error {
|
||||
"input_transcript_paths": req.InputTranscriptPaths,
|
||||
"output_path": req.OutputMergedTranscriptPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.ReportPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
||||
}
|
||||
}
|
||||
@@ -295,7 +296,7 @@ func materializePlaceholders(req MergeRequest) error {
|
||||
|
||||
func materializeTrimPlaceholders(req TrimRequest) error {
|
||||
if req.OutputTrimmedPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write trimmed transcript %q: %w", req.OutputTrimmedPath, err)
|
||||
}
|
||||
}
|
||||
@@ -308,17 +309,17 @@ func materializeTrimPlaceholders(req TrimRequest) error {
|
||||
"output_path": req.OutputTrimmedPath,
|
||||
"keep_selector": req.KeepSelector,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
@@ -327,7 +328,7 @@ func materializeTrimPlaceholders(req TrimRequest) error {
|
||||
|
||||
func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||
if req.OutputNormalizedPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write normalized transcript %q: %w", req.OutputNormalizedPath, err)
|
||||
}
|
||||
}
|
||||
@@ -343,22 +344,22 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||
if req.ReportPath != "" {
|
||||
payload["report_path"] = req.ReportPath
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.ReportPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
||||
}
|
||||
}
|
||||
@@ -367,7 +368,7 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||
|
||||
func materializeRenderPlaceholders(req RenderRequest) error {
|
||||
if req.OutputRenderedPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err)
|
||||
}
|
||||
}
|
||||
@@ -384,17 +385,17 @@ func materializeRenderPlaceholders(req RenderRequest) error {
|
||||
"include_segment_ids": req.IncludeSegmentIDs,
|
||||
"include_metadata": req.IncludeMetadata,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644); err != nil {
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), 0o644); err != nil {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,15 +4,18 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
// MaxOutputFileBytes bounds each Seriatim JSON or rendered-text result.
|
||||
const MaxOutputFileBytes int64 = 64 * 1024 * 1024
|
||||
|
||||
// EnvConfig defines optional Seriatim environment tuning values.
|
||||
type EnvConfig struct {
|
||||
OverlapWordRunGap *float64
|
||||
@@ -129,12 +132,13 @@ func (r *SubprocessRunner) Run(ctx context.Context, req MergeRequest) (MergeResu
|
||||
}
|
||||
|
||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||
Executable: r.binary,
|
||||
Args: args,
|
||||
Timeout: r.timeout,
|
||||
EnvOverrides: env,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
Executable: r.binary,
|
||||
Args: args,
|
||||
Timeout: r.timeout,
|
||||
EnvOverrides: env,
|
||||
DiagnosticOwner: "seriatim",
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
})
|
||||
if err != nil {
|
||||
return MergeResult{
|
||||
@@ -232,11 +236,12 @@ func (r *SubprocessRunner) Trim(ctx context.Context, req TrimRequest) (TrimResul
|
||||
}
|
||||
|
||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||
Executable: binary,
|
||||
Args: args,
|
||||
Timeout: timeout,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
Executable: binary,
|
||||
Args: args,
|
||||
Timeout: timeout,
|
||||
DiagnosticOwner: "seriatim",
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
})
|
||||
if err != nil {
|
||||
return TrimResult{
|
||||
@@ -320,11 +325,12 @@ func (r *SubprocessRunner) Normalize(ctx context.Context, req NormalizeRequest)
|
||||
}
|
||||
|
||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||
Executable: binary,
|
||||
Args: args,
|
||||
Timeout: timeout,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
Executable: binary,
|
||||
Args: args,
|
||||
Timeout: timeout,
|
||||
DiagnosticOwner: "seriatim",
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
})
|
||||
if err != nil {
|
||||
return NormalizeResult{
|
||||
@@ -425,11 +431,12 @@ func (r *SubprocessRunner) Render(ctx context.Context, req RenderRequest) (Rende
|
||||
}
|
||||
|
||||
runRes, err := subprocess.Run(ctx, subprocess.RunRequest{
|
||||
Executable: binary,
|
||||
Args: args,
|
||||
Timeout: timeout,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
Executable: binary,
|
||||
Args: args,
|
||||
Timeout: timeout,
|
||||
DiagnosticOwner: "seriatim",
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
})
|
||||
if err != nil {
|
||||
return RenderResult{
|
||||
@@ -546,7 +553,7 @@ func (r *SubprocessRunner) writeMergeInvocationConfig(req MergeRequest, args []s
|
||||
payload["coalesce_gap"] = *r.coalesceGap
|
||||
}
|
||||
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
}
|
||||
|
||||
func buildTrimArgs(req TrimRequest) []string {
|
||||
@@ -598,7 +605,7 @@ func writeTrimInvocationConfig(req TrimRequest, args []string, binary string, ti
|
||||
"output_path": req.OutputTrimmedPath,
|
||||
"keep_selector": req.KeepSelector,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
}
|
||||
|
||||
func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary string, timeout time.Duration, outputSchema string) error {
|
||||
@@ -613,7 +620,7 @@ func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary
|
||||
"output_schema": outputSchema,
|
||||
"report_path": req.ReportPath,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
}
|
||||
|
||||
func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error {
|
||||
@@ -631,13 +638,13 @@ func writeRenderInvocationConfig(req RenderRequest, args []string, binary string
|
||||
"include_segment_ids": req.IncludeSegmentIDs,
|
||||
"include_metadata": req.IncludeMetadata,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, 0o644)
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
}
|
||||
|
||||
func validateJSONFile(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
data, err := readSeriatimResult(path, "JSON output")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read file: %w", err)
|
||||
return err
|
||||
}
|
||||
var v any
|
||||
if err := json.Unmarshal(data, &v); err != nil {
|
||||
@@ -647,9 +654,9 @@ func validateJSONFile(path string) error {
|
||||
}
|
||||
|
||||
func validateJSONFileWithSegments(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
data, err := readSeriatimResult(path, "transcript JSON output")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read file: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
var payload map[string]any
|
||||
@@ -668,9 +675,9 @@ func validateJSONFileWithSegments(path string) error {
|
||||
}
|
||||
|
||||
func validateNonEmptyTextFile(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
data, err := readSeriatimResult(path, "rendered text output")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read file: %w", err)
|
||||
return err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return fmt.Errorf("file is empty")
|
||||
@@ -683,3 +690,11 @@ func validateNonEmptyTextFile(path string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readSeriatimResult(path, category string) ([]byte, error) {
|
||||
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seriatim %s exceeds or cannot be read within %d-byte limit: %w", category, MaxOutputFileBytes, err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
90
internal/adapters/storage/bounded_read.go
Normal file
90
internal/adapters/storage/bounded_read.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ReadLimitError reports that a remote object exceeded its caller-owned read
|
||||
// limit. The limit is enforced against both available object metadata and the
|
||||
// bytes returned by the opened object body.
|
||||
type ReadLimitError struct {
|
||||
Key string
|
||||
Limit int64
|
||||
Observed int64
|
||||
}
|
||||
|
||||
func (e *ReadLimitError) Error() string {
|
||||
return fmt.Sprintf("object %q exceeds %d-byte read limit (observed at least %d bytes)", e.Key, e.Limit, e.Observed)
|
||||
}
|
||||
|
||||
// ReadObjectBounded opens one object version and retains at most maxBytes of
|
||||
// its content. Object metadata may reject an oversized body early, but a
|
||||
// limit-plus-one read always enforces the boundary when transfer begins.
|
||||
func ReadObjectBounded(ctx context.Context, store ObjectStore, key string, maxBytes int64) (info ObjectInfo, data []byte, err error) {
|
||||
key = strings.TrimSpace(key)
|
||||
if store == nil {
|
||||
return ObjectInfo{}, nil, fmt.Errorf("read bounded object: store is required")
|
||||
}
|
||||
if key == "" {
|
||||
return ObjectInfo{}, nil, fmt.Errorf("read bounded object: key is required")
|
||||
}
|
||||
if maxBytes <= 0 || maxBytes == math.MaxInt64 {
|
||||
return ObjectInfo{}, nil, fmt.Errorf("read bounded object %q: limit must be between 1 and %d bytes", key, int64(math.MaxInt64-1))
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ObjectInfo{}, nil, err
|
||||
}
|
||||
|
||||
info, body, err := store.Read(ctx, key)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, nil, err
|
||||
}
|
||||
if body == nil {
|
||||
return ObjectInfo{}, nil, fmt.Errorf("read bounded object %q: store returned no body", key)
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := body.Close(); closeErr != nil {
|
||||
data = nil
|
||||
err = errors.Join(err, fmt.Errorf("close object %q: %w", key, closeErr))
|
||||
}
|
||||
}()
|
||||
|
||||
if info.Size > maxBytes {
|
||||
return info, nil, &ReadLimitError{Key: key, Limit: maxBytes, Observed: info.Size}
|
||||
}
|
||||
|
||||
data, err = io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, maxBytes+1))
|
||||
if err != nil {
|
||||
return info, nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return info, nil, err
|
||||
}
|
||||
if int64(len(data)) > maxBytes {
|
||||
return info, nil, &ReadLimitError{Key: key, Limit: maxBytes, Observed: int64(len(data))}
|
||||
}
|
||||
return info, data, nil
|
||||
}
|
||||
|
||||
type contextReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (r contextReader) Read(p []byte) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err := r.reader.Read(p)
|
||||
if err == nil {
|
||||
if contextErr := r.ctx.Err(); contextErr != nil {
|
||||
return n, contextErr
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
135
internal/adapters/storage/bounded_read_test.go
Normal file
135
internal/adapters/storage/bounded_read_test.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReadObjectBoundedAcceptsExactLimitWithAbsentSizeMetadata(t *testing.T) {
|
||||
body := &trackingReadCloser{reader: bytes.NewReader([]byte("12345678")), chunkSize: 2}
|
||||
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
|
||||
return ObjectInfo{Key: "control.json", ETag: "generation"}, body, nil
|
||||
}}
|
||||
|
||||
info, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadObjectBounded() error = %v", err)
|
||||
}
|
||||
if string(data) != "12345678" || info.ETag != "generation" {
|
||||
t.Fatalf("ReadObjectBounded() = (%#v, %q), want opened object metadata and bytes", info, data)
|
||||
}
|
||||
if !body.closed {
|
||||
t.Fatal("object body was not closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadObjectBoundedRejectsLimitPlusOneDespiteMissingOrInaccurateMetadata(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
metadataSize int64
|
||||
}{
|
||||
{name: "missing", metadataSize: 0},
|
||||
{name: "inaccurate", metadataSize: 2},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
body := &trackingReadCloser{reader: bytes.NewReader([]byte("123456789")), chunkSize: 1}
|
||||
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
|
||||
return ObjectInfo{Key: "control.json", Size: test.metadataSize}, body, nil
|
||||
}}
|
||||
|
||||
_, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
|
||||
var limitErr *ReadLimitError
|
||||
if !errors.As(err, &limitErr) {
|
||||
t.Fatalf("ReadObjectBounded() error = %v, want ReadLimitError", err)
|
||||
}
|
||||
if data != nil || body.bytesRead != 9 || !body.closed {
|
||||
t.Fatalf("data=%q bytes read=%d closed=%t, want nil, 9, true", data, body.bytesRead, body.closed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadObjectBoundedRejectsOversizedMetadataBeforeTransfer(t *testing.T) {
|
||||
body := &trackingReadCloser{reader: bytes.NewReader([]byte("small"))}
|
||||
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
|
||||
return ObjectInfo{Key: "control.json", Size: 9}, body, nil
|
||||
}}
|
||||
|
||||
_, _, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
|
||||
var limitErr *ReadLimitError
|
||||
if !errors.As(err, &limitErr) {
|
||||
t.Fatalf("ReadObjectBounded() error = %v, want ReadLimitError", err)
|
||||
}
|
||||
if body.bytesRead != 0 || !body.closed {
|
||||
t.Fatalf("bytes read=%d closed=%t, want zero-byte transfer and closed body", body.bytesRead, body.closed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadObjectBoundedPropagatesCancellationAndClosesBody(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
body := &trackingReadCloser{reader: bytes.NewReader([]byte("12345678")), chunkSize: 1, afterRead: cancel}
|
||||
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
|
||||
return ObjectInfo{Key: "control.json"}, body, nil
|
||||
}}
|
||||
|
||||
_, data, err := ReadObjectBounded(ctx, store, "control.json", 8)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("ReadObjectBounded() error = %v, want context cancellation", err)
|
||||
}
|
||||
if data != nil || body.bytesRead != 1 || !body.closed {
|
||||
t.Fatalf("data=%q bytes read=%d closed=%t, want nil, 1, true", data, body.bytesRead, body.closed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadObjectBoundedReturnsCloseFailure(t *testing.T) {
|
||||
closeErr := errors.New("close failed")
|
||||
body := &trackingReadCloser{reader: bytes.NewReader([]byte("ok")), closeErr: closeErr}
|
||||
store := &boundedReadStore{read: func(context.Context, string) (ObjectInfo, io.ReadCloser, error) {
|
||||
return ObjectInfo{Key: "control.json", Size: 2}, body, nil
|
||||
}}
|
||||
|
||||
_, data, err := ReadObjectBounded(context.Background(), store, "control.json", 8)
|
||||
if !errors.Is(err, closeErr) || data != nil || !body.closed {
|
||||
t.Fatalf("data=%q error=%v closed=%t, want close failure and no retained data", data, err, body.closed)
|
||||
}
|
||||
}
|
||||
|
||||
type boundedReadStore struct {
|
||||
ObjectStore
|
||||
read func(context.Context, string) (ObjectInfo, io.ReadCloser, error)
|
||||
}
|
||||
|
||||
func (s *boundedReadStore) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
|
||||
return s.read(ctx, key)
|
||||
}
|
||||
|
||||
type trackingReadCloser struct {
|
||||
reader io.Reader
|
||||
chunkSize int
|
||||
afterRead func()
|
||||
closeErr error
|
||||
bytesRead int
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (r *trackingReadCloser) Read(p []byte) (int, error) {
|
||||
if r.chunkSize > 0 && len(p) > r.chunkSize {
|
||||
p = p[:r.chunkSize]
|
||||
}
|
||||
n, err := r.reader.Read(p)
|
||||
r.bytesRead += n
|
||||
if n > 0 && r.afterRead != nil {
|
||||
r.afterRead()
|
||||
r.afterRead = nil
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *trackingReadCloser) Close() error {
|
||||
r.closed = true
|
||||
return r.closeErr
|
||||
}
|
||||
23
internal/adapters/storage/download_writer.go
Normal file
23
internal/adapters/storage/download_writer.go
Normal file
@@ -0,0 +1,23 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// WriterDownloader is implemented by storage backends that stream an object
|
||||
// into a caller-owned file handle.
|
||||
type WriterDownloader interface {
|
||||
DownloadTo(ctx context.Context, key string, destination io.Writer) error
|
||||
}
|
||||
|
||||
// DownloadTo streams one object into destination. Destination-confined callers
|
||||
// require this capability rather than granting a backend a mutable pathname.
|
||||
func DownloadTo(ctx context.Context, store ObjectStore, key string, destination io.Writer) error {
|
||||
writer, ok := store.(WriterDownloader)
|
||||
if !ok {
|
||||
return fmt.Errorf("object store does not support handle-confined downloads")
|
||||
}
|
||||
return writer.DownloadTo(ctx, key, destination)
|
||||
}
|
||||
@@ -14,16 +14,15 @@ func NewObjectStoreFromConfig(ctx context.Context, cfg *config.Config) (ObjectSt
|
||||
return nil, fmt.Errorf("pipeline config is required")
|
||||
}
|
||||
|
||||
if strings.EqualFold(strings.TrimSpace(cfg.Pipeline.Storage.Backend), "s3") {
|
||||
switch strings.ToLower(strings.TrimSpace(cfg.Pipeline.Storage.Backend)) {
|
||||
case config.StorageBackendS3:
|
||||
if cfg.Pipeline.Storage.S3 == nil {
|
||||
return nil, fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
|
||||
}
|
||||
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
|
||||
case "", config.StorageBackendLocal:
|
||||
return nil, fmt.Errorf("no remote object store backend is configured")
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported pipeline.storage.backend %q", cfg.Pipeline.Storage.Backend)
|
||||
}
|
||||
|
||||
if cfg.Pipeline.Storage.S3 != nil && strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) != "" {
|
||||
return NewS3BackendFromConfig(ctx, *cfg.Pipeline.Storage.S3)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("no remote object store backend is configured")
|
||||
}
|
||||
|
||||
@@ -54,3 +54,35 @@ func TestNewObjectStoreFromConfigNoRemoteBackendConfigured(t *testing.T) {
|
||||
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewObjectStoreFromConfigDoesNotInferS3FromProviderFields(t *testing.T) {
|
||||
called := false
|
||||
original := newS3Client
|
||||
t.Cleanup(func() { newS3Client = original })
|
||||
newS3Client = func(_ context.Context, _ s3ClientOptions) (s3API, error) {
|
||||
called = true
|
||||
return &fakeS3API{}, nil
|
||||
}
|
||||
|
||||
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{
|
||||
Backend: config.StorageBackendLocal,
|
||||
S3: &config.StorageS3Config{Bucket: "my-archive"},
|
||||
}},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "no remote object store backend is configured") {
|
||||
t.Fatalf("NewObjectStoreFromConfig() error = %v, want no-backend error", err)
|
||||
}
|
||||
if called {
|
||||
t.Fatal("NewObjectStoreFromConfig() constructed S3 from incidental provider fields")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewObjectStoreFromConfigRejectsUnknownBackend(t *testing.T) {
|
||||
_, err := NewObjectStoreFromConfig(context.Background(), &config.Config{
|
||||
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{Backend: "s33"}},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported pipeline.storage.backend") {
|
||||
t.Fatalf("NewObjectStoreFromConfig() error = %v, want unsupported-backend error", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,25 +1,35 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FakeBackend provides a deterministic in-memory object store for tests.
|
||||
type FakeBackend struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
Objects map[string]FakeObject
|
||||
Uploads []FakeUploadCall
|
||||
Downloads []FakeDownloadCall
|
||||
Reads []FakeReadCall
|
||||
|
||||
ListErr error
|
||||
DownloadErr error
|
||||
UploadErr error
|
||||
ExistsErr error
|
||||
ListErr error
|
||||
DownloadErr error
|
||||
UploadErr error
|
||||
ExistsErr error
|
||||
UploadHook func(FakeUploadCall) error
|
||||
DownloadHook func(FakeDownloadCall) error
|
||||
}
|
||||
|
||||
// FakeUploadCall captures one upload invocation in call order.
|
||||
@@ -33,6 +43,12 @@ type FakeUploadCall struct {
|
||||
type FakeDownloadCall struct {
|
||||
Key string
|
||||
LocalPath string
|
||||
Bytes int64
|
||||
}
|
||||
|
||||
// FakeReadCall captures one opened object in call order.
|
||||
type FakeReadCall struct {
|
||||
Key string
|
||||
}
|
||||
|
||||
// FakeObject is a deterministic fake object-store record.
|
||||
@@ -46,6 +62,12 @@ type FakeObject struct {
|
||||
|
||||
// SeedObject inserts or replaces an object in the fake object store.
|
||||
func (f *FakeBackend) SeedObject(obj FakeObject) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.seedObject(obj)
|
||||
}
|
||||
|
||||
func (f *FakeBackend) seedObject(obj FakeObject) {
|
||||
if f.Objects == nil {
|
||||
f.Objects = map[string]FakeObject{}
|
||||
}
|
||||
@@ -53,6 +75,9 @@ func (f *FakeBackend) SeedObject(obj FakeObject) {
|
||||
obj.Key = key
|
||||
obj.Data = append([]byte(nil), obj.Data...)
|
||||
obj.Metadata = copyMetadata(obj.Metadata)
|
||||
if obj.ETag == "" {
|
||||
obj.ETag = fakeObjectETag(obj.Data)
|
||||
}
|
||||
f.Objects[key] = obj
|
||||
}
|
||||
|
||||
@@ -65,6 +90,8 @@ func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, er
|
||||
return nil, f.ListErr
|
||||
}
|
||||
|
||||
f.mu.RLock()
|
||||
defer f.mu.RUnlock()
|
||||
normalizedPrefix := normalizeObjectKey(prefix)
|
||||
keys := make([]string, 0, len(f.Objects))
|
||||
for key := range f.Objects {
|
||||
@@ -87,34 +114,77 @@ func (f *FakeBackend) List(ctx context.Context, prefix string) ([]ObjectInfo, er
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Download writes one object to a local path.
|
||||
func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error {
|
||||
// Read returns a stable object body and the generation observed with it.
|
||||
func (f *FakeBackend) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ObjectInfo{}, nil, err
|
||||
}
|
||||
if f.DownloadErr != nil {
|
||||
return ObjectInfo{}, nil, f.DownloadErr
|
||||
}
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
f.mu.RLock()
|
||||
obj, ok := f.Objects[normalizedKey]
|
||||
if ok {
|
||||
obj.Data = append([]byte(nil), obj.Data...)
|
||||
obj.Metadata = copyMetadata(obj.Metadata)
|
||||
}
|
||||
f.mu.RUnlock()
|
||||
if !ok {
|
||||
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, os.ErrNotExist)
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.Reads = append(f.Reads, FakeReadCall{Key: normalizedKey})
|
||||
f.mu.Unlock()
|
||||
return ObjectInfo{Key: obj.Key, Size: int64(len(obj.Data)), ETag: obj.ETag, LastModified: obj.LastModified}, io.NopCloser(bytes.NewReader(obj.Data)), nil
|
||||
}
|
||||
|
||||
// DownloadTo writes one object to a caller-owned destination writer.
|
||||
func (f *FakeBackend) DownloadTo(ctx context.Context, key string, destination io.Writer) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if f.DownloadErr != nil {
|
||||
return f.DownloadErr
|
||||
}
|
||||
if destination == nil {
|
||||
return fmt.Errorf("download object: destination writer is required")
|
||||
}
|
||||
if f.DownloadHook != nil {
|
||||
if err := f.DownloadHook(FakeDownloadCall{Key: normalizeObjectKey(key)}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_, source, err := f.Read(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer source.Close()
|
||||
count, err := io.Copy(destination, source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download object %q: write destination: %w", key, err)
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.Downloads = append(f.Downloads, FakeDownloadCall{Key: normalizeObjectKey(key), Bytes: count})
|
||||
f.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download writes one object to a local path.
|
||||
func (f *FakeBackend) Download(ctx context.Context, key, localPath string) error {
|
||||
if strings.TrimSpace(localPath) == "" {
|
||||
return fmt.Errorf("download object: local path is required")
|
||||
}
|
||||
|
||||
obj, ok := f.Objects[normalizeObjectKey(key)]
|
||||
if !ok {
|
||||
return fmt.Errorf("download object %q: %w", key, os.ErrNotExist)
|
||||
}
|
||||
f.Downloads = append(f.Downloads, FakeDownloadCall{
|
||||
Key: normalizeObjectKey(key),
|
||||
LocalPath: localPath,
|
||||
})
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
||||
}
|
||||
if err := os.WriteFile(localPath, obj.Data, 0o644); err != nil {
|
||||
return fmt.Errorf("download object %q: write local file: %w", key, err)
|
||||
destination, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download object %q: create local file: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
defer destination.Close()
|
||||
return f.DownloadTo(ctx, key, destination)
|
||||
}
|
||||
|
||||
// Upload reads a local file and stores it under key.
|
||||
@@ -132,33 +202,117 @@ func (f *FakeBackend) Upload(ctx context.Context, localPath, key string, opts Up
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(localPath)
|
||||
file, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
|
||||
}
|
||||
defer file.Close()
|
||||
return f.uploadReader(ctx, file, key, opts, localPath)
|
||||
}
|
||||
|
||||
// UploadReader stores content provided by a caller-owned reader.
|
||||
func (f *FakeBackend) UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error) {
|
||||
return f.uploadReader(ctx, source, key, opts, "reader")
|
||||
}
|
||||
|
||||
func (f *FakeBackend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, localPath string) (ObjectInfo, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if f.UploadErr != nil {
|
||||
return ObjectInfo{}, f.UploadErr
|
||||
}
|
||||
if source == nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
|
||||
}
|
||||
if strings.TrimSpace(key) == "" {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(source)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", key, localPath, err)
|
||||
}
|
||||
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
f.Uploads = append(f.Uploads, FakeUploadCall{
|
||||
call := FakeUploadCall{
|
||||
LocalPath: localPath,
|
||||
Key: normalizedKey,
|
||||
Options: UploadOptions{
|
||||
Metadata: copyMetadata(opts.Metadata),
|
||||
ContentType: opts.ContentType,
|
||||
},
|
||||
})
|
||||
now := time.Now().UTC()
|
||||
obj := FakeObject{
|
||||
Key: normalizedKey,
|
||||
Data: data,
|
||||
Metadata: copyMetadata(opts.Metadata),
|
||||
LastModified: &now,
|
||||
}
|
||||
f.SeedObject(obj)
|
||||
return ObjectInfo{
|
||||
Key: normalizedKey,
|
||||
Size: int64(len(data)),
|
||||
LastModified: &now,
|
||||
}, nil
|
||||
f.mu.Lock()
|
||||
f.Uploads = append(f.Uploads, call)
|
||||
f.mu.Unlock()
|
||||
if f.UploadHook != nil {
|
||||
if err := f.UploadHook(call); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
}
|
||||
return f.storeUploadedObject(normalizedKey, data, opts), nil
|
||||
}
|
||||
|
||||
// UploadConditional atomically checks and replaces one mutable object.
|
||||
func (f *FakeBackend) UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if err := validateWriteCondition(condition); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if f.UploadErr != nil {
|
||||
return ObjectInfo{}, f.UploadErr
|
||||
}
|
||||
if source == nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
|
||||
}
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
if normalizedKey == "" {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||
}
|
||||
data, err := io.ReadAll(source)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, err)
|
||||
}
|
||||
call := FakeUploadCall{Key: normalizedKey, Options: UploadOptions{Metadata: copyMetadata(opts.Metadata), ContentType: opts.ContentType}}
|
||||
f.mu.Lock()
|
||||
f.Uploads = append(f.Uploads, call)
|
||||
f.mu.Unlock()
|
||||
if f.UploadHook != nil {
|
||||
if err := f.UploadHook(call); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
existing, found := f.Objects[normalizedKey]
|
||||
if condition.RequireAbsent && found {
|
||||
return ObjectInfo{}, ErrConditionNotMet
|
||||
}
|
||||
if expected := strings.TrimSpace(condition.MatchETag); expected != "" && (!found || existing.ETag != expected) {
|
||||
return ObjectInfo{}, ErrConditionNotMet
|
||||
}
|
||||
return f.storeUploadedObjectLocked(normalizedKey, data, opts), nil
|
||||
}
|
||||
|
||||
func (f *FakeBackend) storeUploadedObject(key string, data []byte, opts UploadOptions) ObjectInfo {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.storeUploadedObjectLocked(key, data, opts)
|
||||
}
|
||||
|
||||
func (f *FakeBackend) storeUploadedObjectLocked(key string, data []byte, opts UploadOptions) ObjectInfo {
|
||||
now := time.Now().UTC()
|
||||
obj := FakeObject{Key: key, Data: append([]byte(nil), data...), Metadata: copyMetadata(opts.Metadata), LastModified: &now}
|
||||
f.seedObject(obj)
|
||||
return ObjectInfo{Key: key, Size: int64(len(data)), ETag: fakeObjectETag(data), LastModified: &now}
|
||||
}
|
||||
|
||||
func fakeObjectETag(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Exists checks object presence.
|
||||
@@ -169,7 +323,9 @@ func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
if f.ExistsErr != nil {
|
||||
return false, f.ExistsErr
|
||||
}
|
||||
f.mu.RLock()
|
||||
_, ok := f.Objects[normalizeObjectKey(key)]
|
||||
f.mu.RUnlock()
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -71,6 +71,21 @@ func TestFakeBackendUploadAndExists(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendConditionalUploadRejectsStaleGeneration(t *testing.T) {
|
||||
fake := &FakeBackend{}
|
||||
fake.SeedObject(FakeObject{Key: "locks.yml", Data: []byte("old")})
|
||||
old := fake.Objects["locks.yml"].ETag
|
||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("new"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); err != nil {
|
||||
t.Fatalf("UploadConditional() error = %v", err)
|
||||
}
|
||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("lost"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); !errors.Is(err, ErrConditionNotMet) {
|
||||
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
||||
}
|
||||
if got := string(fake.Objects["locks.yml"].Data); got != "new" {
|
||||
t.Fatalf("locks object = %q, want successful replacement preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendObjectErrors(t *testing.T) {
|
||||
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}
|
||||
|
||||
|
||||
@@ -2,9 +2,21 @@ package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrConditionNotMet reports that an object changed or already existed before a
|
||||
// conditional write could be committed.
|
||||
var ErrConditionNotMet = errors.New("object write condition not met")
|
||||
|
||||
// ReaderUploader streams caller-owned, already-opened content to object storage.
|
||||
// Callers retain source-selection and filesystem-confinement policy.
|
||||
type ReaderUploader interface {
|
||||
UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error)
|
||||
}
|
||||
|
||||
// ObjectStore is a remote object storage boundary used by prepare, restore, and publish work.
|
||||
//
|
||||
// Key invariant:
|
||||
@@ -12,8 +24,10 @@ import (
|
||||
// infer Narratio session semantics and do not prepend root prefixes.
|
||||
type ObjectStore interface {
|
||||
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
|
||||
Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error)
|
||||
Download(ctx context.Context, key, localPath string) error
|
||||
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
|
||||
UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error)
|
||||
Exists(ctx context.Context, key string) (bool, error)
|
||||
}
|
||||
|
||||
@@ -30,3 +44,10 @@ type UploadOptions struct {
|
||||
Metadata map[string]string
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// WriteCondition protects a mutable object update against a stale snapshot.
|
||||
// Exactly one condition is required by UploadConditional.
|
||||
type WriteCondition struct {
|
||||
MatchETag string
|
||||
RequireAbsent bool
|
||||
}
|
||||
|
||||
@@ -117,6 +117,7 @@ func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, erro
|
||||
normalizedPrefix := normalizeObjectKey(prefix)
|
||||
out := make([]ObjectInfo, 0)
|
||||
var token *string
|
||||
seenTokens := map[string]struct{}{}
|
||||
|
||||
for {
|
||||
resp, err := b.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
@@ -142,44 +143,78 @@ func (b *S3Backend) List(ctx context.Context, prefix string) ([]ObjectInfo, erro
|
||||
})
|
||||
}
|
||||
|
||||
if !valueOrFalseBool(resp.IsTruncated) || resp.NextContinuationToken == nil {
|
||||
if !valueOrFalseBool(resp.IsTruncated) {
|
||||
break
|
||||
}
|
||||
token = resp.NextContinuationToken
|
||||
next := strings.TrimSpace(valueOrEmpty(resp.NextContinuationToken))
|
||||
if next == "" {
|
||||
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response has an empty continuation token", b.bucket, normalizedPrefix)
|
||||
}
|
||||
if _, repeated := seenTokens[next]; repeated {
|
||||
return nil, fmt.Errorf("s3 list objects bucket %q prefix %q: truncated response repeated continuation token", b.bucket, normalizedPrefix)
|
||||
}
|
||||
seenTokens[next] = struct{}{}
|
||||
token = &next
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Read retrieves an object together with the generation observed for its body.
|
||||
func (b *S3Backend) Read(ctx context.Context, key string) (ObjectInfo, io.ReadCloser, error) {
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{Bucket: &b.bucket, Key: &normalizedKey})
|
||||
if err != nil {
|
||||
if isS3NotFound(err) {
|
||||
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, os.ErrNotExist)
|
||||
}
|
||||
return ObjectInfo{}, nil, fmt.Errorf("read object %q: %w", normalizedKey, err)
|
||||
}
|
||||
var lastModified *time.Time
|
||||
if resp.LastModified != nil {
|
||||
t := *resp.LastModified
|
||||
lastModified = &t
|
||||
}
|
||||
return ObjectInfo{
|
||||
Key: normalizedKey, Size: valueOrZeroInt64(resp.ContentLength),
|
||||
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""), LastModified: lastModified,
|
||||
}, resp.Body, nil
|
||||
}
|
||||
|
||||
// DownloadTo retrieves one object into the caller-owned destination writer.
|
||||
func (b *S3Backend) DownloadTo(ctx context.Context, key string, destination io.Writer) error {
|
||||
if destination == nil {
|
||||
return fmt.Errorf("download object: destination writer is required")
|
||||
}
|
||||
_, body, err := b.Read(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download object %q: %w", normalizeObjectKey(key), err)
|
||||
}
|
||||
defer body.Close()
|
||||
|
||||
if _, err := io.Copy(destination, body); err != nil {
|
||||
return fmt.Errorf("download object %q: copy body: %w", normalizeObjectKey(key), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download retrieves one object to localPath, creating parent directories as needed.
|
||||
func (b *S3Backend) Download(ctx context.Context, key, localPath string) error {
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
if strings.TrimSpace(localPath) == "" {
|
||||
return fmt.Errorf("download object: local path is required")
|
||||
}
|
||||
|
||||
resp, err := b.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: &b.bucket,
|
||||
Key: &normalizedKey,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("download object %q: %w", normalizedKey, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||
return fmt.Errorf("download object %q: create parent directory: %w", normalizedKey, err)
|
||||
return fmt.Errorf("download object %q: create parent directory: %w", key, err)
|
||||
}
|
||||
dst, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download object %q: create local file: %w", normalizedKey, err)
|
||||
return fmt.Errorf("download object %q: create local file: %w", key, err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
if _, err := io.Copy(dst, resp.Body); err != nil {
|
||||
return fmt.Errorf("download object %q: copy body: %w", normalizedKey, err)
|
||||
if err := b.DownloadTo(ctx, key, dst); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := dst.Sync(); err != nil {
|
||||
return fmt.Errorf("download object %q: sync local file: %w", normalizedKey, err)
|
||||
return fmt.Errorf("download object %q: sync local file: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -204,28 +239,65 @@ func (b *S3Backend) Upload(ctx context.Context, localPath, key string, opts Uplo
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: stat local file: %w", normalizedKey, localPath, err)
|
||||
}
|
||||
return b.uploadReader(ctx, file, key, opts, stat.Size(), WriteCondition{})
|
||||
}
|
||||
|
||||
// UploadReader sends caller-owned content to key.
|
||||
func (b *S3Backend) UploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions) (ObjectInfo, error) {
|
||||
return b.uploadReader(ctx, source, key, opts, 0, WriteCondition{})
|
||||
}
|
||||
|
||||
// UploadConditional uploads a mutable object only when its observed generation
|
||||
// still matches, or when no object exists yet.
|
||||
func (b *S3Backend) UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error) {
|
||||
if err := validateWriteCondition(condition); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
return b.uploadReader(ctx, source, key, opts, 0, condition)
|
||||
}
|
||||
|
||||
func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key string, opts UploadOptions, size int64, condition WriteCondition) (ObjectInfo, error) {
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
if source == nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: source is required")
|
||||
}
|
||||
if normalizedKey == "" {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object: key is required")
|
||||
}
|
||||
|
||||
input := &s3.PutObjectInput{
|
||||
Bucket: &b.bucket,
|
||||
Key: &normalizedKey,
|
||||
Body: file,
|
||||
Body: source,
|
||||
Metadata: copyMetadata(opts.Metadata),
|
||||
}
|
||||
if strings.TrimSpace(opts.ContentType) != "" {
|
||||
ct := strings.TrimSpace(opts.ContentType)
|
||||
input.ContentType = &ct
|
||||
}
|
||||
if condition.RequireAbsent {
|
||||
wildcard := "*"
|
||||
input.IfNoneMatch = &wildcard
|
||||
} else if expected := strings.TrimSpace(condition.MatchETag); expected != "" {
|
||||
input.IfMatch = &expected
|
||||
}
|
||||
|
||||
resp, err := b.client.PutObject(ctx, input)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q from %q: %w", normalizedKey, localPath, err)
|
||||
if isS3ConditionalConflict(err) {
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, ErrConditionNotMet)
|
||||
}
|
||||
return ObjectInfo{}, fmt.Errorf("upload object %q: %w", normalizedKey, err)
|
||||
}
|
||||
|
||||
return ObjectInfo{
|
||||
info := ObjectInfo{
|
||||
Key: normalizedKey,
|
||||
Size: stat.Size(),
|
||||
ETag: strings.Trim(valueOrEmpty(resp.ETag), "\""),
|
||||
}, nil
|
||||
}
|
||||
if size > 0 {
|
||||
info.Size = size
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// Exists checks whether one object key exists.
|
||||
@@ -239,18 +311,43 @@ func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if isS3NotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
|
||||
}
|
||||
|
||||
func validateWriteCondition(condition WriteCondition) error {
|
||||
if condition.RequireAbsent == (strings.TrimSpace(condition.MatchETag) != "") {
|
||||
return fmt.Errorf("conditional upload requires exactly one of MatchETag or RequireAbsent")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isS3NotFound(err error) bool {
|
||||
var notFound *types.NotFound
|
||||
if errors.As(err, ¬Found) {
|
||||
return false, nil
|
||||
return true
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
switch apiErr.ErrorCode() {
|
||||
case "NotFound", "NoSuchKey", "404":
|
||||
return false, nil
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false, fmt.Errorf("head object %q: %w", normalizedKey, err)
|
||||
return false
|
||||
}
|
||||
|
||||
func isS3ConditionalConflict(err error) bool {
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
switch apiErr.ErrorCode() {
|
||||
case "PreconditionFailed", "ConditionalRequestConflict", "412", "409":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func valueOrEmpty(v *string) string {
|
||||
|
||||
@@ -2,6 +2,7 @@ package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -17,34 +18,81 @@ import (
|
||||
)
|
||||
|
||||
type fakeS3API struct {
|
||||
listOut *s3.ListObjectsV2Output
|
||||
listErr error
|
||||
listOut *s3.ListObjectsV2Output
|
||||
listOutputs []*s3.ListObjectsV2Output
|
||||
listErr error
|
||||
listCalls int
|
||||
|
||||
getBody io.ReadCloser
|
||||
getErr error
|
||||
getBody io.ReadCloser
|
||||
getErr error
|
||||
getSize *int64
|
||||
getETag *string
|
||||
getLastModified *time.Time
|
||||
|
||||
putOut *s3.PutObjectOutput
|
||||
putErr error
|
||||
|
||||
headErr error
|
||||
|
||||
lastList *s3.ListObjectsV2Input
|
||||
lastGet *s3.GetObjectInput
|
||||
lastPut *s3.PutObjectInput
|
||||
lastHead *s3.HeadObjectInput
|
||||
lastList *s3.ListObjectsV2Input
|
||||
lastLists []*s3.ListObjectsV2Input
|
||||
lastGet *s3.GetObjectInput
|
||||
lastPut *s3.PutObjectInput
|
||||
lastHead *s3.HeadObjectInput
|
||||
}
|
||||
|
||||
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
|
||||
f.lastList = params
|
||||
f.lastLists = append(f.lastLists, params)
|
||||
if f.listErr != nil {
|
||||
return nil, f.listErr
|
||||
}
|
||||
if f.listCalls < len(f.listOutputs) {
|
||||
out := f.listOutputs[f.listCalls]
|
||||
f.listCalls++
|
||||
return out, nil
|
||||
}
|
||||
if f.listOut == nil {
|
||||
return &s3.ListObjectsV2Output{}, nil
|
||||
}
|
||||
return f.listOut, nil
|
||||
}
|
||||
|
||||
func TestS3BackendListPaginatesAndRejectsNonProgressingTokens(t *testing.T) {
|
||||
t.Run("multiple pages", func(t *testing.T) {
|
||||
client := &fakeS3API{listOutputs: []*s3.ListObjectsV2Output{
|
||||
{Contents: []types.Object{{Key: strPtr("prefix/a"), Size: int64Ptr(1)}}, IsTruncated: boolPtr(true), NextContinuationToken: strPtr("next")},
|
||||
{Contents: []types.Object{{Key: strPtr("prefix/b"), Size: int64Ptr(2)}}, IsTruncated: boolPtr(false)},
|
||||
}}
|
||||
items, err := (&S3Backend{bucket: "bucket-1", client: client}).List(context.Background(), "prefix/")
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
if len(items) != 2 || items[0].Key != "prefix/a" || items[1].Key != "prefix/b" {
|
||||
t.Fatalf("List() items = %#v", items)
|
||||
}
|
||||
if len(client.lastLists) != 2 || client.lastLists[1].ContinuationToken == nil || *client.lastLists[1].ContinuationToken != "next" {
|
||||
t.Fatalf("continuation calls = %#v", client.lastLists)
|
||||
}
|
||||
})
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
outputs []*s3.ListObjectsV2Output
|
||||
want string
|
||||
}{
|
||||
{name: "empty", outputs: []*s3.ListObjectsV2Output{{IsTruncated: boolPtr(true)}}, want: "empty continuation token"},
|
||||
{name: "repeated", outputs: []*s3.ListObjectsV2Output{{IsTruncated: boolPtr(true), NextContinuationToken: strPtr("again")}, {IsTruncated: boolPtr(true), NextContinuationToken: strPtr("again")}}, want: "repeated continuation token"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := (&S3Backend{bucket: "bucket-1", client: &fakeS3API{listOutputs: test.outputs}}).List(context.Background(), "prefix/")
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) || !strings.Contains(err.Error(), "bucket-1") || !strings.Contains(err.Error(), "prefix/") {
|
||||
t.Fatalf("List() error = %v, want contextual %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
|
||||
f.lastGet = params
|
||||
if f.getErr != nil {
|
||||
@@ -54,7 +102,7 @@ func (f *fakeS3API) GetObject(_ context.Context, params *s3.GetObjectInput, _ ..
|
||||
if body == nil {
|
||||
body = io.NopCloser(strings.NewReader(""))
|
||||
}
|
||||
return &s3.GetObjectOutput{Body: body}, nil
|
||||
return &s3.GetObjectOutput{Body: body, ContentLength: f.getSize, ETag: f.getETag, LastModified: f.getLastModified}, nil
|
||||
}
|
||||
|
||||
func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
|
||||
@@ -126,6 +174,33 @@ func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendReadReturnsOpenedObjectMetadata(t *testing.T) {
|
||||
lastModified := time.Date(2026, 8, 11, 1, 2, 3, 0, time.UTC)
|
||||
client := &fakeS3API{
|
||||
getBody: io.NopCloser(strings.NewReader("locks")),
|
||||
getSize: int64Ptr(5),
|
||||
getETag: strPtr(`"generation"`),
|
||||
getLastModified: &lastModified,
|
||||
}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
|
||||
info, body, err := backend.Read(context.Background(), `sessions\locks.yml`)
|
||||
if err != nil {
|
||||
t.Fatalf("Read() error = %v", err)
|
||||
}
|
||||
data, readErr := io.ReadAll(body)
|
||||
closeErr := body.Close()
|
||||
if readErr != nil || closeErr != nil {
|
||||
t.Fatalf("read body error=%v close error=%v", readErr, closeErr)
|
||||
}
|
||||
if info.Key != "sessions/locks.yml" || info.Size != 5 || info.ETag != "generation" || info.LastModified == nil || !info.LastModified.Equal(lastModified) {
|
||||
t.Fatalf("Read() info = %#v, want opened object metadata", info)
|
||||
}
|
||||
if string(data) != "locks" || client.lastGet == nil || *client.lastGet.Key != "sessions/locks.yml" {
|
||||
t.Fatalf("Read() data=%q request=%#v", data, client.lastGet)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendUploadAndExists(t *testing.T) {
|
||||
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
@@ -160,6 +235,22 @@ func TestS3BackendUploadAndExists(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendConditionalUploadUsesProviderPrecondition(t *testing.T) {
|
||||
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
if _, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: "before"}); err != nil {
|
||||
t.Fatalf("UploadConditional() error = %v", err)
|
||||
}
|
||||
if client.lastPut == nil || client.lastPut.IfMatch == nil || *client.lastPut.IfMatch != "before" || client.lastPut.IfNoneMatch != nil {
|
||||
t.Fatalf("PutObject conditional input = %#v", client.lastPut)
|
||||
}
|
||||
client.putErr = &smithy.GenericAPIError{Code: "PreconditionFailed", Message: "changed"}
|
||||
_, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{RequireAbsent: true})
|
||||
if !errors.Is(err, ErrConditionNotMet) {
|
||||
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendUploadMissingLocalFile(t *testing.T) {
|
||||
backend := &S3Backend{bucket: "bucket-1", client: &fakeS3API{}}
|
||||
_, err := backend.Upload(context.Background(), filepath.Join(t.TempDir(), "missing.txt"), "key.txt", UploadOptions{})
|
||||
@@ -249,5 +340,6 @@ func TestNewS3BackendFromConfigFallsBackWhenCredentialEnvMissing(t *testing.T) {
|
||||
|
||||
func strPtr(v string) *string { return &v }
|
||||
func int64Ptr(v int64) *int64 { return &v }
|
||||
func boolPtr(v bool) *bool { return &v }
|
||||
|
||||
var _ s3API = (*fakeS3API)(nil)
|
||||
|
||||
391
internal/adapters/subprocess/diagnostics.go
Normal file
391
internal/adapters/subprocess/diagnostics.go
Normal file
@@ -0,0 +1,391 @@
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
const (
|
||||
// MaxStdoutDiagnosticBytes bounds persisted stdout from one external command.
|
||||
MaxStdoutDiagnosticBytes int64 = 8 * 1024 * 1024
|
||||
// MaxStderrDiagnosticBytes bounds persisted stderr from one external command.
|
||||
MaxStderrDiagnosticBytes int64 = 8 * 1024 * 1024
|
||||
diagnosticTailBytes = 2048
|
||||
)
|
||||
|
||||
var inheritedEnvironmentNames = map[string]struct{}{
|
||||
"COMSPEC": {},
|
||||
"HOME": {},
|
||||
"PATH": {},
|
||||
"SYSTEMROOT": {},
|
||||
"TMP": {},
|
||||
"TMPDIR": {},
|
||||
"TEMP": {},
|
||||
"WINDIR": {},
|
||||
// These test-only helper destinations let the adapter package tests exercise
|
||||
// real command invocation without widening the production environment.
|
||||
"AUDITA_HELPER_RECORD_PATH": {},
|
||||
"AUDITA_HELPER_MODE": {},
|
||||
"GO_WANT_AUDITA_HELPER": {},
|
||||
"GO_WANT_SCRIPTORIUM_HELPER": {},
|
||||
"GO_WANT_SERIATIM_HELPER": {},
|
||||
"GO_WANT_SUBPROCESS_HELPER": {},
|
||||
"NOTARIUS_CAPTURE_DIR": {},
|
||||
"NOTARIUS_RECEIPT_FIXTURE": {},
|
||||
"SCRIPTORIUM_HELPER_RECORD_PATH": {},
|
||||
"SCRIPTORIUM_HELPER_MODE": {},
|
||||
"SERIATIM_HELPER_RECORD_PATH": {},
|
||||
"SERIATIM_HELPER_MODE": {},
|
||||
}
|
||||
|
||||
var sensitiveEnvironmentNames = map[string]struct{}{
|
||||
"ANTHROPIC_API_KEY": {},
|
||||
"API_KEY": {},
|
||||
"AUDITA_LLM_API_KEY": {},
|
||||
"AWS_ACCESS_KEY_ID": {},
|
||||
"AWS_SECRET_ACCESS_KEY": {},
|
||||
"AWS_SESSION_TOKEN": {},
|
||||
"OPENAI_API_KEY": {},
|
||||
"OPENROUTER_API_KEY": {},
|
||||
}
|
||||
|
||||
type captureLimitError struct {
|
||||
stream string
|
||||
owner string
|
||||
limit int64
|
||||
}
|
||||
|
||||
func (e *captureLimitError) Error() string {
|
||||
return fmt.Sprintf("%s diagnostic capture for %s exceeded %d bytes", e.stream, e.owner, e.limit)
|
||||
}
|
||||
|
||||
type logWriters struct {
|
||||
files []*os.File
|
||||
Stdout io.Writer
|
||||
Stderr io.Writer
|
||||
|
||||
limits chan *captureLimitError
|
||||
mu sync.Mutex
|
||||
limit *captureLimitError
|
||||
stdout *diagnosticWriter
|
||||
stderr *diagnosticWriter
|
||||
}
|
||||
|
||||
type diagnosticWriter struct {
|
||||
logs *logWriters
|
||||
stream string
|
||||
owner string
|
||||
target io.Writer
|
||||
limit int64
|
||||
received int64
|
||||
persisted int64
|
||||
redactor streamRedactor
|
||||
tail []byte
|
||||
}
|
||||
|
||||
func openLogWriters(stdoutPath, stderrPath, owner string, sensitiveValues []string) (*logWriters, error) {
|
||||
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
cleanStdout := cleanLogPath(stdoutPath)
|
||||
cleanStderr := cleanLogPath(stderrPath)
|
||||
|
||||
stdoutFile, err := openDiagnosticFile(cleanStdout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open stdout log: %w", err)
|
||||
}
|
||||
stderrFile := stdoutFile
|
||||
if cleanStdout != cleanStderr {
|
||||
stderrFile, err = openDiagnosticFile(cleanStderr)
|
||||
if err != nil {
|
||||
_ = stdoutFile.Close()
|
||||
return nil, fmt.Errorf("open stderr log: %w", err)
|
||||
}
|
||||
}
|
||||
if cleanStdout == cleanStderr {
|
||||
logs.files = []*os.File{stdoutFile}
|
||||
} else {
|
||||
logs.files = []*os.File{stdoutFile, stderrFile}
|
||||
}
|
||||
|
||||
logs.stdout = newDiagnosticWriter(logs, "stdout", owner, stdoutFile, MaxStdoutDiagnosticBytes, sensitiveValues)
|
||||
logs.stderr = newDiagnosticWriter(logs, "stderr", owner, stderrFile, MaxStderrDiagnosticBytes, sensitiveValues)
|
||||
logs.Stdout = logs.stdout
|
||||
logs.Stderr = logs.stderr
|
||||
return logs, nil
|
||||
}
|
||||
|
||||
func newDiagnosticWriter(logs *logWriters, stream, owner string, target io.Writer, limit int64, sensitiveValues []string) *diagnosticWriter {
|
||||
return &diagnosticWriter{
|
||||
logs: logs,
|
||||
stream: stream,
|
||||
owner: owner,
|
||||
target: target,
|
||||
limit: limit,
|
||||
redactor: newStreamRedactor(sensitiveValues),
|
||||
}
|
||||
}
|
||||
|
||||
func (w *diagnosticWriter) Write(data []byte) (int, error) {
|
||||
if w.received >= w.limit {
|
||||
return len(data), w.reachLimit()
|
||||
}
|
||||
accepted := data
|
||||
if remaining := w.limit - w.received; int64(len(accepted)) > remaining {
|
||||
accepted = accepted[:remaining]
|
||||
}
|
||||
w.received += int64(len(accepted))
|
||||
if err := w.writeRedacted(w.redactor.Write(accepted)); err != nil {
|
||||
return len(data), err
|
||||
}
|
||||
if len(accepted) != len(data) {
|
||||
return len(data), w.reachLimit()
|
||||
}
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
func (w *diagnosticWriter) Flush() error {
|
||||
return w.writeRedacted(w.redactor.Flush())
|
||||
}
|
||||
|
||||
func (w *diagnosticWriter) writeRedacted(data []byte) error {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
w.logs.mu.Lock()
|
||||
remaining := w.limit - w.persisted
|
||||
if remaining <= 0 {
|
||||
w.logs.mu.Unlock()
|
||||
return w.reachLimit()
|
||||
}
|
||||
toWrite := data
|
||||
exceeded := int64(len(data)) > remaining
|
||||
if exceeded {
|
||||
toWrite = toWrite[:remaining]
|
||||
}
|
||||
written, err := w.target.Write(toWrite)
|
||||
w.persisted += int64(written)
|
||||
w.retainTail(toWrite[:written])
|
||||
w.logs.mu.Unlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exceeded {
|
||||
return w.reachLimit()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *diagnosticWriter) Tail() string {
|
||||
w.logs.mu.Lock()
|
||||
defer w.logs.mu.Unlock()
|
||||
return strings.TrimSpace(string(w.tail))
|
||||
}
|
||||
|
||||
func (w *diagnosticWriter) retainTail(data []byte) {
|
||||
if len(data) >= diagnosticTailBytes {
|
||||
if cap(w.tail) < diagnosticTailBytes {
|
||||
w.tail = make([]byte, diagnosticTailBytes)
|
||||
} else {
|
||||
w.tail = w.tail[:diagnosticTailBytes]
|
||||
}
|
||||
copy(w.tail, data[len(data)-diagnosticTailBytes:])
|
||||
return
|
||||
}
|
||||
if cap(w.tail) < diagnosticTailBytes {
|
||||
retained := make([]byte, len(w.tail), diagnosticTailBytes)
|
||||
copy(retained, w.tail)
|
||||
w.tail = retained
|
||||
}
|
||||
if overflow := len(w.tail) + len(data) - diagnosticTailBytes; overflow > 0 {
|
||||
copy(w.tail, w.tail[overflow:])
|
||||
w.tail = w.tail[:len(w.tail)-overflow]
|
||||
}
|
||||
w.tail = append(w.tail, data...)
|
||||
}
|
||||
|
||||
func (w *diagnosticWriter) reachLimit() error {
|
||||
limit := &captureLimitError{stream: w.stream, owner: w.owner, limit: w.limit}
|
||||
w.logs.mu.Lock()
|
||||
if w.logs.limit == nil {
|
||||
w.logs.limit = limit
|
||||
w.logs.limits <- limit
|
||||
}
|
||||
w.logs.mu.Unlock()
|
||||
return limit
|
||||
}
|
||||
|
||||
func (l *logWriters) Limits() <-chan *captureLimitError { return l.limits }
|
||||
|
||||
func (l *logWriters) Limit() *captureLimitError {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.limit
|
||||
}
|
||||
|
||||
func (l *logWriters) Flush() error {
|
||||
return joinErrors(l.stdout.Flush(), l.stderr.Flush())
|
||||
}
|
||||
|
||||
func (l *logWriters) Close() {
|
||||
_ = l.Flush()
|
||||
for _, file := range l.files {
|
||||
_ = file.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func cleanLogPath(path string) string {
|
||||
trimmed := strings.TrimSpace(path)
|
||||
if trimmed == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Clean(trimmed)
|
||||
}
|
||||
|
||||
func openDiagnosticFile(path string) (*os.File, error) {
|
||||
if path == "" {
|
||||
return os.OpenFile(os.DevNull, os.O_WRONLY, 0)
|
||||
}
|
||||
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
|
||||
return nil, fmt.Errorf("create log directory for %q: %w", path, err)
|
||||
}
|
||||
file, err := fileops.OpenFileConfined(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileops.WorkspaceFileMode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open log file %q: %w", path, err)
|
||||
}
|
||||
if err := file.Chmod(fileops.WorkspaceFileMode); err != nil {
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("set log file permissions %q: %w", path, err)
|
||||
}
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func (r RunRequest) diagnosticOwner() string {
|
||||
if owner := strings.TrimSpace(r.DiagnosticOwner); owner != "" {
|
||||
return owner
|
||||
}
|
||||
return "subprocess"
|
||||
}
|
||||
|
||||
func buildChildEnvironment(base []string, overrides map[string]string) []string {
|
||||
values := make(map[string]string, len(inheritedEnvironmentNames)+len(overrides))
|
||||
for _, item := range base {
|
||||
name, value, ok := strings.Cut(item, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
normalized := strings.ToUpper(name)
|
||||
if _, allowed := inheritedEnvironmentNames[normalized]; allowed {
|
||||
values[name] = value
|
||||
}
|
||||
}
|
||||
for name, value := range overrides {
|
||||
values[name] = value
|
||||
}
|
||||
names := make([]string, 0, len(values))
|
||||
for name := range values {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
out := make([]string, 0, len(names))
|
||||
for _, name := range names {
|
||||
out = append(out, name+"="+values[name])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sensitiveEnvironmentValues(environment []string, additionalNames []string) []string {
|
||||
names := make(map[string]struct{}, len(sensitiveEnvironmentNames)+len(additionalNames))
|
||||
for name := range sensitiveEnvironmentNames {
|
||||
names[name] = struct{}{}
|
||||
}
|
||||
for _, name := range additionalNames {
|
||||
if trimmed := strings.ToUpper(strings.TrimSpace(name)); trimmed != "" {
|
||||
names[trimmed] = struct{}{}
|
||||
}
|
||||
}
|
||||
values := make([]string, 0, len(names))
|
||||
for _, item := range environment {
|
||||
name, value, ok := strings.Cut(item, "=")
|
||||
if !ok || strings.TrimSpace(value) == "" {
|
||||
continue
|
||||
}
|
||||
if _, sensitive := names[strings.ToUpper(name)]; sensitive {
|
||||
values = append(values, value)
|
||||
}
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
type streamRedactor struct {
|
||||
values []string
|
||||
buffer []byte
|
||||
maxLen int
|
||||
}
|
||||
|
||||
func newStreamRedactor(values []string) streamRedactor {
|
||||
unique := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
if value != "" {
|
||||
unique[value] = struct{}{}
|
||||
}
|
||||
}
|
||||
sorted := make([]string, 0, len(unique))
|
||||
for value := range unique {
|
||||
sorted = append(sorted, value)
|
||||
}
|
||||
sort.Slice(sorted, func(i, j int) bool { return len(sorted[i]) > len(sorted[j]) })
|
||||
maxLen := 1
|
||||
for _, value := range sorted {
|
||||
if len(value) > maxLen {
|
||||
maxLen = len(value)
|
||||
}
|
||||
}
|
||||
return streamRedactor{values: sorted, maxLen: maxLen}
|
||||
}
|
||||
|
||||
func (r *streamRedactor) Write(data []byte) []byte {
|
||||
r.buffer = append(r.buffer, data...)
|
||||
safeCut := len(r.buffer) - r.maxLen + 1
|
||||
if safeCut <= 0 {
|
||||
return nil
|
||||
}
|
||||
emitCut := safeCut
|
||||
for _, value := range r.values {
|
||||
start := 0
|
||||
for {
|
||||
index := bytes.Index(r.buffer[start:], []byte(value))
|
||||
if index < 0 {
|
||||
break
|
||||
}
|
||||
index += start
|
||||
if index+len(value) > safeCut && index < emitCut {
|
||||
emitCut = index
|
||||
}
|
||||
start = index + 1
|
||||
}
|
||||
}
|
||||
output := redactBytes(r.buffer[:emitCut], r.values)
|
||||
r.buffer = append(r.buffer[:0], r.buffer[emitCut:]...)
|
||||
return output
|
||||
}
|
||||
|
||||
func (r *streamRedactor) Flush() []byte {
|
||||
output := redactBytes(r.buffer, r.values)
|
||||
r.buffer = nil
|
||||
return output
|
||||
}
|
||||
|
||||
func redactBytes(data []byte, values []string) []byte {
|
||||
out := append([]byte(nil), data...)
|
||||
for _, value := range values {
|
||||
out = bytes.ReplaceAll(out, []byte(value), []byte("<redacted>"))
|
||||
}
|
||||
return out
|
||||
}
|
||||
66
internal/adapters/subprocess/process_tree.go
Normal file
66
internal/adapters/subprocess/process_tree.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
gracefulTerminationWait = 2 * time.Second
|
||||
forcefulTerminationWait = 2 * time.Second
|
||||
)
|
||||
|
||||
// ownedProcessTree owns every process started by a command invocation.
|
||||
// Implementations must tolerate a leader that has already exited.
|
||||
type ownedProcessTree interface {
|
||||
Start(*exec.Cmd) error
|
||||
TerminateGracefully() error
|
||||
TerminateForcefully() error
|
||||
Dispose() error
|
||||
}
|
||||
|
||||
func waitForOwnedCommand(ctx context.Context, tree ownedProcessTree, waitCh <-chan error, captureLimits <-chan *captureLimitError) (waitErr, ctxErr error, captureLimit *captureLimitError, cleanupErr error) {
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
return waitErr, nil, nil, nil
|
||||
case <-ctx.Done():
|
||||
ctxErr = ctx.Err()
|
||||
case captureLimit = <-captureLimits:
|
||||
}
|
||||
|
||||
cleanupErr = tree.TerminateGracefully()
|
||||
gracefulTimer := time.NewTimer(gracefulTerminationWait)
|
||||
defer gracefulTimer.Stop()
|
||||
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
// The leader may exit before descendants finish graceful shutdown.
|
||||
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||
return waitErr, ctxErr, captureLimit, cleanupErr
|
||||
case <-gracefulTimer.C:
|
||||
}
|
||||
|
||||
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||
forcefulTimer := time.NewTimer(forcefulTerminationWait)
|
||||
defer forcefulTimer.Stop()
|
||||
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
return waitErr, ctxErr, captureLimit, cleanupErr
|
||||
case <-forcefulTimer.C:
|
||||
return nil, ctxErr, captureLimit, joinErrors(cleanupErr, fmt.Errorf("owned subprocess did not reap within %s after forceful termination", forcefulTerminationWait))
|
||||
}
|
||||
}
|
||||
|
||||
func joinErrors(errs ...error) error {
|
||||
filtered := make([]error, 0, len(errs))
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
filtered = append(filtered, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(filtered...)
|
||||
}
|
||||
227
internal/adapters/subprocess/process_tree_supported_test.go
Normal file
227
internal/adapters/subprocess/process_tree_supported_test.go
Normal file
@@ -0,0 +1,227 @@
|
||||
//go:build linux || darwin || windows
|
||||
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRunCancellationTerminatesProcessTree(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
resultCh := make(chan runOutcome, 1)
|
||||
req, sentinelPath := processTreeRequest(t)
|
||||
go func() {
|
||||
result, err := Run(ctx, req)
|
||||
resultCh <- runOutcome{result: result, err: err}
|
||||
}()
|
||||
|
||||
awaitHelperReady(t, req.EnvOverrides["SUBPROCESS_HELPER_READY_PATH"])
|
||||
cancel()
|
||||
|
||||
outcome := awaitRunOutcome(t, resultCh)
|
||||
if !outcome.result.Canceled {
|
||||
t.Fatalf("Canceled = %v, want true", outcome.result.Canceled)
|
||||
}
|
||||
if !errors.Is(outcome.err, context.Canceled) {
|
||||
t.Fatalf("error = %v, want context cancellation", outcome.err)
|
||||
}
|
||||
assertDescendantDidNotSurvive(t, sentinelPath)
|
||||
}
|
||||
|
||||
func TestRunTimeoutTerminatesProcessTree(t *testing.T) {
|
||||
req, sentinelPath := processTreeRequest(t)
|
||||
req.Timeout = 100 * time.Millisecond
|
||||
|
||||
result, err := Run(context.Background(), req)
|
||||
if !result.TimedOut {
|
||||
t.Fatalf("TimedOut = %v, want true", result.TimedOut)
|
||||
}
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("error = %v, want context deadline exceeded", err)
|
||||
}
|
||||
assertDescendantDidNotSurvive(t, sentinelPath)
|
||||
}
|
||||
|
||||
func TestRunCaptureLimitTerminatesProcessTree(t *testing.T) {
|
||||
req, sentinelPath := processTreeRequest(t)
|
||||
req.Args[len(req.Args)-1] = "tree-spam"
|
||||
|
||||
result, err := Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want capture-limit error")
|
||||
}
|
||||
if result.ExitCode == 0 {
|
||||
t.Fatalf("ExitCode = %d, want terminated process", result.ExitCode)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "stdout diagnostic capture for subprocess exceeded") {
|
||||
t.Fatalf("error = %q, want stdout capture-limit context", err)
|
||||
}
|
||||
info, statErr := os.Stat(req.StdoutLogPath)
|
||||
if statErr != nil {
|
||||
t.Fatalf("stat stdout diagnostic: %v", statErr)
|
||||
}
|
||||
if info.Size() != MaxStdoutDiagnosticBytes {
|
||||
t.Fatalf("stdout diagnostic size = %d, want %d", info.Size(), MaxStdoutDiagnosticBytes)
|
||||
}
|
||||
assertDescendantDidNotSurvive(t, sentinelPath)
|
||||
}
|
||||
|
||||
func TestRunDisposesDescendantsAfterLeaderExit(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mode string
|
||||
wantExitCode int
|
||||
wantWaitDelay bool
|
||||
ignoreTerm bool
|
||||
}{
|
||||
{name: "success retaining streams", mode: "leader-exit-retained", wantExitCode: 0, wantWaitDelay: true},
|
||||
{name: "success redirecting streams", mode: "leader-exit-redirected", wantExitCode: 0},
|
||||
{name: "failed leader", mode: "leader-fail-redirected", wantExitCode: 9, ignoreTerm: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req, sentinelPath, releasePath := leaderExitRequest(t, tt.mode)
|
||||
if tt.ignoreTerm {
|
||||
req.EnvOverrides["SUBPROCESS_HELPER_IGNORE_TERM"] = "1"
|
||||
}
|
||||
outcomes := make(chan runOutcome, 1)
|
||||
go func() {
|
||||
result, err := Run(context.Background(), req)
|
||||
outcomes <- runOutcome{result: result, err: err}
|
||||
}()
|
||||
|
||||
var outcome runOutcome
|
||||
select {
|
||||
case outcome = <-outcomes:
|
||||
case <-time.After(6 * time.Second):
|
||||
t.Fatal("Run() did not complete bounded owned-tree disposal")
|
||||
}
|
||||
if outcome.result.ExitCode != tt.wantExitCode {
|
||||
t.Fatalf("ExitCode = %d, want %d", outcome.result.ExitCode, tt.wantExitCode)
|
||||
}
|
||||
if tt.wantWaitDelay {
|
||||
if !errors.Is(outcome.err, exec.ErrWaitDelay) {
|
||||
t.Fatalf("error = %v, want exec.ErrWaitDelay", outcome.err)
|
||||
}
|
||||
} else if tt.wantExitCode == 0 && outcome.err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", outcome.err)
|
||||
} else if tt.wantExitCode != 0 {
|
||||
var exitErr *exec.ExitError
|
||||
if !errors.As(outcome.err, &exitErr) || exitErr.ExitCode() != tt.wantExitCode {
|
||||
t.Fatalf("error = %v, want exit code %d", outcome.err, tt.wantExitCode)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(req.EnvOverrides["SUBPROCESS_HELPER_READY_PATH"]); err != nil {
|
||||
t.Fatalf("descendant readiness file: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(releasePath, []byte("release"), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(release) error = %v", err)
|
||||
}
|
||||
assertDescendantDidNotSurvive(t, sentinelPath)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type runOutcome struct {
|
||||
result RunResult
|
||||
err error
|
||||
}
|
||||
|
||||
func processTreeRequest(t *testing.T) (RunRequest, string) {
|
||||
t.Helper()
|
||||
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
readyPath := filepath.Join(dir, "ready")
|
||||
sentinelPath := filepath.Join(dir, "descendant-survived")
|
||||
return RunRequest{
|
||||
Executable: executable,
|
||||
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "tree"},
|
||||
EnvOverrides: map[string]string{
|
||||
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||
"SUBPROCESS_HELPER_READY_PATH": readyPath,
|
||||
"SUBPROCESS_HELPER_SENTINEL_PATH": sentinelPath,
|
||||
},
|
||||
StdoutLogPath: filepath.Join(dir, "stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||
}, sentinelPath
|
||||
}
|
||||
|
||||
func leaderExitRequest(t *testing.T, mode string) (RunRequest, string, string) {
|
||||
t.Helper()
|
||||
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
readyPath := filepath.Join(dir, "ready")
|
||||
releasePath := filepath.Join(dir, "release")
|
||||
sentinelPath := filepath.Join(dir, "descendant-survived")
|
||||
return RunRequest{
|
||||
Executable: executable,
|
||||
Args: []string{"-test.run=^TestSubprocessHelper$", "--", mode},
|
||||
EnvOverrides: map[string]string{
|
||||
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||
"SUBPROCESS_HELPER_READY_PATH": readyPath,
|
||||
"SUBPROCESS_HELPER_RELEASE_PATH": releasePath,
|
||||
"SUBPROCESS_HELPER_SENTINEL_PATH": sentinelPath,
|
||||
},
|
||||
StdoutLogPath: filepath.Join(dir, "stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||
}, sentinelPath, releasePath
|
||||
}
|
||||
|
||||
func awaitHelperReady(t *testing.T, readyPath string) {
|
||||
t.Helper()
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := os.Stat(readyPath); err == nil {
|
||||
return
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("stat helper readiness: %v", err)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("helper did not start its descendant")
|
||||
}
|
||||
|
||||
func awaitRunOutcome(t *testing.T, outcomes <-chan runOutcome) runOutcome {
|
||||
t.Helper()
|
||||
|
||||
select {
|
||||
case outcome := <-outcomes:
|
||||
if outcome.err == nil {
|
||||
t.Fatal("Run() error = nil, want cancellation error")
|
||||
}
|
||||
return outcome
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("Run() did not return after cancellation")
|
||||
return runOutcome{}
|
||||
}
|
||||
}
|
||||
|
||||
func assertDescendantDidNotSurvive(t *testing.T, sentinelPath string) {
|
||||
t.Helper()
|
||||
|
||||
time.Sleep(700 * time.Millisecond)
|
||||
if _, err := os.Stat(sentinelPath); err == nil {
|
||||
t.Fatal("descendant survived cancellation and wrote its sentinel")
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("stat descendant sentinel: %v", err)
|
||||
}
|
||||
}
|
||||
104
internal/adapters/subprocess/process_tree_unix.go
Normal file
104
internal/adapters/subprocess/process_tree_unix.go
Normal file
@@ -0,0 +1,104 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
const processGroupPollInterval = 10 * time.Millisecond
|
||||
|
||||
type unixProcessTree struct {
|
||||
processGroupID int
|
||||
}
|
||||
|
||||
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||
return &unixProcessTree{}, nil
|
||||
}
|
||||
|
||||
func (tree *unixProcessTree) Start(cmd *exec.Cmd) error {
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
tree.processGroupID = cmd.Process.Pid
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tree *unixProcessTree) TerminateGracefully() error {
|
||||
return tree.signal(syscall.SIGTERM)
|
||||
}
|
||||
|
||||
func (tree *unixProcessTree) TerminateForcefully() error {
|
||||
return tree.signal(syscall.SIGKILL)
|
||||
}
|
||||
|
||||
func (tree *unixProcessTree) Dispose() error {
|
||||
hasMembers, err := tree.hasMembers()
|
||||
if err != nil || !hasMembers {
|
||||
return err
|
||||
}
|
||||
|
||||
cleanupErr := tree.TerminateGracefully()
|
||||
empty, waitErr := tree.waitUntilEmpty(gracefulTerminationWait)
|
||||
cleanupErr = joinErrors(cleanupErr, waitErr)
|
||||
if empty {
|
||||
return cleanupErr
|
||||
}
|
||||
|
||||
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||
empty, waitErr = tree.waitUntilEmpty(forcefulTerminationWait)
|
||||
cleanupErr = joinErrors(cleanupErr, waitErr)
|
||||
if !empty {
|
||||
cleanupErr = joinErrors(cleanupErr, fmt.Errorf("owned subprocess group did not exit within %s after forceful termination", forcefulTerminationWait))
|
||||
}
|
||||
return cleanupErr
|
||||
}
|
||||
|
||||
func (tree *unixProcessTree) signal(signal syscall.Signal) error {
|
||||
if tree.processGroupID <= 0 {
|
||||
return nil
|
||||
}
|
||||
err := syscall.Kill(-tree.processGroupID, signal)
|
||||
if errors.Is(err, syscall.ESRCH) || errors.Is(err, os.ErrProcessDone) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (tree *unixProcessTree) hasMembers() (bool, error) {
|
||||
if tree.processGroupID <= 0 {
|
||||
return false, nil
|
||||
}
|
||||
err := syscall.Kill(-tree.processGroupID, 0)
|
||||
if err == nil || errors.Is(err, syscall.EPERM) {
|
||||
return true, nil
|
||||
}
|
||||
if errors.Is(err, syscall.ESRCH) || errors.Is(err, os.ErrProcessDone) {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("inspect owned subprocess group: %w", err)
|
||||
}
|
||||
|
||||
func (tree *unixProcessTree) waitUntilEmpty(timeout time.Duration) (bool, error) {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for {
|
||||
hasMembers, err := tree.hasMembers()
|
||||
if err != nil || !hasMembers {
|
||||
return !hasMembers, err
|
||||
}
|
||||
remaining := time.Until(deadline)
|
||||
if remaining <= 0 {
|
||||
return false, nil
|
||||
}
|
||||
if remaining > processGroupPollInterval {
|
||||
remaining = processGroupPollInterval
|
||||
}
|
||||
time.Sleep(remaining)
|
||||
}
|
||||
}
|
||||
12
internal/adapters/subprocess/process_tree_unsupported.go
Normal file
12
internal/adapters/subprocess/process_tree_unsupported.go
Normal file
@@ -0,0 +1,12 @@
|
||||
//go:build !linux && !darwin && !windows
|
||||
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||
return nil, fmt.Errorf("owned subprocess trees are unsupported on %s", runtime.GOOS)
|
||||
}
|
||||
122
internal/adapters/subprocess/process_tree_windows.go
Normal file
122
internal/adapters/subprocess/process_tree_windows.go
Normal file
@@ -0,0 +1,122 @@
|
||||
//go:build windows
|
||||
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
type windowsProcessTree struct {
|
||||
job windows.Handle
|
||||
}
|
||||
|
||||
func newOwnedProcessTree() (ownedProcessTree, error) {
|
||||
return &windowsProcessTree{}, nil
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) Start(cmd *exec.Cmd) error {
|
||||
job, err := windows.CreateJobObject(nil, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create job object: %w", err)
|
||||
}
|
||||
|
||||
limits := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{}
|
||||
limits.BasicLimitInformation.LimitFlags = windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
||||
if _, err := windows.SetInformationJobObject(job, windows.JobObjectExtendedLimitInformation, uintptr(unsafe.Pointer(&limits)), uint32(unsafe.Sizeof(limits))); err != nil {
|
||||
_ = windows.CloseHandle(job)
|
||||
return fmt.Errorf("configure job object: %w", err)
|
||||
}
|
||||
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_SUSPENDED}
|
||||
if err := cmd.Start(); err != nil {
|
||||
_ = windows.CloseHandle(job)
|
||||
return err
|
||||
}
|
||||
|
||||
process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(cmd.Process.Pid))
|
||||
if err == nil {
|
||||
err = windows.AssignProcessToJobObject(job, process)
|
||||
_ = windows.CloseHandle(process)
|
||||
}
|
||||
if err == nil {
|
||||
err = resumeInitialThread(uint32(cmd.Process.Pid))
|
||||
}
|
||||
if err != nil {
|
||||
killErr := cmd.Process.Kill()
|
||||
waitErr := cmd.Wait()
|
||||
_ = windows.CloseHandle(job)
|
||||
return joinErrors(fmt.Errorf("assign process to job object: %w", err), killErr, waitErr)
|
||||
}
|
||||
|
||||
tree.job = job
|
||||
return nil
|
||||
}
|
||||
|
||||
func resumeInitialThread(processID uint32) error {
|
||||
snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("snapshot initial thread: %w", err)
|
||||
}
|
||||
defer func() { _ = windows.CloseHandle(snapshot) }()
|
||||
|
||||
entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))}
|
||||
if err := windows.Thread32First(snapshot, &entry); err != nil {
|
||||
return fmt.Errorf("find initial thread: %w", err)
|
||||
}
|
||||
for {
|
||||
if entry.OwnerProcessID != processID {
|
||||
// Keep enumerating until the suspended process's only initial thread
|
||||
// is found.
|
||||
} else {
|
||||
thread, openErr := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID)
|
||||
if openErr != nil {
|
||||
return fmt.Errorf("open initial thread: %w", openErr)
|
||||
}
|
||||
defer func() { _ = windows.CloseHandle(thread) }()
|
||||
if _, resumeErr := windows.ResumeThread(thread); resumeErr != nil {
|
||||
return fmt.Errorf("resume initial thread: %w", resumeErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := windows.Thread32Next(snapshot, &entry); err != nil {
|
||||
if errors.Is(err, windows.ERROR_NO_MORE_FILES) {
|
||||
break
|
||||
}
|
||||
return fmt.Errorf("find initial thread: %w", err)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("find initial thread: no thread found for process %d", processID)
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) TerminateGracefully() error {
|
||||
// Windows jobs have no portable graceful signal. Terminating the owned job
|
||||
// is the safe fallback and prevents a descendant from escaping cleanup.
|
||||
return tree.terminate()
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) TerminateForcefully() error {
|
||||
return tree.terminate()
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) Dispose() error {
|
||||
if tree.job == 0 {
|
||||
return nil
|
||||
}
|
||||
err := windows.CloseHandle(tree.job)
|
||||
tree.job = 0
|
||||
return err
|
||||
}
|
||||
|
||||
func (tree *windowsProcessTree) terminate() error {
|
||||
if tree.job == 0 {
|
||||
return nil
|
||||
}
|
||||
return windows.TerminateJobObject(tree.job, 1)
|
||||
}
|
||||
@@ -2,28 +2,27 @@ package subprocess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// RunRequest defines a subprocess invocation.
|
||||
type RunRequest struct {
|
||||
Executable string
|
||||
Args []string
|
||||
WorkingDir string
|
||||
EnvOverrides map[string]string
|
||||
Timeout time.Duration
|
||||
StdoutLogPath string
|
||||
StderrLogPath string
|
||||
Executable string
|
||||
Args []string
|
||||
WorkingDir string
|
||||
EnvOverrides map[string]string
|
||||
SensitiveEnvNames []string
|
||||
DiagnosticOwner string
|
||||
Timeout time.Duration
|
||||
StdoutLogPath string
|
||||
StderrLogPath string
|
||||
}
|
||||
|
||||
// RunResult captures subprocess execution details.
|
||||
@@ -54,17 +53,26 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
logs, err := openLogWriters(req.StdoutLogPath, req.StderrLogPath)
|
||||
childEnv := buildChildEnvironment(os.Environ(), req.EnvOverrides)
|
||||
logs, err := openLogWriters(req.StdoutLogPath, req.StderrLogPath, req.diagnosticOwner(), sensitiveEnvironmentValues(childEnv, req.SensitiveEnvNames))
|
||||
if err != nil {
|
||||
return RunResult{}, err
|
||||
}
|
||||
defer logs.Close()
|
||||
|
||||
cmd := exec.CommandContext(runCtx, req.Executable, req.Args...)
|
||||
tree, err := newOwnedProcessTree()
|
||||
if err != nil {
|
||||
return RunResult{}, fmt.Errorf("prepare owned subprocess tree: %w", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command(req.Executable, req.Args...)
|
||||
cmd.Dir = req.WorkingDir
|
||||
cmd.Env = mergeEnv(os.Environ(), req.EnvOverrides)
|
||||
cmd.Env = childEnv
|
||||
cmd.Stdout = logs.Stdout
|
||||
cmd.Stderr = logs.Stderr
|
||||
// Streaming capture uses pipes. Bound their lifetime when a leader exits
|
||||
// while a descendant still holds a stream descriptor.
|
||||
cmd.WaitDelay = forcefulTerminationWait
|
||||
|
||||
started := time.Now().UTC()
|
||||
result := RunResult{
|
||||
@@ -74,45 +82,67 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
if err := runCtx.Err(); err != nil {
|
||||
result.CompletedAt = time.Now().UTC()
|
||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||
return result, fmt.Errorf("command was not started: %w", err)
|
||||
}
|
||||
|
||||
if err := tree.Start(cmd); err != nil {
|
||||
result.CompletedAt = time.Now().UTC()
|
||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||
return result, fmt.Errorf("start command %q with args %v: %w", req.Executable, req.Args, err)
|
||||
}
|
||||
|
||||
waitErr := cmd.Wait()
|
||||
waitCh := make(chan error, 1)
|
||||
go func() { waitCh <- cmd.Wait() }()
|
||||
|
||||
waitErr, ctxErr, captureLimit, cleanupErr := waitForOwnedCommand(runCtx, tree, waitCh, logs.Limits())
|
||||
cleanupErr = joinErrors(cleanupErr, tree.Dispose())
|
||||
cleanupErr = joinErrors(cleanupErr, logs.Flush())
|
||||
if captureLimit == nil {
|
||||
captureLimit = logs.Limit()
|
||||
}
|
||||
result.CompletedAt = time.Now().UTC()
|
||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||
if cmd.ProcessState != nil {
|
||||
result.ExitCode = cmd.ProcessState.ExitCode()
|
||||
}
|
||||
|
||||
ctxErr := runCtx.Err()
|
||||
if errors.Is(ctxErr, context.DeadlineExceeded) {
|
||||
if ctxErr == context.DeadlineExceeded {
|
||||
result.TimedOut = true
|
||||
}
|
||||
if errors.Is(ctxErr, context.Canceled) && !result.TimedOut {
|
||||
if ctxErr == context.Canceled && !result.TimedOut {
|
||||
result.Canceled = true
|
||||
}
|
||||
|
||||
if waitErr == nil {
|
||||
if waitErr == nil && ctxErr == nil && cleanupErr == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
stderrTail := readRedactedTail(req.StderrLogPath, req.EnvOverrides, 2048)
|
||||
stderrTail := logs.stderr.Tail()
|
||||
diagnostics := buildDiagnostics(req, result, stderrTail)
|
||||
|
||||
if captureLimit != nil {
|
||||
if cause := joinErrors(waitErr, cleanupErr); cause != nil {
|
||||
return result, fmt.Errorf("%w (%s): %w", captureLimit, diagnostics, cause)
|
||||
}
|
||||
return result, fmt.Errorf("%w (%s)", captureLimit, diagnostics)
|
||||
}
|
||||
if result.TimedOut {
|
||||
return result, fmt.Errorf("command timed out after %s (%s)", req.Timeout, diagnostics)
|
||||
return result, fmt.Errorf("command timed out after %s (%s): %w", req.Timeout, diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
|
||||
}
|
||||
if result.Canceled {
|
||||
return result, fmt.Errorf("command canceled (%s)", diagnostics)
|
||||
return result, fmt.Errorf("command canceled (%s): %w", diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
|
||||
}
|
||||
if exitErr, ok := waitErr.(*exec.ExitError); ok {
|
||||
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, waitErr)
|
||||
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, joinErrors(waitErr, cleanupErr))
|
||||
}
|
||||
if cleanupErr != nil {
|
||||
return result, fmt.Errorf("command cleanup failed (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
|
||||
}
|
||||
|
||||
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, waitErr)
|
||||
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
|
||||
}
|
||||
|
||||
// WriteYAMLAtomic marshals value as YAML and atomically writes it to path.
|
||||
@@ -127,141 +157,17 @@ func WriteYAMLAtomic(path string, value any, perm os.FileMode) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteFileAtomic writes bytes via same-directory temp file + atomic rename.
|
||||
// WriteFileAtomic writes bytes through the shared durable replacement primitive.
|
||||
func WriteFileAtomic(path string, data []byte, perm os.FileMode) error {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fmt.Errorf("write file: path is required")
|
||||
}
|
||||
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("create parent directory %q: %w", dir, err)
|
||||
if err := fileops.WriteFileAtomic(path, data, perm); err != nil {
|
||||
return fmt.Errorf("write file %q: %w", path, err)
|
||||
}
|
||||
|
||||
base := filepath.Base(path)
|
||||
tmp, err := os.CreateTemp(dir, "."+base+".tmp-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
removeTmp := true
|
||||
defer func() {
|
||||
if removeTmp {
|
||||
_ = os.Remove(tmpPath)
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return fmt.Errorf("write temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Sync(); err != nil {
|
||||
_ = tmp.Close()
|
||||
return fmt.Errorf("sync temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("close temp file: %w", err)
|
||||
}
|
||||
if err := os.Chmod(tmpPath, perm); err != nil {
|
||||
return fmt.Errorf("chmod temp file: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmpPath, path); err != nil {
|
||||
return fmt.Errorf("rename temp file: %w", err)
|
||||
}
|
||||
removeTmp = false
|
||||
return nil
|
||||
}
|
||||
|
||||
type logWriters struct {
|
||||
files []*os.File
|
||||
Stdout io.Writer
|
||||
Stderr io.Writer
|
||||
}
|
||||
|
||||
func (l *logWriters) Close() {
|
||||
for _, f := range l.files {
|
||||
_ = f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func openLogWriters(stdoutPath, stderrPath string) (*logWriters, error) {
|
||||
cleanStdout := cleanLogPath(stdoutPath)
|
||||
cleanStderr := cleanLogPath(stderrPath)
|
||||
|
||||
// Keep stdout/stderr on the same file descriptor when both paths target
|
||||
// the same file to avoid descriptor aliasing surprises across runtimes.
|
||||
if cleanStdout != "" && cleanStdout == cleanStderr {
|
||||
f, err := openLogFile(cleanStdout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open shared stdout/stderr log %q: %w", cleanStdout, err)
|
||||
}
|
||||
return &logWriters{
|
||||
files: []*os.File{f},
|
||||
Stdout: f,
|
||||
Stderr: f,
|
||||
}, nil
|
||||
}
|
||||
|
||||
stdoutFile, stdoutWriter, err := logWriter(cleanStdout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open stdout log: %w", err)
|
||||
}
|
||||
stderrFile, stderrWriter, err := logWriter(cleanStderr)
|
||||
if err != nil {
|
||||
closeFile(stdoutFile)
|
||||
return nil, fmt.Errorf("open stderr log: %w", err)
|
||||
}
|
||||
|
||||
files := make([]*os.File, 0, 2)
|
||||
if stdoutFile != nil {
|
||||
files = append(files, stdoutFile)
|
||||
}
|
||||
if stderrFile != nil {
|
||||
files = append(files, stderrFile)
|
||||
}
|
||||
return &logWriters{
|
||||
files: files,
|
||||
Stdout: stdoutWriter,
|
||||
Stderr: stderrWriter,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func cleanLogPath(path string) string {
|
||||
trimmed := strings.TrimSpace(path)
|
||||
if trimmed == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Clean(trimmed)
|
||||
}
|
||||
|
||||
func logWriter(path string) (*os.File, io.Writer, error) {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return nil, io.Discard, nil
|
||||
}
|
||||
f, err := openLogFile(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return f, f, nil
|
||||
}
|
||||
|
||||
func openLogFile(path string) (*os.File, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create log directory for %q: %w", path, err)
|
||||
}
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open log file %q: %w", path, err)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func closeFile(f *os.File) {
|
||||
if f != nil {
|
||||
_ = f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func buildDiagnostics(req RunRequest, result RunResult, stderrTail string) string {
|
||||
details := fmt.Sprintf(
|
||||
"executable=%q args=%v cwd=%q timeout=%s exit_code=%d timed_out=%t canceled=%t stdout_log=%q stderr_log=%q",
|
||||
@@ -298,87 +204,3 @@ func fdDiagnosticsHint(exitCode int, stderrTail string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func readRedactedTail(path string, envOverrides map[string]string, maxBytes int64) string {
|
||||
if strings.TrimSpace(path) == "" || maxBytes <= 0 {
|
||||
return ""
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
size := info.Size()
|
||||
start := int64(0)
|
||||
if size > maxBytes {
|
||||
start = size - maxBytes
|
||||
}
|
||||
if _, err := f.Seek(start, io.SeekStart); err != nil {
|
||||
return ""
|
||||
}
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
tail := strings.TrimSpace(string(data))
|
||||
if tail == "" {
|
||||
return ""
|
||||
}
|
||||
return redactSensitiveTail(tail, envOverrides)
|
||||
}
|
||||
|
||||
func redactSensitiveTail(tail string, envOverrides map[string]string) string {
|
||||
out := tail
|
||||
for k, v := range envOverrides {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
continue
|
||||
}
|
||||
if looksSensitiveEnvKey(k) {
|
||||
out = strings.ReplaceAll(out, v, "<redacted>")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func looksSensitiveEnvKey(key string) bool {
|
||||
k := strings.ToUpper(strings.TrimSpace(key))
|
||||
return strings.Contains(k, "KEY") ||
|
||||
strings.Contains(k, "TOKEN") ||
|
||||
strings.Contains(k, "SECRET") ||
|
||||
strings.Contains(k, "PASSWORD")
|
||||
}
|
||||
|
||||
func mergeEnv(base []string, overrides map[string]string) []string {
|
||||
if len(overrides) == 0 {
|
||||
return base
|
||||
}
|
||||
|
||||
kv := make(map[string]string, len(base)+len(overrides))
|
||||
for _, item := range base {
|
||||
k, v, ok := strings.Cut(item, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
kv[k] = v
|
||||
}
|
||||
for k, v := range overrides {
|
||||
kv[k] = v
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(kv))
|
||||
for k := range kv {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
out := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
out = append(out, k+"="+kv[k])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -120,6 +127,316 @@ func TestRunFailureRedactsSensitiveTail(t *testing.T) {
|
||||
if !strings.Contains(err.Error(), "<redacted>") {
|
||||
t.Fatalf("error = %q, want redacted stderr tail marker", err.Error())
|
||||
}
|
||||
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath} {
|
||||
data, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read diagnostic %q: %v", path, readErr)
|
||||
}
|
||||
if strings.Contains(string(data), secretValue) {
|
||||
t.Fatalf("diagnostic %q leaked secret: %q", path, data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("creating symlinks requires privileges that are not available on every Windows runner")
|
||||
}
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
targetPath := filepath.Join(dir, "outside.log")
|
||||
const original = "must remain unchanged"
|
||||
if err := os.WriteFile(targetPath, []byte(original), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(target) error = %v", err)
|
||||
}
|
||||
stdoutPath := filepath.Join(dir, "stdout.log")
|
||||
if err := os.Symlink(targetPath, stdoutPath); err != nil {
|
||||
t.Fatalf("Symlink() error = %v", err)
|
||||
}
|
||||
|
||||
_, err = Run(context.Background(), RunRequest{
|
||||
Executable: exe,
|
||||
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "success"},
|
||||
EnvOverrides: map[string]string{"GO_WANT_SUBPROCESS_HELPER": "1"},
|
||||
StdoutLogPath: stdoutPath,
|
||||
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "symbolic link") {
|
||||
t.Fatalf("Run() error = %v, want symbolic-link rejection", err)
|
||||
}
|
||||
data, readErr := os.ReadFile(targetPath)
|
||||
if readErr != nil {
|
||||
t.Fatalf("ReadFile(target) error = %v", readErr)
|
||||
}
|
||||
if string(data) != original {
|
||||
t.Fatalf("target content = %q, want %q", data, original)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFailureUsesOpenedDiagnosticAfterPathReplacement(t *testing.T) {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
readyPath := filepath.Join(dir, "ready")
|
||||
releasePath := filepath.Join(dir, "release")
|
||||
stderrPath := filepath.Join(dir, "stderr.log")
|
||||
openedPath := filepath.Join(dir, "opened-stderr.log")
|
||||
const secretValue = "replacement-api-key-value"
|
||||
const commandContent = "trusted command failure"
|
||||
req := RunRequest{
|
||||
Executable: exe,
|
||||
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "delayed-fail"},
|
||||
EnvOverrides: map[string]string{
|
||||
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||
"API_KEY": secretValue,
|
||||
"SUBPROCESS_HELPER_READY_PATH": readyPath,
|
||||
"SUBPROCESS_HELPER_RELEASE_PATH": releasePath,
|
||||
"SUBPROCESS_HELPER_STDERR": commandContent,
|
||||
},
|
||||
StdoutLogPath: filepath.Join(dir, "stdout.log"),
|
||||
StderrLogPath: stderrPath,
|
||||
}
|
||||
|
||||
resultCh := make(chan error, 1)
|
||||
go func() {
|
||||
_, runErr := Run(context.Background(), req)
|
||||
resultCh <- runErr
|
||||
}()
|
||||
waitForHelperFile(t, readyPath)
|
||||
if err := os.Rename(stderrPath, openedPath); err != nil {
|
||||
t.Fatalf("Rename(stderr log) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(stderrPath, []byte(secretValue), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(replacement) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(releasePath, []byte("continue"), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(release) error = %v", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case runErr := <-resultCh:
|
||||
if runErr == nil {
|
||||
t.Fatal("Run() error = nil, want command failure")
|
||||
}
|
||||
if strings.Contains(runErr.Error(), secretValue) {
|
||||
t.Fatalf("error read replacement-path content: %q", runErr)
|
||||
}
|
||||
if !strings.Contains(runErr.Error(), commandContent) {
|
||||
t.Fatalf("error = %q, want retained command diagnostic", runErr)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("Run() did not return after helper release")
|
||||
}
|
||||
|
||||
openedData, err := os.ReadFile(openedPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(opened diagnostic) error = %v", err)
|
||||
}
|
||||
if !strings.Contains(string(openedData), commandContent) {
|
||||
t.Fatalf("opened diagnostic = %q, want command content", openedData)
|
||||
}
|
||||
replacementData, err := os.ReadFile(stderrPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(replacement diagnostic) error = %v", err)
|
||||
}
|
||||
if string(replacementData) != secretValue {
|
||||
t.Fatalf("replacement diagnostic = %q, want %q", replacementData, secretValue)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRedactsSplitCredentialInSeparateAndSharedDiagnostics(t *testing.T) {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
}
|
||||
const secretValue = "split-super-secret-value"
|
||||
|
||||
for _, shared := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "separate", true: "shared"}[shared], func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
stdoutPath := filepath.Join(dir, "stdout.log")
|
||||
stderrPath := filepath.Join(dir, "stderr.log")
|
||||
if shared {
|
||||
stderrPath = stdoutPath
|
||||
}
|
||||
req := RunRequest{
|
||||
Executable: exe,
|
||||
Args: []string{"-test.run=^TestSubprocessHelper$", "--", "splitsecret"},
|
||||
EnvOverrides: map[string]string{
|
||||
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||
"API_KEY": secretValue,
|
||||
},
|
||||
StdoutLogPath: stdoutPath,
|
||||
StderrLogPath: stderrPath,
|
||||
}
|
||||
|
||||
_, runErr := Run(context.Background(), req)
|
||||
if runErr == nil {
|
||||
t.Fatal("Run() error = nil, want command failure")
|
||||
}
|
||||
if strings.Contains(runErr.Error(), secretValue) || !strings.Contains(runErr.Error(), "<redacted>") {
|
||||
t.Fatalf("error = %q, want redacted credential", runErr)
|
||||
}
|
||||
paths := map[string]struct{}{stdoutPath: {}, stderrPath: {}}
|
||||
for path := range paths {
|
||||
data, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
t.Fatalf("ReadFile(%q) error = %v", path, readErr)
|
||||
}
|
||||
if strings.Contains(string(data), secretValue) || !strings.Contains(string(data), "<redacted>") {
|
||||
t.Fatalf("diagnostic %q = %q, want redacted credential", path, data)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRedactsInheritedSensitiveEnvironment(t *testing.T) {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
}
|
||||
secretValue := "inherited-secret-value"
|
||||
t.Setenv("OPENROUTER_API_KEY", secretValue)
|
||||
dir := t.TempDir()
|
||||
req := RunRequest{
|
||||
Executable: exe,
|
||||
Args: []string{"-test.run=TestSubprocessHelper", "--", "echoenv"},
|
||||
EnvOverrides: map[string]string{
|
||||
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||
"SUBPROCESS_HELPER_ENV_KEY": "OPENROUTER_API_KEY",
|
||||
},
|
||||
StdoutLogPath: filepath.Join(dir, "stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||
}
|
||||
_, err = Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want non-nil")
|
||||
}
|
||||
if strings.Contains(err.Error(), secretValue) {
|
||||
t.Fatalf("error leaked inherited secret: %q", err)
|
||||
}
|
||||
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath} {
|
||||
data, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read diagnostic %q: %v", path, readErr)
|
||||
}
|
||||
if strings.Contains(string(data), secretValue) {
|
||||
t.Fatalf("diagnostic %q leaked inherited secret: %q", path, data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRedactsSensitiveOutputAndErrorTail(t *testing.T) {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
}
|
||||
secretValue := "override-secret-value"
|
||||
dir := t.TempDir()
|
||||
req := RunRequest{
|
||||
Executable: exe,
|
||||
Args: []string{"-test.run=TestSubprocessHelper", "--", "echoenv"},
|
||||
EnvOverrides: map[string]string{
|
||||
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||
"SUBPROCESS_HELPER_ENV_KEY": "OPENROUTER_API_KEY",
|
||||
"OPENROUTER_API_KEY": secretValue,
|
||||
},
|
||||
StdoutLogPath: filepath.Join(dir, "stdout.log"),
|
||||
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||
}
|
||||
_, err = Run(context.Background(), req)
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want non-nil")
|
||||
}
|
||||
if strings.Contains(err.Error(), secretValue) || !strings.Contains(err.Error(), "<redacted>") {
|
||||
t.Fatalf("error = %q, want redacted secret", err)
|
||||
}
|
||||
for _, path := range []string{req.StdoutLogPath, req.StderrLogPath} {
|
||||
data, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read diagnostic %q: %v", path, readErr)
|
||||
}
|
||||
if strings.Contains(string(data), secretValue) || !strings.Contains(string(data), "<redacted>") {
|
||||
t.Fatalf("diagnostic %q = %q, want redacted secret", path, data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamRedactorHandlesSplitAndOverlappingSecrets(t *testing.T) {
|
||||
redactor := newStreamRedactor([]string{"abc", "abcde", "cde", ""})
|
||||
var output bytes.Buffer
|
||||
output.Write(redactor.Write([]byte("start-ab")))
|
||||
output.Write(redactor.Write([]byte("cde-end")))
|
||||
output.Write(redactor.Flush())
|
||||
if got := output.String(); got != "start-<redacted>-end" {
|
||||
t.Fatalf("redacted output = %q, want one redacted marker", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
||||
exactLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
var exactOutput bytes.Buffer
|
||||
exact := newDiagnosticWriter(exactLogs, "stdout", "test", &exactOutput, 5, nil)
|
||||
if _, err := exact.Write([]byte("abcde")); err != nil {
|
||||
t.Fatalf("exact Write() error = %v", err)
|
||||
}
|
||||
if err := exact.Flush(); err != nil {
|
||||
t.Fatalf("exact Flush() error = %v", err)
|
||||
}
|
||||
if got := exactOutput.String(); got != "abcde" {
|
||||
t.Fatalf("exact output = %q, want abcde", got)
|
||||
}
|
||||
if exactLogs.Limit() != nil {
|
||||
t.Fatal("exact write recorded a capture limit")
|
||||
}
|
||||
|
||||
cappedLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
var cappedOutput bytes.Buffer
|
||||
capped := newDiagnosticWriter(cappedLogs, "stderr", "test", &cappedOutput, 5, nil)
|
||||
if _, err := capped.Write([]byte("abcdef")); err == nil {
|
||||
t.Fatal("cap-plus-one Write() error = nil, want capture limit")
|
||||
}
|
||||
if err := capped.Flush(); err != nil {
|
||||
t.Fatalf("cap-plus-one Flush() error = %v", err)
|
||||
}
|
||||
if got := cappedOutput.String(); got != "abcde" {
|
||||
t.Fatalf("capped output = %q, want abcde", got)
|
||||
}
|
||||
if limit := cappedLogs.Limit(); limit == nil || limit.stream != "stderr" || limit.limit != 5 {
|
||||
t.Fatalf("capture limit = %#v, want stderr limit 5", limit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticWriterRetainsBoundedRedactedTail(t *testing.T) {
|
||||
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
var output bytes.Buffer
|
||||
secret := "credential-value"
|
||||
writer := newDiagnosticWriter(logs, "stderr", "test", &output, 16*1024, []string{secret})
|
||||
prefix := strings.Repeat("x", diagnosticTailBytes+512)
|
||||
if _, err := writer.Write([]byte(prefix + secret[:7])); err != nil {
|
||||
t.Fatalf("first Write() error = %v", err)
|
||||
}
|
||||
if _, err := writer.Write([]byte(secret[7:] + "-failure")); err != nil {
|
||||
t.Fatalf("second Write() error = %v", err)
|
||||
}
|
||||
if err := writer.Flush(); err != nil {
|
||||
t.Fatalf("Flush() error = %v", err)
|
||||
}
|
||||
tail := writer.Tail()
|
||||
if len(tail) > diagnosticTailBytes {
|
||||
t.Fatalf("retained tail length = %d, want at most %d", len(tail), diagnosticTailBytes)
|
||||
}
|
||||
if strings.Contains(tail, secret) || !strings.Contains(tail, "<redacted>-failure") {
|
||||
t.Fatalf("retained tail = %q, want bounded redacted content", tail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFailureAddsBadDescriptorHint(t *testing.T) {
|
||||
@@ -184,15 +501,14 @@ func TestRunInheritsParentEnvironmentByDefault(t *testing.T) {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SUBPROCESS_HELPER", "1")
|
||||
t.Setenv("SUBPROCESS_HELPER_ENV_KEY", "SUBPROCESS_PARENT_VALUE")
|
||||
t.Setenv("SUBPROCESS_PARENT_VALUE", "inherited-value")
|
||||
t.Setenv("PATH", "inherited-value")
|
||||
|
||||
dir := t.TempDir()
|
||||
stdoutPath := filepath.Join(dir, "stdout.log")
|
||||
req := RunRequest{
|
||||
Executable: exe,
|
||||
Args: []string{"-test.run=TestSubprocessHelper", "--", "printenv"},
|
||||
EnvOverrides: map[string]string{"GO_WANT_SUBPROCESS_HELPER": "1", "SUBPROCESS_HELPER_ENV_KEY": "PATH"},
|
||||
StdoutLogPath: stdoutPath,
|
||||
}
|
||||
|
||||
@@ -214,16 +530,16 @@ func TestRunEnvOverridesWinOverInheritedValues(t *testing.T) {
|
||||
t.Fatalf("os.Executable() error = %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("GO_WANT_SUBPROCESS_HELPER", "1")
|
||||
t.Setenv("SUBPROCESS_HELPER_ENV_KEY", "SUBPROCESS_PARENT_VALUE")
|
||||
t.Setenv("SUBPROCESS_PARENT_VALUE", "parent-value")
|
||||
|
||||
dir := t.TempDir()
|
||||
stdoutPath := filepath.Join(dir, "stdout.log")
|
||||
req := RunRequest{
|
||||
Executable: exe,
|
||||
Args: []string{"-test.run=TestSubprocessHelper", "--", "printenv"},
|
||||
EnvOverrides: map[string]string{"SUBPROCESS_PARENT_VALUE": "override-value"},
|
||||
Executable: exe,
|
||||
Args: []string{"-test.run=TestSubprocessHelper", "--", "printenv"},
|
||||
EnvOverrides: map[string]string{
|
||||
"GO_WANT_SUBPROCESS_HELPER": "1",
|
||||
"SUBPROCESS_HELPER_ENV_KEY": "SUBPROCESS_PARENT_VALUE",
|
||||
"SUBPROCESS_PARENT_VALUE": "override-value",
|
||||
},
|
||||
StdoutLogPath: stdoutPath,
|
||||
}
|
||||
|
||||
@@ -361,6 +677,30 @@ func TestSubprocessHelper(t *testing.T) {
|
||||
case "failbadfd":
|
||||
_, _ = os.Stderr.WriteString("OSError: [Errno 9] Bad file descriptor\n")
|
||||
os.Exit(120)
|
||||
case "delayed-fail":
|
||||
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
|
||||
os.Exit(4)
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
if _, err := os.Stat(os.Getenv("SUBPROCESS_HELPER_RELEASE_PATH")); err == nil {
|
||||
break
|
||||
} else if !errors.Is(err, os.ErrNotExist) || time.Now().After(deadline) {
|
||||
os.Exit(5)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
_, _ = os.Stderr.WriteString(os.Getenv("SUBPROCESS_HELPER_STDERR"))
|
||||
os.Exit(6)
|
||||
case "splitsecret":
|
||||
secret := os.Getenv("API_KEY")
|
||||
split := len(secret) / 2
|
||||
for _, stream := range []*os.File{os.Stdout, os.Stderr} {
|
||||
_, _ = stream.WriteString(secret[:split])
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
_, _ = stream.WriteString(secret[split:] + "\n")
|
||||
}
|
||||
os.Exit(7)
|
||||
case "sleep":
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
os.Exit(0)
|
||||
@@ -368,7 +708,115 @@ func TestSubprocessHelper(t *testing.T) {
|
||||
key := os.Getenv("SUBPROCESS_HELPER_ENV_KEY")
|
||||
_, _ = os.Stdout.WriteString(os.Getenv(key) + "\n")
|
||||
os.Exit(0)
|
||||
case "echoenv":
|
||||
key := os.Getenv("SUBPROCESS_HELPER_ENV_KEY")
|
||||
value := os.Getenv(key)
|
||||
_, _ = os.Stdout.WriteString(value)
|
||||
_, _ = os.Stderr.WriteString(value)
|
||||
os.Exit(5)
|
||||
case "spam":
|
||||
chunk := strings.Repeat("x", 64*1024)
|
||||
count, _ := strconv.Atoi(os.Getenv("SUBPROCESS_HELPER_CHUNKS"))
|
||||
for range count {
|
||||
_, _ = os.Stdout.WriteString(chunk)
|
||||
}
|
||||
os.Exit(0)
|
||||
case "tree-spam":
|
||||
descendant := exec.Command(os.Args[0], "-test.run=^TestSubprocessHelper$", "--", "descendant")
|
||||
descendant.Env = append(os.Environ(), "GO_WANT_SUBPROCESS_HELPER=1")
|
||||
descendant.Stdout = os.Stdout
|
||||
descendant.Stderr = os.Stderr
|
||||
if err := descendant.Start(); err != nil {
|
||||
os.Exit(3)
|
||||
}
|
||||
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
|
||||
os.Exit(4)
|
||||
}
|
||||
chunk := strings.Repeat("x", 64*1024)
|
||||
for {
|
||||
_, _ = os.Stdout.WriteString(chunk)
|
||||
}
|
||||
case "tree":
|
||||
descendant := exec.Command(os.Args[0], "-test.run=^TestSubprocessHelper$", "--", "descendant")
|
||||
descendant.Env = append(os.Environ(), "GO_WANT_SUBPROCESS_HELPER=1")
|
||||
descendant.Stdout = os.Stdout
|
||||
descendant.Stderr = os.Stderr
|
||||
if err := descendant.Start(); err != nil {
|
||||
os.Exit(3)
|
||||
}
|
||||
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
|
||||
os.Exit(4)
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
os.Exit(0)
|
||||
case "leader-exit-retained", "leader-exit-redirected", "leader-fail-redirected":
|
||||
descendant := exec.Command(os.Args[0], "-test.run=^TestSubprocessHelper$", "--", "descendant-after-release")
|
||||
descendant.Env = append(os.Environ(), "GO_WANT_SUBPROCESS_HELPER=1")
|
||||
if mode == "leader-exit-retained" {
|
||||
descendant.Stdout = os.Stdout
|
||||
descendant.Stderr = os.Stderr
|
||||
}
|
||||
if err := descendant.Start(); err != nil {
|
||||
os.Exit(3)
|
||||
}
|
||||
if !helperFileAppeared(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), 2*time.Second) {
|
||||
os.Exit(4)
|
||||
}
|
||||
if mode == "leader-fail-redirected" {
|
||||
os.Exit(9)
|
||||
}
|
||||
os.Exit(0)
|
||||
case "descendant-after-release":
|
||||
if os.Getenv("SUBPROCESS_HELPER_IGNORE_TERM") == "1" {
|
||||
signal.Ignore(syscall.SIGTERM)
|
||||
}
|
||||
if err := os.WriteFile(os.Getenv("SUBPROCESS_HELPER_READY_PATH"), []byte("ready"), 0o600); err != nil {
|
||||
os.Exit(4)
|
||||
}
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := os.Stat(os.Getenv("SUBPROCESS_HELPER_RELEASE_PATH")); err == nil {
|
||||
_ = os.WriteFile(os.Getenv("SUBPROCESS_HELPER_SENTINEL_PATH"), []byte("survived"), 0o600)
|
||||
os.Exit(0)
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
os.Exit(5)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
os.Exit(0)
|
||||
case "descendant":
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
_ = os.WriteFile(os.Getenv("SUBPROCESS_HELPER_SENTINEL_PATH"), []byte("survived"), 0o600)
|
||||
time.Sleep(10 * time.Second)
|
||||
os.Exit(0)
|
||||
default:
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForHelperFile(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("Stat(%q) error = %v", path, err)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("helper file %q was not created", path)
|
||||
}
|
||||
|
||||
func helperFileAppeared(path string, timeout time.Duration) bool {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return true
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return false
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -2,8 +2,10 @@ package whisperx
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
var minimalTranscriptJSON = []byte(`{"schema":"speaker_transcript.v1","segments":[]}`)
|
||||
@@ -31,7 +33,8 @@ func (n *NoopClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
||||
|
||||
// FakeClient captures requests and returns deterministic responses for tests.
|
||||
type FakeClient struct {
|
||||
Requests []TranscribeRequest
|
||||
requestsMu sync.RWMutex
|
||||
requests []TranscribeRequest
|
||||
Err error
|
||||
Result TranscribeResult
|
||||
TranscribeFn func(ctx context.Context, req TranscribeRequest) (TranscribeResult, error)
|
||||
@@ -42,7 +45,9 @@ func (f *FakeClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
||||
if err := ctx.Err(); err != nil {
|
||||
return TranscribeResult{}, err
|
||||
}
|
||||
f.Requests = append(f.Requests, req)
|
||||
f.requestsMu.Lock()
|
||||
f.requests = append(f.requests, req)
|
||||
f.requestsMu.Unlock()
|
||||
if f.TranscribeFn != nil {
|
||||
return f.TranscribeFn(ctx, req)
|
||||
}
|
||||
@@ -65,12 +70,19 @@ func (f *FakeClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// RequestsSnapshot returns a copy of captured requests safe for concurrent test assertions.
|
||||
func (f *FakeClient) RequestsSnapshot() []TranscribeRequest {
|
||||
f.requestsMu.RLock()
|
||||
defer f.requestsMu.RUnlock()
|
||||
return append([]TranscribeRequest(nil), f.requests...)
|
||||
}
|
||||
|
||||
func writeMinimalJSON(path string) error {
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, minimalTranscriptJSON, 0o644)
|
||||
return fileops.WriteFileAtomic(path, minimalTranscriptJSON, fileops.WorkspaceFileMode)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package whisperx
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -14,8 +15,9 @@ func TestFakeClientCapturesRequestAndReturnsPath(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe() error = %v", err)
|
||||
}
|
||||
if len(fake.Requests) != 1 || fake.Requests[0].SpeakerID != "alice" {
|
||||
t.Fatalf("requests = %#v, want one alice request", fake.Requests)
|
||||
requests := fake.RequestsSnapshot()
|
||||
if len(requests) != 1 || requests[0].SpeakerID != "alice" {
|
||||
t.Fatalf("requests = %#v, want one alice request", requests)
|
||||
}
|
||||
if res.OutputRawTranscriptPath != req.OutputRawTranscriptPath {
|
||||
t.Fatalf("output path = %q, want %q", res.OutputRawTranscriptPath, req.OutputRawTranscriptPath)
|
||||
@@ -29,3 +31,22 @@ func TestFakeClientError(t *testing.T) {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeClientRequestsSnapshotSupportsConcurrentCalls(t *testing.T) {
|
||||
fake := &FakeClient{}
|
||||
const callers = 16
|
||||
var group sync.WaitGroup
|
||||
group.Add(callers)
|
||||
for i := 0; i < callers; i++ {
|
||||
go func() {
|
||||
defer group.Done()
|
||||
if _, err := fake.Transcribe(context.Background(), TranscribeRequest{}); err != nil {
|
||||
t.Errorf("Transcribe() error = %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
group.Wait()
|
||||
if got := len(fake.RequestsSnapshot()); got != callers {
|
||||
t.Fatalf("captured requests = %d, want %d", got, callers)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package whisperx
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -14,10 +13,16 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
const defaultMaxResponseBytes int64 = 10 * 1024 * 1024
|
||||
const (
|
||||
defaultMaxWhisperXResponseBytes int64 = 10 * 1024 * 1024
|
||||
whisperXUploadBufferSize = 32 * 1024
|
||||
)
|
||||
|
||||
// HTTPClientConfig contains parsed, deterministic WhisperX HTTP client settings.
|
||||
type HTTPClientConfig struct {
|
||||
@@ -39,6 +44,7 @@ type HTTPClient struct {
|
||||
retryDelay time.Duration
|
||||
httpClient *http.Client
|
||||
maxResponseBytes int64
|
||||
openAudio func(string) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
// NewHTTPClientFromConfigValues builds a client from config values and parses durations once.
|
||||
@@ -72,11 +78,11 @@ func NewHTTPClient(cfg HTTPClientConfig) (*HTTPClient, error) {
|
||||
return nil, fmt.Errorf("whisperx transcribe_url is required")
|
||||
}
|
||||
u, err := url.Parse(cfg.TranscribeURL)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
if err != nil || !u.IsAbs() || u.Host == "" || !isHTTPURLScheme(u.Scheme) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid whisperx transcribe_url %q: %w", cfg.TranscribeURL, err)
|
||||
}
|
||||
return nil, fmt.Errorf("invalid whisperx transcribe_url %q", cfg.TranscribeURL)
|
||||
return nil, fmt.Errorf("invalid whisperx transcribe_url %q: must be an absolute http or https URL", cfg.TranscribeURL)
|
||||
}
|
||||
if cfg.Timeout <= 0 {
|
||||
return nil, fmt.Errorf("whisperx timeout must be > 0")
|
||||
@@ -93,12 +99,14 @@ func NewHTTPClient(cfg HTTPClientConfig) (*HTTPClient, error) {
|
||||
|
||||
client := cfg.HTTPClient
|
||||
if client == nil {
|
||||
client = &http.Client{}
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.ExpectContinueTimeout = 100 * time.Millisecond
|
||||
client = &http.Client{Transport: transport}
|
||||
}
|
||||
|
||||
maxBytes := cfg.MaxResponseBytes
|
||||
if maxBytes <= 0 {
|
||||
maxBytes = defaultMaxResponseBytes
|
||||
maxBytes = defaultMaxWhisperXResponseBytes
|
||||
}
|
||||
|
||||
return &HTTPClient{
|
||||
@@ -109,6 +117,7 @@ func NewHTTPClient(cfg HTTPClientConfig) (*HTTPClient, error) {
|
||||
retryDelay: cfg.RetryDelay,
|
||||
httpClient: client,
|
||||
maxResponseBytes: maxBytes,
|
||||
openAudio: func(path string) (io.ReadCloser, error) { return os.Open(path) },
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -147,7 +156,7 @@ func (c *HTTPClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
||||
result.Duration = time.Since(start)
|
||||
return result, fmt.Errorf("whisperx attempt %d returned invalid json: %w", attempt, err)
|
||||
}
|
||||
if err := writeFileAtomic(req.OutputRawTranscriptPath, body, 0o644); err != nil {
|
||||
if err := writeFileAtomic(req.OutputRawTranscriptPath, body, fileops.WorkspaceFileMode); err != nil {
|
||||
result.Duration = time.Since(start)
|
||||
return result, fmt.Errorf("whisperx write transcript output %q: %w", req.OutputRawTranscriptPath, err)
|
||||
}
|
||||
@@ -183,52 +192,44 @@ func (c *HTTPClient) Transcribe(ctx context.Context, req TranscribeRequest) (Tra
|
||||
}
|
||||
|
||||
func (c *HTTPClient) doTranscribeAttempt(ctx context.Context, audioPath string) (int, []byte, error) {
|
||||
bodyBuf := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(bodyBuf)
|
||||
upload := newMultipartUpload(ctx, audioPath, c.language, c.openAudio)
|
||||
defer upload.Close()
|
||||
|
||||
fileWriter, err := writer.CreateFormFile("file", filepath.Base(audioPath))
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("create multipart file field: %w", err)
|
||||
}
|
||||
|
||||
audioFile, err := os.Open(audioPath)
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("open audio file %q: %w", audioPath, err)
|
||||
}
|
||||
if _, err := io.Copy(fileWriter, audioFile); err != nil {
|
||||
_ = audioFile.Close()
|
||||
return 0, nil, fmt.Errorf("copy audio file %q: %w", audioPath, err)
|
||||
}
|
||||
if err := audioFile.Close(); err != nil {
|
||||
return 0, nil, fmt.Errorf("close audio file %q: %w", audioPath, err)
|
||||
}
|
||||
|
||||
if err := writer.WriteField("language", c.language); err != nil {
|
||||
return 0, nil, fmt.Errorf("write language form field: %w", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return 0, nil, fmt.Errorf("close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), bodyBuf)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String(), upload)
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("build whisperx request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
req.Header.Set("Content-Type", upload.contentType)
|
||||
req.Header.Set("Expect", "100-continue")
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
_ = upload.Close()
|
||||
if producerErr := upload.Wait(); producerErr != nil {
|
||||
return 0, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
|
||||
}
|
||||
return 0, nil, fmt.Errorf("perform whisperx request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
data, err := readBounded(resp.Body, c.maxResponseBytes)
|
||||
if err != nil {
|
||||
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
_ = upload.Close()
|
||||
if producerErr := upload.Wait(); producerErr != nil {
|
||||
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", producerErr)
|
||||
}
|
||||
if _, err := readWhisperXResponse(resp.Body, c.maxResponseBytes); err != nil {
|
||||
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
||||
}
|
||||
return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return resp.StatusCode, nil, fmt.Errorf("whisperx returned status %d", resp.StatusCode)
|
||||
if err := upload.Wait(); err != nil {
|
||||
return resp.StatusCode, nil, fmt.Errorf("stream whisperx request body: %w", err)
|
||||
}
|
||||
|
||||
data, err := readWhisperXResponse(resp.Body, c.maxResponseBytes)
|
||||
if err != nil {
|
||||
return resp.StatusCode, nil, fmt.Errorf("read whisperx response body: %w", err)
|
||||
}
|
||||
return resp.StatusCode, data, nil
|
||||
}
|
||||
@@ -264,57 +265,188 @@ func (c *HTTPClient) shouldRetry(parent context.Context, err error, status int)
|
||||
return false
|
||||
}
|
||||
|
||||
func readBounded(r io.Reader, maxBytes int64) ([]byte, error) {
|
||||
func readWhisperXResponse(r io.Reader, maxBytes int64) ([]byte, error) {
|
||||
limited := io.LimitReader(r, maxBytes+1)
|
||||
data, err := io.ReadAll(limited)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > maxBytes {
|
||||
return nil, fmt.Errorf("response exceeds max size %d bytes", maxBytes)
|
||||
return nil, fmt.Errorf("whisperx response exceeds configured limit of %d bytes", maxBytes)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func isHTTPURLScheme(scheme string) bool {
|
||||
switch strings.ToLower(scheme) {
|
||||
case "http", "https":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type multipartUpload struct {
|
||||
reader *io.PipeReader
|
||||
writer *io.PipeWriter
|
||||
contentType string
|
||||
done chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
audio io.Closer
|
||||
err error
|
||||
aborted bool
|
||||
}
|
||||
|
||||
func newMultipartUpload(ctx context.Context, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) *multipartUpload {
|
||||
reader, writer := io.Pipe()
|
||||
multipartWriter := multipart.NewWriter(writer)
|
||||
upload := &multipartUpload{
|
||||
reader: reader,
|
||||
writer: writer,
|
||||
contentType: multipartWriter.FormDataContentType(),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
|
||||
go func() {
|
||||
err := upload.write(ctx, multipartWriter, audioPath, language, openAudio)
|
||||
if err != nil {
|
||||
_ = writer.CloseWithError(err)
|
||||
} else {
|
||||
_ = writer.Close()
|
||||
}
|
||||
upload.mu.Lock()
|
||||
upload.err = err
|
||||
upload.audio = nil
|
||||
upload.mu.Unlock()
|
||||
close(upload.done)
|
||||
}()
|
||||
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
upload.abort()
|
||||
case <-upload.done:
|
||||
}
|
||||
}()
|
||||
|
||||
return upload
|
||||
}
|
||||
|
||||
func (u *multipartUpload) Read(p []byte) (int, error) {
|
||||
return u.reader.Read(p)
|
||||
}
|
||||
|
||||
func (u *multipartUpload) Close() error {
|
||||
u.abort()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *multipartUpload) Wait() error {
|
||||
<-u.done
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
return u.err
|
||||
}
|
||||
|
||||
func (u *multipartUpload) write(ctx context.Context, writer *multipart.Writer, audioPath, language string, openAudio func(string) (io.ReadCloser, error)) error {
|
||||
fileWriter, err := writer.CreateFormFile("file", filepath.Base(audioPath))
|
||||
if err != nil {
|
||||
return u.producerError(ctx, fmt.Errorf("create multipart file field: %w", err))
|
||||
}
|
||||
|
||||
audioFile, err := openAudio(audioPath)
|
||||
if err != nil {
|
||||
return u.producerError(ctx, fmt.Errorf("open audio file %q: %w", audioPath, err))
|
||||
}
|
||||
u.setAudio(audioFile)
|
||||
|
||||
_, copyErr := io.CopyBuffer(fileWriter, &contextReader{ctx: ctx, reader: audioFile}, make([]byte, whisperXUploadBufferSize))
|
||||
closeErr := audioFile.Close()
|
||||
u.clearAudio(audioFile)
|
||||
if copyErr != nil {
|
||||
return u.producerError(ctx, fmt.Errorf("copy audio file %q: %w", audioPath, copyErr))
|
||||
}
|
||||
if closeErr != nil {
|
||||
return u.producerError(ctx, fmt.Errorf("close audio file %q: %w", audioPath, closeErr))
|
||||
}
|
||||
|
||||
if err := writer.WriteField("language", language); err != nil {
|
||||
return u.producerError(ctx, fmt.Errorf("write language form field: %w", err))
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return u.producerError(ctx, fmt.Errorf("close multipart writer: %w", err))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *multipartUpload) producerError(ctx context.Context, err error) error {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
u.mu.Lock()
|
||||
aborted := u.aborted
|
||||
u.mu.Unlock()
|
||||
if aborted {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (u *multipartUpload) setAudio(audio io.Closer) {
|
||||
u.mu.Lock()
|
||||
u.audio = audio
|
||||
aborted := u.aborted
|
||||
u.mu.Unlock()
|
||||
if aborted {
|
||||
_ = audio.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (u *multipartUpload) clearAudio(audio io.Closer) {
|
||||
u.mu.Lock()
|
||||
if u.audio == audio {
|
||||
u.audio = nil
|
||||
}
|
||||
u.mu.Unlock()
|
||||
}
|
||||
|
||||
func (u *multipartUpload) abort() {
|
||||
u.mu.Lock()
|
||||
if u.aborted {
|
||||
u.mu.Unlock()
|
||||
return
|
||||
}
|
||||
u.aborted = true
|
||||
audio := u.audio
|
||||
u.mu.Unlock()
|
||||
|
||||
_ = u.reader.Close()
|
||||
if audio != nil {
|
||||
_ = audio.Close()
|
||||
}
|
||||
}
|
||||
|
||||
type contextReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (r *contextReader) Read(p []byte) (int, error) {
|
||||
select {
|
||||
case <-r.ctx.Done():
|
||||
return 0, r.ctx.Err()
|
||||
default:
|
||||
return r.reader.Read(p)
|
||||
}
|
||||
}
|
||||
|
||||
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fmt.Errorf("path is required")
|
||||
}
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("create parent dir %q: %w", dir, err)
|
||||
if err := fileops.WriteFileAtomic(path, data, perm); err != nil {
|
||||
return fmt.Errorf("write file %q: %w", path, err)
|
||||
}
|
||||
|
||||
base := filepath.Base(path)
|
||||
tmp, err := os.CreateTemp(dir, "."+base+".tmp-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
removeTmp := true
|
||||
defer func() {
|
||||
if removeTmp {
|
||||
_ = os.Remove(tmpPath)
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return fmt.Errorf("write temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Sync(); err != nil {
|
||||
_ = tmp.Close()
|
||||
return fmt.Errorf("sync temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("close temp file: %w", err)
|
||||
}
|
||||
if err := os.Chmod(tmpPath, perm); err != nil {
|
||||
return fmt.Errorf("chmod temp file: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmpPath, path); err != nil {
|
||||
return fmt.Errorf("rename temp file: %w", err)
|
||||
}
|
||||
removeTmp = false
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
@@ -19,6 +20,7 @@ func TestHTTPClientTranscribeSuccess(t *testing.T) {
|
||||
var gotLanguage string
|
||||
var gotFileField string
|
||||
var gotFileSize int
|
||||
var gotFileData string
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
@@ -40,6 +42,7 @@ func TestHTTPClientTranscribeSuccess(t *testing.T) {
|
||||
t.Fatalf("ReadAll(file) error = %v", err)
|
||||
}
|
||||
gotFileSize = len(data)
|
||||
gotFileData = string(data)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"schema":"speaker_transcript.v1","segments":[]}`))
|
||||
@@ -77,13 +80,27 @@ func TestHTTPClientTranscribeSuccess(t *testing.T) {
|
||||
if gotFileSize == 0 {
|
||||
t.Fatal("file size = 0, want >0")
|
||||
}
|
||||
if gotFileData != "audio-data" {
|
||||
t.Fatalf("file data = %q, want exact payload", gotFileData)
|
||||
}
|
||||
verifyJSONFile(t, outPath)
|
||||
}
|
||||
|
||||
func TestHTTPClientRetriesOnTransientAndSucceeds(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
var payloads []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n := calls.Add(1)
|
||||
file, _, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
t.Fatalf("FormFile(file) error = %v", err)
|
||||
}
|
||||
data, err := io.ReadAll(file)
|
||||
_ = file.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll(file) error = %v", err)
|
||||
}
|
||||
payloads = append(payloads, string(data))
|
||||
if n == 1 {
|
||||
http.Error(w, "temporary", http.StatusInternalServerError)
|
||||
return
|
||||
@@ -112,6 +129,9 @@ func TestHTTPClientRetriesOnTransientAndSucceeds(t *testing.T) {
|
||||
if calls.Load() != 2 {
|
||||
t.Fatalf("calls = %d, want 2", calls.Load())
|
||||
}
|
||||
if len(payloads) != 2 || payloads[0] != "audio-data" || payloads[1] != "audio-data" {
|
||||
t.Fatalf("retry payloads = %#v, want two exact audio payloads", payloads)
|
||||
}
|
||||
verifyJSONFile(t, outPath)
|
||||
}
|
||||
|
||||
@@ -247,8 +267,245 @@ func TestHTTPClientConstructorValidation(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("expected bad retry_delay error")
|
||||
}
|
||||
for _, endpoint := range []string{"ftp://example.com/transcribe", "file:///tmp/transcribe", "//example.com/transcribe", "https:/missing-host"} {
|
||||
if _, err := NewHTTPClientFromConfigValues(endpoint, "en", "30m", "2s", 1); err == nil {
|
||||
t.Errorf("NewHTTPClientFromConfigValues(%q) error = nil, want endpoint validation error", endpoint)
|
||||
}
|
||||
}
|
||||
for _, endpoint := range []string{"http://example.com/transcribe", "https://example.com/transcribe"} {
|
||||
if _, err := NewHTTPClientFromConfigValues(endpoint, "en", "30m", "2s", 1); err != nil {
|
||||
t.Errorf("NewHTTPClientFromConfigValues(%q) error = %v", endpoint, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPClientStreamsUploadBeforeSourceCompletes(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
source := newGatedReadCloser([]byte("audio-data"), release)
|
||||
firstByteReceived := make(chan struct{})
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
part := firstMultipartFilePart(t, r)
|
||||
buf := make([]byte, 1)
|
||||
if _, err := part.Read(buf); err != nil {
|
||||
t.Errorf("Read(file) error = %v", err)
|
||||
return
|
||||
}
|
||||
close(firstByteReceived)
|
||||
if _, err := io.Copy(io.Discard, part); err != nil {
|
||||
t.Errorf("discard remaining file data: %v", err)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client := newTestHTTPClient(t, srv.URL)
|
||||
client.openAudio = func(string) (io.ReadCloser, error) { return source, nil }
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := client.Transcribe(context.Background(), TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
|
||||
done <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-firstByteReceived:
|
||||
close(release)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("server did not receive streamed audio before source completed")
|
||||
}
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("Transcribe() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPClientSourceReadFailureReachesCaller(t *testing.T) {
|
||||
sourceErr := errors.New("source read failed")
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client := newTestHTTPClient(t, srv.URL)
|
||||
client.openAudio = func(string) (io.ReadCloser, error) {
|
||||
return &failingReadCloser{first: []byte("partial"), err: sourceErr}, nil
|
||||
}
|
||||
|
||||
_, err := client.Transcribe(context.Background(), TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
|
||||
if !errors.Is(err, sourceErr) {
|
||||
t.Fatalf("Transcribe() error = %v, want source read failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPClientEarlyServerResponseReturns(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
source := newGatedReadCloser([]byte("audio-data"), release)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "bad request", http.StatusBadRequest)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client := newTestHTTPClient(t, srv.URL)
|
||||
client.openAudio = func(string) (io.ReadCloser, error) { return source, nil }
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := client.Transcribe(context.Background(), TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
|
||||
done <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("Transcribe() error = nil, want HTTP status error")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Transcribe() did not finish after server closed the request early")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPClientCancellationReleasesBlockedProducer(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
source := newGatedReadCloser([]byte("audio-data"), release)
|
||||
firstByteReceived := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
part := firstMultipartFilePart(t, r)
|
||||
buf := make([]byte, 1)
|
||||
if _, err := part.Read(buf); err != nil {
|
||||
t.Errorf("Read(file) error = %v", err)
|
||||
return
|
||||
}
|
||||
close(firstByteReceived)
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
case <-source.closed:
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client := newTestHTTPClient(t, srv.URL)
|
||||
client.openAudio = func(string) (io.ReadCloser, error) { return source, nil }
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := client.Transcribe(ctx, TranscribeRequest{AudioPath: "audio.flac", OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
|
||||
done <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-firstByteReceived:
|
||||
cancel()
|
||||
case <-time.After(time.Second):
|
||||
cancel()
|
||||
t.Fatal("server did not receive initial streamed audio")
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Transcribe() error = %v, want context cancellation", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Transcribe() did not finish after cancellation")
|
||||
}
|
||||
select {
|
||||
case <-source.closed:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("blocked audio source was not closed on cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPClientBoundsWhisperXResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := NewHTTPClient(HTTPClientConfig{TranscribeURL: srv.URL, Language: "en", Timeout: time.Second, MaxResponseBytes: 4})
|
||||
if err != nil {
|
||||
t.Fatalf("NewHTTPClient() error = %v", err)
|
||||
}
|
||||
audioPath := writeWhisperXTestFile(t, "audio.flac", "audio-data")
|
||||
_, err = client.Transcribe(context.Background(), TranscribeRequest{AudioPath: audioPath, OutputRawTranscriptPath: filepath.Join(t.TempDir(), "raw.json")})
|
||||
if err == nil || !strings.Contains(err.Error(), "whisperx response exceeds configured limit") {
|
||||
t.Fatalf("Transcribe() error = %v, want bounded WhisperX response error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestHTTPClient(t *testing.T, endpoint string) *HTTPClient {
|
||||
t.Helper()
|
||||
client, err := NewHTTPClientFromConfigValues(endpoint, "en", "2s", "1ms", 0)
|
||||
if err != nil {
|
||||
t.Fatalf("NewHTTPClientFromConfigValues() error = %v", err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
func firstMultipartFilePart(t *testing.T, r *http.Request) *multipart.Part {
|
||||
t.Helper()
|
||||
reader, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
t.Fatalf("MultipartReader() error = %v", err)
|
||||
}
|
||||
part, err := reader.NextPart()
|
||||
if err != nil {
|
||||
t.Fatalf("NextPart() error = %v", err)
|
||||
}
|
||||
if part.FormName() != "file" {
|
||||
t.Fatalf("first form field = %q, want file", part.FormName())
|
||||
}
|
||||
return part
|
||||
}
|
||||
|
||||
type gatedReadCloser struct {
|
||||
first []byte
|
||||
release <-chan struct{}
|
||||
closed chan struct{}
|
||||
sent bool
|
||||
once atomic.Bool
|
||||
}
|
||||
|
||||
func newGatedReadCloser(first []byte, release <-chan struct{}) *gatedReadCloser {
|
||||
return &gatedReadCloser{first: first, release: release, closed: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (r *gatedReadCloser) Read(p []byte) (int, error) {
|
||||
if !r.sent {
|
||||
r.sent = true
|
||||
return copy(p, r.first), nil
|
||||
}
|
||||
select {
|
||||
case <-r.release:
|
||||
return 0, io.EOF
|
||||
case <-r.closed:
|
||||
return 0, errors.New("audio source closed")
|
||||
}
|
||||
}
|
||||
|
||||
func (r *gatedReadCloser) Close() error {
|
||||
if r.once.CompareAndSwap(false, true) {
|
||||
close(r.closed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type failingReadCloser struct {
|
||||
first []byte
|
||||
err error
|
||||
sent bool
|
||||
}
|
||||
|
||||
func (r *failingReadCloser) Read(p []byte) (int, error) {
|
||||
if !r.sent {
|
||||
r.sent = true
|
||||
return copy(p, r.first), nil
|
||||
}
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
func (r *failingReadCloser) Close() error { return nil }
|
||||
|
||||
func writeWhisperXTestFile(t *testing.T, name, contents string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), name)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
@@ -47,19 +48,55 @@ func (f *artifactSelectionFlag) Normalize() ([]string, error) {
|
||||
}
|
||||
|
||||
func validateSelectedArtifacts(cfg *config.Config, selected []string) error {
|
||||
if len(selected) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := resolveEffectiveArtifacts(cfg, selected)
|
||||
return err
|
||||
}
|
||||
|
||||
func resolveEffectiveArtifacts(cfg *config.Config, selected []string) (artifacts.EffectiveArtifactSet, error) {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
||||
return fmt.Errorf("--artifacts requires pipeline.scriptorium.artifacts to be configured")
|
||||
if len(selected) == 0 {
|
||||
return artifacts.ResolveEffectiveArtifactSet(nil, nil)
|
||||
}
|
||||
return artifacts.EffectiveArtifactSet{}, fmt.Errorf("--artifacts requires pipeline.scriptorium.artifacts to be configured")
|
||||
}
|
||||
configured := cfg.Pipeline.Scriptorium.Artifacts
|
||||
if len(configured) == 0 {
|
||||
return fmt.Errorf("--artifacts requires at least one configured artifact in pipeline.scriptorium.artifacts")
|
||||
configured := artifacts.ConfiguredArtifactDefinitions(cfg.Pipeline.Scriptorium.Artifacts)
|
||||
if len(selected) > 0 && len(configured) == 0 {
|
||||
return artifacts.EffectiveArtifactSet{}, fmt.Errorf("--artifacts requires at least one configured artifact in pipeline.scriptorium.artifacts")
|
||||
}
|
||||
for _, name := range selected {
|
||||
if _, ok := configured[name]; !ok {
|
||||
return fmt.Errorf("--artifacts includes unknown artifact %q", name)
|
||||
effective, err := artifacts.ResolveEffectiveArtifactSet(configured, selected)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "is not configured") {
|
||||
return artifacts.EffectiveArtifactSet{}, fmt.Errorf("--artifacts includes unknown artifact %q", selectedArtifactName(err))
|
||||
}
|
||||
return artifacts.EffectiveArtifactSet{}, err
|
||||
}
|
||||
if err := validateEffectiveArtifactConfiguration(cfg.Pipeline.Scriptorium.Artifacts, effective); err != nil {
|
||||
return artifacts.EffectiveArtifactSet{}, err
|
||||
}
|
||||
return effective, nil
|
||||
}
|
||||
|
||||
func selectedArtifactName(err error) string {
|
||||
message := err.Error()
|
||||
start := strings.Index(message, "\"")
|
||||
if start < 0 {
|
||||
return ""
|
||||
}
|
||||
end := strings.Index(message[start+1:], "\"")
|
||||
if end < 0 {
|
||||
return ""
|
||||
}
|
||||
return message[start+1 : start+1+end]
|
||||
}
|
||||
|
||||
func validateEffectiveArtifactConfiguration(configured map[string]config.ScriptoriumArtifactConfig, effective artifacts.EffectiveArtifactSet) error {
|
||||
for _, name := range effective.Keys() {
|
||||
artifactCfg := configured[name]
|
||||
if strings.TrimSpace(artifactCfg.PromptID) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.prompt_id is required when selected", name)
|
||||
}
|
||||
if strings.TrimSpace(artifactCfg.OutputPath) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when selected", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -21,7 +21,7 @@ func TestExecuteRunStageArtifactsUnsupportedStageFails(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
[]string{"run-stage", "extract", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
@@ -130,6 +130,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
seed.MarkStageSkipped("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), "notarius_disabled")
|
||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||
t.Fatalf("save manifest: %v", err)
|
||||
}
|
||||
@@ -143,7 +144,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "executed=0 skipped=10") {
|
||||
if !strings.Contains(out.String(), "executed=1 skipped=11") {
|
||||
t.Fatalf("output = %q, want all stages skipped", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,6 +110,20 @@ func TestValidateSelectedArtifacts(t *testing.T) {
|
||||
},
|
||||
selected: []string{"player_handout", "session_recap"},
|
||||
},
|
||||
{
|
||||
name: "selected disabled artifact must be executable",
|
||||
cfg: &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Scriptorium: &config.ScriptoriumConfig{
|
||||
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||
"player_handout": {Enabled: false, OutputPath: "artifacts/player_handout.md"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
selected: []string{"player_handout"},
|
||||
wantErr: "pipeline.scriptorium.artifacts.player_handout.prompt_id is required when selected",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||
)
|
||||
|
||||
func resolveCampaignConfigPath(pipelineCfg *config.PipelineConfig, campaignIDFlag, campaignFileFlag string) (string, error) {
|
||||
@@ -33,12 +34,8 @@ func resolveCampaignConfigPath(pipelineCfg *config.PipelineConfig, campaignIDFla
|
||||
}
|
||||
|
||||
func validateCampaignIDToken(campaignID string) error {
|
||||
if filepath.IsAbs(campaignID) ||
|
||||
strings.Contains(campaignID, "/") ||
|
||||
strings.Contains(campaignID, `\`) ||
|
||||
campaignID == "." ||
|
||||
campaignID == ".." {
|
||||
return fmt.Errorf("campaign id %q must be a single path segment", campaignID)
|
||||
if err := pathsafe.ValidateOpaqueSegment(campaignID); err != nil {
|
||||
return fmt.Errorf("campaign id %q must be a single path segment and opaque identifier: %w", campaignID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
// Clean removes local workspace/spool state while preserving durable cache
|
||||
@@ -39,10 +40,12 @@ func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCac
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("clean: session_id is required unless --all is set")
|
||||
}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
defer func() { _ = loaded.Close() }()
|
||||
cfg := loaded.Config
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return fmt.Errorf("clean: resolved pipeline and session config are required")
|
||||
}
|
||||
@@ -135,7 +138,7 @@ func reportCleanScopedDir(out io.Writer, root, target, policy string, dryRun boo
|
||||
fmt.Fprintf(out, "Missing: %s\n", dir.TargetAbs)
|
||||
return nil
|
||||
}
|
||||
if err := os.RemoveAll(dir.TargetAbs); err != nil {
|
||||
if err := fileops.RemoveAllUnderRoot(dir.RootAbs, dir.TargetAbs); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted: %s\n", dir.TargetAbs)
|
||||
@@ -160,7 +163,7 @@ func reportCleanRootChildren(out io.Writer, root, policy string, dryRun bool) er
|
||||
fmt.Fprintf(out, "Would delete: %s\n", entry)
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(entry); err != nil {
|
||||
if err := fileops.RemoveAllUnderRoot(rootAbs, entry); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, entry, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted: %s\n", entry)
|
||||
@@ -265,7 +268,7 @@ func reportCleanScopedFile(out io.Writer, root, target, policy string, dryRun bo
|
||||
fmt.Fprintf(out, "Missing cache file: %s\n", file.TargetAbs)
|
||||
return false, nil
|
||||
}
|
||||
if err := os.Remove(file.TargetAbs); err != nil {
|
||||
if err := fileops.RemoveAllUnderRoot(file.RootAbs, file.TargetAbs); err != nil {
|
||||
return false, fmt.Errorf("cleanup policy %s: remove %q: %w", policy, file.TargetAbs, err)
|
||||
}
|
||||
fmt.Fprintf(out, "Deleted cache file: %s\n", file.TargetAbs)
|
||||
|
||||
@@ -31,8 +31,8 @@ func TestExecuteValidCommands(t *testing.T) {
|
||||
args []string
|
||||
wantOut string
|
||||
}{
|
||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=10 skipped=0; manifest="},
|
||||
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nrender: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=11 skipped=1; manifest="},
|
||||
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nextract: run\nrender: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
||||
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
||||
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||
}
|
||||
@@ -189,7 +189,7 @@ func TestExecuteRunStagePolishLoadsCredentialFromSecretsDir(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
sessionID := "2026-05-03"
|
||||
secretsDir := filepath.Join(configDir, "secrets")
|
||||
if err := os.MkdirAll(secretsDir, 0o755); err != nil {
|
||||
if err := os.MkdirAll(secretsDir, secretDirectoryPrivateMode); err != nil {
|
||||
t.Fatalf("MkdirAll(%q): %v", secretsDir, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(secretsDir, "OPENROUTER_API_KEY"), []byte("from-secret-file\n"), 0o600); err != nil {
|
||||
@@ -218,7 +218,7 @@ audita:
|
||||
binary: ` + auditaBinary + `
|
||||
llm_api_key_env: OPENROUTER_API_KEY
|
||||
notification:
|
||||
timeout: 10s
|
||||
mode: noop
|
||||
`
|
||||
sessionYAML := `session_id: ` + sessionID + `
|
||||
campaign: sample-campaign
|
||||
@@ -283,7 +283,7 @@ seriatim:
|
||||
audita:
|
||||
binary: audita
|
||||
notification:
|
||||
timeout: 10s
|
||||
mode: noop
|
||||
`
|
||||
sessionYAML := `session_id: 2026-05-03
|
||||
campaign: sample-campaign
|
||||
@@ -308,8 +308,8 @@ inputs:
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "read secrets env_dir") {
|
||||
t.Fatalf("stderr = %q, want secrets read-dir error context", stderr.String())
|
||||
if !strings.Contains(stderr.String(), "validate secrets env_dir") {
|
||||
t.Fatalf("stderr = %q, want secrets validation error context", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -335,7 +335,7 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=10 skipped=0; manifest=") {
|
||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=11 skipped=1; manifest=") {
|
||||
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
||||
}
|
||||
}
|
||||
@@ -476,9 +476,6 @@ func writeValidConfigFiles(t *testing.T, workspaceRoot string, transcribeURL ...
|
||||
seriatimBinary := writeSeriatimAppTestWrapper(t)
|
||||
scriptoriumBinary := writeScriptoriumAppTestWrapper(t)
|
||||
auditaBinary := writeAuditaAppTestWrapper(t)
|
||||
t.Setenv("GO_WANT_APP_SERIATIM_HELPER", "1")
|
||||
t.Setenv("GO_WANT_APP_SCRIPTORIUM_HELPER", "1")
|
||||
t.Setenv("GO_WANT_APP_AUDITA_HELPER", "1")
|
||||
t.Setenv("AUDITA_LLM_API_KEY", "test-audita-key")
|
||||
t.Setenv("PATH", filepath.Dir(scriptoriumBinary)+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
|
||||
@@ -513,7 +510,7 @@ seriatim:
|
||||
audita:
|
||||
binary: ` + auditaBinary + `
|
||||
notification:
|
||||
timeout: 10s
|
||||
mode: noop
|
||||
`
|
||||
|
||||
sessionYAML := `session_id: 2026-05-03
|
||||
@@ -623,7 +620,7 @@ func writeScriptoriumAppTestWrapper(t *testing.T) string {
|
||||
}
|
||||
|
||||
func TestScriptoriumAppHelper(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_APP_SCRIPTORIUM_HELPER") != "1" {
|
||||
if !appHelperInvocation() {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -663,7 +660,7 @@ func TestScriptoriumAppHelper(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSeriatimAppHelper(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_APP_SERIATIM_HELPER") != "1" {
|
||||
if !appHelperInvocation() {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -725,7 +722,7 @@ func writeAuditaAppTestWrapper(t *testing.T) string {
|
||||
}
|
||||
|
||||
func TestAuditaAppHelper(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_APP_AUDITA_HELPER") != "1" {
|
||||
if !appHelperInvocation() {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -779,6 +776,15 @@ func TestAuditaAppHelper(t *testing.T) {
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func appHelperInvocation() bool {
|
||||
for _, arg := range os.Args {
|
||||
if arg == "--" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func appSeriatimFlagValue(args []string, name string) string {
|
||||
for i := 0; i < len(args)-1; i++ {
|
||||
if args[i] == name {
|
||||
|
||||
@@ -2,13 +2,16 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
type pipelineCampaignConfig struct {
|
||||
@@ -18,14 +21,48 @@ type pipelineCampaignConfig struct {
|
||||
Campaign *config.CampaignConfig
|
||||
}
|
||||
|
||||
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (*config.Config, error) {
|
||||
var downloadObjectToTempFn = storage.DownloadObjectToTemp
|
||||
|
||||
type commandConfig struct {
|
||||
Config *config.Config
|
||||
cleanup func() error
|
||||
}
|
||||
|
||||
func (c *commandConfig) Close() error {
|
||||
if c == nil || c.cleanup == nil {
|
||||
return nil
|
||||
}
|
||||
cleanup := c.cleanup
|
||||
c.cleanup = nil
|
||||
return cleanup()
|
||||
}
|
||||
|
||||
func retainedCommandConfig(cfg *config.Config) *commandConfig {
|
||||
return &commandConfig{Config: cfg}
|
||||
}
|
||||
|
||||
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (loaded *commandConfig, err error) {
|
||||
var cleanup func() error
|
||||
defer func() {
|
||||
if err == nil || cleanup == nil {
|
||||
return
|
||||
}
|
||||
if cleanupErr := cleanup(); cleanupErr != nil {
|
||||
err = errors.Join(err, cleanupErr)
|
||||
}
|
||||
}()
|
||||
|
||||
base, err := loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if explicitSession := strings.TrimSpace(sessionFlag); explicitSession != "" {
|
||||
return config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, explicitSession, sessionOpts)
|
||||
cfg, err := config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, explicitSession, sessionOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return retainedCommandConfig(cfg), nil
|
||||
}
|
||||
|
||||
discoveredSession, err := discoverSessionConfigPathWithCandidates(config.DefaultSessionConfigSearchPaths)
|
||||
@@ -33,7 +70,11 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
return nil, err
|
||||
}
|
||||
if discoveredSession.Path != "" {
|
||||
return config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, discoveredSession.Path, sessionOpts)
|
||||
cfg, err := config.LoadWithSessionOptions(base.PipelinePath, base.CampaignPath, discoveredSession.Path, sessionOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return retainedCommandConfig(cfg), nil
|
||||
}
|
||||
|
||||
sessionID := strings.TrimSpace(sessionOpts.SessionID)
|
||||
@@ -41,7 +82,11 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, "remote session loading requires a session_id")
|
||||
}
|
||||
|
||||
sessionPrefix := artifacts.S3SessionPrefix(base.Pipeline.Storage.S3.RootPrefix, config.CampaignID(base.Campaign), sessionID)
|
||||
rootPrefix := ""
|
||||
if base.Pipeline.Storage.S3 != nil {
|
||||
rootPrefix = base.Pipeline.Storage.S3.RootPrefix
|
||||
}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(rootPrefix, config.CampaignID(base.Campaign), sessionID)
|
||||
remoteKey := artifacts.S3SessionConfigKey(sessionPrefix)
|
||||
partialCfg := &config.Config{
|
||||
Pipeline: base.Pipeline,
|
||||
@@ -58,20 +103,26 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
|
||||
}
|
||||
sessionTempPath, err := storage.DownloadObjectToTemp(ctx, store, remoteKey, "narratio-session-*.yml")
|
||||
sessionTempPath, err := downloadObjectToTempFn(ctx, store, remoteKey, "narratio-session-*.yml")
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
||||
}
|
||||
cleanup = func() error {
|
||||
if err := fileops.RemoveAllUnderRoot(filepath.Dir(sessionTempPath), sessionTempPath); err != nil {
|
||||
return fmt.Errorf("remove downloaded remote session config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
sessionBytes, err := os.ReadFile(sessionTempPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read downloaded remote session %q: %w", sessionTempPath, err)
|
||||
return nil, fmt.Errorf("read downloaded remote session config: %w", err)
|
||||
}
|
||||
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return config.Resolve(
|
||||
cfg, err := config.Resolve(
|
||||
base.PipelinePath,
|
||||
base.Pipeline,
|
||||
base.CampaignPath,
|
||||
@@ -85,9 +136,14 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
S3Key: remoteKey,
|
||||
S3Size: sessionInfo.Size,
|
||||
S3ETag: sessionInfo.ETag,
|
||||
SpoolPath: sessionTempPath,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
loaded = &commandConfig{Config: cfg, cleanup: cleanup}
|
||||
cleanup = nil
|
||||
return loaded, nil
|
||||
}
|
||||
|
||||
func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag string) (*pipelineCampaignConfig, error) {
|
||||
|
||||
442
internal/app/extract_lifecycle_test.go
Normal file
442
internal/app/extract_lifecycle_test.go
Normal file
@@ -0,0 +1,442 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type materializingNotariusRunner struct {
|
||||
cfg *config.NotariusConfig
|
||||
requests []notarius.RunRequest
|
||||
failuresRemaining int
|
||||
}
|
||||
|
||||
func (r *materializingNotariusRunner) Run(_ context.Context, req notarius.RunRequest) (notarius.RunResult, error) {
|
||||
r.requests = append(r.requests, req)
|
||||
if r.failuresRemaining > 0 {
|
||||
r.failuresRemaining--
|
||||
return notarius.RunResult{}, errors.New("notarius execution failed")
|
||||
}
|
||||
externalRunID := fmt.Sprintf("notarius-run-%d", len(r.requests))
|
||||
bundle := filepath.Join(req.OutputRoot, externalRunID)
|
||||
lanesDir := filepath.Join(bundle, "lanes")
|
||||
if err := os.MkdirAll(lanesDir, 0o755); err != nil {
|
||||
return notarius.RunResult{}, err
|
||||
}
|
||||
for path, content := range map[string]string{
|
||||
filepath.Join(bundle, "index.json"): `{"manifest_file":"manifest.json"}`,
|
||||
filepath.Join(bundle, "manifest.json"): `{}`,
|
||||
filepath.Join(bundle, "rejected.json"): `{"rejected":[]}`,
|
||||
filepath.Join(bundle, "warnings.json"): `{"warnings":[]}`,
|
||||
filepath.Join(lanesDir, "npcs.json"): `{"npcs":[]}`,
|
||||
} {
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
return notarius.RunResult{}, err
|
||||
}
|
||||
}
|
||||
output := r.cfg.Outputs["npc_registry"]
|
||||
return notarius.RunResult{
|
||||
Receipt: notarius.Receipt{
|
||||
SchemaVersion: notarius.ReceiptSchemaVersion, RunID: externalRunID,
|
||||
PipelineID: req.PipelineID, OutputDirectory: bundle, IndexFile: "index.json",
|
||||
NormalizedOutputCount: 1, ValidationStatus: "valid",
|
||||
},
|
||||
BundleRoot: bundle,
|
||||
Index: notarius.Index{
|
||||
Path: filepath.Join(bundle, "index.json"), RejectedPath: filepath.Join(bundle, "rejected.json"),
|
||||
WarningsPath: filepath.Join(bundle, "warnings.json"),
|
||||
Lanes: []notarius.LaneDescriptor{{
|
||||
LaneID: output.LaneID, File: "lanes/npcs.json", Path: filepath.Join(lanesDir, "npcs.json"),
|
||||
MediaType: output.MediaType, SchemaID: output.SchemaID,
|
||||
SchemaVersion: output.SchemaVersion, ModuleKey: output.ModuleKey,
|
||||
}},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestExtractLifecycleDisabledThenEnabled(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||
plan, err := BuildSingleStagePlan("extract")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
|
||||
}
|
||||
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("disabled executeStages() error = %v", err)
|
||||
}
|
||||
if len(first.Executed) != 1 || len(first.Skipped) != 1 || first.Skipped[0] != "extract" || len(runner.requests) != 0 {
|
||||
t.Fatalf("disabled summary = %#v requests=%d", first, len(runner.requests))
|
||||
}
|
||||
|
||||
cfg.Pipeline.Notarius.Enabled = true
|
||||
second, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("enabled executeStages() error = %v", err)
|
||||
}
|
||||
if len(second.Executed) != 1 || len(second.Skipped) != 0 || len(runner.requests) != 1 {
|
||||
t.Fatalf("enabled summary = %#v requests=%d", second, len(runner.requests))
|
||||
}
|
||||
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), second.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if loaded.Stages["extract"] == nil || loaded.Stages["extract"].Status != manifest.StatusSucceeded || len(loaded.Stages["extract"].Outputs) != 2 {
|
||||
t.Fatalf("extract record = %#v, want succeeded manifest-ready outputs", loaded.Stages["extract"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleChangedOutcomeRerunsSucceededDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("disabled executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 1 || len(runner.requests) != 0 {
|
||||
t.Fatalf("disabled run analyze=%d Notarius=%d, want 1 and 0", analyzeRuns, len(runner.requests))
|
||||
}
|
||||
|
||||
cfg.Pipeline.Notarius.Enabled = true
|
||||
summary, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("enabled executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 2 || len(runner.requests) != 1 {
|
||||
t.Fatalf("enabled run analyze=%d Notarius=%d, want 2 and 1", analyzeRuns, len(runner.requests))
|
||||
}
|
||||
if len(summary.Executed) != 2 || len(summary.Skipped) != 0 {
|
||||
t.Fatalf("enabled summary = %#v, want extract and analyze executed", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleFailureInvalidatesAndOrdinaryRetryRerunsDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
runner.failuresRemaining = 1
|
||||
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil || !strings.Contains(err.Error(), "notarius execution failed") {
|
||||
t.Fatalf("failed executeStages() error = %v", err)
|
||||
}
|
||||
failed := loadLifecycleManifest(t, cfg)
|
||||
if failed.Stages["extract"].Status != manifest.StatusFailed || failed.Stages["analyze"].Status != manifest.StatusStale {
|
||||
t.Fatalf("failed lifecycle extract=%#v analyze=%#v", failed.Stages["extract"], failed.Stages["analyze"])
|
||||
}
|
||||
if failed.Stages["analyze"].Error == nil || failed.Stages["analyze"].Error.Message != staleReasonFailure {
|
||||
t.Fatalf("analyze stale reason = %#v, want %q", failed.Stages["analyze"].Error, staleReasonFailure)
|
||||
}
|
||||
|
||||
summary, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("retry executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 1 || len(runner.requests) != 2 || len(summary.Executed) != 2 {
|
||||
t.Fatalf("retry analyze=%d Notarius=%d summary=%#v", analyzeRuns, len(runner.requests), summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleForcedSelfSkipInvalidatesDownstream(t *testing.T) {
|
||||
cfg, env, _ := extractionLifecycleFixture(t, false)
|
||||
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||
plan, _ := BuildSingleStagePlan("extract")
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
if loaded.Stages["extract"].Status != manifest.StatusSkipped || loaded.Stages["analyze"].Status != manifest.StatusStale {
|
||||
t.Fatalf("forced self-skip extract=%#v analyze=%#v", loaded.Stages["extract"], loaded.Stages["analyze"])
|
||||
}
|
||||
if loaded.Stages["analyze"].Error == nil || loaded.Stages["analyze"].Error.Message != staleReasonForcedReplacement {
|
||||
t.Fatalf("analyze stale reason = %#v, want %q", loaded.Stages["analyze"].Error, staleReasonForcedReplacement)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleForcedFailureInvalidatesDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
plan, _ := BuildSingleStagePlan("extract")
|
||||
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("initial executeStages() error = %v", err)
|
||||
}
|
||||
succeeded := loadLifecycleManifest(t, cfg).Stages["extract"]
|
||||
if succeeded == nil || succeeded.Status != manifest.StatusSucceeded || len(succeeded.Outputs) == 0 || len(succeeded.Logs) == 0 || len(succeeded.Metadata) == 0 {
|
||||
t.Fatalf("initial extraction result = %#v, want succeeded result details", succeeded)
|
||||
}
|
||||
|
||||
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||
runner.failuresRemaining = 1
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true}); err == nil {
|
||||
t.Fatal("executeStages() error = nil, want forced extraction failure")
|
||||
}
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
if loaded.Stages["extract"].Status != manifest.StatusFailed || loaded.Stages["analyze"].Status != manifest.StatusStale {
|
||||
t.Fatalf("forced failure extract=%#v analyze=%#v", loaded.Stages["extract"], loaded.Stages["analyze"])
|
||||
}
|
||||
if loaded.Stages["analyze"].Error == nil || loaded.Stages["analyze"].Error.Message != staleReasonForcedReplacement {
|
||||
t.Fatalf("analyze stale reason = %#v, want %q", loaded.Stages["analyze"].Error, staleReasonForcedReplacement)
|
||||
}
|
||||
failed := loaded.Stages["extract"]
|
||||
if len(failed.Outputs) != 0 || len(failed.Logs) != 0 || len(failed.GeneratedConfigs) != 0 || len(failed.Metadata) != 0 {
|
||||
t.Fatalf("failed replacement inherited extraction result details: %#v", failed)
|
||||
}
|
||||
historical, err := (&manifest.LocalStore{}).LoadRun(context.Background(), first.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadRun(initial) error = %v", err)
|
||||
}
|
||||
historicalExtract := historical.Stages["extract"]
|
||||
if historicalExtract == nil || historicalExtract.Status != manifest.StatusSucceeded || len(historicalExtract.Outputs) == 0 || len(historicalExtract.Logs) == 0 || len(historicalExtract.Metadata) == 0 {
|
||||
t.Fatalf("historical extraction result = %#v, want preserved succeeded details", historicalExtract)
|
||||
}
|
||||
if _, err := os.Stat(succeeded.Outputs[0].LocalPath); err != nil {
|
||||
t.Fatalf("durable extraction output was not preserved: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleRepeatedSelfSkipPreservesSucceededDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
second, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("second executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 1 || len(runner.requests) != 0 {
|
||||
t.Fatalf("repeated disabled run analyze=%d Notarius=%d, want 1 and 0", analyzeRuns, len(runner.requests))
|
||||
}
|
||||
if len(second.Executed) != 1 || len(second.Skipped) != 2 {
|
||||
t.Fatalf("second summary = %#v, want executed self-skip and skipped analyze", second)
|
||||
}
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
if loaded.Stages["analyze"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("analyze = %#v, want succeeded", loaded.Stages["analyze"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleSkipsCurrentResumableResult(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
resumed, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("resume executeStages() error = %v", err)
|
||||
}
|
||||
if len(resumed.Executed) != 0 || len(resumed.Skipped) != 2 || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||
t.Fatalf("resume summary = %#v requests=%d analyze=%d", resumed, len(runner.requests), analyzeRuns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleRerunsAfterDirectTranscriptChange(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
persisted := loadLifecycleManifest(t, cfg)
|
||||
trimmed := persisted.Stages["trim"].Outputs[0].LocalPath
|
||||
if err := os.WriteFile(trimmed, []byte(`{"segments":[{"id":"changed"}]}`), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(trimmed transcript) error = %v", err)
|
||||
}
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPathFor(cfg), persisted); err != nil {
|
||||
t.Fatalf("Save(mutated) error = %v", err)
|
||||
}
|
||||
|
||||
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("rerun executeStages() error = %v", err)
|
||||
}
|
||||
if len(rerun.Executed) != 2 || len(rerun.Skipped) != 0 || len(runner.requests) != 2 || analyzeRuns != 2 {
|
||||
t.Fatalf("rerun summary = %#v requests=%d analyze=%d", rerun, len(runner.requests), analyzeRuns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleResumesAndRerunsObsoleteResults(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mutate func(*testing.T, *config.Config, *manifest.Manifest)
|
||||
}{
|
||||
{name: "configuration changed", mutate: func(_ *testing.T, cfg *config.Config, _ *manifest.Manifest) {
|
||||
output := cfg.Pipeline.Notarius.Outputs["npc_registry"]
|
||||
output.SchemaVersion = "v2"
|
||||
cfg.Pipeline.Notarius.Outputs["npc_registry"] = output
|
||||
}},
|
||||
{name: "payload missing", mutate: func(t *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
||||
t.Fatalf("Remove() error = %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "payload tampered", mutate: func(t *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||
if err := os.WriteFile(m.Stages["extract"].Outputs[0].LocalPath, []byte(`{"npcs":["tampered"]}`), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "record incompatible", mutate: func(_ *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].Contract.SchemaID = "incompatible"
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
plan, _ := BuildSingleStagePlan("extract")
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
persisted, err := (&manifest.LocalStore{}).Load(context.Background(), first.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
test.mutate(t, cfg, persisted)
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), first.ManifestPath, persisted); err != nil {
|
||||
t.Fatalf("Save(mutated) error = %v", err)
|
||||
}
|
||||
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("rerun executeStages() error = %v", err)
|
||||
}
|
||||
if len(rerun.Executed) != 1 || len(rerun.Skipped) != 0 || len(runner.requests) != 2 {
|
||||
t.Fatalf("rerun summary = %#v requests=%d", rerun, len(runner.requests))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleUnsafeResumeErrorPreservesSuccess(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
persisted, err := store.Load(context.Background(), first.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
persisted.Stages["extract"].Outputs[0].LocalPath = filepath.Join(cfg.Pipeline.Workspace.Root, "outside.json")
|
||||
if err := store.Save(context.Background(), first.ManifestPath, persisted); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
before, _ := json.Marshal(map[string]*manifest.StageRecord{
|
||||
"extract": persisted.Stages["extract"],
|
||||
"analyze": persisted.Stages["analyze"],
|
||||
})
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil || !strings.Contains(err.Error(), "unsafe") {
|
||||
t.Fatalf("executeStages() error = %v, want unsafe resume failure", err)
|
||||
}
|
||||
afterManifest, err := store.Load(context.Background(), first.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load(after) error = %v", err)
|
||||
}
|
||||
after, _ := json.Marshal(map[string]*manifest.StageRecord{
|
||||
"extract": afterManifest.Stages["extract"],
|
||||
"analyze": afterManifest.Stages["analyze"],
|
||||
})
|
||||
if string(before) != string(after) || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||
t.Fatalf("successful records changed: before=%s after=%s requests=%d analyze=%d", before, after, len(runner.requests), analyzeRuns)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionLifecyclePlan(t *testing.T, analyzeRuns *int) []stage.Stage {
|
||||
t.Helper()
|
||||
plan, err := BuildSingleStagePlan("extract")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
|
||||
}
|
||||
return append(plan, countingStage{name: "analyze", runs: analyzeRuns})
|
||||
}
|
||||
|
||||
func loadLifecycleManifest(t *testing.T, cfg *config.Config) *manifest.Manifest {
|
||||
t.Helper()
|
||||
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
return loaded
|
||||
}
|
||||
|
||||
func markLifecycleStageSucceeded(t *testing.T, cfg *config.Config, name string) {
|
||||
t.Helper()
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
loaded.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPathFor(cfg), loaded); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionLifecycleFixture(t *testing.T, enabled bool) (*config.Config, *stage.Env, *materializingNotariusRunner) {
|
||||
t.Helper()
|
||||
cfg := testConfig(t)
|
||||
root := cfg.Pipeline.Workspace.Root
|
||||
binary := filepath.Join(root, "notarius")
|
||||
configPath := filepath.Join(root, "notarius.yml")
|
||||
workingDirectory := filepath.Join(root, "notarius-work")
|
||||
if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(binary) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(config) error = %v", err)
|
||||
}
|
||||
if err := os.Mkdir(workingDirectory, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(working directory) error = %v", err)
|
||||
}
|
||||
cfg.Pipeline.Notarius = &config.NotariusConfig{
|
||||
Enabled: enabled, Binary: binary, ConfigPath: configPath, PipelineID: "dnd-session",
|
||||
Timeout: "45m", WorkingDirectory: workingDirectory,
|
||||
Outputs: map[string]config.NotariusOutputConfig{
|
||||
"npc_registry": {
|
||||
LaneID: "npc-registry", MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry",
|
||||
SchemaVersion: "v1", ModuleKey: "dnd/npc-registry",
|
||||
},
|
||||
},
|
||||
}
|
||||
paths, err := artifacts.NewLocalStore(root).EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureLayoutFor() error = %v", err)
|
||||
}
|
||||
inputPath := filepath.Join(paths.ArtifactsDir, "trimmed.from-manifest.json")
|
||||
if err := os.WriteFile(inputPath, []byte(`{"segments":[]}`), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(input) error = %v", err)
|
||||
}
|
||||
m := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
m.Campaign = cfg.Session.Campaign
|
||||
m.MarkStageSucceeded("trim", time.Now().UTC(), []manifest.ArtifactRecord{{
|
||||
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
|
||||
LocalPath: inputPath,
|
||||
}})
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), paths.ManifestPath, m); err != nil {
|
||||
t.Fatalf("Save(seed) error = %v", err)
|
||||
}
|
||||
runner := &materializingNotariusRunner{cfg: cfg.Pipeline.Notarius}
|
||||
return cfg, &stage.Env{Notarius: runner}, runner
|
||||
}
|
||||
@@ -142,24 +142,3 @@ func commandObjectStoreTestConfig(secretsDir string) *config.Config {
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func restoreEnvAfterTest(t *testing.T, names ...string) {
|
||||
t.Helper()
|
||||
originals := make(map[string]string, len(names))
|
||||
present := make(map[string]bool, len(names))
|
||||
for _, name := range names {
|
||||
value, ok := os.LookupEnv(name)
|
||||
originals[name] = value
|
||||
present[name] = ok
|
||||
_ = os.Unsetenv(name)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, name := range names {
|
||||
if present[name] {
|
||||
_ = os.Setenv(name, originals[name])
|
||||
} else {
|
||||
_ = os.Unsetenv(name)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -10,22 +10,27 @@ import (
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog, error) {
|
||||
catalog := artifacts.NewArtifactCatalog()
|
||||
if err := catalog.RegisterBuiltIns(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
configured := map[string]artifacts.ConfiguredArtifactDefinition{}
|
||||
func buildHelperArtifactCatalog(cfg *config.Config, m *manifest.Manifest) (*artifacts.ArtifactCatalog, error) {
|
||||
configured := artifacts.ConfiguredArtifactDefinitions(nil)
|
||||
if cfg.Pipeline.Scriptorium != nil {
|
||||
for key, item := range cfg.Pipeline.Scriptorium.Artifacts {
|
||||
configured[key] = artifacts.ConfiguredArtifactDefinition{Enabled: item.Enabled, OutputPath: item.OutputPath}
|
||||
}
|
||||
configured = artifacts.ConfiguredArtifactDefinitions(cfg.Pipeline.Scriptorium.Artifacts)
|
||||
}
|
||||
if err := catalog.RegisterConfiguredArtifacts(configured, nil); err != nil {
|
||||
extractionDefinitions := artifacts.ExtractionDefinitionsFromConfig(cfg.Pipeline.Notarius)
|
||||
effective, err := artifacts.ResolveEffectiveArtifactSet(configured, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
catalog, err := artifacts.BootstrapRuntimeCatalog(configured, effective, extractionDefinitions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
if cfg.Pipeline.Notarius != nil && cfg.Pipeline.Notarius.Enabled {
|
||||
catalog.HydrateExtractionArtifacts(paths, m, extractionDefinitions)
|
||||
}
|
||||
return catalog, nil
|
||||
}
|
||||
|
||||
@@ -40,9 +45,19 @@ func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.Art
|
||||
for _, entry := range catalog.ListConfigured() {
|
||||
writeArtifactLine(out, entry.SourceID, lockSet)
|
||||
}
|
||||
fmt.Fprintln(out, "Extraction:")
|
||||
for _, entry := range catalog.ListExtraction() {
|
||||
state := "unavailable"
|
||||
if entry.Available {
|
||||
state = "available"
|
||||
}
|
||||
writeExtractionArtifactLine(out, entry.SourceID, state, entry.Provenance, lockSet)
|
||||
}
|
||||
fmt.Fprintln(out, "Previous-session:")
|
||||
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
|
||||
fmt.Fprintf(out, "- %s required=%t\n", artifactpolicy.PreviousSessionSourceID(req.Name), req.Required)
|
||||
if effective, err := resolveEffectiveArtifacts(cfg, nil); err == nil {
|
||||
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg), effective) {
|
||||
fmt.Fprintf(out, "- %s required=%t\n", artifactpolicy.PreviousSessionSourceID(req.Name), req.Required)
|
||||
}
|
||||
}
|
||||
fmt.Fprintln(out, "Published:")
|
||||
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||
@@ -50,6 +65,17 @@ func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.Art
|
||||
}
|
||||
}
|
||||
|
||||
func writeExtractionArtifactLine(out io.Writer, source, state, provenance string, lockSet map[string]config.PublishLockRule) {
|
||||
parts := []string{source, "planned", state}
|
||||
if strings.TrimSpace(provenance) != "" {
|
||||
parts = append(parts, "provenance="+strings.TrimSpace(provenance))
|
||||
}
|
||||
if _, ok := lockSet[source]; ok {
|
||||
parts = append(parts, "locked")
|
||||
}
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.PublishLockRule) {
|
||||
parts := []string{source}
|
||||
if _, ok := lockSet[source]; ok {
|
||||
@@ -101,9 +127,55 @@ func remotePublishedOutputAvailability(ctx context.Context, cfg *config.Config,
|
||||
return out
|
||||
}
|
||||
|
||||
func remotePublishedOutputAvailabilityForCurrent(
|
||||
ctx context.Context,
|
||||
cfg *config.Config,
|
||||
store storage.ObjectStore,
|
||||
catalog *artifacts.ArtifactCatalog,
|
||||
current *RemoteCurrentState,
|
||||
) map[string]string {
|
||||
if current == nil || current.Commit == nil {
|
||||
return remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
||||
}
|
||||
out := map[string]string{}
|
||||
runPrefix := artifacts.S3RunPrefix(current.SessionPrefix, current.RunID)
|
||||
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
dest, _, err := helperPublishedOutputDest(rule, catalog)
|
||||
if err != nil {
|
||||
out[publishedOutputRemoteStateKey(source, "")] = "remote=error"
|
||||
continue
|
||||
}
|
||||
key := artifacts.S3RunRelativeDestinationKey(runPrefix, dest)
|
||||
if committedPublishedOutput(current.Commit, key) {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = "remote=published"
|
||||
} else {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = "remote=missing"
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func committedPublishedOutput(commit *artifacts.RemoteCommitManifest, key string) bool {
|
||||
if commit == nil {
|
||||
return false
|
||||
}
|
||||
for _, artifact := range commit.Artifacts {
|
||||
if artifact.Type == artifacts.RemoteArtifactTypePublishedOutput && artifact.DestinationKey == key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
normalized, err := artifactpolicy.ResolvePublishedDestination(source, rule.Dest, helperConfiguredOutputPathMap(catalog))
|
||||
normalized, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(
|
||||
source,
|
||||
rule.Dest,
|
||||
helperConfiguredOutputPathMap(catalog),
|
||||
helperExtractionOutputSet(catalog),
|
||||
)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
@@ -112,6 +184,19 @@ func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts
|
||||
return normalized, showDest, nil
|
||||
}
|
||||
|
||||
func helperExtractionOutputSet(catalog *artifacts.ArtifactCatalog) map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
if catalog == nil {
|
||||
return out
|
||||
}
|
||||
for _, entry := range catalog.ListExtraction() {
|
||||
if strings.TrimSpace(entry.ExtractionKey) != "" {
|
||||
out[entry.ExtractionKey] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func helperConfiguredOutputPathMap(catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||
out := map[string]string{}
|
||||
if catalog == nil {
|
||||
|
||||
@@ -22,11 +22,12 @@ func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("artifacts list: session_id is required")
|
||||
}
|
||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, remote)
|
||||
cfg, store, locks, m, cleanup, err := loadHelperContext(ctx, flags, remote)
|
||||
if err != nil {
|
||||
return fmt.Errorf("artifacts list: %w", err)
|
||||
}
|
||||
catalog, err := buildHelperArtifactCatalog(cfg)
|
||||
defer cleanup()
|
||||
catalog, err := buildHelperArtifactCatalog(cfg, m)
|
||||
if err != nil {
|
||||
return fmt.Errorf("artifacts list: %w", err)
|
||||
}
|
||||
|
||||
@@ -90,33 +90,41 @@ func Artifacts(ctx context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
}
|
||||
|
||||
func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore bool) (*config.Config, storage.ObjectStore, *effectiveLocks, *manifest.Manifest, error) {
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore bool) (*config.Config, storage.ObjectStore, *effectiveLocks, *manifest.Manifest, func(), error) {
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
release := true
|
||||
defer func() {
|
||||
if release {
|
||||
_ = loaded.Close()
|
||||
}
|
||||
}()
|
||||
cfg := loaded.Config
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
var store storage.ObjectStore
|
||||
if needStore {
|
||||
store, err = newCommandObjectStore(ctx, cfg, nil)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
} else {
|
||||
store, _ = objectStoreIfConfigured(ctx, cfg)
|
||||
}
|
||||
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
m, err := loadLocalManifest(ctx, paths.ManifestPath)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, err
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
return cfg, store, locks, m, nil
|
||||
release = false
|
||||
return cfg, store, locks, m, func() { _ = loaded.Close() }, nil
|
||||
}
|
||||
|
||||
func objectStoreIfConfigured(ctx context.Context, cfg *config.Config) (storage.ObjectStore, error) {
|
||||
|
||||
@@ -3,13 +3,17 @@ package app
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
@@ -187,8 +191,8 @@ func TestExecuteSessionInitRemoteLoadsSecretsBeforeObjectStoreInit(t *testing.T)
|
||||
secretKeyEnv := "NARRATIO_TEST_SESSION_INIT_OBJECT_SECRET"
|
||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
|
||||
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
||||
|
||||
fake := &storage.FakeBackend{}
|
||||
@@ -415,8 +419,8 @@ func TestExecuteSessionValidateLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||
secretKeyEnv := "NARRATIO_TEST_VALIDATE_OBJECT_SECRET"
|
||||
restoreEnvAfterTest(t, accessKeyEnv, secretKeyEnv)
|
||||
secretsDir := t.TempDir()
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
|
||||
mustWriteTestFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, accessKeyEnv), "test-key-id\n")
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "test-secret\n")
|
||||
addSecretsToPipelineConfig(t, pipelinePath, secretsDir, accessKeyEnv, secretKeyEnv)
|
||||
if err := os.WriteFile(sessionPath, []byte(`session_id: 2026-05-03
|
||||
inputs:
|
||||
@@ -508,7 +512,7 @@ func TestExecuteLocksAddListAndRemoveUseRemoteLockStore(t *testing.T) {
|
||||
if code != 0 {
|
||||
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
||||
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
||||
}
|
||||
@@ -520,6 +524,123 @@ func TestExecuteLocksAddListAndRemoveUseRemoteLockStore(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutateRemoteLockStoreRetainsConcurrentUpdates(t *testing.T) {
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{S3: &config.StorageS3Config{Bucket: "bucket", RootPrefix: "root"}}},
|
||||
Session: &config.SessionConfig{Campaign: "campaign", SessionID: "session"},
|
||||
}
|
||||
fake := &storage.FakeBackend{}
|
||||
arrived := make(chan struct{}, 2)
|
||||
release := make(chan struct{})
|
||||
var hookMu sync.Mutex
|
||||
hookCalls := 0
|
||||
fake.UploadHook = func(storage.FakeUploadCall) error {
|
||||
hookMu.Lock()
|
||||
hookCalls++
|
||||
call := hookCalls
|
||||
hookMu.Unlock()
|
||||
if call <= 2 {
|
||||
arrived <- struct{}{}
|
||||
<-release
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
mutate := func(source string) error {
|
||||
return mutateRemoteLockStore(context.Background(), cfg, fake, func(lockStore *config.PublishLockStore) error {
|
||||
set := lockSourceSet(lockStore.Locks)
|
||||
set[source] = config.PublishLockRule{Source: source}
|
||||
lockStore.Locks = lockMapValues(set)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
errs := make(chan error, 2)
|
||||
go func() { errs <- mutate("narratio.transcript.final") }()
|
||||
go func() { errs <- mutate("narratio.transcript.final_trimmed") }()
|
||||
<-arrived
|
||||
<-arrived
|
||||
close(release)
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatalf("first concurrent mutation error = %v", err)
|
||||
}
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatalf("second concurrent mutation error = %v", err)
|
||||
}
|
||||
|
||||
locks, _, _, err := loadRemoteLockStore(context.Background(), cfg, fake)
|
||||
if err != nil {
|
||||
t.Fatalf("loadRemoteLockStore() error = %v", err)
|
||||
}
|
||||
if len(locks.Locks) != 2 || locks.Locks[0].Source != "narratio.transcript.final" || locks.Locks[1].Source != "narratio.transcript.final_trimmed" {
|
||||
t.Fatalf("remote locks = %#v, want both concurrent updates", locks.Locks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRemoteLockStoreAcceptsExactLimitAndRejectsLimitPlusOne(t *testing.T) {
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{S3: &config.StorageS3Config{Bucket: "bucket", RootPrefix: "root"}}},
|
||||
Session: &config.SessionConfig{Campaign: "campaign", SessionID: "session"},
|
||||
}
|
||||
key, err := remoteLocksKey(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("remoteLocksKey() error = %v", err)
|
||||
}
|
||||
encoded, err := config.MarshalPublishLockStore(&config.PublishLockStore{})
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalPublishLockStore() error = %v", err)
|
||||
}
|
||||
exact := append(append([]byte(nil), encoded...), bytes.Repeat([]byte(" "), int(MaxRemoteLockStoreBytes)-len(encoded))...)
|
||||
store := &storage.FakeBackend{}
|
||||
store.SeedObject(storage.FakeObject{Key: key, Data: exact, ETag: "lock-generation"})
|
||||
|
||||
locks, gotKey, generation, err := loadRemoteLockStore(context.Background(), cfg, store)
|
||||
if err != nil {
|
||||
t.Fatalf("loadRemoteLockStore() exact-limit error = %v", err)
|
||||
}
|
||||
if locks == nil || gotKey != key || generation != "lock-generation" {
|
||||
t.Fatalf("loadRemoteLockStore() = (%#v, %q, %q), want decoded locks and opened generation", locks, gotKey, generation)
|
||||
}
|
||||
if len(store.Downloads) != 0 || len(store.Reads) != 1 || store.Reads[0].Key != key {
|
||||
t.Fatalf("lock transfers reads=%#v downloads=%#v, want one direct read", store.Reads, store.Downloads)
|
||||
}
|
||||
|
||||
store.SeedObject(storage.FakeObject{Key: key, Data: append(exact, ' '), ETag: "new-generation"})
|
||||
_, _, _, err = loadRemoteLockStore(context.Background(), cfg, store)
|
||||
if err == nil || !strings.Contains(err.Error(), "remote lock control object") || !strings.Contains(err.Error(), key) || !strings.Contains(err.Error(), fmt.Sprint(MaxRemoteLockStoreBytes)) {
|
||||
t.Fatalf("loadRemoteLockStore() limit-plus-one error = %v, want category, key, and limit", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRemoteLockStoreRejectsMalformedYAML(t *testing.T) {
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{S3: &config.StorageS3Config{Bucket: "bucket", RootPrefix: "root"}}},
|
||||
Session: &config.SessionConfig{Campaign: "campaign", SessionID: "session"},
|
||||
}
|
||||
key, err := remoteLocksKey(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("remoteLocksKey() error = %v", err)
|
||||
}
|
||||
store := &storage.FakeBackend{}
|
||||
store.SeedObject(storage.FakeObject{Key: key, Data: []byte("locks: [\n"), ETag: "lock-generation"})
|
||||
|
||||
if _, _, _, err := loadRemoteLockStore(context.Background(), cfg, store); err == nil {
|
||||
t.Fatal("loadRemoteLockStore() error = nil, want malformed YAML failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutateRemoteLockStoreHonorsCancellation(t *testing.T) {
|
||||
cfg := &config.Config{
|
||||
Pipeline: &config.PipelineConfig{Storage: config.StorageConfig{S3: &config.StorageS3Config{Bucket: "bucket", RootPrefix: "root"}}},
|
||||
Session: &config.SessionConfig{Campaign: "campaign", SessionID: "session"},
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
err := mutateRemoteLockStore(ctx, cfg, &storage.FakeBackend{}, func(*config.PublishLockStore) error { return nil })
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("mutateRemoteLockStore() error = %v, want context cancellation", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteLocksAddDuplicateRequiresForce(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
@@ -772,6 +893,71 @@ func TestExecuteArtifactsListRemoteReportsPublishedAvailability(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteArtifactsListReportsExtractionLifecycleWithoutPayload(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
addExtractionOutputToPipeline(t, pipelinePath)
|
||||
addPublishOutputsToPipeline(t, pipelinePath, `
|
||||
outputs:
|
||||
- source: narratio.extraction.encounters
|
||||
dest: artifacts/encounters.json
|
||||
required: true
|
||||
`)
|
||||
lanePath := writeOperatorExtractionManifest(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
publishedKey := artifacts.S3PublishedOutputKey(
|
||||
artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"),
|
||||
"artifacts/encounters.json",
|
||||
)
|
||||
fake.SeedObject(storage.FakeObject{Key: publishedKey, Data: []byte(`{"secret":"DO_NOT_PRINT"}`)})
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{
|
||||
"session", "artifacts", "2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--remote",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
out := stdout.String()
|
||||
for _, want := range []string{
|
||||
"Extraction:",
|
||||
"narratio.extraction.encounters planned available provenance=manifest.current_extract_run",
|
||||
"narratio.extraction.encounters dest=artifacts/encounters.json remote=published",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("stdout = %q, want %q", out, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out, "DO_NOT_PRINT") {
|
||||
t.Fatalf("operator output exposed extraction payload: %q", out)
|
||||
}
|
||||
|
||||
if err := os.Remove(lanePath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = Execute([]string{
|
||||
"session", "artifacts", "2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("unavailable exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio.extraction.encounters planned unavailable") {
|
||||
t.Fatalf("stdout = %q, want unavailable extraction state", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteArtifactsListRemoteUsesPublishOutputDestinations(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
@@ -912,9 +1098,6 @@ func TestExecuteStatusReportsRemoteArtifactCatalogErrorsWithoutFailing(t *testin
|
||||
if !strings.Contains(out, "Remote outputs:") || !strings.Contains(out, "narratio.transcript.final_trimmed remote=error") {
|
||||
t.Fatalf("stdout = %q, want remote output error state", out)
|
||||
}
|
||||
if !strings.Contains(out, "Publish locks: error:") {
|
||||
t.Fatalf("stdout = %q, want publish locks error", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStatusReportsMissingRemoteCurrentStateWithoutFailing(t *testing.T) {
|
||||
@@ -965,6 +1148,36 @@ func TestExecuteStatusReportsPreviousStateReadinessWithoutFailing(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStatusDetectsMissingRequiredPreviousArtifact(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
replaceInFileOrFatal(t, pipelinePath, "source: narratio.artifact.session_recap", "source: narratio.previous_session.artifact.session_recap")
|
||||
replaceInFileOrFatal(t, sessionPath, "session_id: 2026-05-03\n", "session_id: 2026-05-03\nprevious_session_id: 2026-04-26\n")
|
||||
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||
}
|
||||
fake := &storage.FakeBackend{}
|
||||
seedRestorePreviousCurrentManifestOnly(t, fake, cfg)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{
|
||||
"session", "status", "2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), `Previous-session artifacts: unavailable: remote required previous-session artifact "session_recap" object missing`) {
|
||||
t.Fatalf("stdout = %q, want missing required previous artifact", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSessionValidateReportsPreviousStateFindingAndReturnsFindingError(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
@@ -1006,7 +1219,7 @@ func TestExecutePublishLoadsRemoteLocks(t *testing.T) {
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||
// The publish stage only checks the manifest statuses and source files.
|
||||
_ = stageName
|
||||
}
|
||||
@@ -1041,6 +1254,85 @@ func addPublishOutputsToPipeline(t *testing.T, pipelinePath, publishYAML string)
|
||||
}
|
||||
}
|
||||
|
||||
func addExtractionOutputToPipeline(t *testing.T, pipelinePath string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data = append(data, []byte(`notarius:
|
||||
enabled: true
|
||||
config_path: notarius.yml
|
||||
pipeline_id: campaign.extract
|
||||
outputs:
|
||||
encounters:
|
||||
lane_id: encounters
|
||||
media_type: application/json
|
||||
schema_id: encounters
|
||||
schema_version: "1"
|
||||
module_key: encounters
|
||||
`)...)
|
||||
if err := os.WriteFile(pipelinePath, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(filepath.Dir(pipelinePath), "notarius.yml"), []byte("{}\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeOperatorExtractionManifest(t *testing.T, workspaceRoot string) string {
|
||||
t.Helper()
|
||||
paths := artifacts.NewLocalStore(workspaceRoot).SessionPathsFor("sample-campaign", "2026-05-03")
|
||||
bundleRoot := filepath.Join(paths.ArtifactsDir, "notarius", "extract-run-1")
|
||||
lanePath := filepath.Join(bundleRoot, "lanes", "encounters.json")
|
||||
indexPath := filepath.Join(bundleRoot, "index.json")
|
||||
trimmedPath := filepath.Join(paths.Root, filepath.FromSlash(artifacts.TranscriptPathFinalTrimmed))
|
||||
mustWriteTestFile(t, lanePath, `{"secret":"DO_NOT_PRINT"}`)
|
||||
mustWriteTestFile(t, indexPath, `{"lanes":[]}`)
|
||||
mustWriteTestFile(t, trimmedPath, `{"segments":[]}`)
|
||||
laneChecksum, err := artifacts.SHA256File(lanePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
indexChecksum, err := artifacts.SHA256File(indexPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := manifest.New("2026-05-03", time.Now().UTC())
|
||||
m.Campaign = "sample-campaign"
|
||||
m.Stages["trim"] = &manifest.StageRecord{
|
||||
Name: "trim", Status: manifest.StatusSucceeded,
|
||||
Outputs: []manifest.ArtifactRecord{{
|
||||
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, LocalPath: trimmedPath, ProducerRunID: "trim-run-1",
|
||||
}},
|
||||
}
|
||||
input, err := artifacts.ResolveExtractionInputIdentity(paths, m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Stages["extract"] = &manifest.StageRecord{
|
||||
Name: "extract", Status: manifest.StatusSucceeded,
|
||||
Metadata: map[string]any{
|
||||
"narratio_run_id": "extract-run-1", "bundle_root": bundleRoot,
|
||||
"receipt": map[string]any{"run_id": "notarius-run-1", "pipeline_id": "campaign.extract"},
|
||||
"direct_input": input.Metadata(),
|
||||
},
|
||||
Outputs: []manifest.ArtifactRecord{
|
||||
{
|
||||
Kind: "notarius_lane", SourceID: artifacts.ExtractionArtifactSourceID("encounters"), LocalPath: lanePath,
|
||||
ProducerRunID: "extract-run-1", Checksum: laneChecksum,
|
||||
Contract: &artifactmodel.ContractMetadata{MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters"},
|
||||
ExternalProvenance: &artifactmodel.ExternalProvenance{System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: "encounters"},
|
||||
},
|
||||
{Kind: "notarius_index", LocalPath: indexPath, ProducerRunID: "extract-run-1", Checksum: indexChecksum},
|
||||
},
|
||||
}
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), paths.ManifestPath, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return lanePath
|
||||
}
|
||||
|
||||
func replaceInFileOrFatal(t *testing.T, path, old, new string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
@@ -1076,7 +1368,7 @@ func writeValidPublishRunConfigFiles(t *testing.T, workspaceRoot string) (string
|
||||
m := manifest.New("2026-05-03", nowUTC())
|
||||
m.Campaign = "sample-campaign"
|
||||
m.RunID = "20260521T160000Z-test"
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||
m.MarkStageSucceeded(name, nowUTC(), nil)
|
||||
}
|
||||
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
|
||||
@@ -2,6 +2,7 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/previouscache"
|
||||
)
|
||||
|
||||
type stableInputCheck struct {
|
||||
@@ -34,9 +36,9 @@ type remoteAudioCheck struct {
|
||||
}
|
||||
|
||||
type previousArtifactReadiness struct {
|
||||
Requirements []artifacts.PreviousArtifactRequirement
|
||||
MissingID bool
|
||||
Err error
|
||||
Requirements []artifacts.PreviousArtifactRequirement
|
||||
SkippedMissing []string
|
||||
Err error
|
||||
}
|
||||
|
||||
type remoteCurrentStateCheck struct {
|
||||
@@ -152,23 +154,27 @@ func inspectPreviousArtifactReadiness(
|
||||
if len(requirements) == 0 {
|
||||
return out
|
||||
}
|
||||
if strings.TrimSpace(cfg.Session.PreviousSessionID) == "" {
|
||||
out.MissingID = true
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
out.Err = fmt.Errorf("resolved config with pipeline/session is required")
|
||||
return out
|
||||
}
|
||||
if store == nil {
|
||||
out.Err = fmt.Errorf("previous-session artifacts cannot be checked because storage is unavailable")
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
plan, err := previouscache.Resolve(ctx, cfg, paths, requirements, store)
|
||||
if err != nil {
|
||||
var pointerMissing *artifacts.CurrentRunPointerMissingError
|
||||
var manifestMissing *artifacts.CurrentManifestMissingError
|
||||
if errors.As(err, &pointerMissing) {
|
||||
out.Err = fmt.Errorf("remote %w", pointerMissing)
|
||||
return out
|
||||
}
|
||||
if errors.As(err, &manifestMissing) {
|
||||
out.Err = fmt.Errorf("remote %w", manifestMissing)
|
||||
return out
|
||||
}
|
||||
out.Err = fmt.Errorf("remote %w", err)
|
||||
return out
|
||||
}
|
||||
|
||||
prefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.PreviousSessionID)
|
||||
if _, err := artifacts.LoadCurrentState(ctx, store, prefix, artifacts.CurrentStateValidation{
|
||||
ExpectedSessionID: strings.TrimSpace(cfg.Session.PreviousSessionID),
|
||||
ExpectedCampaign: strings.TrimSpace(cfg.Session.Campaign),
|
||||
ValidateRunID: true,
|
||||
}); err != nil {
|
||||
out.Err = fmt.Errorf("remote %v", err)
|
||||
}
|
||||
out.SkippedMissing = append([]string(nil), plan.SkippedMissing...)
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -38,10 +38,11 @@ func LocksList(ctx context.Context, args []string, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks: session_id is required")
|
||||
}
|
||||
cfg, _, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||
cfg, _, locks, _, cleanup, err := loadHelperContext(ctx, flags, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks: %w", err)
|
||||
}
|
||||
defer cleanup()
|
||||
writeLocks(out, cfg, locks)
|
||||
return nil
|
||||
}
|
||||
@@ -63,26 +64,31 @@ func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks add: session_id is required")
|
||||
}
|
||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||
cfg, store, locks, _, cleanup, err := loadHelperContext(ctx, flags, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
|
||||
defer cleanup()
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks add"); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
||||
return fmt.Errorf("locks add: source %q is locked by pipeline config and cannot be modified remotely", source)
|
||||
}
|
||||
remoteSet := lockSourceSet(locks.Remote)
|
||||
if _, exists := remoteSet[source]; exists && !force {
|
||||
return fmt.Errorf("locks add: remote lock for %q already exists; pass --force to update", source)
|
||||
}
|
||||
remoteSet[source] = config.PublishLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
||||
remoteLocks := lockMapValues(remoteSet)
|
||||
if _, err := config.ValidatePublishLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
||||
if err := mutateRemoteLockStore(ctx, cfg, store, func(lockStore *config.PublishLockStore) error {
|
||||
remoteSet := lockSourceSet(lockStore.Locks)
|
||||
if _, exists := remoteSet[source]; exists && !force {
|
||||
return fmt.Errorf("remote lock for %q already exists; pass --force to update", source)
|
||||
}
|
||||
remoteSet[source] = config.PublishLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
||||
lockStore.Locks = lockMapValues(remoteSet)
|
||||
normalized, err := config.ValidatePublishLockRules(lockStore.Locks, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lockStore.Locks = normalized
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "narratio session locks add: locked %s\n", source)
|
||||
@@ -102,23 +108,26 @@ func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks remove: session_id is required")
|
||||
}
|
||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, true)
|
||||
cfg, store, locks, _, cleanup, err := loadHelperContext(ctx, flags, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
|
||||
defer cleanup()
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks remove"); err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
remoteSet := lockSourceSet(locks.Remote)
|
||||
if _, ok := remoteSet[source]; !ok {
|
||||
if _, static := lockSourceSet(locks.Static)[source]; static {
|
||||
return fmt.Errorf("locks remove: source %q is locked by pipeline config and cannot be unlocked remotely", source)
|
||||
if err := mutateRemoteLockStore(ctx, cfg, store, func(lockStore *config.PublishLockStore) error {
|
||||
remoteSet := lockSourceSet(lockStore.Locks)
|
||||
if _, ok := remoteSet[source]; !ok {
|
||||
if _, static := lockSourceSet(locks.Static)[source]; static {
|
||||
return fmt.Errorf("source %q is locked by pipeline config and cannot be unlocked remotely", source)
|
||||
}
|
||||
return fmt.Errorf("remote lock for %q does not exist", source)
|
||||
}
|
||||
return fmt.Errorf("locks remove: remote lock for %q does not exist", source)
|
||||
}
|
||||
delete(remoteSet, source)
|
||||
remoteLocks := lockMapValues(remoteSet)
|
||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
||||
delete(remoteSet, source)
|
||||
lockStore.Locks = lockMapValues(remoteSet)
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "narratio session locks remove: unlocked %s\n", source)
|
||||
|
||||
52
internal/app/operator_session_init_test.go
Normal file
52
internal/app/operator_session_init_test.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSessionInitRejectsRenderedTemplateThatOmitsExpectedPreviousSession(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
campaignDir := filepath.Dir(campaignPath)
|
||||
templatePath := filepath.Join(campaignDir, "session.template.yml")
|
||||
template := `session_id: {{ session_id }}
|
||||
campaign: sample-campaign
|
||||
inputs:
|
||||
audio_dir: ./audio
|
||||
`
|
||||
if err := os.WriteFile(templatePath, []byte(template), 0o644); err != nil {
|
||||
t.Fatalf("write session template: %v", err)
|
||||
}
|
||||
campaign := `campaign_id: sample-campaign
|
||||
session_template_file: session.template.yml
|
||||
inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(campaignPath, []byte(campaign), 0o644); err != nil {
|
||||
t.Fatalf("write campaign config: %v", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := SessionInit(context.Background(), []string{
|
||||
"2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--output", filepath.Join(t.TempDir(), "session.yml"),
|
||||
"--previous-session-id", "2026-04-26",
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unused template variable value(s): previous_session_id") {
|
||||
t.Fatalf("SessionInit() error = %q, want missing previous-session template error", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -25,11 +25,13 @@ func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
|
||||
findings := []finding{}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
findings = append(findings, errorFinding("config", err.Error()))
|
||||
return renderFindings(out, "", "", findings)
|
||||
}
|
||||
defer func() { _ = loaded.Close() }()
|
||||
cfg := loaded.Config
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
findings = append(findings, errorFinding("config", err.Error()))
|
||||
} else {
|
||||
@@ -53,16 +55,23 @@ func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
}
|
||||
|
||||
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg))
|
||||
effective, effectiveErr := resolveEffectiveArtifacts(cfg, nil)
|
||||
if effectiveErr != nil {
|
||||
findings = append(findings, errorFinding("config", effectiveErr.Error()))
|
||||
return renderFindings(out, cfg.Session.Campaign, cfg.Session.SessionID, findings)
|
||||
}
|
||||
requirements := artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg), effective)
|
||||
previous := inspectPreviousArtifactReadiness(ctx, cfg, store, requirements)
|
||||
if len(previous.Requirements) == 0 {
|
||||
findings = append(findings, okFinding("previous", "no previous-session artifacts required"))
|
||||
} else if previous.MissingID {
|
||||
findings = append(findings, errorFinding("previous", "previous_session_id is required by configured previous-session artifacts"))
|
||||
} else if previous.Err != nil {
|
||||
findings = append(findings, errorFinding("previous", previous.Err.Error()))
|
||||
} else {
|
||||
for _, req := range previous.Requirements {
|
||||
if !req.Required && previousRequirementSkipped(previous.SkippedMissing, req.Name) {
|
||||
findings = append(findings, okFinding("previous", fmt.Sprintf("%s required=false unavailable", req.Name)))
|
||||
continue
|
||||
}
|
||||
findings = append(findings, okFinding("previous", fmt.Sprintf("%s required=%t", req.Name, req.Required)))
|
||||
}
|
||||
}
|
||||
@@ -82,3 +91,12 @@ func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
return renderFindings(out, cfg.Session.Campaign, cfg.Session.SessionID, findings)
|
||||
}
|
||||
|
||||
func previousRequirementSkipped(values []string, name string) bool {
|
||||
for _, value := range values {
|
||||
if value == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
// Status reports effective local/remote session state.
|
||||
@@ -25,10 +26,12 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("status: session_id is required")
|
||||
}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return fmt.Errorf("status: %w", err)
|
||||
}
|
||||
defer func() { _ = loaded.Close() }()
|
||||
cfg := loaded.Config
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return fmt.Errorf("status: %w", err)
|
||||
}
|
||||
@@ -40,16 +43,19 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
writeStatusStableInputs(out, inspectStableInputs(cfg))
|
||||
writeStatusLocalAudio(out, inspectLocalAudioPresence(cfg))
|
||||
|
||||
var localManifest *manifest.Manifest
|
||||
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
|
||||
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
|
||||
} else if m == nil {
|
||||
fmt.Fprintln(out, "Local manifest: missing")
|
||||
} else {
|
||||
localManifest = m
|
||||
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
|
||||
writeStageStatuses(out, m)
|
||||
}
|
||||
|
||||
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||
var remoteCurrent *RemoteCurrentState
|
||||
if storeErr != nil {
|
||||
fmt.Fprintf(out, "Remote publish: unavailable: %v\n", storeErr)
|
||||
} else if store != nil {
|
||||
@@ -57,22 +63,27 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
if current.Err != nil {
|
||||
fmt.Fprintf(out, "Remote publish: missing or unavailable: %v\n", current.Err)
|
||||
} else {
|
||||
remoteCurrent = current.State
|
||||
fmt.Fprintf(out, "Remote publish: current run %s\n", current.State.RunID)
|
||||
fmt.Fprintf(out, "Remote manifest: %s\n", current.State.CurrentManifestKey)
|
||||
}
|
||||
}
|
||||
writeStatusRemoteAudio(ctx, out, cfg, store, storeErr)
|
||||
effective, effectiveErr := resolveEffectiveArtifacts(cfg, nil)
|
||||
if effectiveErr != nil {
|
||||
return fmt.Errorf("status: resolve effective artifacts: %w", effectiveErr)
|
||||
}
|
||||
writeStatusPreviousArtifacts(out, inspectPreviousArtifactReadiness(
|
||||
ctx,
|
||||
cfg,
|
||||
store,
|
||||
artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)),
|
||||
artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg), effective),
|
||||
))
|
||||
|
||||
lockChecks := inspectEffectiveLocks(ctx, cfg, store)
|
||||
locks := lockChecks.Locks
|
||||
lockErr := lockChecks.Err
|
||||
if catalog, catalogErr := buildHelperArtifactCatalog(cfg); catalogErr != nil {
|
||||
if catalog, catalogErr := buildHelperArtifactCatalog(cfg, localManifest); catalogErr != nil {
|
||||
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
|
||||
} else if storeErr == nil {
|
||||
catalogLocks := locks
|
||||
@@ -84,7 +95,7 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
publishedRemoteState := map[string]string{}
|
||||
if store != nil {
|
||||
publishedRemoteState = remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
||||
publishedRemoteState = remotePublishedOutputAvailabilityForCurrent(ctx, cfg, store, catalog, remoteCurrent)
|
||||
}
|
||||
fmt.Fprintln(out, "Remote outputs:")
|
||||
writeArtifactList(out, cfg, catalog, catalogLocks, publishedRemoteState)
|
||||
@@ -149,10 +160,6 @@ func writeStatusPreviousArtifacts(out io.Writer, readiness previousArtifactReadi
|
||||
fmt.Fprintln(out, "Previous-session artifacts: not required")
|
||||
return
|
||||
}
|
||||
if readiness.MissingID {
|
||||
fmt.Fprintln(out, "Previous-session artifacts: unavailable: previous_session_id is required by configured previous-session artifacts")
|
||||
return
|
||||
}
|
||||
if readiness.Err != nil {
|
||||
fmt.Fprintf(out, "Previous-session artifacts: unavailable: %v\n", readiness.Err)
|
||||
return
|
||||
@@ -162,5 +169,9 @@ func writeStatusPreviousArtifacts(out io.Writer, readiness previousArtifactReadi
|
||||
names = append(names, fmt.Sprintf("%s(required=%t)", req.Name, req.Required))
|
||||
}
|
||||
sort.Strings(names)
|
||||
if len(readiness.SkippedMissing) > 0 {
|
||||
fmt.Fprintf(out, "Previous-session artifacts: ready: %s; optional unavailable: %s\n", strings.Join(names, ", "), strings.Join(readiness.SkippedMissing, ", "))
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, "Previous-session artifacts: ready: %s\n", strings.Join(names, ", "))
|
||||
}
|
||||
|
||||
16
internal/app/paths_test_helpers_test.go
Normal file
16
internal/app/paths_test_helpers_test.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
)
|
||||
|
||||
func mustPreviousArtifactPathForCampaign(t *testing.T, root, campaign, sessionID, relative string) string {
|
||||
t.Helper()
|
||||
path, err := artifacts.SessionPreviousArtifactPathForCampaign(root, campaign, sessionID, relative)
|
||||
if err != nil {
|
||||
t.Fatalf("SessionPreviousArtifactPathForCampaign() error = %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -22,7 +22,7 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
var flags commonConfigFlags
|
||||
var force bool
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||
fs.BoolVar(&force, "force", false, "show all stages as scheduled to rerun")
|
||||
|
||||
if err := parseSessionAwareFlags("plan", fs, args, &flags.sessionID); err != nil {
|
||||
return err
|
||||
@@ -30,10 +30,12 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
if flags.sessionID == "" {
|
||||
return fmt.Errorf("plan: session_id is required")
|
||||
}
|
||||
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
if err != nil {
|
||||
return fmt.Errorf("plan: %w", err)
|
||||
}
|
||||
defer func() { _ = loaded.Close() }()
|
||||
cfg := loaded.Config
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return fmt.Errorf("plan: %w", err)
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user