Compare commits
104 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b68435128b | |||
| 5fb454a714 | |||
| 79cf9f1139 | |||
| 9bb203476d | |||
| 0726786016 | |||
| 204caf9d24 | |||
| 85663f5bc9 | |||
| f51b9bb046 | |||
| dd905c4daa | |||
| c3307ab5fb | |||
| 1484f51f48 | |||
| 51d21676c1 | |||
| c2806a1615 | |||
| 078f661e3a | |||
| adc60aa588 | |||
| 8d9eb5cc40 | |||
| 563acce012 | |||
| 49c359feb8 | |||
| 38202d4472 | |||
| 6d07dafc9f | |||
| 6c66e18c4b | |||
| 4048996652 | |||
| 470450e509 | |||
| 7e78c22fec | |||
| 610eaefb27 | |||
| 4bc33773fa | |||
| b74e31e1cc | |||
| 2f1e4407db | |||
| 4ccb4a73ca | |||
| 145baac393 | |||
| be59bff690 | |||
| 14b4fee214 | |||
| 7f84fc5d1e | |||
| a8b47f6185 | |||
| 5b62a8d60d | |||
| 0d4d71b9dc | |||
| 979d8049b1 | |||
| abd405015b | |||
| fddf78dc80 | |||
| 3136c26d4e | |||
| 4b84b4223d | |||
| 7a0e2cb11b | |||
| 74ab9218ff | |||
| da99857fd0 | |||
| a520cfdcc8 | |||
| 7febccf550 | |||
| 74977b4064 | |||
| 172e2d6a4f | |||
| 5c520dc0d4 | |||
| 6d18a3126e | |||
| 1e222a4db3 | |||
| 6a15052d77 | |||
| 2e69c14898 | |||
| dbda2a0161 | |||
| 2ee64f2eb1 | |||
| 8afe9427ca | |||
| e0f19833a1 | |||
| ca1b694aa9 | |||
| 9331373245 | |||
| ec072dc163 | |||
| f7ce1dcd56 | |||
| df275874e3 | |||
| 1da34ab5cd | |||
| c0fbc14f2d | |||
| 2d20900cbf | |||
| 7997f81f47 | |||
| 6d748d65d3 | |||
| 49324d3f35 | |||
| c98b4ef4ef | |||
| c8bae3a4a3 | |||
| ccbd7ee820 | |||
| e9c4873030 | |||
| 57719b7d81 | |||
| 62b1f9b0ae | |||
| 6c7a45d8d0 | |||
| 82d75e7beb | |||
| 61095e69ce | |||
| 2cfca710ec | |||
| 9679117e94 | |||
| d67a0389e0 | |||
| 9f84446509 | |||
| ec9b0e919d | |||
| 9bb7c1f4e3 | |||
| 3c092fbe34 | |||
| 3a7e2e5175 | |||
| 45b27ad562 | |||
| 1e418d6929 | |||
| 39b2665d8b | |||
| ba858cc997 | |||
| 0d57422b61 | |||
| b7d989f61f | |||
| 6813340f51 | |||
| eb788f17b8 | |||
| ebd3f415d3 | |||
| f15d8d8ce0 | |||
| 6c7bfd5e31 | |||
| 7c782a2603 | |||
| f1eef5b5e9 | |||
| f14583321d | |||
| 04283e99d4 | |||
| 7284228bec | |||
| da74d7ad18 | |||
| af5fbc1c4f | |||
| 5e4d2016e8 |
@@ -3,12 +3,12 @@ when:
|
||||
|
||||
steps:
|
||||
validate-release:
|
||||
image: golang:1.25.5
|
||||
image: golang:1.26.7
|
||||
commands:
|
||||
- ./scripts/check-release-candidate.sh "$CI_COMMIT_TAG"
|
||||
|
||||
build-release-assets:
|
||||
image: golang:1.25.5
|
||||
image: golang:1.26.7
|
||||
depends_on:
|
||||
- validate-release
|
||||
commands:
|
||||
|
||||
@@ -3,6 +3,6 @@ when:
|
||||
|
||||
steps:
|
||||
shuffled-race-tests:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
commands:
|
||||
- go test -race -shuffle=on -count=3 ./...
|
||||
|
||||
@@ -3,37 +3,37 @@ when:
|
||||
|
||||
steps:
|
||||
tests:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
commands:
|
||||
- go test ./...
|
||||
|
||||
race-tests:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go test -race ./...
|
||||
|
||||
static-analysis:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go vet ./...
|
||||
|
||||
build:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go build ./...
|
||||
|
||||
documentation-and-examples:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
depends_on: tests
|
||||
commands:
|
||||
- go test ./internal/doccheck
|
||||
- go test ./internal/config -run '^TestExamplesLoadAndValidate$'
|
||||
|
||||
cross-build:
|
||||
image: golang:1.25
|
||||
image: golang:1.26
|
||||
depends_on: [race-tests, static-analysis, build, documentation-and-examples]
|
||||
commands:
|
||||
- |
|
||||
|
||||
42
docs/cli.md
42
docs/cli.md
@@ -106,10 +106,10 @@ family declarations, and runtime provenance.
|
||||
|
||||
`config sources` reports the same fully validated resolution without printing
|
||||
effective values. Its header identifies the root, ordered imports, selected
|
||||
profile and overlay, selected campaign, party mode/source, and digest. The
|
||||
profile and overlay, selected campaign, party source, and digest. The
|
||||
remaining tab-separated records are sorted as `path`, `role`, and `source`.
|
||||
Roles distinguish root, import, profile, centralized default, campaign, party,
|
||||
legacy-player, and generated family ownership. A generated party member has
|
||||
and generated family ownership. A generated party member has
|
||||
one family record and one party record at the same logical path. The output
|
||||
never reads or prints secret values.
|
||||
|
||||
@@ -153,7 +153,7 @@ Behavior:
|
||||
|
||||
- evaluates one inclusive contiguous range of the canonical stage order;
|
||||
- defaults an omitted `--from` to `prepare` and an omitted `--through` to
|
||||
`notify`, so omitting both retains full-pipeline behavior;
|
||||
`cleanup`, so omitting both retains full-pipeline behavior;
|
||||
- rejects unknown endpoints and a `--from` endpoint after `--through`;
|
||||
- runs `render` before `extract`; an omitted or disabled Notarius
|
||||
configuration records an explicit `notarius_disabled` self-skip;
|
||||
@@ -163,7 +163,9 @@ Behavior:
|
||||
- rejects repeated `--from`, `--through`, or `--force` options, including
|
||||
`--name=value` spellings;
|
||||
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- writes session and run manifests.
|
||||
- writes the session manifest as the sole durable progress ledger. Each
|
||||
invocation also creates its run-local directory and returns an immediate
|
||||
command summary; neither creates another durable progress record.
|
||||
- reports the resolved profile (or `none`) and effective configuration digest.
|
||||
|
||||
When `--artifacts` is present, the selected range must contain `analyze` or
|
||||
@@ -211,6 +213,7 @@ Valid stage names:
|
||||
- `analyze`
|
||||
- `publish`
|
||||
- `notify`
|
||||
- `cleanup`
|
||||
|
||||
Rules:
|
||||
|
||||
@@ -240,6 +243,10 @@ Equivalent to:
|
||||
narratio run-stage publish <session_id> --force [...common config flags]
|
||||
```
|
||||
|
||||
This command selects only `publish`; it does not invoke cleanup. Use a normal
|
||||
full run to include configured cleanup, or `run-stage cleanup` to retry cleanup
|
||||
from valid publication evidence.
|
||||
|
||||
### `clean`
|
||||
|
||||
```bash
|
||||
@@ -253,8 +260,8 @@ Behavior:
|
||||
- `--all` removes all local session work and spool state;
|
||||
- cache remains unless `--clear-cache` is provided.
|
||||
|
||||
See [Operations: Cleanup](./operations.md#cleanup) for deletion scope and
|
||||
post-publish cleanup behavior.
|
||||
See [Operations: Cleanup](./operations.md#cleanup) for manual and pipeline-stage
|
||||
deletion scope.
|
||||
|
||||
### `session plan`
|
||||
|
||||
@@ -279,7 +286,8 @@ execution; the preview does not predict output identities.
|
||||
narratio session validate <session_id> [...common config flags]
|
||||
```
|
||||
|
||||
Read-only preflight checks for config validity, required inputs, audio mode, previous-session requirements, publish outputs, and effective locks.
|
||||
Read-only preflight checks for config validity, required inputs, audio mode,
|
||||
previous-session requirements, publish outputs, and independent remote locks.
|
||||
|
||||
### `session status`
|
||||
|
||||
@@ -288,7 +296,12 @@ narratio session status <session_id> [...common config flags]
|
||||
```
|
||||
|
||||
Prints local manifest state and, when storage is available, status for the
|
||||
pointer-selected remote commit and its declared published outputs.
|
||||
pointer-selected remote commit and its declared published outputs. Current
|
||||
publish metadata reports `published`, `carried_locked`, `locked_unpublished`,
|
||||
`optional_missing`, and `unselected`; carried output lines include the run
|
||||
derived from their exact declared key, the key itself, and its declared
|
||||
destination. A separately displayed `dest` is the destination requested by the
|
||||
current configuration and may differ while an older declaration is locked.
|
||||
|
||||
### `session init`
|
||||
|
||||
@@ -346,7 +359,8 @@ narratio session artifacts <session_id> [--remote] [...common config flags]
|
||||
Lists effective built-in, configured Scriptorium, and configured extraction
|
||||
sources; reports planned, available, unavailable, and published state without
|
||||
reading payload bodies; and includes publish rules, lock state, and optional
|
||||
remote published-state availability.
|
||||
remote published-state availability. Remote inspection uses exact commit
|
||||
declarations and displays the five current publish outcomes when recorded.
|
||||
|
||||
### `session locks`
|
||||
|
||||
@@ -358,12 +372,14 @@ narratio session locks remove <session_id> <source> [...common config flags]
|
||||
|
||||
Behavior:
|
||||
|
||||
- list mode reports the effective merge of static and remote locks;
|
||||
- add/remove mutate only remote locks;
|
||||
- static locks from pipeline config cannot be removed by CLI commands.
|
||||
- list mode validates and reports independent remote locks sorted by source;
|
||||
- add creates one lock and reports an existing lock unless `--force` is used;
|
||||
- `--force` performs one generation-matched replacement and reports a
|
||||
concurrent change for operator retry; and
|
||||
- remove is idempotent and reports when the source was not locked.
|
||||
|
||||
See [Operations: Publish Locks](./operations.md#publish-locks) for lock storage
|
||||
and precedence.
|
||||
and publication behavior.
|
||||
|
||||
## `--artifacts` Selection Rules
|
||||
|
||||
|
||||
@@ -62,8 +62,7 @@ effective pipeline YAML, with defaults and concrete expansion included but
|
||||
composition and family declarations omitted. `config sources` prints a stable
|
||||
source projection instead of effective values: root/import/profile/default
|
||||
ownership plus campaign/party and generated-family records. Canonical derived
|
||||
players trace to the party; a legacy configured players file is explicitly
|
||||
marked as a legacy player source. The [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff)
|
||||
players trace to the party. The [CLI reference](cli.md#config-validate-config-show-config-sources-and-config-diff)
|
||||
owns command syntax and output conventions.
|
||||
|
||||
`config diff` compares normalized field values rather than YAML text or source
|
||||
@@ -80,11 +79,11 @@ Use the [production/testing bundle](../examples/production-testing/pipeline.yml)
|
||||
as the complete copyable migration reference. Split stable pipeline settings
|
||||
into explicit additive imports, place production/testing differences in one
|
||||
selected overlay, and retain a production `default_profile`. Convert campaign
|
||||
rosters to [canonical party input](integrations/party.md), remove a separate
|
||||
`players_file`, then express character work as families. Inspect the result
|
||||
rosters to [canonical party input](integrations/party.md), then express
|
||||
character work as families. Inspect the result
|
||||
with `config validate`, `config show`, and `config sources`; use `config diff`
|
||||
to review profiles before running a session. Unversioned parties and their
|
||||
`players_file` remain a clearly bounded legacy compatibility path.
|
||||
to review profiles before running a session. Unversioned party documents are
|
||||
rejected.
|
||||
|
||||
### Identity segments
|
||||
|
||||
@@ -113,15 +112,13 @@ remote state with an unsafe legacy identity must be migrated before use.
|
||||
- Pipeline defaults are applied before validation.
|
||||
- Campaign and session identities must agree.
|
||||
- Required stable files (`speakers_file`, `autocorrect_file`, `glossary_file`,
|
||||
`party_file`) and the optional `spell_catalog_file` resolve from session
|
||||
overrides when provided, otherwise from campaign defaults. An empty or
|
||||
omitted session spell-catalog value inherits the campaign value.
|
||||
- `party_file` is classified when pipeline and campaign configuration are
|
||||
combined. A versioned [canonical party](integrations/party.md) is
|
||||
campaign-owned, derives the players input internally, and forbids both a
|
||||
separate `players_file` and a session `party_file` override. An unversioned
|
||||
party remains a bounded legacy input and requires `players_file`; its normal
|
||||
campaign/session overrides continue to apply.
|
||||
`party_file`) are campaign-owned; the first three may be overridden by the
|
||||
session. The optional `spell_catalog_file` also inherits a campaign value
|
||||
when omitted by the session.
|
||||
- `party_file` is parsed when pipeline and campaign configuration are combined.
|
||||
It must be a versioned [canonical party](integrations/party.md), is
|
||||
campaign-owned and derives the players input internally. `players_file` and
|
||||
session `party_file` are unsupported configuration keys.
|
||||
- Exactly one audio mode must be configured in session input:
|
||||
- local (`audio_dir` or `audio_files`), or
|
||||
- S3 (`audio_s3.prefix`).
|
||||
@@ -213,7 +210,6 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
```
|
||||
|
||||
@@ -259,9 +255,6 @@ publish:
|
||||
- source: narratio.artifact.session_recap
|
||||
dest: artifacts/session_recap.md
|
||||
required: true
|
||||
locks:
|
||||
- source: narratio.artifact.session_recap
|
||||
reason: manual post-publish edits
|
||||
```
|
||||
|
||||
Rules:
|
||||
@@ -272,7 +265,8 @@ Rules:
|
||||
a rule names that source; the Notarius index and complete bundle are not
|
||||
publish sources.
|
||||
- `outputs[].required` defaults to `true`.
|
||||
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
||||
- publish locks are independent remote session objects managed with
|
||||
`narratio session locks`; they are not pipeline configuration fields.
|
||||
|
||||
## Full Schema
|
||||
|
||||
@@ -284,7 +278,7 @@ Rules:
|
||||
| `composition.default_profile` | string | Conditional | selected when profiles exist and no caller explicitly selects one; must name a declared profile |
|
||||
| `composition.profiles.<name>.overlay` | string | Conditional | required for every declared profile; one confined `.yml` or `.yaml` overlay relative to the root pipeline directory |
|
||||
| `pipeline.workspace.root` | string | No | `/var/lib/narratio` |
|
||||
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false` |
|
||||
| `pipeline.workspace.cleanup_after_publish` | bool | No | `false`; selects the published run's workspace target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||
| `pipeline.campaigns.root` | string | No | `/usr/local/share/narratio/campaigns` |
|
||||
| `pipeline.campaigns.default_campaign_id` | string | No | empty |
|
||||
| `pipeline.secrets.env_dir` | string | No | empty |
|
||||
@@ -297,7 +291,7 @@ Rules:
|
||||
| `pipeline.storage.s3.access_key_id_env` | string | No | `OBJECT_STORAGE_KEY_ID` |
|
||||
| `pipeline.storage.s3.secret_access_key_env` | string | No | `OBJECT_STORAGE_KEY` |
|
||||
| `pipeline.spool.root` | string | No | `/var/spool/narratio` |
|
||||
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false` |
|
||||
| `pipeline.spool.delete_audio_after_publish` | bool | No | `false`; selects the published run's spool-audio target for final cleanup and requires effective `pipeline.publish.enabled=true` and `pipeline.publish.upload_run=true` |
|
||||
| `pipeline.cache.root` | string | No | `/var/cache/narratio` |
|
||||
| `pipeline.cache.s3_audio` | bool | No | `true` |
|
||||
| `pipeline.publish.enabled` | bool | No | `true` |
|
||||
@@ -306,9 +300,6 @@ Rules:
|
||||
| `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source |
|
||||
| `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation |
|
||||
| `pipeline.publish.outputs[].required` | bool | No | `true` |
|
||||
| `pipeline.publish.locks[]` | list | No | empty |
|
||||
| `pipeline.publish.locks[].source` | string | Yes (per lock) | must reference supported publish source |
|
||||
| `pipeline.publish.locks[].reason` | string | No | empty |
|
||||
| `pipeline.whisperx.transcribe_url` | string | Yes | absolute `http` or `https` URL |
|
||||
| `pipeline.whisperx.language` | string | No | `en` |
|
||||
| `pipeline.whisperx.timeout` | duration | No | `30m` |
|
||||
@@ -454,7 +445,7 @@ Without `--artifacts`, analyze executes enabled configured artifacts. With an
|
||||
explicit `--artifacts` list, the exact named configured artifacts are the
|
||||
one-invocation targets even if their `enabled` values are false. Analyze closes
|
||||
those targets over `depends_on`: a current prerequisite is reused, while a
|
||||
stale, missing, failed, or legacy prerequisite is rebuilt before its dependent.
|
||||
stale, missing, or failed prerequisite is rebuilt before its dependent.
|
||||
Unrelated artifacts are not executed. Named targets and any prerequisite that
|
||||
may require rebuilding must therefore have valid executable fields. This
|
||||
override affects analyze planning only; publish uses the list only to filter
|
||||
@@ -476,7 +467,8 @@ does not provide adapter-specific input passthrough fields.
|
||||
`pipeline.scriptorium.artifact_families` declares a shared artifact template
|
||||
for every canonical campaign character. Configuration resolution expands each
|
||||
family into ordinary `pipeline.scriptorium.artifacts` entries before analyze
|
||||
planning or Scriptorium invocation. A legacy party cannot be used for a family.
|
||||
planning or Scriptorium invocation. Party parsing rejects unsupported document
|
||||
schemas before family expansion.
|
||||
|
||||
For each `pipeline.scriptorium.artifact_families.<name>`:
|
||||
|
||||
@@ -524,7 +516,6 @@ integration.
|
||||
| `inputs.speakers_file` | string | Yes | stable input default |
|
||||
| `inputs.autocorrect_file` | string | Yes | stable input default |
|
||||
| `inputs.glossary_file` | string | Yes | stable input default |
|
||||
| `inputs.players_file` | string | Conditional | required only with an unversioned legacy `party_file`; forbidden for a canonical party |
|
||||
| `inputs.party_file` | string | Yes | stable campaign party source; relative paths resolve from `campaign.yml` |
|
||||
| `inputs.spell_catalog_file` | string | No | optional spell-catalog overlay default; required when a Notarius reference selects `narratio.input.spell_catalog` |
|
||||
|
||||
@@ -540,8 +531,6 @@ integration.
|
||||
| `inputs.speakers_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.autocorrect_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.glossary_file` | string | No | overrides campaign stable input |
|
||||
| `inputs.players_file` | string | No | legacy-party override; forbidden for a canonical party |
|
||||
| `inputs.party_file` | string | No | legacy-party override; forbidden for a canonical campaign party |
|
||||
| `inputs.spell_catalog_file` | string | No | overrides the optional campaign spell catalog; empty or omitted inherits the campaign value |
|
||||
| `inputs.audio_dir` | string | Conditional | local audio mode |
|
||||
| `inputs.audio_files[]` | list[string] | Conditional | local audio mode |
|
||||
|
||||
@@ -10,6 +10,11 @@ polished transcripts and generated artifacts. Start with the
|
||||
[Architecture](policy/architecture.md) for normative system boundaries, and the
|
||||
[Internal Overview](internal/overview.md) for implemented component ownership.
|
||||
|
||||
## Go Prerequisite
|
||||
|
||||
Maintained development and verification require Go 1.26. Use the official Go
|
||||
1.26 toolchain before running repository commands.
|
||||
|
||||
## What To Read
|
||||
|
||||
| When working on | Read | Why |
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# Canonical Party Input
|
||||
|
||||
`party.yml` is a campaign-owned roster input. Narratio recognizes the
|
||||
versioned `narratio.party.v1` document below when it resolves a pipeline,
|
||||
campaign, and session together.
|
||||
`party.yml` is a campaign-owned roster input. It must be a single, versioned
|
||||
`narratio.party.v1` document when Narratio resolves a pipeline, campaign, and
|
||||
session together.
|
||||
|
||||
```yaml
|
||||
schema_version: narratio.party.v1
|
||||
@@ -34,19 +34,17 @@ non-empty, trimmed display strings without control characters. Character names
|
||||
and aliases must be unique across the full roster under Unicode-aware
|
||||
case-insensitive comparison; player names may repeat.
|
||||
|
||||
The document has exactly one YAML document and accepts no unknown fields. A
|
||||
wrong or malformed `schema_version` is an error.
|
||||
The document has exactly one YAML document and accepts no unknown fields.
|
||||
`schema_version` is required: a missing, wrong, or malformed value is an error.
|
||||
|
||||
## Legacy migration boundary
|
||||
## Migration
|
||||
|
||||
An unversioned party input remains supported only as opaque legacy reference
|
||||
material while campaigns migrate. It requires a separate `players_file` and
|
||||
retains the existing session override behavior. It cannot be mixed with a
|
||||
canonical party: canonical campaigns must omit `players_file`, and sessions
|
||||
must not override their party or players inputs.
|
||||
Unversioned party inputs are rejected. Replace each legacy party document with
|
||||
the versioned structure above, move each roster entry under `characters`, and
|
||||
remove separate player and session-party settings: those keys are unsupported.
|
||||
|
||||
Use the canonical document for new campaigns. The configuration rules and
|
||||
source-relative path behavior are defined in the [Configuration Reference](../config.md).
|
||||
The configuration rules and source-relative path behavior are defined in the
|
||||
[Configuration Reference](../config.md).
|
||||
|
||||
## Derived players document
|
||||
|
||||
|
||||
@@ -35,8 +35,9 @@ Adapters do not own:
|
||||
|
||||
## Default Wiring
|
||||
|
||||
`internal/app/runner.go` initializes default adapters when not injected and
|
||||
only when the selected execution plan needs them:
|
||||
The application composes default adapters per invocation when they were not
|
||||
injected through the stage environment, and only when the selected execution
|
||||
plan needs them:
|
||||
|
||||
- WhisperX HTTP client for `transcribe`.
|
||||
- Seriatim subprocess runner for `merge`, `normalize`, `trim`, or `render`.
|
||||
@@ -49,8 +50,10 @@ only when the selected execution plan needs them:
|
||||
Remote publish locks are loaded only for a selected, enabled publish that
|
||||
uploads a run. Shared session lifecycle setup still applies to every selected
|
||||
range, but an unselected integration is neither initialized nor validated by
|
||||
runner composition. Each selected stage retains its own fail-fast configuration
|
||||
and input validation.
|
||||
application-owned composition. Constructor, clock, randomness, remote-lock,
|
||||
and post-publish deletion boundaries are scoped to the invocation; stages still
|
||||
receive their collaborators through the stage environment. Each selected stage
|
||||
retains its own fail-fast configuration and input validation.
|
||||
|
||||
`session plan` is outside production adapter composition. It performs
|
||||
resume validation and models selected transitions against cloned manifest
|
||||
@@ -63,15 +66,19 @@ prepared reference resolution, receipt, bundle, and configured-lane policy.
|
||||
The adapter validates the ordered selector/absolute-path pairs and is the sole
|
||||
owner of serializing them as repeated `--reference` arguments before `--json`.
|
||||
|
||||
Object-store construction goes through `newCommandObjectStore`, which loads
|
||||
configured filesystem secrets before adapter initialization.
|
||||
Application-owned object-store construction loads configured filesystem
|
||||
secrets before invoking the invocation's storage factory. Production commands
|
||||
use the concrete storage adapter constructor; focused tests can supply an
|
||||
in-memory backend without changing package state.
|
||||
|
||||
## Failure Semantics
|
||||
|
||||
- Constructor errors fail stage execution setup early.
|
||||
- Runtime adapter errors propagate to stage code and then manifest failure handling.
|
||||
- Subprocess adapters persist stage logs/generated configs through stage-managed paths.
|
||||
- Shared subprocess execution starts an owned process group on Linux/macOS or a
|
||||
- Shared subprocess execution keeps private lifecycle mechanics separate from
|
||||
Narratio's environment, diagnostics, redaction, and error policy. It starts
|
||||
an owned process group on Linux/macOS or a
|
||||
kill-on-close job object on Windows. Every terminal path disposes of that
|
||||
owned tree before returning. After a natural leader exit, Unix checks for
|
||||
remaining group members and uses bounded graceful then forceful termination;
|
||||
@@ -86,7 +93,8 @@ configured filesystem secrets before adapter initialization.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Composition: `internal/app/runner.go`, `internal/app/object_store.go`
|
||||
- Composition: `internal/app/runner_composition.go`,
|
||||
`internal/app/runner_dependencies.go`, `internal/app/object_store.go`
|
||||
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
||||
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
|
||||
- `internal/adapters/whisperx/http_test.go`
|
||||
|
||||
@@ -20,8 +20,8 @@ The internal registry recognizes these stable built-in source IDs:
|
||||
- `narratio.transcript.final_trimmed_markdown`
|
||||
- `narratio.bounds.session`
|
||||
|
||||
Registry entries bind each ID to its producer, output kind, canonical fallback,
|
||||
and content validator. The focused stage documents own their input/output flow;
|
||||
Registry entries bind each ID to its producer, output kind, and content
|
||||
validator. The focused stage documents own their input/output flow;
|
||||
[Configuration](../config.md) owns where operators may select these IDs.
|
||||
|
||||
## Configured, Extraction, And Previous-Session Sources
|
||||
@@ -69,8 +69,9 @@ publish rules during configuration resolution.
|
||||
Configured outputs, including non-executable prerequisites, become available
|
||||
only when the versioned analyze state identifies a current result whose source,
|
||||
contract, canonical configured path, size, and checksum match a confined
|
||||
no-follow regular file. An incidental canonical file and a legacy aggregate
|
||||
analyze output are unavailable.
|
||||
no-follow regular file. An incidental canonical file is unavailable, and a
|
||||
manifest with an unsupported aggregate-only representation is rejected before
|
||||
catalog construction.
|
||||
Extraction entries are registered from configuration and become available only
|
||||
after compatible extraction evidence is hydrated.
|
||||
|
||||
@@ -91,8 +92,13 @@ Current provenance values:
|
||||
|
||||
Built-ins:
|
||||
|
||||
1. manifest producer outputs (when present)
|
||||
2. canonical session-path fallback
|
||||
- resolve only from the current manifest's succeeded registered producer;
|
||||
- require exactly one output with the registered source ID and output kind, a
|
||||
safe session-rooted path, producer run ID, canonical checksum, and valid
|
||||
content; and
|
||||
- verify the registered checksum against the durable bytes. Incidental files,
|
||||
canonical filenames, duplicate outputs, failed/stale producer records, and
|
||||
malformed evidence are unavailable.
|
||||
|
||||
Configured sources (`narratio.artifact.*`):
|
||||
|
||||
@@ -103,7 +109,7 @@ Configured sources (`narratio.artifact.*`):
|
||||
record for the exact configured key and source ID, a complete contract, the
|
||||
configured canonical relative path, positive stored size, and stored
|
||||
checksum matching bytes read from a confined no-follow regular file; and
|
||||
- treat non-current statuses, legacy or malformed records, removed keys,
|
||||
- treat non-current statuses, malformed records, removed keys,
|
||||
unsafe or missing files, and size/checksum mismatches as unavailable without
|
||||
rewriting manifest state. Catalog construction iterates current
|
||||
configuration, so removed or renamed records are not advertised.
|
||||
@@ -135,9 +141,12 @@ Extraction sources (`narratio.extraction.*`):
|
||||
|
||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||
|
||||
- resolve only from local `previous/` cache state;
|
||||
- prefer manifest-backed previous-input paths;
|
||||
- fallback to existing previous-cache filesystem paths.
|
||||
- resolve only from one matching `previous_artifact` input record in the
|
||||
current manifest, including its exact source ID and checksum;
|
||||
- are hydrated only from the unique `published_output` declaration for their
|
||||
canonical source in the selected remote commit; and
|
||||
- never infer a path from a previous manifest, current configuration, basename,
|
||||
or existing cache file.
|
||||
|
||||
Source absence is evaluated by the consuming artifact input. An optional input
|
||||
is omitted from that invocation; a required input fails resolution. This is
|
||||
@@ -165,25 +174,34 @@ Validation by content type:
|
||||
Artifacts package owns shared remote current-state loading mechanics used by
|
||||
restore, status and validation checks, and previous-cache planning.
|
||||
|
||||
The selected session manifest is accepted only with
|
||||
`schema_version: narratio.session-manifest.v1` and complete stage identity.
|
||||
An unsupported local or downloaded manifest must be replaced by regeneration,
|
||||
verified restore, or republication; artifact resolution never fills in missing
|
||||
authority markers.
|
||||
|
||||
For a new-protocol current state, the pointer-selected immutable commit is the
|
||||
complete restore authority. Callers receive its declared object identities and
|
||||
must not supplement them by listing mutable session prefixes. The legacy reader
|
||||
is intentionally separate and remains migration-only support.
|
||||
must not supplement them by listing mutable session prefixes.
|
||||
|
||||
A `published_output` declaration must use a canonical publish-compatible source
|
||||
ID and an exact `runs/{run_id}/published/{destination}` key in the same session.
|
||||
Only this declaration type may refer to an earlier run; other declared objects
|
||||
remain confined to the selected commit's run.
|
||||
|
||||
The reader opens each small control object directly and enforces owner-specific
|
||||
limits before decoding: 64 KiB for the mutable commit pointer, 4 MiB for the
|
||||
immutable commit manifest, and 8 MiB for the selected session manifest. Legacy
|
||||
compatibility applies a 4 KiB limit to `current/run_id.txt` and the same 8 MiB
|
||||
manifest limit to `current/manifest.json`. These are exposed as
|
||||
`MaxCurrentCommitPointerBytes`, `MaxRemoteCommitManifestBytes`,
|
||||
`MaxRemoteSessionManifestBytes`, `MaxLegacyCurrentRunPointerBytes`, and
|
||||
`MaxLegacyCurrentManifestBytes`.
|
||||
immutable commit manifest, and 8 MiB for the selected session manifest. These
|
||||
are exposed as `MaxCurrentCommitPointerBytes`,
|
||||
`MaxRemoteCommitManifestBytes`, and `MaxRemoteSessionManifestBytes`.
|
||||
|
||||
Each read uses the generation and size metadata returned with its opened body.
|
||||
Actual bytes remain subject to a limit-plus-one read even if size metadata is
|
||||
absent or inaccurate. Immutable selections then retain their declared-size,
|
||||
checksum, generation, and identity checks. No current-state control object is
|
||||
downloaded through a temporary file.
|
||||
downloaded through a temporary file. Artifact and previous-session downloads
|
||||
validate their complete sibling temporary file against available committed
|
||||
identity before replacing a canonical local path.
|
||||
|
||||
Core helpers:
|
||||
|
||||
@@ -191,10 +209,9 @@ Core helpers:
|
||||
- `ValidateCurrentStateIdentity`
|
||||
- `RemoteCommitManifest` and `CurrentCommitPointer`
|
||||
|
||||
Typed missing-state errors:
|
||||
Typed missing-state error:
|
||||
|
||||
- `CurrentRunPointerMissingError` (`ErrCurrentRunPointerMissing`)
|
||||
- `CurrentManifestMissingError` (`ErrCurrentManifestMissing`)
|
||||
- `CurrentCommitPointerMissingError` (`ErrCurrentCommitPointerMissing`)
|
||||
|
||||
Identity validation supports caller-provided expectations:
|
||||
|
||||
@@ -208,6 +225,21 @@ Caller policy is intentionally outside artifacts helpers:
|
||||
- some callers downgrade missing state to status/findings;
|
||||
- some callers skip optional behavior when state is missing.
|
||||
|
||||
## Remote Lock Objects
|
||||
|
||||
The artifacts package defines the independent remote-lock payload and identity
|
||||
contract. Each canonical publish source maps to one object below the validated
|
||||
session `locks/` prefix; its filename is the lowercase SHA-256 digest of the
|
||||
exact source followed by `.lock`.
|
||||
|
||||
Lock payloads are strict versioned JSON containing the canonical source, an
|
||||
optional reason, and a UTC creation time. Reads are bounded to
|
||||
`MaxRemoteLockObjectBytes` (64 KiB), reject unknown fields and trailing values,
|
||||
and verify that the payload source matches the key digest. Exact lookup reports
|
||||
presence explicitly, prefix listing validates every object and sorts by source,
|
||||
creation uses an absence precondition, forced replacement uses one observed
|
||||
generation, and removal is idempotent.
|
||||
|
||||
## Key Path Helpers
|
||||
|
||||
`internal/artifacts/paths.go` and S3-key helpers define canonical helpers for:
|
||||
@@ -220,8 +252,13 @@ Caller policy is intentionally outside artifacts helpers:
|
||||
New publication creates run-scoped immutable objects, including
|
||||
`runs/{run_id}/commit.json` and `runs/{run_id}/session-manifest.json`. The sole
|
||||
mutable selector is `current/commit-pointer.json`; readers verify its selected
|
||||
commit and declared object generations/checksums. Legacy current-pair loading
|
||||
is confined to `current_state_legacy.go` for migration only.
|
||||
commit and declared object generations/checksums.
|
||||
|
||||
A `published_output` declaration may identify its commit's run or an earlier
|
||||
immutable run in the same campaign/session namespace. The key parser validates
|
||||
that namespace, rejects mutable and malformed keys, and derives the originating
|
||||
run from the exact declaration. All other commit artifacts remain confined to
|
||||
the commit's own run prefix.
|
||||
|
||||
Campaign, session, and Narratio run IDs are validated as portable opaque
|
||||
segments at configuration and artifact boundaries before they can be used in a
|
||||
@@ -252,7 +289,6 @@ physical layout.
|
||||
`internal/artifacts/prepared_input.go`
|
||||
- Current state: `internal/artifacts/current_state.go`,
|
||||
`internal/artifacts/current_state_commit.go`,
|
||||
`internal/artifacts/current_state_legacy.go`
|
||||
- Paths and keys: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/s3_keys.go`
|
||||
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
||||
|
||||
@@ -11,6 +11,11 @@ physical restore scope belong in
|
||||
Restore separates remote authority, local conflict policy, and filesystem
|
||||
mutation so each remains testable independently.
|
||||
|
||||
The public command supplies production defaults to one private,
|
||||
invocation-scoped dependency value. Discovery, planning, and execution use that
|
||||
same value, so focused sequencing tests can replace one boundary without
|
||||
mutable package hooks.
|
||||
|
||||
## Discovery Contract
|
||||
|
||||
Discovery delegates current-state pointer and manifest loading to
|
||||
@@ -34,18 +39,25 @@ Planner behavior:
|
||||
|
||||
- a new-protocol restore uses only the selected commit's declared artifact set;
|
||||
each action carries that artifact's immutable key, checksum, size, and
|
||||
generation. Coherent legacy state remains on the isolated compatibility path;
|
||||
generation;
|
||||
- remote-to-local mapping is traversal-safe;
|
||||
- actions are sorted by local relative path and then remote key;
|
||||
- planning opens the selected session root and inspects every target by its
|
||||
normalized session-relative path. Missing ancestors produce a missing target;
|
||||
an ancestor or leaf symlink is a non-regular conflict and is never opened for
|
||||
hashing;
|
||||
- force converts differing eligible regular files from conflicts to downloads;
|
||||
directories and other non-regular targets remain conflicts.
|
||||
directories, symlinks, and other non-regular targets remain conflicts.
|
||||
|
||||
For a non-dry-run restore, planning/classification happens only after acquiring
|
||||
the session lock. Runner manifest/reuse checks acquire that same lock first.
|
||||
|
||||
Previous-cache readiness is resolved through `previouscache.Resolve` for restore,
|
||||
prepare, status, and validation. A committed source is selected only by its
|
||||
exact source identity; legacy fallback remains isolated and rejects ambiguity.
|
||||
prepare, status, and validation. Each requirement selects the unique
|
||||
`published_output` for its canonical source in the pointer-selected commit,
|
||||
then strips only that run's immutable `published/` prefix for its safe local
|
||||
cache path. It never probes candidate objects or derives paths from manifests,
|
||||
configuration, or basenames.
|
||||
|
||||
## Execution Contract
|
||||
|
||||
@@ -53,10 +65,14 @@ Execution order and safety:
|
||||
|
||||
- non-manifest downloads happen before manifest install;
|
||||
- `manifest.json` installs last;
|
||||
- downloads use sibling temp files plus atomic rename;
|
||||
- downloads use sibling temp files, validate the complete object, then use an
|
||||
atomic rename;
|
||||
- manifest replacement is validated before rename;
|
||||
- each committed object is verified against its declared checksum, size, and
|
||||
generation before installation;
|
||||
- installation reopens the selected session root, creates only validated
|
||||
missing directory components with workspace modes, and rejects symlinked or
|
||||
non-regular ancestors and leaves before replacement;
|
||||
- a committed manifest already verified during discovery is retained for the
|
||||
matching restore action and revalidated before installation, avoiding a
|
||||
second body transfer;
|
||||
@@ -67,15 +83,17 @@ Execution order and safety:
|
||||
- restored manifest local references are rebased beneath the selected local
|
||||
session root. Unsafe relative references and producer-machine absolute paths
|
||||
outside the manifest's producer session root are rejected; producer-local
|
||||
spool/cache and cleanup locations are not restored as authority.
|
||||
spool paths and run-local diagnostic references are not restored as
|
||||
authority. Cleanup remains ordinary restored stage state and is not reset or
|
||||
synthesized by restore.
|
||||
|
||||
Audio restore path:
|
||||
|
||||
- uses `audio.MaterializeS3Audio`;
|
||||
- integrates spool and S3 audio cache paths;
|
||||
- reuses cached audio only when its no-follow regular file, content digest, and
|
||||
- reuses cached audio only when its opened regular file, content digest, and
|
||||
identity sidecar all match the selected remote object version; otherwise it
|
||||
refreshes through the durable download path.
|
||||
refreshes through the validated download path.
|
||||
|
||||
## Reporting Contract
|
||||
|
||||
@@ -92,9 +110,7 @@ Audio restore path:
|
||||
loaded at discovery; later pointer changes cannot add objects or substitute a
|
||||
different run into its plan;
|
||||
- a verified `current/commit-pointer.json` and its selected immutable commit
|
||||
establish new-protocol remote commitment; coherent legacy
|
||||
`current/run_id.txt` plus `current/manifest.json` remains read-only migration
|
||||
support;
|
||||
establish remote commitment; when absent, republish the session;
|
||||
- restore does not execute pipeline stages.
|
||||
|
||||
## Implementation And Tests
|
||||
@@ -107,4 +123,5 @@ Audio restore path:
|
||||
- Tests: `internal/app/restore_discovery_test.go`,
|
||||
`internal/app/restore_plan_test.go`,
|
||||
`internal/app/restore_execution_test.go`,
|
||||
`internal/app/restore_recovery_test.go`,
|
||||
`internal/app/restore_workflow_test.go`
|
||||
|
||||
@@ -11,10 +11,11 @@ pipeline-loading boundary implemented by `internal/config`.
|
||||
1. Parse the root YAML into a presence-aware composition tree. The tree retains
|
||||
source names, full field paths, node kinds, declaration order, and explicit
|
||||
zero, false, empty-map, and empty-list values.
|
||||
2. Remove the root-only `composition` envelope and validate its explicit
|
||||
`imports`, `default_profile`, and named `profiles` declarations. A load
|
||||
option retains the difference between omitted and explicitly empty profile
|
||||
selection.
|
||||
2. Decode the root-only `composition` envelope, validate its declaration shapes
|
||||
and cross-field profile semantics, then clone the root without that envelope
|
||||
as the ordinary pipeline body. The parsed source document is not mutated. A
|
||||
load option retains the difference between omitted and explicitly empty
|
||||
profile selection.
|
||||
3. Open each import relative to the root pipeline directory through the
|
||||
confined regular-file boundary. Imports must use a `.yml` or `.yaml`
|
||||
extension and cannot traverse, use symlinks, repeat a file, import the root,
|
||||
@@ -75,8 +76,7 @@ each contributor as root, import, profile, or centralized default. The
|
||||
projection replaces generated concrete member paths with paired family and
|
||||
canonical-party records, and does the same for generated publish rules.
|
||||
Campaign records identify campaign-owned fields and party inputs; canonical
|
||||
derived players point to the party source, while legacy players retain a
|
||||
dedicated legacy-player role. The application command only joins these sorted
|
||||
derived players point to the party source. The application command only joins these sorted
|
||||
records with selection metadata and never reparses configuration files.
|
||||
|
||||
`config diff` uses a paired profile loader that parses the root, imports, and
|
||||
@@ -90,28 +90,45 @@ Values are compact deterministic JSON representations for command output, not
|
||||
raw YAML fragments, ownership records, or secret material. A differing digest
|
||||
with no projected difference is treated as an internal consistency error.
|
||||
|
||||
Campaign context construction also reads and classifies the campaign-owned
|
||||
party source through `ParseParty`. A canonical party retains its raw bytes and
|
||||
normalized roster in runtime-only `ResolvedParty` provenance, while a legacy
|
||||
party remains opaque. Canonical resolution creates a virtual
|
||||
`derived_from_party` players input and rejects competing campaign or session
|
||||
players files and session party overrides. The compact legacy compatibility
|
||||
path resolves the effective campaign/session party and players files together.
|
||||
Campaign context construction reads the campaign-owned party source through
|
||||
`ParseParty`. The required canonical party retains its raw bytes and normalized
|
||||
roster in runtime-only `ResolvedParty` provenance. Canonical resolution creates
|
||||
a deterministic `derived_from_party` players projection. The party is the only
|
||||
configured roster authority; player and session-party fields are unsupported.
|
||||
|
||||
## Canonical Party Domain
|
||||
|
||||
`ParseParty` is the package-owned boundary for classifying a party source.
|
||||
When a top-level `schema_version` is present, it strictly validates the
|
||||
`ParseParty` is the package-owned boundary for parsing a party source. It
|
||||
requires a top-level `schema_version` and strictly validates the
|
||||
`narratio.party.v1` contract into ordered character domain values. The
|
||||
canonical value retains a separate exact byte copy of its source so consumers
|
||||
can materialize the authored party document without reserializing it. Its
|
||||
`PlayersYAML` method deterministically derives the versioned players-only
|
||||
projection.
|
||||
|
||||
An unversioned source is classified by the small legacy compatibility boundary
|
||||
in `party_legacy.go`; it deliberately exposes no parsed roster information.
|
||||
That boundary exists solely to isolate removable compatibility behavior from
|
||||
the canonical parser.
|
||||
An unversioned source is rejected before resolution, family expansion, or
|
||||
prepare-stage materialization.
|
||||
|
||||
## Validation Ownership
|
||||
|
||||
Configuration validation remains within `internal/config`, with the public
|
||||
entry points and pipeline-domain sequencing separated from the private owners
|
||||
for core pipeline/session rules, external adapters, publish/storage, Notarius,
|
||||
Scriptorium, and shared scalar/path checks. These are package-local ownership
|
||||
boundaries rather than extension points or a general validation framework.
|
||||
|
||||
Validation fails fast and does not define which of several independent invalid
|
||||
fields is reported first. Map-backed declarations are nevertheless traversed
|
||||
deterministically. Notarius validates scalar policy, builds normalized reference
|
||||
bindings and output lanes independently, and installs all candidate values only
|
||||
after every check succeeds. Scriptorium validates artifact declarations and
|
||||
outputs, dependencies and inputs, variables, referenced outputs, and the final
|
||||
dependency graph in deterministic artifact order.
|
||||
|
||||
Diagnostics identify the relevant semantic field. Composition conflicts also
|
||||
retain every claiming source needed to correct an additive ownership conflict.
|
||||
Complete prose, punctuation, formatting, and private validation choreography are
|
||||
not compatibility contracts.
|
||||
|
||||
## Diagnostics And Runtime Metadata
|
||||
|
||||
@@ -153,8 +170,7 @@ configuration tests continue to protect defaults and validation after assembly.
|
||||
`party_test.go` protects the versioned party schema, domain invariants, and
|
||||
deterministic players projection without involving campaign or runtime wiring.
|
||||
`party_resolution_test.go` protects campaign-owned party loading, canonical
|
||||
input restrictions, legacy overrides, source provenance, and virtual players
|
||||
input selection.
|
||||
input restrictions, source provenance, and virtual players input selection.
|
||||
|
||||
## Artifact Family Resolution
|
||||
|
||||
|
||||
@@ -1,71 +1,55 @@
|
||||
# Internal: File Operations
|
||||
|
||||
`internal/fileops` owns the narrow mechanics for durable replacement of one
|
||||
`internal/fileops` owns the narrow mechanics for atomically visible replacement of one
|
||||
byte file. Callers keep ownership of serialization, validation, cancellation,
|
||||
and destination-directory policy.
|
||||
|
||||
## Destination Confinement
|
||||
## Destination Paths
|
||||
|
||||
Before it creates, replaces, or installs a destination file, `fileops` opens
|
||||
each ancestor from the filesystem root and rejects symbolic links or components
|
||||
that change during traversal. The resulting parent-directory handle is retained
|
||||
for sibling temporary-file creation and rename, so a later pathname swap cannot
|
||||
redirect the replacement. Existing destination symlinks are replaced as leaf
|
||||
entries; their targets are never followed.
|
||||
Ordinary workspace replacement uses standard operating-system path semantics.
|
||||
Callers with a root-scoped safety contract own that validation at their boundary.
|
||||
|
||||
Remote object acquisition uses a writer supplied by the storage owner. The
|
||||
writer receives a `fileops`-owned, already-open sibling temporary file rather
|
||||
than a mutable destination path. Callers still own remote object selection,
|
||||
validation, conflict handling, and final mode.
|
||||
Remote object acquisition is owned by its domain caller. The caller writes a
|
||||
sibling temporary file, validates the complete result, and passes that file to
|
||||
`InstallSiblingTempFile` only when it is ready to become canonical. Callers own
|
||||
remote object selection, validation, conflict handling, and temporary-file
|
||||
cleanup.
|
||||
|
||||
Directory promotion keeps the verified destination parent open while it creates
|
||||
the temporary tree, copies regular source entries, and performs the platform
|
||||
no-replace rename. Platforms without a verified handle-relative atomic
|
||||
no-replace primitive reject promotion before writing a temporary tree.
|
||||
Directory bundles are not promoted by `internal/fileops`; extraction owns its
|
||||
validated Notarius bundle rename.
|
||||
|
||||
## Cleanup Contract
|
||||
|
||||
`RemoveAllUnderRoot` accepts an explicit root and a proper descendant. It opens
|
||||
the root and each target ancestor without following symlinks, then removes the
|
||||
tree through those directory handles. It rejects root deletion and any symlink
|
||||
encountered in the target path or tree; repeated removal of a missing target is
|
||||
successful. Command and post-publish policy remains owned by `internal/app`.
|
||||
`RemoveAllUnderRoot` accepts an explicit root and a proper descendant, opens
|
||||
that root with `os.OpenRoot`, and removes the normalized root-relative target
|
||||
with `Root.RemoveAll`. It rejects root, traversal, and out-of-root targets; a
|
||||
missing target is successful. `RemoveFileUnderRoot` additionally requires an
|
||||
existing leaf to be regular. Command and post-publish policy remains owned by
|
||||
`internal/app`.
|
||||
|
||||
## Confined Reads
|
||||
## Bounded Reads
|
||||
|
||||
`ReadRegularFileUnderRoot` is the no-follow, bounded read primitive for a
|
||||
caller-selected root and relative file path; `ReadRegularFile` is its
|
||||
path-based convenience wrapper. They verify every ancestor through directory
|
||||
handles and admit only a stable regular-file handle. Callers enforce their own
|
||||
byte limits and access policy. Credential mode policy and environment
|
||||
precedence remain owned by `internal/app`.
|
||||
`ReadFileBounded` opens a normal path and reads at most its caller-supplied
|
||||
limit. Credential mode policy and environment precedence remain owned by
|
||||
`internal/app`.
|
||||
|
||||
## Replacement Contract
|
||||
|
||||
`ReplaceFileAtomic` requires an existing destination directory. It creates a
|
||||
sibling temporary file, writes the complete byte sequence, applies the
|
||||
caller-supplied mode, syncs and closes the file, runs an optional pre-rename
|
||||
check, replaces the destination with a rename, then syncs the containing
|
||||
directory.
|
||||
caller-supplied mode, closes the file, runs an optional pre-rename check, and
|
||||
replaces the destination with a rename.
|
||||
|
||||
The pre-rename check is the last point at which a caller can cancel without
|
||||
installing a new destination. A failure before the rename leaves the old
|
||||
destination unchanged and removes the temporary file; any cleanup failure is
|
||||
returned alongside the primary failure. A failure after the rename may leave
|
||||
the new file visible, but it is not reported as crash-durable.
|
||||
destination unchanged and removes the temporary file. Successful replacement
|
||||
provides atomic visibility; it makes no crash- or power-loss durability claim.
|
||||
|
||||
Replacement follows the operating system's same-filesystem rename semantics.
|
||||
If a platform cannot replace an existing destination, the operation returns an
|
||||
error and never removes the old file as an emulation step.
|
||||
|
||||
## Directory-Sync Support
|
||||
|
||||
Linux and macOS attempt to sync the destination directory. Windows opens the
|
||||
directory with backup semantics and flushes its buffers. If either operation
|
||||
is unavailable for the platform, directory handle, or filesystem,
|
||||
`ErrDirectorySyncUnsupported` is returned. Narratio does not treat that result
|
||||
as successful crash-durable replacement.
|
||||
|
||||
`WriteFileAtomic`, copy helpers, and downloaded temporary-file installation
|
||||
retain their compatibility behavior of creating the destination parent with
|
||||
the repository's workspace permissions before using this contract.
|
||||
`WriteFileAtomic` and the copy helpers create the destination parent with the
|
||||
repository's workspace permissions. `InstallSiblingTempFile` requires the
|
||||
caller to provide an existing temporary file in the existing destination
|
||||
directory.
|
||||
|
||||
@@ -2,29 +2,38 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
Explain the session-progress and invocation-audit models implemented by
|
||||
`internal/manifest`. Physical manifest placement belongs in
|
||||
Explain the session-progress model implemented by `internal/manifest`. Physical
|
||||
manifest placement belongs in
|
||||
[Operations](../operations.md#local-state-layout).
|
||||
|
||||
## Session Manifest
|
||||
|
||||
`manifest.Manifest` records:
|
||||
|
||||
- required `schema_version: narratio.session-manifest.v1`
|
||||
- identity (`session_id`, `campaign`, `run_id`)
|
||||
- local path metadata (`local_workdir`, `local_spool_dir`)
|
||||
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
|
||||
- `inputs` records
|
||||
- durable `artifacts` records
|
||||
- per-stage `stages` map
|
||||
- an optional `post_publish_cleanup` obligation, which binds a committed run,
|
||||
remote commit identity, and each exact root-confined local target to its
|
||||
completion evidence
|
||||
- per-stage `stages` map, including the ordinary cleanup stage record
|
||||
|
||||
Session, campaign, and run identities in local and downloaded manifests must be
|
||||
portable opaque segments. Unsafe legacy identities are rejected with migration
|
||||
guidance rather than being normalized into a different workspace or remote
|
||||
namespace.
|
||||
|
||||
Decoding is strict at every typed manifest field. Unknown fields are rejected;
|
||||
the schema version remains `narratio.session-manifest.v1` and there is no
|
||||
compatibility reader that silently drops retired state. Cleanup progress uses
|
||||
only the cleanup entry in `stages`.
|
||||
|
||||
Every persisted session manifest has a non-nil `stages` map. Each entry is
|
||||
non-nil and its embedded `name` exactly matches its map key; loading never
|
||||
invents either marker. A missing or unsupported schema version, or incomplete
|
||||
stage identity, is unsupported local state: move it aside and regenerate it,
|
||||
or restore a verified current committed snapshot when that is authoritative.
|
||||
|
||||
Prepare records independent `party` and `players` input checksums. In canonical
|
||||
party mode, the party record retains its campaign source identity while the
|
||||
players record uses `derived_from_party`; raw roster content is never embedded
|
||||
@@ -43,11 +52,11 @@ The model admits these stage states:
|
||||
|
||||
### Analyze-owned artifact state
|
||||
|
||||
The `analyze` stage record may carry `analyze_state_version: 1` and an
|
||||
Every `analyze` stage record carries `analyze_state_version: 1` and an
|
||||
`analyze_artifacts` map keyed by normalized configured artifact key. The
|
||||
version is the authority marker: version 1 with no entries is a valid evaluated
|
||||
empty set, while an absent version is legacy aggregate-only state and provides
|
||||
no current configured-artifact evidence.
|
||||
empty set. An absent or unsupported version is unsupported local state and is
|
||||
rejected rather than being treated as aggregate-only authority.
|
||||
|
||||
Each analyze artifact record has one disposition:
|
||||
|
||||
@@ -72,18 +81,15 @@ do not replace the concrete configured key or infer family membership from a
|
||||
name, so older records without them remain valid.
|
||||
|
||||
The session-stage collection is the reconciled authority across invocations.
|
||||
The corresponding collection on an invocation's `analyze` stage record is an
|
||||
audit of only the artifacts evaluated or attempted by that run. These records
|
||||
remain analyze-owned data inside the fixed stage; they are not dynamic stages
|
||||
or generic subtasks.
|
||||
These records remain analyze-owned data inside the fixed stage; they are not
|
||||
dynamic stages or generic subtasks.
|
||||
|
||||
The stage result contract has one analyze-specific projection boundary. On
|
||||
success, the runner validates and deep-copies the complete reconciled session
|
||||
collection and the invocation subset. Aggregate session outputs are rebuilt in
|
||||
configured-key order from current session records only; invocation outputs are
|
||||
limited to current records produced by that invocation's run ID. Ordinary
|
||||
stage outputs cannot accompany this projection, so there is one source of
|
||||
artifact authority.
|
||||
Analyze checkpoints one complete validated candidate collection after each
|
||||
successful artifact and before exposing that artifact to later work. It saves a
|
||||
detached manifest candidate and installs its version, collection, and updated
|
||||
timestamp into live state only after the store reports success. Aggregate stage
|
||||
outputs are ordinary result outputs rebuilt in configured-key order from current
|
||||
session records only.
|
||||
|
||||
Successful incremental execution replaces only evaluated artifact records and
|
||||
preserves valid unrelated current records. Rebuilt outputs are compared by
|
||||
@@ -92,43 +98,31 @@ the same recomputed fingerprint to remain current, while a changed identity
|
||||
removes output authority from every unselected transitive dependent by marking
|
||||
it stale. A partial analyze invocation can therefore succeed while unrelated
|
||||
configured records remain stale. Existing canonical files never create current
|
||||
records without validated execution and projection.
|
||||
records without validated execution and a successful checkpoint.
|
||||
|
||||
Aggregate analyze status is deliberately coarser than this collection. Resume
|
||||
validation may skip a succeeded aggregate record when the selected artifact
|
||||
closure is current even if unrelated records are stale. Conversely, a stale
|
||||
aggregate record may cross the ordinary runner boundary and perform zero
|
||||
Scriptorium calls when reconciliation proves every selected artifact current;
|
||||
the successful projection then restores the aggregate status.
|
||||
ordinary successful stage completion then restores the aggregate status.
|
||||
|
||||
Analyze may return a projection together with an error. That restricted result
|
||||
cannot carry ordinary outputs, skip state, aggregate logs, generated configs,
|
||||
or metadata. The runner persists only the validated per-artifact collections,
|
||||
then marks the aggregate analyze and run state failed and invalidates delivery
|
||||
dependents conservatively. Unrelated current records survive because the
|
||||
session projection is complete. A malformed projection is not applied, and a
|
||||
failed session projection save restores the prior per-artifact authority before
|
||||
terminal failure persistence.
|
||||
On an artifact error, analyze installs a complete retry collection on the live
|
||||
manifest before returning. The active record is failed without output, current
|
||||
transitive dependents are stale, unrelated current records survive, and only
|
||||
earlier checkpointed completions remain current. The runner's ordinary failure
|
||||
path marks aggregate analyze failed, clears aggregate outputs, invalidates
|
||||
delivery dependents, and saves that collection. A terminal session-save failure
|
||||
is joined with the original adapter, validation, filesystem, cancellation, or
|
||||
checkpoint cause.
|
||||
|
||||
The incremental executor constructs this restricted projection at each
|
||||
scheduled artifact boundary. The active record is failed without output,
|
||||
current transitive dependents are stale, unrelated current records survive, and
|
||||
only earlier validated and materialized completions remain current in the
|
||||
invocation subset. Session failure state is persisted before invocation failure
|
||||
state. If either terminal save fails, its persistence error is joined with the
|
||||
original adapter, validation, or filesystem cause; a failed projection save
|
||||
does not turn incidental canonical bytes into manifest authority.
|
||||
|
||||
## Run Manifest
|
||||
|
||||
`manifest.RunManifest` is created for each invocation and records:
|
||||
|
||||
- invocation identity and `force` flag
|
||||
- the selected profile (when any) and secret-free effective configuration digest
|
||||
- requested stages
|
||||
- per-stage action (`run` or `skip`)
|
||||
- per-stage status
|
||||
- overall run status (`running`, `succeeded`, `failed`)
|
||||
If an artifact checkpoint fails, analyze rolls its working collection back to
|
||||
the last committed collection before constructing the active failed record.
|
||||
Newly materialized canonical bytes therefore remain incidental and cannot
|
||||
become current authority. A later ordinary run reconciles the durable records,
|
||||
reuses earlier current artifacts, and resumes with the failed or stale work.
|
||||
The same recovery applies after process interruption: the aggregate record may
|
||||
be non-successful, but every completed artifact checkpoint remains reusable.
|
||||
|
||||
## Remote Commit Manifest
|
||||
|
||||
@@ -145,38 +139,51 @@ binds that object by checksum, size, and generation. Readers strictly reject
|
||||
unknown fields, version mismatches, pointer/commit identity mismatches, and
|
||||
objects that do not match their declaration.
|
||||
|
||||
The reader retains a temporary, clearly isolated compatibility path for a
|
||||
coherent legacy `current/manifest.json` plus `current/run_id.txt` pair. That
|
||||
path is removable after migration and is never used to write new state.
|
||||
An absent commit pointer means no remote state is published. Republish to
|
||||
establish a current immutable snapshot.
|
||||
|
||||
## Persistence Semantics
|
||||
|
||||
`manifest.LocalStore`:
|
||||
|
||||
- validates loaded documents;
|
||||
- normalizes missing maps/stage records;
|
||||
- writes through a sibling temporary file, syncing the completed file and
|
||||
destination directory after atomic replacement;
|
||||
- accepts only the exact supported session-manifest schema version;
|
||||
- rejects unknown fields and multiple JSON values;
|
||||
- validates stage-map identity without synthesizing missing maps, records, or
|
||||
names;
|
||||
- retains only representation-preserving collection normalization;
|
||||
- writes through a sibling temporary file and atomic replacement visibility;
|
||||
- updates `updated_at` on save.
|
||||
|
||||
If the operating system or filesystem cannot sync a directory, save returns an
|
||||
explicit error instead of claiming crash-durable replacement. A returned error
|
||||
after the rename can therefore leave the new manifest visible but not confirmed
|
||||
durable; callers must reload it before retrying.
|
||||
Manifest replacement provides atomic visibility, not a crash- or power-loss
|
||||
durability guarantee. Callers reload after a returned save error before retrying.
|
||||
An analyze checkpoint error leaves the prior manifest as per-artifact authority.
|
||||
The runner then attempts to persist the bounded failed/stale retry collection;
|
||||
if that terminal save also fails, its error is joined with the originating
|
||||
failure and the next invocation must reconcile from the last readable manifest.
|
||||
|
||||
## Execution Semantics
|
||||
|
||||
The application runner marks an executing stage running and then succeeded or
|
||||
failed in both manifests, persisting each transition. On success it records
|
||||
outputs, logs, generated configuration references, metadata, and—when the
|
||||
stage implements the optional contract—a versioned semantic-configuration
|
||||
fingerprint. Artifact
|
||||
Planning and execution share one pure pre-execution decision over the recorded
|
||||
stage status, force selection, and normalized resume-validation result. That
|
||||
decision identifies run versus reuse and any required stale and dependent
|
||||
invalidation transition. Planning applies it only to a cloned manifest.
|
||||
Execution remains the sole owner of observed stage results and persistence.
|
||||
|
||||
For each selected stage, the application runner computes semantic evidence,
|
||||
performs any eligible resume validation, applies the pre-execution decision,
|
||||
and persists required invalidation before either reusing the existing success
|
||||
or beginning an attempt. The same one-stage execution owner marks the attempt
|
||||
running before invoking the stage and then records and persists its failed,
|
||||
skipped, or successful terminal state before a later stage may start. On
|
||||
success it records outputs, logs, generated configuration references, metadata,
|
||||
and—when the stage implements the optional contract—a versioned
|
||||
semantic-configuration fingerprint. Artifact
|
||||
records may include optional contract and external provenance objects; old
|
||||
manifests remain compatible when those fields are absent. A successful forced
|
||||
rerun marks only succeeded transitive dependent session-stage records stale.
|
||||
The application owns a fixed dependency relation distinct from execution order;
|
||||
dependents are returned in canonical order. Render and extract therefore never
|
||||
stale one another, while either can stale analyze, publish, and notify.
|
||||
stale one another, while either can stale analyze, publish, notify, and cleanup.
|
||||
|
||||
Starting an execution clears the current session-stage record's prior outputs,
|
||||
logs, generated configuration references, metadata, and semantic fingerprint.
|
||||
@@ -184,8 +191,7 @@ Failed and skipped
|
||||
transitions enforce the same clearing rule directly, while success repopulates
|
||||
only fields returned by the new result. Marking a record stale does not clear
|
||||
those details because resume validation and diagnosis may still require them
|
||||
before execution begins. Invocation run manifests remain immutable audit
|
||||
records of their own outcomes.
|
||||
before execution begins.
|
||||
|
||||
Aggregate lifecycle clearing deliberately preserves the analyze-owned
|
||||
per-artifact collection. This lets later reconciliation replace only evaluated
|
||||
@@ -194,30 +200,27 @@ existing aggregate-only lifecycle behavior and are forbidden from carrying the
|
||||
analyze-specific fields.
|
||||
|
||||
A stage may explicitly return a skipped disposition and stable reason. The
|
||||
runner persists that outcome in both manifests, clears older outputs for the
|
||||
session-stage record along with older logs, generated configuration references,
|
||||
and metadata, then applies any bounded details from the current skip and
|
||||
continues. This self-skip is distinct from deciding not to execute an
|
||||
runner persists that outcome in the session manifest, clears older outputs,
|
||||
logs, generated configuration references, and metadata, then applies any
|
||||
bounded details from the current skip and continues. This self-skip is distinct from deciding not to execute an
|
||||
already-succeeded stage and is reconsidered on later runs. Skipped results
|
||||
cannot contain outputs. An intentional self-skip records the current semantic
|
||||
fingerprint because it is a completed, reusable stage result; failed or
|
||||
interrupted work never promotes one.
|
||||
|
||||
When an already-succeeded stage is skipped, the invocation run manifest records
|
||||
the `skip` action and reason. The session manifest deliberately retains its
|
||||
existing succeeded record because it remains the cross-invocation progress
|
||||
authority. If a stage supplies semantic configuration evidence, reuse first
|
||||
When an already-succeeded stage is skipped, the session manifest deliberately
|
||||
retains its existing succeeded record because it remains the cross-invocation
|
||||
progress authority. If a stage supplies semantic configuration evidence, reuse first
|
||||
requires the persisted positive schema version and lowercase SHA-256 digest to
|
||||
match the current resolved stage semantics. Missing legacy evidence, malformed
|
||||
match the current resolved stage semantics. Missing semantic evidence, malformed
|
||||
evidence, or a mismatch makes the stage and its fixed transitive dependents
|
||||
stale. The invocation skip copies the matched fingerprint for provenance but
|
||||
does not rewrite session authority. The existing stage-specific resume
|
||||
stale. The existing stage-specific resume
|
||||
validator runs only after this semantic check succeeds; both checks are
|
||||
required. Extraction and analyze have resume validators and may reject an
|
||||
otherwise eligible skip when their selected durable evidence is obsolete; the
|
||||
runner marks the aggregate record stale and executes it. Analyze's validator
|
||||
can still accept a partial selection when only unrelated artifact records are
|
||||
stale.
|
||||
required. Extraction, analyze, and cleanup have resume validators and may
|
||||
reject an otherwise eligible skip when their selected durable evidence is
|
||||
obsolete; the runner marks the aggregate record stale and executes it.
|
||||
Analyze's validator can still accept a partial selection when only unrelated
|
||||
artifact records are stale.
|
||||
|
||||
Implemented reuse coverage is deliberately split between aggregate semantic
|
||||
evidence and focused durable validators:
|
||||
@@ -235,6 +238,7 @@ evidence and focused durable validators:
|
||||
| analyze artifacts | per-artifact fingerprint, reconciliation, and output validator | [analyze](stage-analyze.md), [Scriptorium](../integrations/scriptorium.md) |
|
||||
| publish | aggregate semantic fingerprint plus immediate lock/commit checks | [publish](stage-publish.md), [storage adapter](adapters.md) |
|
||||
| notify | aggregate delivery-mode fingerprint | [pipeline overview](overview.md), [configuration](../config.md#notifications) |
|
||||
| cleanup | aggregate semantic fingerprint plus publish-evidence and target-absence validator | [cleanup](stage-cleanup.md), [workspace](workspace.md) |
|
||||
|
||||
These contracts record resolved choices Narratio can observe, not operational
|
||||
runner tuning. External model, module, prompt, profile, and configuration-file
|
||||
@@ -242,10 +246,8 @@ contents that a tool privately loads remain outside the contract when their
|
||||
configured identifier is unchanged; operators must force the affected work
|
||||
after such a private content change.
|
||||
|
||||
Session manifest is the authoritative stage-progress ledger across invocations.
|
||||
Run manifest is invocation-scoped audit state.
|
||||
|
||||
Both manifests retain the most recently resolved invocation's bounded
|
||||
The session manifest is the authoritative stage-progress ledger across
|
||||
invocations. It retains the most recently resolved invocation's bounded
|
||||
configuration provenance. It identifies the selected profile name and source
|
||||
(`default` or `cli`) plus the effective configuration digest, but never a raw
|
||||
secret or profile content. This provenance is informational: it does not
|
||||
@@ -254,9 +256,9 @@ invalidates only stages whose semantic configuration changed. When a private
|
||||
external-tool model, module, prompt, or profile changes behind an unchanged
|
||||
configured identifier, use `--force` for the affected work.
|
||||
|
||||
`session plan` computes the same current fingerprint and applies the same
|
||||
comparison and invalidation rules to a cloned manifest. It predicts the runner
|
||||
decision without persisting session or invocation state. The shared helper
|
||||
`session plan` computes the same current fingerprint and applies the shared
|
||||
decision and invalidation rules to a cloned manifest. It predicts the runner
|
||||
decision without persisting session state. The semantic fingerprint owner
|
||||
hashes deterministic JSON from stage-owned typed structs; stage providers must
|
||||
exclude secrets, complete effective-configuration dumps, and operational
|
||||
values that cannot affect canonical results. Concrete coverage is owned by the
|
||||
@@ -266,7 +268,7 @@ Before an explicitly bounded execution starts after `prepare`, the application
|
||||
reads the session manifest and accepts only `succeeded` or `skipped` for every
|
||||
excluded canonical prefix stage. The first other status or absent record fails
|
||||
the request before layout mutation, adapter initialization, session-manifest
|
||||
writes, or run-manifest creation. Excluded prefix records are not passed to
|
||||
writes, or run-directory creation. Excluded prefix records are not passed to
|
||||
resume validators. Records after the selected end are not prerequisites and
|
||||
may be made stale by selected work without being scheduled.
|
||||
|
||||
@@ -283,33 +285,30 @@ session manifest must agree on campaign and session identity before execution;
|
||||
the current projection is refreshed for every invocation while stage progress,
|
||||
inputs, and durable artifacts remain session history.
|
||||
|
||||
For handled failures after an invocation record is created, the runner records
|
||||
the failure on the session ledger and persists it before persisting the failed
|
||||
run audit record. This preserves the resume authority while making a partial
|
||||
persistence disagreement visible. Abrupt process death remains an accepted case
|
||||
where a durable running record can require operator interpretation.
|
||||
For handled failures after the session identity transition, the runner records
|
||||
the failure on the session ledger and atomically attempts its save while
|
||||
preserving the originating error. Abrupt process death remains an accepted case
|
||||
where the last session transition can require operator interpretation.
|
||||
|
||||
## Invariants
|
||||
|
||||
- stage resume/skip decisions are session-manifest driven.
|
||||
- semantic fingerprint comparison precedes stage-specific resume validation.
|
||||
- only successful and intentional-skipped results promote current semantic
|
||||
evidence; invocation reuse copies evidence without replacing session state.
|
||||
evidence.
|
||||
- running, failed, and self-skipped stages do not retain result payloads from
|
||||
an earlier success.
|
||||
- stale stages retain prior details until replacement execution starts.
|
||||
- force reruns stale succeeded stages in the fixed dependency relation.
|
||||
- run manifest does not replace session manifest as progress authority.
|
||||
- the session manifest is the only durable local progress ledger.
|
||||
- remote commitment is established by a verified current pointer and remote
|
||||
commit relationship, never by a mutable session-manifest boolean.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Models and transitions: `internal/manifest/manifest.go`,
|
||||
`internal/manifest/run_manifest.go`
|
||||
- Models and transitions: `internal/manifest/manifest.go`
|
||||
- Remote commit model and readers: `internal/artifacts/remote_commit.go`,
|
||||
`internal/artifacts/current_state_commit.go`,
|
||||
`internal/artifacts/current_state_legacy.go`
|
||||
- Persistence and validation: `internal/manifest/store.go`
|
||||
- Package tests: `internal/manifest/*_test.go`
|
||||
- Assembled execution behavior: `internal/app/runner_test.go`,
|
||||
|
||||
@@ -22,20 +22,29 @@ session safety controls, and runs commands. Pipeline commands execute the
|
||||
canonical stage sequence through adapter interfaces, while manifests record
|
||||
progress and artifact services resolve durable inputs and outputs.
|
||||
|
||||
The pipeline runner follows one explicit invocation lifecycle: inspect bounded
|
||||
prerequisites without mutation, prepare the workspace layout, acquire one
|
||||
session lock, load or create the authoritative session manifest, compose only
|
||||
the collaborators needed by the selected range, execute each selected stage in
|
||||
order, and then return the summary. Cleanup, when selected, is the ordinary
|
||||
final stage rather than runner finalization work.
|
||||
The lock remains held from manifest initialization through finalization, and a
|
||||
failed stage or required manifest save prevents later stages from starting.
|
||||
|
||||
## Components
|
||||
|
||||
| Area | Implemented owners | Responsibility |
|
||||
| --- | --- | --- |
|
||||
| Executable | `cmd/narratio` | Process entry, standard stream wiring, argument handoff, and exit status. |
|
||||
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, cleanup gates, and user-facing reporting. |
|
||||
| Application orchestration | `internal/app` | Command dispatch, configuration selection, secret-file environment loading, production composition, session locking, planning, execution, restore, and user-facing reporting. |
|
||||
| Configuration | [`internal/config`](configuration.md) | Presence-aware root/import/profile composition, canonical party and family expansion, strict YAML loading, defaults, normalization, session templating, and validation. |
|
||||
| Pipeline stages | `internal/stage` | Canonical stage registry, shared stage contract, execution dependencies, and implemented stage behavior. |
|
||||
| External boundaries | `internal/adapters`, `internal/audio` | WhisperX HTTP, downstream subprocesses, notification, object storage, and S3 audio materialization behind Narratio contracts. |
|
||||
| Manifests | `internal/manifest` | Durable session progress, invocation audit state, stage transitions, validation, and atomic persistence. |
|
||||
| Manifests | `internal/manifest` | Durable session progress, stage transitions, validation, and atomic persistence. |
|
||||
| Artifacts and paths | `internal/artifacts`, `internal/pathsafe` | Artifact identities and resolution, local and remote path/key models, current-state discovery, and confined relative destinations. |
|
||||
| Previous-session cache | `internal/previouscache` | Deterministic planning and materialization requirements for configured previous-session inputs. |
|
||||
| Artifact policy | `internal/artifactpolicy` | Source and destination policy, configured artifact identity validation, and publish destination safety. |
|
||||
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, [`internal/fileops`](fileops.md) | Transcript and artifact data contracts plus durable single-file replacement helpers; unsupported directory syncing is reported explicitly. |
|
||||
| Shared models and file operations | `internal/artifactmodel`, `internal/contracts`, [`internal/fileops`](fileops.md) | Transcript and artifact data contracts plus atomic single-file replacement, bounded reads, workspace modes, and root-scoped cleanup helpers. |
|
||||
| Logging | `internal/logging` | Application logger construction and shared structured logging behavior. |
|
||||
|
||||
The application boundary composes concrete implementations. Stages depend on
|
||||
@@ -43,10 +52,19 @@ Narratio-level contracts; external transport and SDK details remain in
|
||||
adapters. The normative rules for these relationships remain in
|
||||
[Architecture](../policy/architecture.md).
|
||||
|
||||
Command execution normalizes one private dependency value per invocation.
|
||||
Configuration reads, secret-directory reads, remote downloads, object-store
|
||||
construction, and restore phase functions use that value throughout the
|
||||
command. Public command entry points supply production defaults, while focused
|
||||
tests replace only the external or sequencing boundary they exercise without
|
||||
mutating package state.
|
||||
|
||||
Pipeline execution and `session plan` share the same inclusive contiguous-range
|
||||
model. Planning clones session state and applies selected-stage transitions and
|
||||
resume validation in memory; it does not create invocation state or initialize
|
||||
stage-execution adapters. Command configuration loading can still retrieve a
|
||||
model and the same pure pre-execution decision over recorded status, force, and
|
||||
normalized resume validation. Planning applies that decision and modeled
|
||||
selected-stage transitions to cloned session state; it does not persist, create
|
||||
invocation state, initialize stage-execution adapters, or claim to observe an
|
||||
execution result. Command configuration loading can still retrieve a
|
||||
missing session file through configured remote storage. It retains the initially
|
||||
composed pipeline and selected campaign while resolving either a local or
|
||||
downloaded remote session, so one invocation cannot mix pipeline revisions.
|
||||
@@ -68,6 +86,7 @@ The implemented canonical order is:
|
||||
9. [`analyze`](stage-analyze.md)
|
||||
10. [`publish`](stage-publish.md)
|
||||
11. `notify` (no-op)
|
||||
12. [`cleanup`](stage-cleanup.md)
|
||||
|
||||
`notify` currently has no persisted pipeline outputs and uses the explicit
|
||||
`noop` notification mode. Its versioned semantic evidence records that delivery
|
||||
@@ -81,7 +100,8 @@ registry owns the flat execution sequence. The application orchestration owner
|
||||
uses a fixed, validated dependency relation to find transitive dependents in
|
||||
canonical order. In particular, `render` and `extract` are sibling consumers of
|
||||
trimmed transcript state: neither invalidates the other, while either can stale
|
||||
`analyze`, `publish`, and `notify`.
|
||||
`analyze`, `publish`, `notify`, and `cleanup`. Cleanup is a dependent of notify,
|
||||
so any earlier replacement can stale a prior cleanup result.
|
||||
|
||||
## Focused Documentation
|
||||
|
||||
@@ -91,8 +111,8 @@ trimmed transcript state: neither invalidates the other, while either can stale
|
||||
failure behavior, and test surfaces.
|
||||
- [Artifact Internals](artifacts.md): source identities, runtime catalog,
|
||||
resolution, previous requirements, and current-state helpers.
|
||||
- [Manifest Internals](manifest.md): session and run records, persistence, and
|
||||
execution transitions.
|
||||
- [Manifest Internals](manifest.md): session records, persistence, and execution
|
||||
transitions.
|
||||
- [Storage Internals](storage.md): object-store interface and S3 behavior.
|
||||
- [Workspace Internals](workspace.md): local layout, locking, and cleanup
|
||||
guardrails.
|
||||
@@ -108,6 +128,7 @@ trimmed transcript state: neither invalidates the other, while either can stale
|
||||
- [`extract`](stage-extract.md)
|
||||
- [`analyze`](stage-analyze.md)
|
||||
- [`publish`](stage-publish.md)
|
||||
- [`cleanup`](stage-cleanup.md)
|
||||
|
||||
Use this map to find an owner, then read its focused documentation and tests
|
||||
before changing behavior.
|
||||
|
||||
@@ -31,12 +31,14 @@ Supported source families:
|
||||
## Key Behavior
|
||||
|
||||
- when `pipeline.scriptorium` is absent or no configured artifact is
|
||||
executable, completes successfully with no outputs and records explanatory
|
||||
metadata. This is not an explicit self-skip: both manifests record success,
|
||||
satisfy publish's prerequisite, and an ordinary later run reuses the result
|
||||
while the effective set remains empty. Enabling or selecting an artifact
|
||||
later makes missing versioned evidence non-resumable and schedules it without
|
||||
requiring force.
|
||||
executable, completes successfully and records explanatory metadata. Before
|
||||
returning, it reconciles and checkpoints the complete session collection:
|
||||
records removed from configuration are dropped, while valid records for
|
||||
configured but unselected artifacts remain authoritative. This is not an
|
||||
explicit self-skip: the session manifest records success, satisfies publish's
|
||||
prerequisite, and an ordinary later run reuses the result while the effective
|
||||
set remains empty. Enabling or selecting an artifact later makes missing
|
||||
versioned evidence non-resumable and schedules it without requiring force.
|
||||
- builds a runtime artifact catalog containing built-ins, configured artifacts,
|
||||
and configured extraction lanes. Extraction availability is hydrated only
|
||||
from compatible successful extraction evidence.
|
||||
@@ -75,7 +77,7 @@ Supported source families:
|
||||
validated current dependency-output identities, and sorted effective
|
||||
Scriptorium variables (including Narratio's sticky session variable).
|
||||
- provides read-only reconciliation that classifies each configured record as
|
||||
current, stale, missing, failed, legacy, or otherwise non-resumable, and
|
||||
current, stale, missing, failed, or otherwise non-resumable, and
|
||||
separately identifies manifest records removed from current configuration.
|
||||
A record is current only when its fingerprint version and value match and its
|
||||
configured output still passes manifest-authoritative evidence validation.
|
||||
@@ -89,13 +91,14 @@ Supported source families:
|
||||
rebuilt when required, while unrelated disabled artifacts are excluded.
|
||||
- the work plan carries explicit targets, prerequisite-only work, deterministic
|
||||
execution and reuse lists, invalidated and removed records, and a cloned
|
||||
projected record collection. Valid unrelated configured records survive the
|
||||
projection, removed records are omitted, and legacy files never become
|
||||
current without regeneration.
|
||||
reconciled starting collection. Valid unrelated configured records survive
|
||||
reconciliation, removed records are omitted, and incidental files never
|
||||
become current without manifest-backed regeneration.
|
||||
- implements aggregate resume validation by running the same read-only catalog,
|
||||
fingerprint reconciliation, and work planner used by execution. A succeeded
|
||||
aggregate record is reusable exactly when the selected closure schedules no
|
||||
artifact work; stale unrelated records do not block a partial selection.
|
||||
artifact work and no removed configuration records require a checkpoint;
|
||||
stale unrelated records do not block a partial selection.
|
||||
- exposes the typed artifact decision to `session plan`. Planning applies it to
|
||||
a cloned manifest after modeling earlier selected stage transitions, so
|
||||
aggregate run/skip and artifact execute/reuse decisions match the ordinary
|
||||
@@ -104,6 +107,10 @@ Supported source families:
|
||||
prerequisites remain available through the runtime catalog without invoking
|
||||
Scriptorium; newly produced prerequisites enter that catalog with the same
|
||||
contract, checksum, and size identity used for persisted current evidence.
|
||||
- before the first adapter call, checkpoints the reconciled starting collection
|
||||
when it removed configuration records or cleared obsolete authority. After
|
||||
each successful artifact, checkpoints the complete collection before making
|
||||
that artifact available to the next scheduled dependent.
|
||||
- keeps adapter output in the invocation's run-local analyze directory until
|
||||
it is a safe, non-empty, bounded regular file with a calculated checksum and
|
||||
complete output contract. Canonical replacement uses the shared atomic file
|
||||
@@ -112,16 +119,15 @@ Supported source families:
|
||||
- records each successful artifact's freshly computed fingerprint, canonical
|
||||
relative output path, contract, checksum, size, producer run ID, bounded
|
||||
Scriptorium provenance, logs, and generated configuration references in the
|
||||
analyze-owned projection.
|
||||
analyze-owned manifest collection.
|
||||
- preserves valid unrelated current records during partial execution. If a
|
||||
rebuilt output's bytes and contract are unchanged, unselected dependents may
|
||||
remain current. If that semantic identity changes, unselected transitive
|
||||
dependents become stale without being executed; dependents included in the
|
||||
invocation are evaluated in dependency order instead.
|
||||
- reports all evaluated targets and prerequisites in invocation state. The
|
||||
runner reconstructs aggregate session outputs from every current session
|
||||
record and invocation outputs from only records produced by the current run.
|
||||
Unrelated stale records do not make an otherwise successful partial
|
||||
- returns aggregate outputs from every current session record in normalized-key
|
||||
order after the complete reconciled collection has been checkpointed;
|
||||
unrelated stale records do not make an otherwise successful partial
|
||||
invocation fail.
|
||||
- resolves previous-session sources from local `previous/` cache only.
|
||||
- runs optional render-debug, then artifact execution.
|
||||
@@ -137,25 +143,31 @@ Supported source families:
|
||||
guidance.
|
||||
- dependency cycles or unavailable required dependencies fail.
|
||||
- adapter validation failures fail stage.
|
||||
- a scheduled artifact failure returns the restricted analyze-state projection
|
||||
- a scheduled artifact failure installs a retry collection on the live manifest
|
||||
with the active artifact marked `failed`, a bounded error, and no output
|
||||
authority. Current transitive dependents become stale without execution.
|
||||
- earlier artifacts from the invocation remain current only after their
|
||||
run-local output passed validation and canonical materialization. They remain
|
||||
in invocation history; unattempted later artifacts do not appear there.
|
||||
- earlier artifacts remain current only after their run-local output passed
|
||||
validation, canonical materialization, and a successful complete-collection
|
||||
manifest checkpoint; unattempted later artifacts retain their reconciled
|
||||
session state.
|
||||
- unrelated current records survive a partial failure. Old canonical bytes for
|
||||
the failed artifact and newly materialized bytes whose projection cannot be
|
||||
persisted are incidental, not current evidence.
|
||||
- the runner persists a valid partial projection before it marks aggregate
|
||||
analyze failed and invalidates publish and notify through the application
|
||||
dependency relation. Projection-persistence errors retain the last durable
|
||||
per-artifact authority and are joined with the original failure context.
|
||||
the failed artifact and newly materialized bytes whose checkpoint fails are
|
||||
incidental, not current evidence.
|
||||
- a checkpoint failure stops the queue immediately. The in-memory retry
|
||||
collection starts from the last successful checkpoint, marks the active
|
||||
artifact failed without output authority, and stales its current transitive
|
||||
dependents.
|
||||
- the runner's ordinary error path marks aggregate analyze failed, clears its
|
||||
aggregate outputs, invalidates publish, notify, and cleanup through the
|
||||
application dependency relation, and persists the live retry collection.
|
||||
Terminal-save errors are joined with the original artifact or checkpoint
|
||||
failure.
|
||||
|
||||
## Invariants
|
||||
|
||||
- `analyze` performs no remote storage calls for previous-session source resolution.
|
||||
- input-identity resolution is read-only: it does not invoke adapters,
|
||||
materialize outputs, update status, or create run records.
|
||||
materialize outputs, update status, or create run-local files.
|
||||
- fingerprints exclude timeouts, retries, timestamps, producer and Narratio run
|
||||
IDs, executable and config paths, workspace roots, diagnostic locations, and
|
||||
executable or private transitive configuration contents. A change that is
|
||||
@@ -164,6 +176,9 @@ Supported source families:
|
||||
- output provenance and metadata are deterministic per execution.
|
||||
- a canonical file without current per-artifact manifest evidence is never
|
||||
promoted to current state.
|
||||
- an ordinary retry after interruption or partial failure reuses every valid
|
||||
earlier checkpoint and schedules only failed, stale, missing, or otherwise
|
||||
non-current work in dependency order.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
@@ -173,13 +188,18 @@ Supported source families:
|
||||
- [CLI](../cli.md) owns user-visible artifact selection.
|
||||
- [Scriptorium](../integrations/scriptorium.md) owns the subprocess contract.
|
||||
- Implementation and tests: `internal/stage/analyze.go`,
|
||||
`internal/stage/analyze_input_identity.go`, `internal/stage/analyze_test.go`,
|
||||
`internal/stage/analyze_input_identity_test.go`,
|
||||
`internal/stage/analyze_artifact_execution.go`,
|
||||
`internal/stage/analyze_execution_state.go`,
|
||||
`internal/stage/analyze_input_identity.go`,
|
||||
`internal/stage/analyze_fingerprint.go`,
|
||||
`internal/stage/analyze_reconciliation.go`, `internal/stage/analyze_plan.go`,
|
||||
`internal/stage/analyze_resume.go`, `internal/stage/analyze_test.go`,
|
||||
`internal/stage/analyze_checkpoint_test.go`,
|
||||
`internal/stage/analyze_input_identity_test.go`,
|
||||
`internal/stage/analyze_fingerprint_test.go`,
|
||||
`internal/stage/analyze_reconciliation.go`, and
|
||||
`internal/stage/analyze_reconciliation_test.go`,
|
||||
`internal/stage/analyze_plan.go`, `internal/stage/analyze_plan_test.go`, and
|
||||
`internal/stage/analyze_incremental_execution_test.go`, and
|
||||
`internal/stage/analyze_plan_test.go`,
|
||||
`internal/stage/analyze_incremental_execution_test.go`,
|
||||
`internal/stage/analyze_failure_test.go`,
|
||||
`internal/stage/analyze_resume.go`, and `internal/stage/analyze_resume_test.go`
|
||||
`internal/stage/analyze_resume_test.go`, and
|
||||
`internal/app/analyze_checkpoint_recovery_test.go`.
|
||||
|
||||
54
docs/internal/stage-cleanup.md
Normal file
54
docs/internal/stage-cleanup.md
Normal file
@@ -0,0 +1,54 @@
|
||||
# Stage: cleanup
|
||||
|
||||
## Purpose
|
||||
|
||||
Remove explicitly configured local data for the run selected by a successful
|
||||
publish commit. Cleanup is the twelfth and final canonical stage, after
|
||||
`notify`.
|
||||
|
||||
## Authorization And Targets
|
||||
|
||||
Cleanup is disabled unless at least one of these policies is true:
|
||||
|
||||
- `pipeline.spool.delete_audio_after_publish`
|
||||
- `pipeline.workspace.cleanup_after_publish`
|
||||
|
||||
An enabled cleanup requires the manifest's publish stage to be succeeded with
|
||||
`uploaded=true`, a canonical `published_run_id`, and the exact remote commit and
|
||||
current commit-pointer keys for the manifest campaign/session prefix. The
|
||||
configured campaign and session must match the manifest. Cleanup derives its
|
||||
only possible targets through the canonical artifact path helpers:
|
||||
|
||||
- the published run's session spool-audio directory; and
|
||||
- the published run's workspace run directory.
|
||||
|
||||
It does not scan parent directories, infer the newest run, or read object
|
||||
storage. Every deletion uses the root-confined file operation, which rejects a
|
||||
root itself, traversal, symlink escape, and out-of-root targets.
|
||||
|
||||
## Execution And Resume
|
||||
|
||||
Disabled cleanup records an explicit skipped result. Enabled cleanup removes
|
||||
targets in deterministic policy order and treats an already-missing target as
|
||||
success. A successful record contains the published run ID and completed policy
|
||||
names.
|
||||
|
||||
A previous cleanup success is reusable only while its semantic configuration
|
||||
matches and every selected target remains absent. A recreated target makes the
|
||||
stage non-resumable. A failure is recorded against cleanup; publish remains
|
||||
succeeded and remote current state is unchanged. On a later normal full run,
|
||||
valid upstream stages are skipped and cleanup is retried without republishing.
|
||||
|
||||
`narratio publish` selects publish only and never invokes cleanup. A bounded run
|
||||
through `notify` or any earlier endpoint also excludes it. Operators may run
|
||||
`narratio run-stage cleanup <session_id>` directly; missing, failed, malformed,
|
||||
or identity-mismatched publication evidence fails before deletion.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Planning and evidence validation: `internal/stage/cleanup_plan.go`
|
||||
- Execution and resume: `internal/stage/cleanup.go`
|
||||
- Root-confined deletion: `internal/fileops/cleanup.go`
|
||||
- Focused tests: `internal/stage/cleanup_plan_test.go`,
|
||||
`internal/stage/cleanup_test.go`, and
|
||||
`internal/app/cleanup_orchestration_test.go`
|
||||
@@ -46,8 +46,8 @@ invocation-local snapshot path, never payload contents. Snapshot bytes must
|
||||
match the prepared identity both before and after Notarius runs, so a concurrent
|
||||
prepared-file replacement cannot make recorded provenance describe different
|
||||
bytes from those supplied to Notarius.
|
||||
Validation completes before
|
||||
promotion, so a rejected result cannot expose a partial durable bundle.
|
||||
Validation completes before the single durable rename, so a rejected result
|
||||
cannot expose a partial durable bundle.
|
||||
|
||||
Any executed extraction outcome that replaces a different effective outcome
|
||||
marks succeeded analysis and delivery dependents stale. Render is an independent
|
||||
@@ -92,17 +92,17 @@ identifier.
|
||||
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
|
||||
index compatibility, inconsistent warning or diagnostic envelopes,
|
||||
required-lane rejection or incomplete validation, payload inspection,
|
||||
checksum, or promotion errors fail the stage through ordinary manifest
|
||||
checksum, or installation errors fail the stage through ordinary manifest
|
||||
transition handling.
|
||||
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
|
||||
not given selectable extraction sources unless the complete configured result
|
||||
has passed validation and promotion.
|
||||
has passed validation and installation.
|
||||
|
||||
When a replacement attempt begins, the current session-stage record no longer
|
||||
advertises payload from the previous success. A failed replacement therefore
|
||||
has no current outputs, logs, generated configuration references, or metadata,
|
||||
while the earlier invocation manifest and immutable promoted bundle remain
|
||||
available for audit and recovery.
|
||||
while the earlier immutable promoted bundle remains available for audit and
|
||||
recovery.
|
||||
|
||||
## Implementation And Focused Tests
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ Normalize polished transcript into final transcript using Seriatim.
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- resolves polished transcript from manifest outputs/canonical fallback.
|
||||
- resolves polished transcript only from verified manifest output evidence.
|
||||
- applies `pipeline.normalize` config or default normalize config.
|
||||
- runs Seriatim normalize with configured timeout/binary.
|
||||
- validates normalized transcript and optional report.
|
||||
|
||||
@@ -16,7 +16,7 @@ Run Audita polishing on base transcript and produce polished transcript.
|
||||
|
||||
## Key Behavior
|
||||
|
||||
- resolves base transcript from merge outputs/canonical fallback.
|
||||
- resolves base transcript only from verified merge manifest output evidence.
|
||||
- invokes an Audita runner configured with static model/runtime options; the
|
||||
invocation supplies paths and modules.
|
||||
- validates processed transcript structure (`segments` array required).
|
||||
|
||||
@@ -39,9 +39,8 @@ Materialize canonical current-session inputs before processing stages.
|
||||
- materializes a configured spell catalog with checksum and provenance, or
|
||||
safely removes an obsolete canonical spell catalog and its manifest record
|
||||
when the effective input is omitted.
|
||||
- in canonical party mode, copies the validated raw party bytes unchanged and
|
||||
deterministically generates the prepared players projection; legacy mode
|
||||
continues to copy its opaque party and explicit players sources.
|
||||
- copies the validated canonical party bytes unchanged and deterministically
|
||||
generates the prepared players projection.
|
||||
- scans enabled configured artifact inputs for `narratio.previous_session.artifact.*` requirements.
|
||||
- clears managed `previous/` state on every invocation, then, when requirements exist:
|
||||
- resolves the pointer-selected previous source through the shared resolver;
|
||||
@@ -49,8 +48,10 @@ Materialize canonical current-session inputs before processing stages.
|
||||
- records previous inputs in `manifest.inputs`.
|
||||
|
||||
Required previous-session inputs fail when unavailable; optional missing inputs
|
||||
are typed skipped results. Committed sources use their exact source-to-destination
|
||||
mapping, while the isolated legacy reader rejects ambiguous fallback matches.
|
||||
are typed skipped results. Each requirement resolves only from its unique
|
||||
`published_output` declaration in the pointer-selected remote commit. Prepare
|
||||
strips that selected run's immutable `published/` prefix to derive the local
|
||||
cache path, then validates the committed checksum, size, and generation.
|
||||
|
||||
## Invariants
|
||||
|
||||
|
||||
@@ -7,15 +7,16 @@ Upload run/session outputs to object storage and atomically advance remote curre
|
||||
## Inputs
|
||||
|
||||
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
|
||||
- invocation-scoped run files
|
||||
- declared run-local diagnostic files
|
||||
- resolved publish output rules
|
||||
- effective publish locks (static + remote merged lock set), revalidated at the
|
||||
remote commit boundary
|
||||
- independent per-source remote lock objects read at planning start
|
||||
- the validated prior current commit when an initially locked source may carry
|
||||
- durable previous-session cache files when present
|
||||
|
||||
## Outputs
|
||||
|
||||
- uploaded invocation record and selected publish outputs;
|
||||
- uploaded session manifest, declared run-local diagnostics, and selected
|
||||
publish outputs;
|
||||
- uploaded durable previous-session cache files when present;
|
||||
- immutable run-scoped commit manifest; and
|
||||
- current commit pointer, written last.
|
||||
@@ -25,14 +26,27 @@ Exact remote placement and the operator workflow belong in
|
||||
|
||||
## Key Behavior
|
||||
|
||||
`publish.go` retains the transaction-owning `Run` path. Source discovery,
|
||||
manifest-derived archive metadata, confined opening, and checksum validation
|
||||
live in `publish_sources.go`; artifact-catalog matching, selection, required
|
||||
and optional rules, destination resolution, and collision inputs live in
|
||||
`publish_outputs.go`. Copying, hashing, immutable upload, reuse checks, and
|
||||
remote verification live in `publish_transfer.go`; committed-manifest,
|
||||
immutable-commit, and current-pointer encoding and writes live in
|
||||
`publish_commit.go`. `publish_output_plan.go` owns the deterministic, I/O-free
|
||||
five-outcome model and its secret-free metadata projection.
|
||||
|
||||
- when publishing or run upload is disabled, completes successfully with no
|
||||
outputs and records explanatory metadata. This is not an explicit self-skip:
|
||||
both manifests record success. Enablement and upload policy are fingerprinted,
|
||||
so changing either automatically makes the prior result non-resumable.
|
||||
the session manifest records success. Enablement and upload policy are
|
||||
fingerprinted, so changing either automatically makes the prior result
|
||||
non-resumable.
|
||||
- validates prerequisite stage success and object-store availability.
|
||||
- derives a deterministic run-archive allowlist from the validated run
|
||||
`manifest.json`: declared run-local outputs, logs, generated configs, and the
|
||||
manifest itself. Unlisted workspace files are not archive candidates.
|
||||
- derives a deterministic run-archive allowlist from the session manifest's
|
||||
declared current-run outputs, logs, and generated configurations. References
|
||||
outside the current run root, including canonical session files and prior-run
|
||||
diagnostics, are not archive candidates. No distinguished diagnostic file is
|
||||
read or uploaded.
|
||||
- opens each archive candidate beneath its archive root without following
|
||||
symlinked ancestors or leaf entries, verifies that it is a regular file and
|
||||
checks a declared checksum when present, then streams the opened descriptor.
|
||||
@@ -41,26 +55,34 @@ Exact remote placement and the operator workflow belong in
|
||||
- resolves publish output sources through runtime artifact catalog and
|
||||
manifest-aware resolution. Configured Scriptorium outputs are publishable
|
||||
only from validated `current` per-artifact analyze evidence; an incidental
|
||||
canonical file, legacy aggregate output, stale/failed/unselected record, or
|
||||
mismatched path, size, or checksum remains unavailable. This does not change
|
||||
the explicit compatibility policies owned by built-in, extraction, or
|
||||
previous-session sources.
|
||||
canonical file, aggregate output, stale/failed/unselected record, or
|
||||
mismatched path, size, or checksum remains unavailable. Built-in and
|
||||
extraction sources likewise require their own current manifest evidence.
|
||||
- publishes extraction lanes only through explicit configured output rules;
|
||||
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
|
||||
- selected artifact filter applies to configured artifact sources only.
|
||||
- locked outputs are skipped intentionally (including required ones).
|
||||
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
||||
- plans every configured output into exactly one source-sorted outcome:
|
||||
`published`, `carried_locked`, `locked_unpublished`, `optional_missing`, or
|
||||
`unselected`.
|
||||
- an initially locked selected output carries its exact prior
|
||||
`published_output` declaration when one exists. The declaration may point to
|
||||
an earlier run in the same session and retains its source, key, checksum,
|
||||
size, and generation; the bytes are not copied.
|
||||
- verifies a carried object's exact key, size, and generation through object
|
||||
metadata without downloading or opening the local output source.
|
||||
- an initially locked source with no prior declaration remains absent even
|
||||
when the output rule is required. Optional missing unlocked outputs remain
|
||||
absent; required missing unlocked outputs fail.
|
||||
- creates one complete immutable source-to-destination mapping before upload;
|
||||
- uploads and verifies every declared immutable object and the commit manifest;
|
||||
- updates `current/commit-pointer.json` exactly once, last; and
|
||||
- does not write the legacy `current/manifest.json` or `current/run_id.txt` pair.
|
||||
- rechecks remote lock state immediately before the pointer update. A newly
|
||||
committed lock aborts selection, leaving any uploaded immutable attempt
|
||||
unselected.
|
||||
- reads the mutable remote lock document through a direct limit-plus-one read
|
||||
capped by `MaxRemoteLockStoreBytes` (1 MiB), retaining the generation returned
|
||||
with the opened body for conditional updates. Oversized lock documents fail
|
||||
before YAML decoding; published artifact payloads do not use this limit.
|
||||
- writes the immutable commit from the combined new and exact carried
|
||||
declarations.
|
||||
- rereads every initially unlocked participating source after the immutable
|
||||
commit. A newly present lock aborts selection, leaving the uploaded immutable
|
||||
attempt unselected. Removing an initial lock may conservatively produce one
|
||||
carry because the initial snapshot controls the plan.
|
||||
- updates `current/commit-pointer.json` exactly once, last, and writes no other
|
||||
mutable publication control record.
|
||||
|
||||
## Metadata Signals
|
||||
|
||||
@@ -68,9 +90,8 @@ Includes counts/lists for:
|
||||
- run uploads
|
||||
- published output uploads
|
||||
- previous uploads
|
||||
- skipped optional outputs
|
||||
- skipped unselected outputs
|
||||
- locked outputs
|
||||
- one `publish_output_outcomes` collection containing all five output states,
|
||||
exact declared keys, and key-derived origin runs where applicable
|
||||
- remote commit and current-pointer key paths
|
||||
- the run identifier selected by the commit
|
||||
|
||||
@@ -83,35 +104,44 @@ Includes counts/lists for:
|
||||
from verified descriptors; symlinks, special files, replacement races, and
|
||||
undeclared entries are rejected or ignored before uploads begin.
|
||||
- run-local diagnostics, including Notarius receipt and stderr files, are
|
||||
archived only when recorded by the run manifest.
|
||||
- publish locks are not overridden by `--force`; remote locks are revalidated
|
||||
immediately before current-state selection.
|
||||
- post-commit local cleanup is authorized by the committed publish metadata and
|
||||
is durably recorded by the application lifecycle before any local deletion.
|
||||
archived only when recorded by the session manifest. A run with no declared
|
||||
current-run diagnostics legitimately uploads none.
|
||||
- independent publish locks are not overridden by `--force`; every initially
|
||||
unlocked relevant source is reread immediately before current-state
|
||||
selection.
|
||||
- successful uploaded publication records the exact run ID, remote commit key,
|
||||
and current-pointer key consumed by the later cleanup stage.
|
||||
|
||||
## Resume Semantics
|
||||
|
||||
The versioned semantic fingerprint covers enabled behavior, run-upload policy,
|
||||
normalized source/destination/required output rules, static lock policy, and
|
||||
the remote backend, bucket, region, endpoint, and root-prefix identity. Rule
|
||||
and lock ordering is canonicalized. Credential environment names,
|
||||
path-addressing transport mode, local workspace placement, and run identifiers
|
||||
are excluded. Remote locks remain mutable state and are still revalidated at
|
||||
the commit boundary; semantic evidence does not replace that safety check.
|
||||
normalized source/destination/required output rules, and the remote backend,
|
||||
bucket, region, endpoint, and root-prefix identity. Rule ordering is
|
||||
canonicalized. Credential
|
||||
environment names, path-addressing transport mode, local workspace placement,
|
||||
and run identifiers are excluded. Independent remote locks remain mutable state
|
||||
and are read directly by publish at planning and commit boundaries; semantic
|
||||
evidence does not replace those safety checks.
|
||||
|
||||
The commit boundary and cleanup gate are normative architecture invariants; see
|
||||
The commit boundary and cleanup authorization are normative architecture invariants; see
|
||||
[Architecture](../policy/architecture.md#publish-commit-boundary).
|
||||
|
||||
Publish never deletes local state. The standalone `publish` command selects
|
||||
only this stage; a full run selects notify and cleanup afterward.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
- [Configuration](../config.md#publish-configuration-summary) owns output and
|
||||
static-lock fields.
|
||||
- [Configuration](../config.md#publish-configuration-summary) owns output
|
||||
rules.
|
||||
- [Operations](../operations.md#publish-locks) owns remote lock lifecycle and
|
||||
physical remote state.
|
||||
- [Artifact Internals](artifacts.md) explains source resolution and current-state
|
||||
helpers.
|
||||
- Implementation and tests: `internal/stage/publish.go`,
|
||||
`internal/stage/publish_sources.go`, `internal/stage/publish_outputs.go`,
|
||||
`internal/stage/publish_output_plan.go`,
|
||||
`internal/stage/publish_transfer.go`, `internal/stage/publish_commit.go`,
|
||||
`internal/stage/publish_test.go`,
|
||||
`internal/stage/semantic_contracts_delivery.go`,
|
||||
`internal/app/operator_helpers_test.go`, and
|
||||
`internal/app/post_publish_cleanup_test.go`
|
||||
`internal/app/cleanup_orchestration_test.go`
|
||||
|
||||
@@ -21,7 +21,7 @@ records that may consume rendered transcripts.
|
||||
## Key Behavior
|
||||
|
||||
- uses `pipeline.render` settings (enabled/format/title/booleans).
|
||||
- resolves inputs manifest-first, then canonical fallback.
|
||||
- resolves inputs only from verified manifest output evidence.
|
||||
- writes run-local outputs first, then materializes canonical session outputs.
|
||||
- records input provenance, output paths, adapter metadata, logs, and generated config refs.
|
||||
- when `pipeline.render.enabled=false`, completes successfully with no outputs
|
||||
|
||||
@@ -16,7 +16,7 @@ Generate raw per-speaker transcripts from prepared audio using WhisperX.
|
||||
|
||||
- discovers prepared audio from manifest inputs or canonical audio directory.
|
||||
- derives the transcript identity from the prepared `.flac` filename.
|
||||
- dispatches WhisperX requests through a bounded worker pool.
|
||||
- dispatches one WhisperX request per prepared file through a bounded errgroup.
|
||||
- validates each output as JSON.
|
||||
- writes run-local outputs then materializes canonical transcript outputs only
|
||||
after every planned request succeeds.
|
||||
|
||||
@@ -16,6 +16,7 @@ operator-selected storage fields and credential mechanisms belong in
|
||||
- `Download(ctx, key, localPath)`
|
||||
- `Upload(ctx, localPath, key, opts)`
|
||||
- `UploadConditional(ctx, source, key, opts, condition)`
|
||||
- `Delete(ctx, key)` removes one object and succeeds when it is already absent
|
||||
- `Exists(ctx, key)`
|
||||
|
||||
Key invariant:
|
||||
@@ -47,6 +48,8 @@ not own discovery, defaults, or configuration validation.
|
||||
- `Read` binds a returned body to its S3 ETag. `UploadConditional` maps an ETag
|
||||
match or absence precondition directly to the provider request and reports a
|
||||
failed precondition without performing a local check-then-write replacement.
|
||||
- `Delete` maps one normalized key to the provider delete-object operation and
|
||||
treats an absent object as success.
|
||||
- `Exists` maps not-found responses to `false`.
|
||||
|
||||
## Invariants
|
||||
@@ -55,6 +58,11 @@ not own discovery, defaults, or configuration validation.
|
||||
- bounded reads never retain more than the caller's limit plus one byte and do
|
||||
not replace owner-specific size policy.
|
||||
- publish ordering semantics are owned by stage/app code, not storage adapters.
|
||||
- A caller that installs remote content into a canonical workspace path first
|
||||
writes a sibling temporary file, validates the complete object against its
|
||||
owner-specific contract, and only then replaces the destination. Temporary
|
||||
files used solely for remote comparisons are removed by their callers and do
|
||||
not represent installed workspace state.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
|
||||
@@ -14,8 +14,8 @@ path model, and layout creation goes through `EnsureLayoutFor`. Callers should
|
||||
consume those helpers instead of rebuilding relative paths.
|
||||
|
||||
`internal/pathsafe` validates relative destinations. `internal/fileops` opens
|
||||
cleanup roots and their descendants through no-follow directory handles before
|
||||
removing them.
|
||||
explicit cleanup roots and removes only normalized proper descendants through
|
||||
the resulting root handle.
|
||||
|
||||
`internal/fileops` owns the ordinary workspace mode contract. On POSIX,
|
||||
`WorkspaceDirectoryMode` is setgid `02775` and `WorkspaceFileMode` is `0664`.
|
||||
@@ -33,13 +33,8 @@ previous-session cache paths remain session-durable and are never redirected
|
||||
into run-local output space.
|
||||
|
||||
Extraction uses run-local receipt, stderr, and output-root helpers, then
|
||||
promotes the validated external bundle to the unique immutable Notarius bundle
|
||||
path supplied by `internal/artifacts`. `internal/fileops.PromoteDirectory`
|
||||
copies only regular files and directories to a same-filesystem temporary
|
||||
sibling. Source traversal uses confined directory handles and identity checks
|
||||
so replacing an inspected root, directory, or file is rejected rather than
|
||||
followed. The completed tree is atomically renamed without replacing an
|
||||
existing destination. Exact physical paths belong in
|
||||
validates and renames the external bundle to the unique immutable Notarius
|
||||
bundle path supplied by `internal/artifacts`. Exact physical paths belong in
|
||||
[Operations](../operations.md#extraction-workflow).
|
||||
|
||||
## Locking
|
||||
@@ -50,21 +45,25 @@ file retains owner metadata after release or process death; its existence is
|
||||
not evidence that a lock is active. Command and restore flows wait for this
|
||||
lock only while their context remains active, and report a release failure.
|
||||
|
||||
The runner assigns every invocation a run-workspace identity but creates the
|
||||
physical directory only before an executing stage that requires it. Fully
|
||||
reused invocations and cleanup-only invocations therefore do not accumulate
|
||||
empty run directories.
|
||||
|
||||
## Cleanup Semantics
|
||||
|
||||
Automatic post-publish cleanup:
|
||||
The canonical cleanup stage:
|
||||
|
||||
- is created only after a successful publish commit with complete publish
|
||||
metadata, then is persisted before any deletion;
|
||||
- requires `uploaded=true`, a remote commit key, and a current commit-pointer
|
||||
key in publish metadata;
|
||||
- runs after notify only when selected by the invocation range;
|
||||
- requires a succeeded publish record with `uploaded=true`, the published run
|
||||
ID, remote commit key, and current commit-pointer key;
|
||||
- consumes the resolved cleanup policy described in
|
||||
[Configuration](../config.md);
|
||||
- refuses unsafe deletes (root delete, out-of-root delete, and symlinked
|
||||
ancestors or entries);
|
||||
- retries any recorded incomplete target on later invocations even when no
|
||||
publish work is selected. Missing targets are a successful, idempotent
|
||||
cleanup result only after the completion evidence is saved.
|
||||
- refuses unsafe deletes (root delete, traversal, and out-of-root targets);
|
||||
- treats missing targets as successful, idempotent deletion; and
|
||||
- uses its ordinary failed/succeeded manifest stage record for retry. A later
|
||||
full run can skip reusable upstream stages and retry cleanup without
|
||||
republishing.
|
||||
|
||||
Manual cleanup uses the same root-confined deletion mechanism. Invocation
|
||||
syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
||||
@@ -75,6 +74,8 @@ syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
||||
- campaign-aware session root is mandatory.
|
||||
- manifest-driven stage state is durable across runs.
|
||||
- cleanup guardrails prevent destructive root/out-of-scope deletion.
|
||||
- restore installation is confined to validated relative paths beneath the
|
||||
selected session root.
|
||||
- ordinary workspace paths retain group-writable directory and file modes across
|
||||
nested creation, replacement, and Notarius promotion.
|
||||
|
||||
@@ -83,12 +84,14 @@ syntax and exact deletion scope belong in [CLI](../cli.md#clean) and
|
||||
- Path model and local store: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/local.go`
|
||||
- Run-local materialization: `internal/stage/run_local.go`
|
||||
- Immutable bundle promotion: `internal/fileops/directory.go`
|
||||
- Immutable bundle installation: `internal/stage/extract.go`
|
||||
- Workspace modes: `internal/fileops/modes.go`
|
||||
- Cleanup confinement: `internal/fileops/cleanup.go`,
|
||||
`internal/app/cleanup_targets.go`, `internal/app/post_publish_cleanup.go`
|
||||
`internal/stage/cleanup.go`, `internal/stage/cleanup_plan.go`, and
|
||||
`internal/app/cleanup_targets.go`
|
||||
- Tests: `internal/artifacts/paths_model_test.go`,
|
||||
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
||||
`internal/fileops/directory_test.go`, `internal/fileops/modes_posix_test.go`,
|
||||
`internal/fileops/modes_posix_test.go`,
|
||||
`internal/fileops/cleanup_test.go`, `internal/app/cleanup_targets_test.go`,
|
||||
`internal/app/post_publish_cleanup_test.go`
|
||||
`internal/stage/cleanup_test.go`, and
|
||||
`internal/app/cleanup_orchestration_test.go`
|
||||
|
||||
@@ -98,12 +98,12 @@ Canonical stage order:
|
||||
9. `analyze`
|
||||
10. `publish`
|
||||
11. `notify`
|
||||
12. `cleanup`
|
||||
|
||||
Execution rules:
|
||||
|
||||
- succeeded stages are skipped unless `--force` is set; stages with semantic
|
||||
configuration contracts additionally require matching versioned evidence,
|
||||
and missing legacy evidence causes a safe one-time rerun;
|
||||
configuration contracts additionally require matching versioned evidence;
|
||||
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- forcing a stage marks succeeded transitive dependents as `stale` before the
|
||||
replacement runs; render and extract are independent siblings; and
|
||||
@@ -155,7 +155,7 @@ narratio run 2026-04-04 --from extract --through analyze --force
|
||||
```
|
||||
|
||||
Omitting `--from` selects from `prepare`; omitting `--through` selects through
|
||||
`notify`. Force applies only within the selected range. Repeating `--from`,
|
||||
`cleanup`. Force applies only within the selected range. Repeating `--from`,
|
||||
`--through`, or `--force` is rejected instead of resolving by argument order.
|
||||
The plan command uses the same selection contract and prints only the selected
|
||||
range. Planning is read-only: it clones the loaded manifest, models selected
|
||||
@@ -169,11 +169,11 @@ selected artifact evidence is still semantically current.
|
||||
Before a bounded run or plan whose range starts after `prepare`, every excluded
|
||||
prefix stage must already have a session-manifest status of `succeeded` or
|
||||
`skipped`. Narratio reports the first absent, pending, running, failed, stale,
|
||||
or interrupted prerequisite without creating a run record or changing session
|
||||
state. Widen `--from` to include that stage, or recover it explicitly before
|
||||
retrying. Excluded prefix stages are not resume-validated or repaired as part
|
||||
of the bounded invocation; selected stages still reject missing, unsafe, or
|
||||
manifest-inconsistent inputs at their owning boundary.
|
||||
or interrupted prerequisite without creating a run-local directory or changing
|
||||
session state. Widen `--from` to include that stage, or recover it explicitly
|
||||
before retrying. Excluded prefix stages are not resume-validated or repaired as
|
||||
part of the bounded invocation; selected stages still reject missing, unsafe,
|
||||
or manifest-inconsistent inputs at their owning boundary.
|
||||
|
||||
Stages after `--through` are not prerequisites and are never scheduled by the
|
||||
bounded invocation. A selected forced stage can mark one of those succeeded
|
||||
@@ -195,8 +195,8 @@ This command is a transparent expansion to a forced bounded `run` from
|
||||
`extract` through `analyze`. Extraction always rebuilds its complete configured
|
||||
bundle. Analysis rebuilds the selected targets and their required analysis
|
||||
prerequisites, or uses the normal default selection when no artifact names are
|
||||
given. The command does not run publish or notify; delivery remains a separate
|
||||
operator action.
|
||||
given. The command does not run publish, notify, or cleanup; delivery and
|
||||
retention remain separate operator actions.
|
||||
|
||||
Inspect current artifact evidence, then publish explicitly when the regenerated
|
||||
set is ready:
|
||||
@@ -206,11 +206,17 @@ narratio session artifacts 2026-04-04
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
If planning or execution reports stale, missing, failed, legacy, or tampered
|
||||
If planning or execution reports stale, missing, failed, or tampered
|
||||
analysis evidence, regenerate the affected target instead of copying an older
|
||||
canonical file into place or editing the manifest. See
|
||||
[Troubleshooting: Analysis artifact evidence is not current](./troubleshooting.md#analysis-artifact-evidence-is-not-current).
|
||||
|
||||
Analyze checkpoints each completed configured artifact before starting later
|
||||
artifact work. After a partial failure or interruption, retry normally: valid
|
||||
earlier checkpoints are reused and execution resumes with failed, stale, or
|
||||
missing work in dependency order. `--force` is unnecessary unless the operator
|
||||
intends to replace otherwise-current artifacts.
|
||||
|
||||
## Artifact Selection
|
||||
|
||||
For a configured artifact family, selecting its family key expands to every
|
||||
@@ -280,15 +286,13 @@ narratio run-stage prepare 2026-04-04 --force
|
||||
|
||||
Starting a replacement clears the previous extraction payload from the current
|
||||
session-stage record. If that replacement fails or self-skips, the current
|
||||
record does not fall back to the earlier outputs. The earlier run manifest and
|
||||
immutable bundle remain available for inspection, but downstream resolution
|
||||
requires a new current successful extraction record.
|
||||
record does not fall back to the earlier outputs. The earlier immutable bundle
|
||||
remains available for inspection, but downstream resolution requires a new
|
||||
current successful extraction record.
|
||||
|
||||
Atomic Notarius bundle promotion is supported on Linux and macOS. On Windows
|
||||
and other operating systems, extraction fails before copying the bundle into a
|
||||
temporary promotion tree because Narratio has no verified atomic no-replace
|
||||
directory primitive there. This is an extraction limitation, not a broader
|
||||
platform-support guarantee for every Narratio workflow.
|
||||
Notarius extraction validates its run-local bundle and moves it once to the
|
||||
unique durable bundle path. A pre-existing durable destination is a collision;
|
||||
no duplicate promotion tree is created.
|
||||
|
||||
## External Command Lifecycle
|
||||
|
||||
@@ -311,12 +315,15 @@ Run-local diagnostics are:
|
||||
- `runs/{run_id}/extract/notarius.stderr.log`
|
||||
- `runs/{run_id}/extract/notarius-output/` before durable promotion
|
||||
|
||||
The run-record upload is an allowlist derived from the validated run manifest,
|
||||
not a workspace scan. Each declared source is opened without following
|
||||
symlinked ancestors or the leaf, verified as a regular file, and streamed from
|
||||
that verified descriptor. Unlisted files and unsafe entries are never uploaded.
|
||||
The durable bundle is never scanned for implicit publication; only lanes named
|
||||
by explicit `pipeline.publish.outputs` rules are uploaded.
|
||||
The run upload is an allowlist derived from the session manifest's declared
|
||||
current-run outputs, logs, and generated configurations, not a workspace scan.
|
||||
References to canonical session files and prior-run paths are excluded. Each
|
||||
declared source is opened without following symlinked ancestors or the leaf,
|
||||
verified as a regular file, and streamed from that verified descriptor.
|
||||
Unlisted files and unsafe entries are never uploaded; a run with no declared
|
||||
current-run diagnostics uploads no run-local files. No distinguished diagnostic
|
||||
file is uploaded. The durable bundle is never scanned for implicit publication;
|
||||
only lanes named by explicit `pipeline.publish.outputs` rules are uploaded.
|
||||
|
||||
To intentionally replace the current extraction result, run:
|
||||
|
||||
@@ -339,10 +346,11 @@ outcome changes also stale affected downstream stages, while an identical
|
||||
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
||||
|
||||
Publish reuse additionally tracks enabled/run-upload behavior, normalized
|
||||
output rules, static locks, and remote backend/bucket/region/endpoint/root
|
||||
identity. Credential environment names, local workspace placement, and run IDs
|
||||
are excluded. Regardless of semantic reuse evidence, executing publish still
|
||||
revalidates mutable remote locks immediately before commit selection.
|
||||
output rules, and remote backend/bucket/region/endpoint/root identity.
|
||||
Credential environment names, local workspace placement, and run IDs are
|
||||
excluded. Regardless of semantic reuse evidence, executing publish reads
|
||||
independent remote locks directly and rechecks initially unlocked sources
|
||||
before commit selection.
|
||||
|
||||
## Publish Workflow
|
||||
|
||||
@@ -358,6 +366,9 @@ Equivalent:
|
||||
narratio run-stage publish 2026-04-04 --force
|
||||
```
|
||||
|
||||
This standalone command stops after publish and does not invoke cleanup. A
|
||||
default full `run` continues through notify and cleanup.
|
||||
|
||||
Publish commit model:
|
||||
|
||||
- uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
|
||||
@@ -381,31 +392,44 @@ commit by version, checksum, size, and storage generation; committed artifacts
|
||||
are also checksum- and generation-bound. Readers accept this contract now and
|
||||
strictly reject mismatched or unknown data.
|
||||
|
||||
Legacy reads are limited to a coherent `current/manifest.json` and
|
||||
`current/run_id.txt` pair; a torn pair is rejected. New publication does not
|
||||
write that pair and remote commit state does not carry local
|
||||
`current_pointer_written` metadata.
|
||||
A selected commit may carry a `published_output` declaration from an earlier
|
||||
run in the same campaign/session. Restore, previous-session caching, status,
|
||||
and artifact inspection use that declaration's exact immutable key and stable
|
||||
source; they do not reconstruct a current-run key or scan the session prefix.
|
||||
Other artifact types remain confined to the selected commit's own run.
|
||||
|
||||
When the pointer is absent, remote state is not published; republish the
|
||||
session to establish a committed snapshot. Remote commit state does not carry
|
||||
local `current_pointer_written` metadata.
|
||||
|
||||
## Publish Locks
|
||||
|
||||
Lock sources:
|
||||
Each mutable lock is an independent strict JSON object at
|
||||
`{session_prefix}/locks/{sha256(source)}.lock`. The lowercase digest is computed
|
||||
from the exact canonical source. Listing validates every object in that
|
||||
reserved prefix and reports locks sorted by source.
|
||||
|
||||
- static locks in `pipeline.publish.locks`
|
||||
- mutable remote locks in `{session_prefix}/locks.yml`
|
||||
Upgrades do not read, migrate, or delete the retired `{session_prefix}/locks.yml`
|
||||
object. It has no effect on publication and may be removed manually after
|
||||
verifying lock commands and publication behavior with the upgraded Narratio
|
||||
version. No migration command is required.
|
||||
|
||||
Effective lock rules:
|
||||
Operational rules:
|
||||
|
||||
- static and remote locks are merged;
|
||||
- static locks win on source collisions;
|
||||
- locked outputs are intentional skips;
|
||||
- lock add/remove commands mutate only remote lock state through generation-bound
|
||||
conditional writes. A command retries a bounded number of concurrent
|
||||
conflicts while its invocation context remains active, so it never replaces a
|
||||
different lock-document generation; and
|
||||
- a publish re-reads remote locks immediately before it writes the current
|
||||
commit pointer. A lock committed before that recheck prevents selecting the
|
||||
new snapshot, even though its already-uploaded immutable objects may remain
|
||||
available for a later retry.
|
||||
- add uses conditional creation and reports an existing lock;
|
||||
- add with `--force` observes the current generation and attempts one
|
||||
generation-matched replacement. A concurrent change is reported for an
|
||||
explicit operator retry;
|
||||
- remove is idempotent and reports whether a lock was present;
|
||||
- the initial publish lock snapshot controls output planning. A locked source
|
||||
carries its exact declaration from the prior current commit when available;
|
||||
a never-published locked source remains absent even when required;
|
||||
- carried bytes are not copied. Publish verifies the prior object's exact key,
|
||||
size, and generation without downloading it; and
|
||||
- after immutable uploads and the immutable commit, publish re-reads every
|
||||
initially unlocked participating source. A newly added lock prevents the
|
||||
current pointer from advancing, although the immutable attempt may remain.
|
||||
A removed initial lock may conservatively cause one carry.
|
||||
|
||||
Examples:
|
||||
|
||||
@@ -439,13 +463,22 @@ Default restore scope:
|
||||
|
||||
- the committed session manifest and the committed transcript/artifact objects
|
||||
declared by the selected remote commit
|
||||
- `previous/**` when needed by configured previous-session artifact inputs
|
||||
- `previous/**` when needed by configured previous-session artifact inputs;
|
||||
each is selected from its exact committed `published_output` declaration
|
||||
|
||||
Optional:
|
||||
|
||||
- `--include-audio` to include `audio/**`
|
||||
- `--force` to overwrite eligible conflicting regular files; it never replaces
|
||||
directories or other non-regular local targets
|
||||
directories, symlinks, or other non-regular local targets
|
||||
|
||||
Restore treats a local leaf symlink or any symlinked ancestor beneath the
|
||||
selected session root as a conflict, even when the linked content matches the
|
||||
selected remote object. `--force` does not override that conflict.
|
||||
|
||||
Before each install, restore validates the destination again beneath the
|
||||
selected session root. A target changed into a symlink, directory, or other
|
||||
non-regular entry is not overwritten.
|
||||
|
||||
Restore writes an execution report at `reports/restore-latest.json`.
|
||||
|
||||
@@ -474,6 +507,12 @@ Durable session paths:
|
||||
- `config/**`
|
||||
- `runs/**`
|
||||
|
||||
`manifest.json` is the versioned session-progress document. Narratio accepts
|
||||
only `narratio.session-manifest.v1`. Do not edit unsupported session state to
|
||||
add missing markers. Move it aside and regenerate it, restore a verified
|
||||
current committed snapshot, or republish healthy regenerated state as the
|
||||
relevant workflow requires.
|
||||
|
||||
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
||||
stderr, and pre-promotion output remain in the producing run's `extract`
|
||||
directory as described in [Extraction Workflow](#extraction-workflow).
|
||||
@@ -498,6 +537,8 @@ Cache layout (durable S3 audio cache):
|
||||
Each cached audio file has an adjacent managed identity record. It binds the
|
||||
file to its remote object version and verified digest; deleting or altering the
|
||||
record simply causes Narratio to download and verify the object again.
|
||||
Remote audio and restored files are fully transferred into sibling temporary
|
||||
files and validated before they replace their canonical workspace paths.
|
||||
|
||||
### Workspace Permissions
|
||||
|
||||
@@ -553,17 +594,29 @@ Rules:
|
||||
- `clean` deletes work/spool session state;
|
||||
- cache is preserved unless `--clear-cache` is set;
|
||||
- each deletion is confined beneath its configured workspace, spool, or cache
|
||||
root and refuses symlinked paths;
|
||||
- automatic post-publish cleanup is gated by successful publish commit plus:
|
||||
root; root, traversal, and out-of-root targets are refused;
|
||||
- the final cleanup stage is gated by successful publish evidence plus:
|
||||
- `pipeline.spool.delete_audio_after_publish=true`
|
||||
- `pipeline.workspace.cleanup_after_publish=true`
|
||||
- Narratio first records the exact run-scoped cleanup obligation. If cleanup
|
||||
reports incomplete, the remote committed snapshot remains current; rerun
|
||||
publish to retry only the outstanding confined local cleanup.
|
||||
- the manifest publish record has `uploaded=true`, the published run ID, the
|
||||
exact remote commit key, and the current commit-pointer key.
|
||||
|
||||
Post-publish cleanup is evaluated only when `publish` actually executes in the
|
||||
current invocation. A bounded range that excludes publish does not replay a
|
||||
cleanup obligation as an unrelated side effect.
|
||||
Cleanup derives only the configured run-scoped spool and workspace targets,
|
||||
uses root-confined deletion, and does not read or mutate object storage. A
|
||||
failure leaves publish succeeded and the remote committed snapshot current.
|
||||
Retry with a normal full run: valid upstream work, including publish and notify,
|
||||
is skipped while the failed cleanup stage runs again. `run-stage cleanup` is
|
||||
also available when the manifest has valid publish evidence. A range ending at
|
||||
`notify` or earlier and the standalone `publish` command never delete local
|
||||
state.
|
||||
|
||||
Older local manifests that contain the retired cleanup-obligation field are
|
||||
intentionally incompatible with strict manifest decoding. If such an
|
||||
obligation is still outstanding, complete it with the pre-upgrade binary.
|
||||
Otherwise verify that the remote publication is committed, move aside or
|
||||
regenerate the incompatible local manifest (or restore the verified remote
|
||||
snapshot), and then use the cleanup stage. Manual JSON editing is not the
|
||||
normal migration path.
|
||||
|
||||
## Operational Caveats
|
||||
|
||||
|
||||
@@ -116,8 +116,11 @@ adapter and publish metadata.
|
||||
|
||||
Resume and skip decisions are manifest-driven. Filesystem state may be
|
||||
inspected and validated, but file presence alone does not replace recorded
|
||||
stage state. Invocation-scoped run records provide an audit of one execution;
|
||||
they do not replace the session manifest as progress authority.
|
||||
stage state. Each invocation retains an in-memory command summary and a
|
||||
run-local workspace identity, but neither is a durable progress ledger. The
|
||||
physical run directory is created on demand only when an executing stage
|
||||
requires it; a fully reused invocation or a cleanup-only invocation does not
|
||||
leave an empty run directory.
|
||||
|
||||
Restore treats committed remote current state as its authority. It must plan
|
||||
deterministically, confine remote-to-local paths, protect local conflicts, and
|
||||
@@ -185,8 +188,8 @@ availability.
|
||||
Writes, moves, replacements, and deletions must use narrow, explicit,
|
||||
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
||||
relative destinations must not expand the scope of an operation. Cleanup is
|
||||
permitted only through explicit operator action or configured post-publish
|
||||
gates, and it must preserve durable cache unless cache removal is explicitly
|
||||
permitted only through explicit operator action or the configured final cleanup
|
||||
stage, and it must preserve durable cache unless cache removal is explicitly
|
||||
requested.
|
||||
|
||||
Physical layout, retention, and operational lifecycle belong in
|
||||
@@ -202,16 +205,26 @@ the immutable commit manifest, and finally the current commit pointer.
|
||||
`current/commit-pointer.json` is the sole mutable selector and must be written
|
||||
exactly once, last. Failed, incomplete, skipped, or uncommitted publish attempts
|
||||
must not be presented as current remote state. Publish locks remain authoritative
|
||||
and are not bypassed by a forced run. Mutable remote locks use provider-enforced
|
||||
generation preconditions and are revalidated immediately before pointer
|
||||
selection; loss of that check leaves the prior committed snapshot current.
|
||||
and are not bypassed by a forced run. Each mutable remote lock is an independent
|
||||
per-source object; lock mutations use provider-enforced generation preconditions.
|
||||
Publish revalidates every initially unlocked participating source after writing
|
||||
the immutable commit and before pointer selection. A lock observed then leaves
|
||||
the prior committed snapshot current. A lock created after that final check may
|
||||
race the pointer write; the commit protocol does not claim a stronger distributed
|
||||
transaction boundary.
|
||||
|
||||
Automatic local cleanup is permitted only after a successful publish commit,
|
||||
only when explicitly configured, and only through the path-safety guardrails.
|
||||
It is a durable local obligation bound to that committed run and its exact
|
||||
targets, not an inferred side effect of the current stage list. A cleanup
|
||||
failure makes the invocation incomplete while leaving the committed remote
|
||||
snapshot authoritative; later invocations resume the recorded obligation.
|
||||
It is the final ordinary pipeline stage after `notify`. The stage derives its
|
||||
exact run-scoped targets from validated publish metadata and the configured
|
||||
campaign/session identity; it never infers a target from directory contents or
|
||||
rereads remote storage. A cleanup failure makes the invocation incomplete while
|
||||
leaving publish succeeded and the committed remote snapshot authoritative. A
|
||||
normal later full run reuses valid upstream stages and retries cleanup.
|
||||
|
||||
The standalone `publish` command selects only publish and therefore never
|
||||
invokes cleanup. Operators may select cleanup directly only when the manifest
|
||||
contains valid successful publication evidence.
|
||||
|
||||
## Security, Privacy, And Diagnostics
|
||||
|
||||
|
||||
@@ -5,6 +5,13 @@ binary release. The synchronous release boundary is a successful push of one
|
||||
new tag to `origin`; Woodpecker and Gitea publication happen later and do not
|
||||
change that result.
|
||||
|
||||
## Go prerequisite
|
||||
|
||||
Release preparation requires the maintained Go 1.26 baseline. Tag CI currently
|
||||
uses the explicitly pinned `golang:1.26.7` image for candidate validation and
|
||||
asset construction; that asynchronous work does not change the tag-push
|
||||
completion boundary.
|
||||
|
||||
## Choose a version and write its note
|
||||
|
||||
Narratio is past `v1.0.0`. Use an unused stable tag in the exact form
|
||||
|
||||
100
docs/releases/v1.7.0.md
Normal file
100
docs/releases/v1.7.0.md
Normal file
@@ -0,0 +1,100 @@
|
||||
# Narratio v1.7.0
|
||||
|
||||
Narratio v1.7.0 completes the maintainability refactor begun after v1.6.0. It
|
||||
simplifies filesystem and process handling, makes resumable work more precise,
|
||||
replaces legacy local and remote state with explicit authorities, and turns
|
||||
cleanup into the final ordinary pipeline stage.
|
||||
|
||||
## Summary
|
||||
|
||||
The session manifest is now the sole durable local progress ledger. Analyze
|
||||
checkpoints successful artifacts individually, allowing an interrupted or
|
||||
partially failed analysis to resume at the first incomplete artifact without
|
||||
rerunning successful siblings unless forced.
|
||||
|
||||
Remote publication now commits immutable run state before advancing one
|
||||
verified current pointer. Publish locks are independent per-source objects;
|
||||
locked outputs retain their previously committed declarations rather than
|
||||
being overwritten or silently omitted. Local cleanup runs after notification
|
||||
as a separately recorded, safely retryable stage.
|
||||
|
||||
The release also adopts Go 1.26, simplifies ordinary file operations around
|
||||
Narratio's trusted group-owned workspace model, improves subprocess lifecycle
|
||||
handling, decomposes several large orchestration and validation components,
|
||||
and removes mutable global test seams.
|
||||
|
||||
## Compatibility
|
||||
|
||||
This is a minor release with deliberate removal of compatibility paths that
|
||||
were deprecated and documented as subject to removal:
|
||||
|
||||
- Campaigns must use one versioned `narratio.party.v1` `party.yml`. Separate
|
||||
players files, session-level party overrides, and unversioned party documents
|
||||
are rejected.
|
||||
- Session manifests must declare `schema_version:
|
||||
narratio.session-manifest.v1` and satisfy the current strict schema. Older
|
||||
unversioned state and manifests containing the retired
|
||||
`post_publish_cleanup` obligation are rejected rather than migrated.
|
||||
- Remote current state must be selected by
|
||||
`current/commit-pointer.json` and its referenced immutable run commit. Legacy
|
||||
current-state layouts are no longer inferred.
|
||||
- The retired aggregate `locks.yml` object is ignored. Publish locks now live
|
||||
as independent strict objects under the session lock prefix.
|
||||
- Run manifests are no longer produced or consumed. The session manifest owns
|
||||
durable stage progress; invocation summaries and run workspaces remain
|
||||
operational data rather than a second progress authority.
|
||||
|
||||
The CLI and current pipeline/configuration formats otherwise remain compatible.
|
||||
Building and maintaining Narratio now requires Go 1.26. Cross-built Linux,
|
||||
macOS, and Windows assets remain available for `amd64` and `arm64`.
|
||||
|
||||
## Upgrade
|
||||
|
||||
Before upgrading a campaign that still uses compatibility data:
|
||||
|
||||
1. Replace legacy party/player inputs with the documented
|
||||
[`narratio.party.v1`](../integrations/party.md) campaign roster.
|
||||
2. If a local manifest contains an outstanding `post_publish_cleanup`
|
||||
obligation, complete it with the pre-upgrade binary. Otherwise verify the
|
||||
remote commit, move aside or regenerate the incompatible manifest, or
|
||||
restore a verified committed snapshot. Do not migrate state by manually
|
||||
editing JSON.
|
||||
3. Republish sessions that do not yet have a valid immutable run commit and
|
||||
`current/commit-pointer.json`; legacy remote layouts are not current state in
|
||||
v1.7.0.
|
||||
4. Recreate any required publication locks with `narratio session locks add`.
|
||||
After verifying the new locks and publication behavior, the ignored legacy
|
||||
`locks.yml` object may be removed manually.
|
||||
|
||||
Existing compatible session manifests and already committed remote snapshots
|
||||
need no special action. Consult [Operations](../operations.md) and
|
||||
[Troubleshooting](../troubleshooting.md) for recovery procedures.
|
||||
|
||||
## Changes
|
||||
|
||||
- Upgraded the maintained development and release baseline to Go 1.26.
|
||||
- Reworked subprocess execution and Windows process-tree supervision while
|
||||
preserving contextual wait and cleanup failures.
|
||||
- Replaced custom confined ordinary I/O and directory-promotion machinery with
|
||||
simpler standard-library operations while retaining bounded reads, atomic
|
||||
single-file visibility, workspace modes, validated downloads, and
|
||||
root-confined destructive operations.
|
||||
- Removed legacy party, manifest, artifact-discovery, remote-current, and lock
|
||||
compatibility paths.
|
||||
- Consolidated durable progress in the session manifest and removed the run
|
||||
manifest domain model.
|
||||
- Unified runner planning and execution decisions, separated invocation
|
||||
lifecycle responsibilities, and avoided creating empty run workspaces for
|
||||
fully reused or cleanup-only invocations.
|
||||
- Added per-artifact analyze execution, checkpointing, reconciliation, and
|
||||
recovery.
|
||||
- Decomposed configuration validation by domain and made validation diagnostics
|
||||
deterministic without treating incidental ordering or wording as a contract.
|
||||
- Introduced immutable remote publish commits, one final current pointer,
|
||||
explicit output planning, safe cross-run carried outputs, and independent
|
||||
per-artifact remote locks.
|
||||
- Added `cleanup` as the final canonical stage after `notify`, with validated
|
||||
publish evidence, safe retry, explicit selection, and no separate cleanup
|
||||
obligation.
|
||||
- Replaced mutable command test hooks with invocation-scoped dependencies and
|
||||
improved structural test diagnostics.
|
||||
@@ -56,7 +56,8 @@ Symptom:
|
||||
|
||||
Likely cause:
|
||||
|
||||
- stale field name, typo, invalid enum, or invalid duration/path format.
|
||||
- stale field name, typo, invalid enum, invalid duration/path format, or an
|
||||
unsupported local manifest schema/state marker.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
@@ -68,9 +69,39 @@ Safe fix:
|
||||
|
||||
- align config with [Configuration](./config.md) and the
|
||||
[maintained examples](../examples/README.md).
|
||||
- for a manifest schema error, move the unsupported local state aside and
|
||||
regenerate it; restore a verified current committed snapshot or republish
|
||||
healthy regenerated state when appropriate. Do not add version or stage
|
||||
markers by hand.
|
||||
- if an older manifest is rejected for its retired cleanup-obligation field,
|
||||
finish outstanding cleanup with the pre-upgrade binary. Otherwise verify the
|
||||
remote commit, move aside or regenerate the incompatible manifest, and use
|
||||
the cleanup stage. Do not treat manual JSON editing as the normal migration
|
||||
path.
|
||||
|
||||
Relevant reference: [Configuration](./config.md).
|
||||
|
||||
## Party schema version rejected
|
||||
|
||||
Symptom:
|
||||
|
||||
- configuration loading reports that `party.schema_version` is required or
|
||||
unsupported.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- the campaign `party.yml` is an unversioned legacy document, or names a
|
||||
schema other than `narratio.party.v1`.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- replace the file with one exact, single-document
|
||||
[`narratio.party.v1` party](./integrations/party.md); do not add a version
|
||||
marker to an otherwise incompatible legacy structure. The versioned party is
|
||||
the only configured roster authority.
|
||||
|
||||
Relevant reference: [Canonical Party Input](./integrations/party.md).
|
||||
|
||||
## Unexpected imported or profile value
|
||||
|
||||
Symptom:
|
||||
@@ -171,8 +202,9 @@ Safe fix:
|
||||
|
||||
- widen the bounded range to include the first reported stage, or recover that
|
||||
stage explicitly with `run-stage` before retrying. The failed check does not
|
||||
create a run record or modify the manifest. Narratio does not resume-validate
|
||||
excluded prefix stages, and stages after `--through` are not prerequisites.
|
||||
create a run-local directory or modify the manifest. Narratio does not
|
||||
resume-validate excluded prefix stages, and stages after `--through` are not
|
||||
prerequisites.
|
||||
|
||||
If prerequisite statuses are terminal but a selected stage reports a missing,
|
||||
unsafe, or checksum-inconsistent artifact, repair the artifact at the stage
|
||||
@@ -282,29 +314,6 @@ Narratio validates selector structure and prepared evidence, while Notarius
|
||||
owns slot declarations, media compatibility, and generated-handoff conflicts.
|
||||
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Atomic Notarius promotion unsupported
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails with `atomic no-replace directory promotion is unsupported`
|
||||
before a durable bundle or temporary promotion tree is created.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Narratio is running on an operating system other than Linux, macOS, or
|
||||
Windows, where the required atomic no-replace directory primitive has not
|
||||
been implemented and verified.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- run extraction on Linux, macOS, or Windows. Do not replace the atomic commit
|
||||
with a manual copy or move; the session manifest must never observe a partial
|
||||
or overwritten bundle.
|
||||
|
||||
This is an extraction-specific platform boundary, not a support statement for
|
||||
unrelated Narratio workflows. See
|
||||
[Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Notarius receipt or index incompatible
|
||||
|
||||
Symptom:
|
||||
@@ -362,8 +371,8 @@ Likely causes:
|
||||
Safe fix:
|
||||
|
||||
- allow the automatic rerun after verifying the current configuration. Treat
|
||||
an unsafe path or symlink error as filesystem corruption or tampering and
|
||||
investigate it rather than replacing files manually.
|
||||
an unsafe path or bundle symlink error as filesystem corruption or tampering
|
||||
and investigate it rather than replacing files manually.
|
||||
|
||||
## Notarius transitive configuration changed
|
||||
|
||||
@@ -395,8 +404,7 @@ Symptom:
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the per-artifact record is stale, missing, failed, unselected, malformed, or
|
||||
from the legacy aggregate-only manifest contract;
|
||||
- the per-artifact record is stale, missing, failed, unselected, or malformed;
|
||||
- a configured prompt/profile, dependency, input identity, output path, or
|
||||
effective variable changed; or
|
||||
- the recorded output is missing, unsafe, empty, or has a size/checksum that no
|
||||
@@ -425,33 +433,82 @@ narratio analyze 2026-04-04 --artifacts session_recap
|
||||
Relevant references: [Operations: Artifact Selection](./operations.md#artifact-selection)
|
||||
and [Artifact Internals](./internal/artifacts.md#resolution-rules).
|
||||
|
||||
## Legacy aggregate analysis requires regeneration
|
||||
## Analysis manifest checkpoint failed
|
||||
|
||||
Symptom:
|
||||
|
||||
- a manifest from an older Narratio version reports aggregate analyze success
|
||||
and the old files are present, but configured artifact sources remain
|
||||
unavailable.
|
||||
- analyze reports that it could not save the session manifest after completing
|
||||
an artifact.
|
||||
|
||||
Likely cause:
|
||||
Likely causes:
|
||||
|
||||
- the manifest has no supported per-artifact analyze state. Aggregate output
|
||||
lists do not establish current configured-artifact authority.
|
||||
- the workspace is unwritable or out of space;
|
||||
- the manifest path or its parent directory has unsafe permissions or type; or
|
||||
- a transient local filesystem error interrupted atomic manifest replacement.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- regenerate the required artifacts. A partial selection makes only its
|
||||
targets and prerequisites eligible for current state; unselected legacy
|
||||
files intentionally remain unavailable. Run full analysis later when every
|
||||
enabled configured artifact must become current.
|
||||
- correct the reported filesystem or permission problem;
|
||||
- inspect `session artifacts` to identify the last durable current records; and
|
||||
- rerun analyze normally. Narratio reuses earlier successful checkpoints and
|
||||
retries the failed or non-current artifact and its required dependents.
|
||||
|
||||
Do not treat a newly created canonical artifact file as current when its
|
||||
checkpoint failed, and do not edit the manifest to advertise it. If the error
|
||||
recurs, preserve the manifest and run-local diagnostics before repairing the
|
||||
workspace.
|
||||
|
||||
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
|
||||
and [Manifest Internals](./internal/manifest.md#persistence-semantics).
|
||||
|
||||
## Built-in transcript or bounds source is unavailable
|
||||
|
||||
Symptom:
|
||||
|
||||
- analyze, extract, render, or publish says a built-in source is unavailable
|
||||
even though a file exists at its usual session path.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- the producing stage has not succeeded, its output lacks the exact registered
|
||||
source, run ID, or checksum, its evidence is duplicated or stale, or the
|
||||
durable bytes no longer match the manifest checksum.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- do not copy or rename the file to make it discoverable. Rerun the producing
|
||||
stage to regenerate both the bytes and current manifest evidence, then retry
|
||||
the consumer.
|
||||
|
||||
## An older manifest schema is rejected
|
||||
|
||||
Symptom:
|
||||
|
||||
- Narratio rejects a local session manifest because `schema_version` is missing
|
||||
or unsupported; or
|
||||
- restore rejects a committed remote session manifest for the same reason.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- the manifest predates the supported canonical state contract. Narratio
|
||||
rejects it at the schema boundary rather than interpreting aggregate output
|
||||
lists or incidental files as current authority.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- for unsupported local state, move the old manifest aside and regenerate, or
|
||||
restore a verified current committed snapshot when one is authoritative;
|
||||
- for unsupported committed remote state, regenerate healthy local state as
|
||||
needed and republish the session.
|
||||
|
||||
```bash
|
||||
narratio analyze 2026-04-04 --artifacts session_recap
|
||||
narratio analyze 2026-04-04
|
||||
narratio run 2026-04-04
|
||||
narratio publish 2026-04-04
|
||||
```
|
||||
|
||||
After current records exist, inspect them and publish explicitly. Do not delete
|
||||
the legacy files merely to influence selection; availability is manifest-owned.
|
||||
After current records exist, inspect them and publish explicitly. Do not copy
|
||||
old files into canonical paths merely to influence selection; availability is
|
||||
manifest-owned.
|
||||
|
||||
Relevant references: [Operations: Stage Execution and Continuation Behavior](./operations.md#stage-execution-and-continuation-behavior)
|
||||
and [Manifest Internals](./internal/manifest.md#analyze-owned-artifact-state).
|
||||
@@ -491,7 +548,9 @@ Symptom:
|
||||
Likely causes:
|
||||
|
||||
- missing `session.previous_session_id`;
|
||||
- previous artifact not restored/published for source session.
|
||||
- the selected previous-session commit does not declare the required canonical
|
||||
source as a `published_output`; or
|
||||
- the declared published object is unavailable or fails identity validation.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
@@ -549,6 +608,8 @@ Symptom:
|
||||
Likely cause:
|
||||
|
||||
- local durable files differ from remote restore sources.
|
||||
- a restore target or one of its ancestors is a symlink or another non-regular
|
||||
filesystem entry.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
@@ -559,7 +620,11 @@ narratio session restore 2026-04-04 --dry-run
|
||||
Safe fix:
|
||||
|
||||
- review conflicts;
|
||||
- rerun with `--force` only when remote state should overwrite local.
|
||||
- rerun with `--force` only when remote state should overwrite an eligible
|
||||
regular local file; repair a symlink or other non-regular conflict first.
|
||||
|
||||
If the filesystem changed after the plan was printed, rerun restore after
|
||||
repairing the destination; installation performs the same safety check again.
|
||||
|
||||
Relevant reference: [Operations: Restore Workflow](./operations.md#restore-workflow).
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ with the sample campaign and a compatible local- or S3-audio session.
|
||||
[party](campaigns/sample-campaign/party.yml) fixture, plus an optional
|
||||
[spell-catalog overlay](campaigns/sample-campaign/spell_catalog.json) that
|
||||
follows the Notarius v0.6 contract. Narratio derives the players projection
|
||||
from this party source; the campaign deliberately has no `players_file`.
|
||||
from this party source.
|
||||
- [Sample speaker audio](audio/sample-speaker.flac) is a text placeholder that
|
||||
reserves the expected filename and directory shape. Replace it with a real
|
||||
FLAC file before running transcription.
|
||||
|
||||
4
go.mod
4
go.mod
@@ -1,12 +1,14 @@
|
||||
module gitea.maximumdirect.net/eric/narratio
|
||||
|
||||
go 1.25.0
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.17
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
github.com/google/go-cmp v0.7.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/sys v0.47.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
4
go.sum
4
go.sum
@@ -34,6 +34,10 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
@@ -84,17 +83,17 @@ func materializePlaceholders(req PolishRequest) error {
|
||||
"merged_transcript_path": req.MergedTranscriptPath,
|
||||
"output_path": req.OutputProcessedPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("audita noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("audita noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
@@ -124,7 +123,7 @@ func writeJSONIfRequested(path string, payload any) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal placeholder json for %q: %w", path, err)
|
||||
}
|
||||
if err := subprocess.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write placeholder json %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
|
||||
19
internal/adapters/audita/generated_config.go
Normal file
19
internal/adapters/audita/generated_config.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package audita
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func writeGeneratedYAML(path string, payload any) error {
|
||||
data, err := yaml.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
|
||||
}
|
||||
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated YAML %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -378,7 +378,7 @@ func (r *SubprocessRunner) writeInvocationConfig(req PolishRequest, args []strin
|
||||
"credential_env_var": r.llmAPIKeyEnv,
|
||||
"credential_present": credentialPresent,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
}
|
||||
|
||||
func validateProcessedOutput(path string) error {
|
||||
@@ -425,7 +425,7 @@ func validateJSONFile(path string) error {
|
||||
}
|
||||
|
||||
func readAuditaResult(path string, limit int64, category string) ([]byte, error) {
|
||||
data, err := fileops.ReadRegularFile(path, limit)
|
||||
data, err := fileops.ReadFileBounded(path, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("audita %s result exceeds or cannot be read within %d-byte limit: %w", category, limit, err)
|
||||
}
|
||||
|
||||
@@ -671,7 +671,7 @@ func sumWarningOccurrences(values []WarningSummary) (int, error) {
|
||||
}
|
||||
|
||||
func decodeBoundedJSON(path string, limit int64, destination any) error {
|
||||
data, err := fileops.ReadRegularFile(path, limit)
|
||||
data, err := fileops.ReadFileBounded(path, limit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("notarius JSON result exceeds or cannot be read within %d-byte limit: %w", limit, err)
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
@@ -143,7 +142,7 @@ func (f *FakeRunner) RenderArtifact(ctx context.Context, req RenderArtifactReque
|
||||
|
||||
func materializeRunPlaceholders(req RunArtifactRequest) error {
|
||||
if req.OutputPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputPath, []byte("scriptorium noop/fake run artifact\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write run output %q: %w", req.OutputPath, err)
|
||||
}
|
||||
}
|
||||
@@ -155,17 +154,17 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
|
||||
"prompt_id": req.PromptID,
|
||||
"output_path": req.OutputPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake run stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake run stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
@@ -174,7 +173,7 @@ func materializeRunPlaceholders(req RunArtifactRequest) error {
|
||||
|
||||
func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
||||
if req.OutputPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputPath, []byte("{\"schema\":\"scriptorium.render.v1\",\"placeholder\":true}\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write render output %q: %w", req.OutputPath, err)
|
||||
}
|
||||
}
|
||||
@@ -186,17 +185,17 @@ func materializeRenderPlaceholders(req RenderArtifactRequest) error {
|
||||
"prompt_id": req.PromptID,
|
||||
"output_path": req.OutputPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("scriptorium noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("scriptorium noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
19
internal/adapters/scriptorium/generated_config.go
Normal file
19
internal/adapters/scriptorium/generated_config.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func writeGeneratedYAML(path string, payload any) error {
|
||||
data, err := yaml.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
|
||||
}
|
||||
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated YAML %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -342,11 +342,11 @@ func writeInvocationConfig(path string, payload invocationPayload) error {
|
||||
"render_format": payload.RenderFormat,
|
||||
"render_prompt_logged": payload.RenderPromptStore,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(path, data, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(path, data)
|
||||
}
|
||||
|
||||
func validateNonEmptyOutput(path string) error {
|
||||
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
|
||||
data, err := fileops.ReadFileBounded(path, MaxOutputFileBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scriptorium artifact output exceeds or cannot be read within %d-byte limit: %w", MaxOutputFileBytes, err)
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
@@ -261,7 +260,7 @@ func (f *FakeRunner) Render(ctx context.Context, req RenderRequest) (RenderResul
|
||||
|
||||
func materializePlaceholders(req MergeRequest) error {
|
||||
if req.OutputMergedTranscriptPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputMergedTranscriptPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write merged transcript %q: %w", req.OutputMergedTranscriptPath, err)
|
||||
}
|
||||
}
|
||||
@@ -272,22 +271,22 @@ func materializePlaceholders(req MergeRequest) error {
|
||||
"input_transcript_paths": req.InputTranscriptPaths,
|
||||
"output_path": req.OutputMergedTranscriptPath,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.ReportPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
||||
}
|
||||
}
|
||||
@@ -296,7 +295,7 @@ func materializePlaceholders(req MergeRequest) error {
|
||||
|
||||
func materializeTrimPlaceholders(req TrimRequest) error {
|
||||
if req.OutputTrimmedPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputTrimmedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write trimmed transcript %q: %w", req.OutputTrimmedPath, err)
|
||||
}
|
||||
}
|
||||
@@ -309,17 +308,17 @@ func materializeTrimPlaceholders(req TrimRequest) error {
|
||||
"output_path": req.OutputTrimmedPath,
|
||||
"keep_selector": req.KeepSelector,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake trim stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake trim stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
@@ -328,7 +327,7 @@ func materializeTrimPlaceholders(req TrimRequest) error {
|
||||
|
||||
func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||
if req.OutputNormalizedPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputNormalizedPath, []byte(`{"schema":"seriatim.intermediate.v1","segments":[]}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write normalized transcript %q: %w", req.OutputNormalizedPath, err)
|
||||
}
|
||||
}
|
||||
@@ -344,22 +343,22 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||
if req.ReportPath != "" {
|
||||
payload["report_path"] = req.ReportPath
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake normalize stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake normalize stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.ReportPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.ReportPath, []byte(`{"schema":"seriatim.report.v1","placeholder":true}`), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write report %q: %w", req.ReportPath, err)
|
||||
}
|
||||
}
|
||||
@@ -368,7 +367,7 @@ func materializeNormalizePlaceholders(req NormalizeRequest) error {
|
||||
|
||||
func materializeRenderPlaceholders(req RenderRequest) error {
|
||||
if req.OutputRenderedPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.OutputRenderedPath, []byte("# Transcript\n\nRendered markdown placeholder.\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write rendered transcript %q: %w", req.OutputRenderedPath, err)
|
||||
}
|
||||
}
|
||||
@@ -385,17 +384,17 @@ func materializeRenderPlaceholders(req RenderRequest) error {
|
||||
"include_segment_ids": req.IncludeSegmentIDs,
|
||||
"include_metadata": req.IncludeMetadata,
|
||||
}
|
||||
if err := subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode); err != nil {
|
||||
if err := writeGeneratedYAML(req.GeneratedConfigPath, payload); err != nil {
|
||||
return fmt.Errorf("write generated config %q: %w", req.GeneratedConfigPath, err)
|
||||
}
|
||||
}
|
||||
if req.StdoutLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StdoutLogPath, []byte("seriatim noop/fake render stdout placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stdout log %q: %w", req.StdoutLogPath, err)
|
||||
}
|
||||
}
|
||||
if req.StderrLogPath != "" {
|
||||
if err := subprocess.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
if err := fileops.WriteFileAtomic(req.StderrLogPath, []byte("seriatim noop/fake render stderr placeholder\n"), fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write stderr log %q: %w", req.StderrLogPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
19
internal/adapters/seriatim/generated_config.go
Normal file
19
internal/adapters/seriatim/generated_config.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package seriatim
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func writeGeneratedYAML(path string, payload any) error {
|
||||
data, err := yaml.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal generated YAML for %q: %w", path, err)
|
||||
}
|
||||
if err := fileops.WriteFileAtomic(path, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("write generated YAML %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -553,7 +553,7 @@ func (r *SubprocessRunner) writeMergeInvocationConfig(req MergeRequest, args []s
|
||||
payload["coalesce_gap"] = *r.coalesceGap
|
||||
}
|
||||
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
}
|
||||
|
||||
func buildTrimArgs(req TrimRequest) []string {
|
||||
@@ -605,7 +605,7 @@ func writeTrimInvocationConfig(req TrimRequest, args []string, binary string, ti
|
||||
"output_path": req.OutputTrimmedPath,
|
||||
"keep_selector": req.KeepSelector,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
}
|
||||
|
||||
func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary string, timeout time.Duration, outputSchema string) error {
|
||||
@@ -620,7 +620,7 @@ func writeNormalizeInvocationConfig(req NormalizeRequest, args []string, binary
|
||||
"output_schema": outputSchema,
|
||||
"report_path": req.ReportPath,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
}
|
||||
|
||||
func writeRenderInvocationConfig(req RenderRequest, args []string, binary string, timeout time.Duration, format string) error {
|
||||
@@ -638,7 +638,7 @@ func writeRenderInvocationConfig(req RenderRequest, args []string, binary string
|
||||
"include_segment_ids": req.IncludeSegmentIDs,
|
||||
"include_metadata": req.IncludeMetadata,
|
||||
}
|
||||
return subprocess.WriteYAMLAtomic(req.GeneratedConfigPath, payload, fileops.WorkspaceFileMode)
|
||||
return writeGeneratedYAML(req.GeneratedConfigPath, payload)
|
||||
}
|
||||
|
||||
func validateJSONFile(path string) error {
|
||||
@@ -692,7 +692,7 @@ func validateNonEmptyTextFile(path string) error {
|
||||
}
|
||||
|
||||
func readSeriatimResult(path, category string) ([]byte, error) {
|
||||
data, err := fileops.ReadRegularFile(path, MaxOutputFileBytes)
|
||||
data, err := fileops.ReadFileBounded(path, MaxOutputFileBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("seriatim %s exceeds or cannot be read within %d-byte limit: %w", category, MaxOutputFileBytes, err)
|
||||
}
|
||||
|
||||
@@ -19,14 +19,17 @@ import (
|
||||
type FakeBackend struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
Objects map[string]FakeObject
|
||||
Uploads []FakeUploadCall
|
||||
Downloads []FakeDownloadCall
|
||||
Reads []FakeReadCall
|
||||
Objects map[string]FakeObject
|
||||
Uploads []FakeUploadCall
|
||||
Downloads []FakeDownloadCall
|
||||
Reads []FakeReadCall
|
||||
Deletes []string
|
||||
ExistsKeys []string
|
||||
|
||||
ListErr error
|
||||
DownloadErr error
|
||||
UploadErr error
|
||||
DeleteErr error
|
||||
ExistsErr error
|
||||
UploadHook func(FakeUploadCall) error
|
||||
DownloadHook func(FakeDownloadCall) error
|
||||
@@ -315,6 +318,25 @@ func fakeObjectETag(data []byte) string {
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Delete removes one object. Removing an absent object succeeds.
|
||||
func (f *FakeBackend) Delete(ctx context.Context, key string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if f.DeleteErr != nil {
|
||||
return f.DeleteErr
|
||||
}
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
if normalizedKey == "" {
|
||||
return fmt.Errorf("delete object: key is required")
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.Deletes = append(f.Deletes, normalizedKey)
|
||||
delete(f.Objects, normalizedKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks object presence.
|
||||
func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -323,8 +345,12 @@ func (f *FakeBackend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
if f.ExistsErr != nil {
|
||||
return false, f.ExistsErr
|
||||
}
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
f.mu.Lock()
|
||||
f.ExistsKeys = append(f.ExistsKeys, normalizedKey)
|
||||
f.mu.Unlock()
|
||||
f.mu.RLock()
|
||||
_, ok := f.Objects[normalizeObjectKey(key)]
|
||||
_, ok := f.Objects[normalizedKey]
|
||||
f.mu.RUnlock()
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
@@ -71,23 +71,55 @@ func TestFakeBackendUploadAndExists(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendDeleteIsIdempotent(t *testing.T) {
|
||||
fake := &FakeBackend{}
|
||||
fake.SeedObject(FakeObject{Key: "locks/source.lock", Data: []byte("lock")})
|
||||
|
||||
for range 2 {
|
||||
if err := fake.Delete(context.Background(), `locks\source.lock`); err != nil {
|
||||
t.Fatalf("Delete() error = %v", err)
|
||||
}
|
||||
}
|
||||
if _, ok := fake.Objects["locks/source.lock"]; ok {
|
||||
t.Fatal("Delete() left the object present")
|
||||
}
|
||||
if len(fake.Deletes) != 2 || fake.Deletes[0] != "locks/source.lock" || fake.Deletes[1] != "locks/source.lock" {
|
||||
t.Fatalf("Delete() calls = %#v, want two normalized keys", fake.Deletes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendDeleteHonorsContextAndInjectedError(t *testing.T) {
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
fake := &FakeBackend{}
|
||||
if err := fake.Delete(canceled, "locks/source.lock"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Delete() canceled error = %v, want context.Canceled", err)
|
||||
}
|
||||
|
||||
wantErr := errors.New("delete fail")
|
||||
fake.DeleteErr = wantErr
|
||||
if err := fake.Delete(context.Background(), "locks/source.lock"); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Delete() injected error = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendConditionalUploadRejectsStaleGeneration(t *testing.T) {
|
||||
fake := &FakeBackend{}
|
||||
fake.SeedObject(FakeObject{Key: "locks.yml", Data: []byte("old")})
|
||||
old := fake.Objects["locks.yml"].ETag
|
||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("new"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); err != nil {
|
||||
fake.SeedObject(FakeObject{Key: "control.json", Data: []byte("old")})
|
||||
old := fake.Objects["control.json"].ETag
|
||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("new"), "control.json", UploadOptions{}, WriteCondition{MatchETag: old}); err != nil {
|
||||
t.Fatalf("UploadConditional() error = %v", err)
|
||||
}
|
||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("lost"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: old}); !errors.Is(err, ErrConditionNotMet) {
|
||||
if _, err := fake.UploadConditional(context.Background(), strings.NewReader("lost"), "control.json", UploadOptions{}, WriteCondition{MatchETag: old}); !errors.Is(err, ErrConditionNotMet) {
|
||||
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
||||
}
|
||||
if got := string(fake.Objects["locks.yml"].Data); got != "new" {
|
||||
t.Fatalf("locks object = %q, want successful replacement preserved", got)
|
||||
if got := string(fake.Objects["control.json"].Data); got != "new" {
|
||||
t.Fatalf("control object = %q, want successful replacement preserved", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeBackendObjectErrors(t *testing.T) {
|
||||
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), ExistsErr: errors.New("exists fail")}
|
||||
fake := &FakeBackend{DownloadErr: errors.New("download fail"), UploadErr: errors.New("upload fail"), ListErr: errors.New("list fail"), DeleteErr: errors.New("delete fail"), ExistsErr: errors.New("exists fail")}
|
||||
|
||||
if _, err := fake.List(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "list fail") {
|
||||
t.Fatalf("List() error = %v, want list fail", err)
|
||||
@@ -100,6 +132,9 @@ func TestFakeBackendObjectErrors(t *testing.T) {
|
||||
if _, err := fake.Upload(context.Background(), local, "x", UploadOptions{}); err == nil || !strings.Contains(err.Error(), "upload fail") {
|
||||
t.Fatalf("Upload() error = %v, want upload fail", err)
|
||||
}
|
||||
if err := fake.Delete(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "delete fail") {
|
||||
t.Fatalf("Delete() error = %v, want delete fail", err)
|
||||
}
|
||||
if _, err := fake.Exists(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "exists fail") {
|
||||
t.Fatalf("Exists() error = %v, want exists fail", err)
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ type ObjectStore interface {
|
||||
Download(ctx context.Context, key, localPath string) error
|
||||
Upload(ctx context.Context, localPath, key string, opts UploadOptions) (ObjectInfo, error)
|
||||
UploadConditional(ctx context.Context, source io.Reader, key string, opts UploadOptions, condition WriteCondition) (ObjectInfo, error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
Exists(ctx context.Context, key string) (bool, error)
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ type s3API interface {
|
||||
ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
|
||||
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
|
||||
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
||||
DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
|
||||
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
|
||||
}
|
||||
|
||||
@@ -300,6 +301,25 @@ func (b *S3Backend) uploadReader(ctx context.Context, source io.Reader, key stri
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// Delete removes one object. S3 deletion is idempotent when the key is absent.
|
||||
func (b *S3Backend) Delete(ctx context.Context, key string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
if normalizedKey == "" {
|
||||
return fmt.Errorf("delete object: key is required")
|
||||
}
|
||||
_, err := b.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: &b.bucket,
|
||||
Key: &normalizedKey,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete object %q: %w", normalizedKey, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists checks whether one object key exists.
|
||||
func (b *S3Backend) Exists(ctx context.Context, key string) (bool, error) {
|
||||
normalizedKey := normalizeObjectKey(key)
|
||||
|
||||
@@ -29,16 +29,18 @@ type fakeS3API struct {
|
||||
getETag *string
|
||||
getLastModified *time.Time
|
||||
|
||||
putOut *s3.PutObjectOutput
|
||||
putErr error
|
||||
putOut *s3.PutObjectOutput
|
||||
putErr error
|
||||
deleteErr error
|
||||
|
||||
headErr error
|
||||
|
||||
lastList *s3.ListObjectsV2Input
|
||||
lastLists []*s3.ListObjectsV2Input
|
||||
lastGet *s3.GetObjectInput
|
||||
lastPut *s3.PutObjectInput
|
||||
lastHead *s3.HeadObjectInput
|
||||
lastList *s3.ListObjectsV2Input
|
||||
lastLists []*s3.ListObjectsV2Input
|
||||
lastGet *s3.GetObjectInput
|
||||
lastPut *s3.PutObjectInput
|
||||
lastDelete *s3.DeleteObjectInput
|
||||
lastHead *s3.HeadObjectInput
|
||||
}
|
||||
|
||||
func (f *fakeS3API) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
|
||||
@@ -116,6 +118,14 @@ func (f *fakeS3API) PutObject(_ context.Context, params *s3.PutObjectInput, _ ..
|
||||
return f.putOut, nil
|
||||
}
|
||||
|
||||
func (f *fakeS3API) DeleteObject(_ context.Context, params *s3.DeleteObjectInput, _ ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {
|
||||
f.lastDelete = params
|
||||
if f.deleteErr != nil {
|
||||
return nil, f.deleteErr
|
||||
}
|
||||
return &s3.DeleteObjectOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeS3API) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
|
||||
f.lastHead = params
|
||||
if f.headErr != nil {
|
||||
@@ -177,14 +187,14 @@ func TestS3BackendDownloadCreatesParentDirectory(t *testing.T) {
|
||||
func TestS3BackendReadReturnsOpenedObjectMetadata(t *testing.T) {
|
||||
lastModified := time.Date(2026, 8, 11, 1, 2, 3, 0, time.UTC)
|
||||
client := &fakeS3API{
|
||||
getBody: io.NopCloser(strings.NewReader("locks")),
|
||||
getBody: io.NopCloser(strings.NewReader("state")),
|
||||
getSize: int64Ptr(5),
|
||||
getETag: strPtr(`"generation"`),
|
||||
getLastModified: &lastModified,
|
||||
}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
|
||||
info, body, err := backend.Read(context.Background(), `sessions\locks.yml`)
|
||||
info, body, err := backend.Read(context.Background(), `sessions\control.json`)
|
||||
if err != nil {
|
||||
t.Fatalf("Read() error = %v", err)
|
||||
}
|
||||
@@ -193,10 +203,10 @@ func TestS3BackendReadReturnsOpenedObjectMetadata(t *testing.T) {
|
||||
if readErr != nil || closeErr != nil {
|
||||
t.Fatalf("read body error=%v close error=%v", readErr, closeErr)
|
||||
}
|
||||
if info.Key != "sessions/locks.yml" || info.Size != 5 || info.ETag != "generation" || info.LastModified == nil || !info.LastModified.Equal(lastModified) {
|
||||
if info.Key != "sessions/control.json" || info.Size != 5 || info.ETag != "generation" || info.LastModified == nil || !info.LastModified.Equal(lastModified) {
|
||||
t.Fatalf("Read() info = %#v, want opened object metadata", info)
|
||||
}
|
||||
if string(data) != "locks" || client.lastGet == nil || *client.lastGet.Key != "sessions/locks.yml" {
|
||||
if string(data) != "state" || client.lastGet == nil || *client.lastGet.Key != "sessions/control.json" {
|
||||
t.Fatalf("Read() data=%q request=%#v", data, client.lastGet)
|
||||
}
|
||||
}
|
||||
@@ -238,14 +248,14 @@ func TestS3BackendUploadAndExists(t *testing.T) {
|
||||
func TestS3BackendConditionalUploadUsesProviderPrecondition(t *testing.T) {
|
||||
client := &fakeS3API{putOut: &s3.PutObjectOutput{ETag: strPtr(`"etag123"`)}}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
if _, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{MatchETag: "before"}); err != nil {
|
||||
if _, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "control.json", UploadOptions{}, WriteCondition{MatchETag: "before"}); err != nil {
|
||||
t.Fatalf("UploadConditional() error = %v", err)
|
||||
}
|
||||
if client.lastPut == nil || client.lastPut.IfMatch == nil || *client.lastPut.IfMatch != "before" || client.lastPut.IfNoneMatch != nil {
|
||||
t.Fatalf("PutObject conditional input = %#v", client.lastPut)
|
||||
}
|
||||
client.putErr = &smithy.GenericAPIError{Code: "PreconditionFailed", Message: "changed"}
|
||||
_, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "locks.yml", UploadOptions{}, WriteCondition{RequireAbsent: true})
|
||||
_, err := backend.UploadConditional(context.Background(), strings.NewReader("payload"), "control.json", UploadOptions{}, WriteCondition{RequireAbsent: true})
|
||||
if !errors.Is(err, ErrConditionNotMet) {
|
||||
t.Fatalf("UploadConditional() error = %v, want ErrConditionNotMet", err)
|
||||
}
|
||||
@@ -275,6 +285,37 @@ func TestS3BackendExistsNotFound(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendDeleteMapsRequestAndErrors(t *testing.T) {
|
||||
client := &fakeS3API{}
|
||||
backend := &S3Backend{bucket: "bucket-1", client: client}
|
||||
if err := backend.Delete(context.Background(), `locks\source.lock`); err != nil {
|
||||
t.Fatalf("Delete() error = %v", err)
|
||||
}
|
||||
if client.lastDelete == nil || *client.lastDelete.Bucket != "bucket-1" || *client.lastDelete.Key != "locks/source.lock" {
|
||||
t.Fatalf("DeleteObject input = %#v, want normalized bucket and key", client.lastDelete)
|
||||
}
|
||||
|
||||
wantErr := errors.New("permission denied")
|
||||
client.deleteErr = wantErr
|
||||
err := backend.Delete(context.Background(), "locks/source.lock")
|
||||
if !errors.Is(err, wantErr) || !strings.Contains(err.Error(), "locks/source.lock") {
|
||||
t.Fatalf("Delete() error = %v, want contextual backend error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BackendDeleteHonorsCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
client := &fakeS3API{}
|
||||
err := (&S3Backend{bucket: "bucket-1", client: client}).Delete(ctx, "locks/source.lock")
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Delete() error = %v, want context.Canceled", err)
|
||||
}
|
||||
if client.lastDelete != nil {
|
||||
t.Fatal("Delete() called S3 after cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3BackendFromConfigUsesClientOptions(t *testing.T) {
|
||||
original := newS3Client
|
||||
t.Cleanup(func() { newS3Client = original })
|
||||
|
||||
@@ -12,9 +12,9 @@ import (
|
||||
|
||||
func TestDownloadObjectToTempSuccess(t *testing.T) {
|
||||
store := &FakeBackend{}
|
||||
store.SeedObject(FakeObject{Key: "sessions/a/current/run_id.txt", Data: []byte("run-123\n")})
|
||||
store.SeedObject(FakeObject{Key: "sessions/a/runs/run-123/metadata.txt", Data: []byte("run-123\n")})
|
||||
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/runs/run-123/metadata.txt", "narratio-test-*.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("DownloadObjectToTemp() error = %v", err)
|
||||
}
|
||||
@@ -38,7 +38,7 @@ func TestDownloadObjectToTempFailedDownloadRemovesTempFile(t *testing.T) {
|
||||
t.Fatalf("Glob(before) error = %v", err)
|
||||
}
|
||||
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", pattern)
|
||||
path, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/runs/run-123/metadata.txt", pattern)
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("DownloadObjectToTemp() error = %v, want %v", err, sentinel)
|
||||
}
|
||||
@@ -58,7 +58,7 @@ func TestDownloadObjectToTempCallerContextWrappingPreservesCause(t *testing.T) {
|
||||
sentinel := errors.New("object missing")
|
||||
store := &FakeBackend{DownloadErr: sentinel}
|
||||
|
||||
_, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/current/run_id.txt", "narratio-test-*.txt")
|
||||
_, err := DownloadObjectToTemp(context.Background(), store, "sessions/a/runs/run-123/metadata.txt", "narratio-test-*.txt")
|
||||
if err == nil {
|
||||
t.Fatal("DownloadObjectToTemp() error = nil, want error")
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ type logWriters struct {
|
||||
Stdout io.Writer
|
||||
Stderr io.Writer
|
||||
|
||||
limits chan *captureLimitError
|
||||
limits chan error
|
||||
mu sync.Mutex
|
||||
limit *captureLimitError
|
||||
stdout *diagnosticWriter
|
||||
@@ -92,7 +92,7 @@ type diagnosticWriter struct {
|
||||
}
|
||||
|
||||
func openLogWriters(stdoutPath, stderrPath, owner string, sensitiveValues []string) (*logWriters, error) {
|
||||
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
logs := &logWriters{limits: make(chan error, 1)}
|
||||
cleanStdout := cleanLogPath(stdoutPath)
|
||||
cleanStderr := cleanLogPath(stderrPath)
|
||||
|
||||
@@ -221,7 +221,7 @@ func (w *diagnosticWriter) reachLimit() error {
|
||||
return limit
|
||||
}
|
||||
|
||||
func (l *logWriters) Limits() <-chan *captureLimitError { return l.limits }
|
||||
func (l *logWriters) Limits() <-chan error { return l.limits }
|
||||
|
||||
func (l *logWriters) Limit() *captureLimitError {
|
||||
l.mu.Lock()
|
||||
@@ -255,7 +255,7 @@ func openDiagnosticFile(path string) (*os.File, error) {
|
||||
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(path)); err != nil {
|
||||
return nil, fmt.Errorf("create log directory for %q: %w", path, err)
|
||||
}
|
||||
file, err := fileops.OpenFileConfined(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileops.WorkspaceFileMode)
|
||||
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileops.WorkspaceFileMode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open log file %q: %w", path, err)
|
||||
}
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
// Package subprocess provides reusable process execution and generated-config helpers.
|
||||
// Package subprocess provides internal process execution and Narratio-specific
|
||||
// subprocess policy. Its private execution mechanism owns process lifecycle;
|
||||
// the Run boundary owns environments, diagnostics, redaction, and errors.
|
||||
package subprocess
|
||||
|
||||
91
internal/adapters/subprocess/execution.go
Normal file
91
internal/adapters/subprocess/execution.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
type executionReason uint8
|
||||
|
||||
const (
|
||||
executionNotStarted executionReason = iota
|
||||
executionNaturalExit
|
||||
executionCanceled
|
||||
executionDeadline
|
||||
executionAborted
|
||||
)
|
||||
|
||||
type executionOutcome struct {
|
||||
reason executionReason
|
||||
startedAt time.Time
|
||||
completedAt time.Time
|
||||
exitCode int
|
||||
startErr error
|
||||
waitErr error
|
||||
contextErr error
|
||||
abortErr error
|
||||
cleanupErr error
|
||||
}
|
||||
|
||||
func executeOwnedCommand(ctx context.Context, cmd *exec.Cmd, abort <-chan error) (outcome executionOutcome) {
|
||||
outcome = executionOutcome{startedAt: time.Now().UTC(), exitCode: -1}
|
||||
defer func() { outcome.completedAt = time.Now().UTC() }()
|
||||
if outcome.contextErr = ctx.Err(); outcome.contextErr != nil {
|
||||
outcome.reason = executionNotStarted
|
||||
return outcome
|
||||
}
|
||||
tree, err := newOwnedProcessTree()
|
||||
if err != nil {
|
||||
outcome.reason, outcome.startErr = executionNotStarted, err
|
||||
return outcome
|
||||
}
|
||||
defer func() { outcome.cleanupErr = joinErrors(outcome.cleanupErr, tree.Dispose()) }()
|
||||
cmd.WaitDelay = forcefulTerminationWait
|
||||
if outcome.startErr = tree.Start(cmd); outcome.startErr != nil {
|
||||
outcome.reason = executionNotStarted
|
||||
return outcome
|
||||
}
|
||||
waitCh := make(chan error, 1)
|
||||
go func() { waitCh <- cmd.Wait() }()
|
||||
select {
|
||||
case outcome.waitErr = <-waitCh:
|
||||
outcome.reason = executionNaturalExit
|
||||
case <-ctx.Done():
|
||||
outcome.contextErr = ctx.Err()
|
||||
if outcome.contextErr == context.DeadlineExceeded {
|
||||
outcome.reason = executionDeadline
|
||||
} else {
|
||||
outcome.reason = executionCanceled
|
||||
}
|
||||
outcome.waitErr, outcome.cleanupErr = terminateAndReap(tree, waitCh)
|
||||
case outcome.abortErr = <-abort:
|
||||
outcome.reason = executionAborted
|
||||
outcome.waitErr, outcome.cleanupErr = terminateAndReap(tree, waitCh)
|
||||
}
|
||||
if cmd.ProcessState != nil {
|
||||
outcome.exitCode = cmd.ProcessState.ExitCode()
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
func terminateAndReap(tree ownedProcessTree, waitCh <-chan error) (waitErr, cleanupErr error) {
|
||||
cleanupErr = tree.TerminateGracefully()
|
||||
timer := time.NewTimer(gracefulTerminationWait)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
return waitErr, joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||
case <-timer.C:
|
||||
}
|
||||
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||
forcefulTimer := time.NewTimer(forcefulTerminationWait)
|
||||
defer forcefulTimer.Stop()
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
return waitErr, cleanupErr
|
||||
case <-forcefulTimer.C:
|
||||
return nil, joinErrors(cleanupErr, fmt.Errorf("owned subprocess did not reap within %s after forceful termination", forcefulTerminationWait))
|
||||
}
|
||||
}
|
||||
54
internal/adapters/subprocess/execution_test.go
Normal file
54
internal/adapters/subprocess/execution_test.go
Normal file
@@ -0,0 +1,54 @@
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os/exec"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTerminateAndReapPreservesWaitAndCleanupErrors(t *testing.T) {
|
||||
waitErr := errors.New("wait failed")
|
||||
gracefulErr := errors.New("graceful termination failed")
|
||||
forcefulErr := errors.New("forceful termination failed")
|
||||
tree := &recordingProcessTree{
|
||||
gracefulErr: gracefulErr,
|
||||
forcefulErr: forcefulErr,
|
||||
}
|
||||
waitCh := make(chan error, 1)
|
||||
waitCh <- waitErr
|
||||
|
||||
gotWaitErr, gotCleanupErr := terminateAndReap(tree, waitCh)
|
||||
if !errors.Is(gotWaitErr, waitErr) {
|
||||
t.Fatalf("wait error = %v, want %v", gotWaitErr, waitErr)
|
||||
}
|
||||
for _, want := range []error{gracefulErr, forcefulErr} {
|
||||
if !errors.Is(gotCleanupErr, want) {
|
||||
t.Fatalf("cleanup error = %v, want wrapped %v", gotCleanupErr, want)
|
||||
}
|
||||
}
|
||||
if tree.gracefulCalls != 1 || tree.forcefulCalls != 1 {
|
||||
t.Fatalf("termination calls = graceful %d, forceful %d; want 1 each", tree.gracefulCalls, tree.forcefulCalls)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingProcessTree struct {
|
||||
gracefulErr error
|
||||
forcefulErr error
|
||||
|
||||
gracefulCalls int
|
||||
forcefulCalls int
|
||||
}
|
||||
|
||||
func (*recordingProcessTree) Start(*exec.Cmd) error { return nil }
|
||||
|
||||
func (tree *recordingProcessTree) TerminateGracefully() error {
|
||||
tree.gracefulCalls++
|
||||
return tree.gracefulErr
|
||||
}
|
||||
|
||||
func (tree *recordingProcessTree) TerminateForcefully() error {
|
||||
tree.forcefulCalls++
|
||||
return tree.forcefulErr
|
||||
}
|
||||
|
||||
func (*recordingProcessTree) Dispose() error { return nil }
|
||||
@@ -1,9 +1,7 @@
|
||||
package subprocess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
@@ -22,39 +20,6 @@ type ownedProcessTree interface {
|
||||
Dispose() error
|
||||
}
|
||||
|
||||
func waitForOwnedCommand(ctx context.Context, tree ownedProcessTree, waitCh <-chan error, captureLimits <-chan *captureLimitError) (waitErr, ctxErr error, captureLimit *captureLimitError, cleanupErr error) {
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
return waitErr, nil, nil, nil
|
||||
case <-ctx.Done():
|
||||
ctxErr = ctx.Err()
|
||||
case captureLimit = <-captureLimits:
|
||||
}
|
||||
|
||||
cleanupErr = tree.TerminateGracefully()
|
||||
gracefulTimer := time.NewTimer(gracefulTerminationWait)
|
||||
defer gracefulTimer.Stop()
|
||||
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
// The leader may exit before descendants finish graceful shutdown.
|
||||
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||
return waitErr, ctxErr, captureLimit, cleanupErr
|
||||
case <-gracefulTimer.C:
|
||||
}
|
||||
|
||||
cleanupErr = joinErrors(cleanupErr, tree.TerminateForcefully())
|
||||
forcefulTimer := time.NewTimer(forcefulTerminationWait)
|
||||
defer forcefulTimer.Stop()
|
||||
|
||||
select {
|
||||
case waitErr = <-waitCh:
|
||||
return waitErr, ctxErr, captureLimit, cleanupErr
|
||||
case <-forcefulTimer.C:
|
||||
return nil, ctxErr, captureLimit, joinErrors(cleanupErr, fmt.Errorf("owned subprocess did not reap within %s after forceful termination", forcefulTerminationWait))
|
||||
}
|
||||
}
|
||||
|
||||
func joinErrors(errs ...error) error {
|
||||
filtered := make([]error, 0, len(errs))
|
||||
for _, err := range errs {
|
||||
|
||||
@@ -39,25 +39,48 @@ func (tree *windowsProcessTree) Start(cmd *exec.Cmd) error {
|
||||
return err
|
||||
}
|
||||
|
||||
process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(cmd.Process.Pid))
|
||||
if err == nil {
|
||||
err = windows.AssignProcessToJobObject(job, process)
|
||||
_ = windows.CloseHandle(process)
|
||||
var assignErr error
|
||||
withHandleErr := cmd.Process.WithHandle(func(handle uintptr) {
|
||||
assignErr = windows.AssignProcessToJobObject(job, windows.Handle(handle))
|
||||
})
|
||||
if withHandleErr != nil {
|
||||
return failWindowsProcessStart(cmd, job, fmt.Errorf("acquire stable process handle for job assignment: %w", withHandleErr))
|
||||
}
|
||||
if err == nil {
|
||||
err = resumeInitialThread(uint32(cmd.Process.Pid))
|
||||
if assignErr != nil {
|
||||
return failWindowsProcessStart(cmd, job, fmt.Errorf("assign process to job object: %w", assignErr))
|
||||
}
|
||||
if err != nil {
|
||||
killErr := cmd.Process.Kill()
|
||||
waitErr := cmd.Wait()
|
||||
_ = windows.CloseHandle(job)
|
||||
return joinErrors(fmt.Errorf("assign process to job object: %w", err), killErr, waitErr)
|
||||
if err := resumeInitialThread(uint32(cmd.Process.Pid)); err != nil {
|
||||
return failWindowsProcessStart(cmd, job, fmt.Errorf("resume initial process thread: %w", err))
|
||||
}
|
||||
|
||||
tree.job = job
|
||||
return nil
|
||||
}
|
||||
|
||||
func failWindowsProcessStart(cmd *exec.Cmd, job windows.Handle, setupErr error) error {
|
||||
killErr := cmd.Process.Kill()
|
||||
waitErr := cmd.Wait()
|
||||
var exitErr *exec.ExitError
|
||||
if killErr == nil && errors.As(waitErr, &exitErr) {
|
||||
// A killed suspended child normally reports a non-zero exit from Wait;
|
||||
// successful reaping is not an additional setup failure.
|
||||
waitErr = nil
|
||||
}
|
||||
return joinErrors(
|
||||
setupErr,
|
||||
wrapWindowsProcessCleanupError("kill suspended child", killErr),
|
||||
wrapWindowsProcessCleanupError("reap suspended child", waitErr),
|
||||
wrapWindowsProcessCleanupError("close unowned job object", windows.CloseHandle(job)),
|
||||
)
|
||||
}
|
||||
|
||||
func wrapWindowsProcessCleanupError(operation string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s: %w", operation, err)
|
||||
}
|
||||
|
||||
func resumeInitialThread(processID uint32) error {
|
||||
snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0)
|
||||
if err != nil {
|
||||
|
||||
@@ -7,9 +7,6 @@ import (
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// RunRequest defines a subprocess invocation.
|
||||
@@ -60,63 +57,23 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
}
|
||||
defer logs.Close()
|
||||
|
||||
tree, err := newOwnedProcessTree()
|
||||
if err != nil {
|
||||
return RunResult{}, fmt.Errorf("prepare owned subprocess tree: %w", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command(req.Executable, req.Args...)
|
||||
cmd.Dir = req.WorkingDir
|
||||
cmd.Env = childEnv
|
||||
cmd.Stdout = logs.Stdout
|
||||
cmd.Stderr = logs.Stderr
|
||||
// Streaming capture uses pipes. Bound their lifetime when a leader exits
|
||||
// while a descendant still holds a stream descriptor.
|
||||
cmd.WaitDelay = forcefulTerminationWait
|
||||
|
||||
started := time.Now().UTC()
|
||||
result := RunResult{
|
||||
ExitCode: -1,
|
||||
StartedAt: started,
|
||||
StdoutLogPath: req.StdoutLogPath,
|
||||
StderrLogPath: req.StderrLogPath,
|
||||
}
|
||||
|
||||
if err := runCtx.Err(); err != nil {
|
||||
result.CompletedAt = time.Now().UTC()
|
||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||
return result, fmt.Errorf("command was not started: %w", err)
|
||||
}
|
||||
|
||||
if err := tree.Start(cmd); err != nil {
|
||||
result.CompletedAt = time.Now().UTC()
|
||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||
return result, fmt.Errorf("start command %q with args %v: %w", req.Executable, req.Args, err)
|
||||
}
|
||||
|
||||
waitCh := make(chan error, 1)
|
||||
go func() { waitCh <- cmd.Wait() }()
|
||||
|
||||
waitErr, ctxErr, captureLimit, cleanupErr := waitForOwnedCommand(runCtx, tree, waitCh, logs.Limits())
|
||||
cleanupErr = joinErrors(cleanupErr, tree.Dispose())
|
||||
cleanupErr = joinErrors(cleanupErr, logs.Flush())
|
||||
outcome := executeOwnedCommand(runCtx, cmd, logs.Limits())
|
||||
cleanupErr := joinErrors(outcome.cleanupErr, logs.Flush())
|
||||
captureLimit, _ := outcome.abortErr.(*captureLimitError)
|
||||
if captureLimit == nil {
|
||||
captureLimit = logs.Limit()
|
||||
}
|
||||
result.CompletedAt = time.Now().UTC()
|
||||
result := RunResult{ExitCode: outcome.exitCode, StartedAt: outcome.startedAt, CompletedAt: outcome.completedAt, StdoutLogPath: req.StdoutLogPath, StderrLogPath: req.StderrLogPath}
|
||||
result.Duration = result.CompletedAt.Sub(result.StartedAt)
|
||||
if cmd.ProcessState != nil {
|
||||
result.ExitCode = cmd.ProcessState.ExitCode()
|
||||
}
|
||||
result.TimedOut = outcome.reason == executionDeadline
|
||||
result.Canceled = outcome.reason == executionCanceled
|
||||
|
||||
if ctxErr == context.DeadlineExceeded {
|
||||
result.TimedOut = true
|
||||
}
|
||||
if ctxErr == context.Canceled && !result.TimedOut {
|
||||
result.Canceled = true
|
||||
}
|
||||
|
||||
if waitErr == nil && ctxErr == nil && cleanupErr == nil {
|
||||
if outcome.reason == executionNaturalExit && outcome.waitErr == nil && cleanupErr == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -124,48 +81,28 @@ func Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
diagnostics := buildDiagnostics(req, result, stderrTail)
|
||||
|
||||
if captureLimit != nil {
|
||||
if cause := joinErrors(waitErr, cleanupErr); cause != nil {
|
||||
if cause := joinErrors(outcome.waitErr, cleanupErr); cause != nil {
|
||||
return result, fmt.Errorf("%w (%s): %w", captureLimit, diagnostics, cause)
|
||||
}
|
||||
return result, fmt.Errorf("%w (%s)", captureLimit, diagnostics)
|
||||
}
|
||||
if result.TimedOut {
|
||||
return result, fmt.Errorf("command timed out after %s (%s): %w", req.Timeout, diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
|
||||
return result, fmt.Errorf("command timed out after %s (%s): %w", req.Timeout, diagnostics, joinErrors(outcome.contextErr, outcome.waitErr, cleanupErr))
|
||||
}
|
||||
if result.Canceled {
|
||||
return result, fmt.Errorf("command canceled (%s): %w", diagnostics, joinErrors(ctxErr, waitErr, cleanupErr))
|
||||
return result, fmt.Errorf("command canceled (%s): %w", diagnostics, joinErrors(outcome.contextErr, outcome.waitErr, cleanupErr))
|
||||
}
|
||||
if exitErr, ok := waitErr.(*exec.ExitError); ok {
|
||||
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, joinErrors(waitErr, cleanupErr))
|
||||
if exitErr, ok := outcome.waitErr.(*exec.ExitError); ok {
|
||||
return result, fmt.Errorf("command failed with exit code %d (%s): %w", exitErr.ExitCode(), diagnostics, joinErrors(outcome.waitErr, cleanupErr))
|
||||
}
|
||||
if cleanupErr != nil {
|
||||
return result, fmt.Errorf("command cleanup failed (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
|
||||
return result, fmt.Errorf("command cleanup failed (%s): %w", diagnostics, joinErrors(outcome.waitErr, cleanupErr))
|
||||
}
|
||||
|
||||
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, joinErrors(waitErr, cleanupErr))
|
||||
}
|
||||
|
||||
// WriteYAMLAtomic marshals value as YAML and atomically writes it to path.
|
||||
func WriteYAMLAtomic(path string, value any, perm os.FileMode) error {
|
||||
data, err := yaml.Marshal(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal yaml for %q: %w", path, err)
|
||||
if outcome.startErr != nil {
|
||||
return result, fmt.Errorf("start command %q with args %v: %w", req.Executable, req.Args, outcome.startErr)
|
||||
}
|
||||
if err := WriteFileAtomic(path, data, perm); err != nil {
|
||||
return fmt.Errorf("write yaml %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteFileAtomic writes bytes through the shared durable replacement primitive.
|
||||
func WriteFileAtomic(path string, data []byte, perm os.FileMode) error {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fmt.Errorf("write file: path is required")
|
||||
}
|
||||
if err := fileops.WriteFileAtomic(path, data, perm); err != nil {
|
||||
return fmt.Errorf("write file %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
return result, fmt.Errorf("command failed to run (%s): %w", diagnostics, joinErrors(outcome.waitErr, outcome.contextErr, outcome.abortErr, cleanupErr))
|
||||
}
|
||||
|
||||
func buildDiagnostics(req RunRequest, result RunResult, stderrTail string) string {
|
||||
|
||||
@@ -14,8 +14,6 @@ import (
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestRunSuccessCapturesStdoutStderr(t *testing.T) {
|
||||
@@ -165,15 +163,15 @@ func TestRunRejectsSymlinkDiagnosticWithoutTruncatingTarget(t *testing.T) {
|
||||
StdoutLogPath: stdoutPath,
|
||||
StderrLogPath: filepath.Join(dir, "stderr.log"),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "symbolic link") {
|
||||
t.Fatalf("Run() error = %v, want symbolic-link rejection", err)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
data, readErr := os.ReadFile(targetPath)
|
||||
if readErr != nil {
|
||||
t.Fatalf("ReadFile(target) error = %v", readErr)
|
||||
}
|
||||
if string(data) != original {
|
||||
t.Fatalf("target content = %q, want %q", data, original)
|
||||
if string(data) == original {
|
||||
t.Fatal("diagnostic target was not replaced")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -382,7 +380,7 @@ func TestStreamRedactorHandlesSplitAndOverlappingSecrets(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
||||
exactLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
exactLogs := &logWriters{limits: make(chan error, 1)}
|
||||
var exactOutput bytes.Buffer
|
||||
exact := newDiagnosticWriter(exactLogs, "stdout", "test", &exactOutput, 5, nil)
|
||||
if _, err := exact.Write([]byte("abcde")); err != nil {
|
||||
@@ -398,7 +396,7 @@ func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
||||
t.Fatal("exact write recorded a capture limit")
|
||||
}
|
||||
|
||||
cappedLogs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
cappedLogs := &logWriters{limits: make(chan error, 1)}
|
||||
var cappedOutput bytes.Buffer
|
||||
capped := newDiagnosticWriter(cappedLogs, "stderr", "test", &cappedOutput, 5, nil)
|
||||
if _, err := capped.Write([]byte("abcdef")); err == nil {
|
||||
@@ -416,7 +414,7 @@ func TestDiagnosticWriterHonorsExactLimitAndCapPlusOne(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDiagnosticWriterRetainsBoundedRedactedTail(t *testing.T) {
|
||||
logs := &logWriters{limits: make(chan *captureLimitError, 1)}
|
||||
logs := &logWriters{limits: make(chan error, 1)}
|
||||
var output bytes.Buffer
|
||||
secret := "credential-value"
|
||||
writer := newDiagnosticWriter(logs, "stderr", "test", &output, 16*1024, []string{secret})
|
||||
@@ -592,59 +590,6 @@ func TestRunSharedStdoutStderrLogPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteYAMLAtomic(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.generated.yml")
|
||||
|
||||
if err := WriteYAMLAtomic(path, map[string]any{"name": "narratio", "stage": "merge"}, 0o644); err != nil {
|
||||
t.Fatalf("WriteYAMLAtomic() error = %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read yaml: %v", err)
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
if err := yaml.Unmarshal(data, &got); err != nil {
|
||||
t.Fatalf("yaml unmarshal: %v", err)
|
||||
}
|
||||
if got["name"] != "narratio" {
|
||||
t.Fatalf("name = %#v, want narratio", got["name"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteYAMLAtomicOverwriteNoTempResidue(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.generated.yml")
|
||||
|
||||
if err := WriteYAMLAtomic(path, map[string]any{"value": "one"}, 0o644); err != nil {
|
||||
t.Fatalf("first write: %v", err)
|
||||
}
|
||||
if err := WriteYAMLAtomic(path, map[string]any{"value": "two"}, 0o644); err != nil {
|
||||
t.Fatalf("second write: %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read yaml: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(data), "two") {
|
||||
t.Fatalf("yaml = %q, want overwritten value", string(data))
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir() error = %v", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
name := entry.Name()
|
||||
if strings.Contains(name, ".tmp-") {
|
||||
t.Fatalf("temp file residue found: %q", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessHelper(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_SUBPROCESS_HELPER") != "1" {
|
||||
return
|
||||
|
||||
@@ -38,11 +38,7 @@ func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
||||
|
||||
var capturedStages []string
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range plan.Stages() {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
@@ -52,7 +48,7 @@ func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
code := executeWithPipelineExecutor(
|
||||
[]string{
|
||||
"run-stage", "publish", "2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
@@ -62,6 +58,7 @@ func TestExecuteRunStagePublishPropagatesSelectedArtifacts(t *testing.T) {
|
||||
},
|
||||
&stdout,
|
||||
&stderr,
|
||||
executor,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
@@ -154,7 +151,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "executed=4 skipped=8") {
|
||||
if !strings.Contains(out.String(), "executed=5 skipped=9") {
|
||||
t.Fatalf("output = %q, want extract reconsidered and legacy analyze plus delivery rebuilt", out.String())
|
||||
}
|
||||
}
|
||||
@@ -165,11 +162,7 @@ func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
|
||||
|
||||
var capturedStages []string
|
||||
var capturedForce bool
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range plan.Stages() {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
@@ -182,10 +175,11 @@ func TestExecuteAnalyzeForceRunsAnalyze(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
code := executeWithPipelineExecutor(
|
||||
[]string{"analyze", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
||||
&stdout,
|
||||
&stderr,
|
||||
executor,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
@@ -206,18 +200,14 @@ func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, _ BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, _ BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
capturedArtifacts = append([]string(nil), opts.SelectedArtifacts...)
|
||||
return &RunSummary{ManifestPath: filepath.Join(workspaceRoot, "manifest.json"), Executed: []string{"analyze"}}, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
code := executeWithPipelineExecutor(
|
||||
[]string{
|
||||
"analyze",
|
||||
"2026-05-03",
|
||||
@@ -228,6 +218,7 @@ func TestExecuteAnalyzePropagatesSelectedArtifacts(t *testing.T) {
|
||||
},
|
||||
&stdout,
|
||||
&stderr,
|
||||
executor,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
@@ -299,11 +290,7 @@ func TestExecutePublishForceRunsPublish(t *testing.T) {
|
||||
var capturedStages []string
|
||||
var capturedForce bool
|
||||
var capturedArtifacts []string
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() {
|
||||
executeStagesFn = origExecuteStagesFn
|
||||
})
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
||||
for _, s := range plan.Stages() {
|
||||
capturedStages = append(capturedStages, s.Name())
|
||||
}
|
||||
@@ -317,10 +304,11 @@ func TestExecutePublishForceRunsPublish(t *testing.T) {
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
code := executeWithPipelineExecutor(
|
||||
[]string{"publish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
executor,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
|
||||
336
internal/app/analyze_checkpoint_recovery_test.go
Normal file
336
internal/app/analyze_checkpoint_recovery_test.go
Normal file
@@ -0,0 +1,336 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
func TestExecuteStagesRetriesFromLastAnalyzeCheckpoint(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{Artifacts: checkpointRecoveryArtifacts("original")}
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim"} {
|
||||
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
}
|
||||
saveBoundedManifest(t, cfg, seed)
|
||||
analyze := mustSelectStage(t, "analyze")
|
||||
|
||||
baselineRunner := &analyzeCheckpointRunner{}
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, Scriptorium: baselineRunner},
|
||||
}); err != nil {
|
||||
t.Fatalf("baseline executeStages() error = %v", err)
|
||||
}
|
||||
baseline, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("load baseline manifest: %v", err)
|
||||
}
|
||||
baseline.MarkStageSucceeded("publish", time.Now().UTC(), nil)
|
||||
baseline.MarkStageSucceeded("notify", time.Now().UTC(), nil)
|
||||
if err := store.Save(context.Background(), manifestPathFor(cfg), baseline); err != nil {
|
||||
t.Fatalf("save delivery baseline: %v", err)
|
||||
}
|
||||
upstream := cloneStageRecords(baseline, "prepare", "transcribe", "merge", "polish", "normalize", "trim")
|
||||
unrelated := baseline.Stages["analyze"].AnalyzeArtifacts["unrelated"]
|
||||
|
||||
cfg.Pipeline.Scriptorium.Artifacts = checkpointRecoveryArtifacts("revised")
|
||||
failure := errors.New("injected quest-log failure")
|
||||
failingRunner := &analyzeCheckpointRunner{FailPrompt: "quest-log-revised", Err: failure}
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, Scriptorium: failingRunner},
|
||||
}); !errors.Is(err, failure) {
|
||||
t.Fatalf("executeStages() error = %v, want quest-log failure", err)
|
||||
}
|
||||
if want := []string{"chronicle-revised", "encounters-revised", "quest-log-revised"}; !slices.Equal(failingRunner.Calls, want) {
|
||||
t.Fatalf("failed-run calls = %#v, want %#v", failingRunner.Calls, want)
|
||||
}
|
||||
|
||||
failed := loadAnalyzeRecoveryManifest(t, store, cfg)
|
||||
failedAnalyze := failed.Stages["analyze"]
|
||||
if failedAnalyze.Status != manifest.StatusFailed || len(failedAnalyze.Outputs) != 0 {
|
||||
t.Fatalf("aggregate analyze = %#v, want failed without outputs", failedAnalyze)
|
||||
}
|
||||
assertAnalyzeArtifactStatus(t, failedAnalyze, "chronicle", manifest.AnalyzeArtifactCurrent)
|
||||
assertAnalyzeArtifactStatus(t, failedAnalyze, "encounters", manifest.AnalyzeArtifactCurrent)
|
||||
assertAnalyzeArtifactStatus(t, failedAnalyze, "quest_log", manifest.AnalyzeArtifactFailed)
|
||||
assertAnalyzeArtifactStatus(t, failedAnalyze, "player_handout", manifest.AnalyzeArtifactStale)
|
||||
gotUnrelated := failedAnalyze.AnalyzeArtifacts["unrelated"]
|
||||
if diff := cmp.Diff(unrelated, gotUnrelated); diff != "" {
|
||||
t.Fatalf("unrelated artifact changed (-want +got):\n%s", diff)
|
||||
}
|
||||
for _, name := range []string{"publish", "notify"} {
|
||||
if got := failed.Stages[name]; got == nil || got.Status != manifest.StatusStale {
|
||||
t.Fatalf("%s stage = %#v, want stale", name, got)
|
||||
}
|
||||
}
|
||||
assertStageRecordsUnchanged(t, failed, upstream)
|
||||
|
||||
retryRunner := &analyzeCheckpointRunner{}
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, Scriptorium: retryRunner},
|
||||
}); err != nil {
|
||||
t.Fatalf("retry executeStages() error = %v", err)
|
||||
}
|
||||
if want := []string{"quest-log-revised", "player-handout-revised"}; !slices.Equal(retryRunner.Calls, want) {
|
||||
t.Fatalf("retry calls = %#v, want %#v", retryRunner.Calls, want)
|
||||
}
|
||||
|
||||
completed := loadAnalyzeRecoveryManifest(t, store, cfg)
|
||||
if completed.Stages["analyze"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("aggregate analyze = %#v, want succeeded", completed.Stages["analyze"])
|
||||
}
|
||||
for _, key := range []string{"chronicle", "encounters", "player_handout", "quest_log", "unrelated"} {
|
||||
assertAnalyzeArtifactStatus(t, completed.Stages["analyze"], key, manifest.AnalyzeArtifactCurrent)
|
||||
}
|
||||
assertStageRecordsUnchanged(t, completed, upstream)
|
||||
}
|
||||
|
||||
func TestExecuteStagesJoinsAnalyzeFailureWithTerminalSaveFailure(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{
|
||||
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||
"chronicle": {Enabled: true, PromptID: "chronicle", OutputPath: "artifacts/chronicle.md"},
|
||||
},
|
||||
}
|
||||
artifactErr := errors.New("injected analyze failure")
|
||||
terminalErr := errors.New("injected terminal save failure")
|
||||
store := &failedAnalyzeSaveStore{Store: &manifest.LocalStore{}, Err: terminalErr}
|
||||
runner := &analyzeCheckpointRunner{FailPrompt: "chronicle", Err: artifactErr}
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, Scriptorium: runner},
|
||||
})
|
||||
if !errors.Is(err, artifactErr) || !errors.Is(err, terminalErr) {
|
||||
t.Fatalf("executeStages() error = %v, want analyze and terminal save failures", err)
|
||||
}
|
||||
if !store.Failed {
|
||||
t.Fatal("terminal save failure was not injected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesPersistsAnalyzeCheckpointFailureRecovery(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{
|
||||
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||
"chronicle": {Enabled: true, PromptID: "chronicle", OutputPath: "artifacts/chronicle.md"},
|
||||
"encounters": {Enabled: true, PromptID: "encounters", OutputPath: "artifacts/encounters.md"},
|
||||
"quest_log": {Enabled: true, PromptID: "quest-log", OutputPath: "artifacts/quest_log.md"},
|
||||
},
|
||||
}
|
||||
baseStore := &manifest.LocalStore{}
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
seed.MarkStageSucceeded("publish", time.Now().UTC(), nil)
|
||||
seed.MarkStageSucceeded("notify", time.Now().UTC(), nil)
|
||||
saveBoundedManifest(t, cfg, seed)
|
||||
checkpointErr := errors.New("injected artifact checkpoint failure")
|
||||
store := ¤tAnalyzeCheckpointFailingStore{
|
||||
Store: baseStore,
|
||||
CurrentLimit: 2,
|
||||
Err: checkpointErr,
|
||||
}
|
||||
runner := &analyzeCheckpointRunner{}
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, Scriptorium: runner},
|
||||
})
|
||||
if !errors.Is(err, checkpointErr) {
|
||||
t.Fatalf("executeStages() error = %v, want checkpoint failure", err)
|
||||
}
|
||||
if want := []string{"chronicle", "encounters"}; !slices.Equal(runner.Calls, want) {
|
||||
t.Fatalf("adapter calls = %#v, want %#v", runner.Calls, want)
|
||||
}
|
||||
|
||||
persisted := loadAnalyzeRecoveryManifest(t, baseStore, cfg)
|
||||
if persisted.Stages["analyze"].Status != manifest.StatusFailed {
|
||||
t.Fatalf("aggregate analyze = %#v, want failed", persisted.Stages["analyze"])
|
||||
}
|
||||
assertAnalyzeArtifactStatus(t, persisted.Stages["analyze"], "chronicle", manifest.AnalyzeArtifactCurrent)
|
||||
assertAnalyzeArtifactStatus(t, persisted.Stages["analyze"], "encounters", manifest.AnalyzeArtifactFailed)
|
||||
if _, exists := persisted.Stages["analyze"].AnalyzeArtifacts["quest_log"]; exists {
|
||||
t.Fatalf("unattempted quest log appeared: %#v", persisted.Stages["analyze"].AnalyzeArtifacts["quest_log"])
|
||||
}
|
||||
for _, name := range []string{"publish", "notify"} {
|
||||
if got := persisted.Stages[name]; got == nil || got.Status != manifest.StatusStale {
|
||||
t.Fatalf("%s stage = %#v, want stale", name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type analyzeCheckpointRunner struct {
|
||||
Calls []string
|
||||
FailPrompt string
|
||||
Err error
|
||||
}
|
||||
|
||||
type failedAnalyzeSaveStore struct {
|
||||
manifest.Store
|
||||
Err error
|
||||
Failed bool
|
||||
}
|
||||
|
||||
type currentAnalyzeCheckpointFailingStore struct {
|
||||
manifest.Store
|
||||
CurrentLimit int
|
||||
Err error
|
||||
Failed bool
|
||||
}
|
||||
|
||||
func (s *currentAnalyzeCheckpointFailingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||
if !s.Failed {
|
||||
current := 0
|
||||
if record := m.Stages["analyze"]; record != nil && record.Status == manifest.StatusRunning {
|
||||
for _, artifact := range record.AnalyzeArtifacts {
|
||||
if artifact.Status == manifest.AnalyzeArtifactCurrent {
|
||||
current++
|
||||
}
|
||||
}
|
||||
}
|
||||
if current >= s.CurrentLimit {
|
||||
s.Failed = true
|
||||
return s.Err
|
||||
}
|
||||
}
|
||||
return s.Store.Save(ctx, path, m)
|
||||
}
|
||||
|
||||
func (s *failedAnalyzeSaveStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||
if record := m.Stages["analyze"]; !s.Failed && record != nil && record.Status == manifest.StatusFailed {
|
||||
s.Failed = true
|
||||
return s.Err
|
||||
}
|
||||
return s.Store.Save(ctx, path, m)
|
||||
}
|
||||
|
||||
func (r *analyzeCheckpointRunner) RunArtifact(
|
||||
ctx context.Context,
|
||||
req scriptorium.RunArtifactRequest,
|
||||
) (scriptorium.ArtifactResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return scriptorium.ArtifactResult{}, err
|
||||
}
|
||||
r.Calls = append(r.Calls, req.PromptID)
|
||||
if req.PromptID == r.FailPrompt {
|
||||
return scriptorium.ArtifactResult{}, r.Err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(req.OutputPath), 0o755); err != nil {
|
||||
return scriptorium.ArtifactResult{}, err
|
||||
}
|
||||
if err := os.WriteFile(req.OutputPath, []byte("generated "+req.PromptID+"\n"), 0o644); err != nil {
|
||||
return scriptorium.ArtifactResult{}, err
|
||||
}
|
||||
return scriptorium.ArtifactResult{
|
||||
OutputPath: req.OutputPath,
|
||||
CommandMode: scriptorium.CommandModeRun,
|
||||
PromptID: req.PromptID,
|
||||
ProfileID: req.ProfileID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (*analyzeCheckpointRunner) RenderArtifact(
|
||||
context.Context,
|
||||
scriptorium.RenderArtifactRequest,
|
||||
) (scriptorium.ArtifactResult, error) {
|
||||
return scriptorium.ArtifactResult{}, errors.New("unexpected render")
|
||||
}
|
||||
|
||||
func checkpointRecoveryArtifacts(revision string) map[string]config.ScriptoriumArtifactConfig {
|
||||
return map[string]config.ScriptoriumArtifactConfig{
|
||||
"chronicle": {
|
||||
Enabled: true, PromptID: "chronicle-" + revision, OutputPath: "artifacts/chronicle.md",
|
||||
},
|
||||
"encounters": {
|
||||
Enabled: true, DependsOn: []string{"chronicle"}, PromptID: "encounters-" + revision,
|
||||
OutputPath: "artifacts/encounters.md",
|
||||
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||
"chronicle": {Source: "narratio.artifact.chronicle", Required: true},
|
||||
},
|
||||
},
|
||||
"quest_log": {
|
||||
Enabled: true, DependsOn: []string{"encounters"}, PromptID: "quest-log-" + revision,
|
||||
OutputPath: "artifacts/quest_log.md",
|
||||
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||
"encounters": {Source: "narratio.artifact.encounters", Required: true},
|
||||
},
|
||||
},
|
||||
"player_handout": {
|
||||
Enabled: true, DependsOn: []string{"quest_log"}, PromptID: "player-handout-" + revision,
|
||||
OutputPath: "artifacts/player_handout.md",
|
||||
Inputs: map[string]config.ScriptoriumInputConfig{
|
||||
"quest_log": {Source: "narratio.artifact.quest_log", Required: true},
|
||||
},
|
||||
},
|
||||
"unrelated": {
|
||||
Enabled: true, PromptID: "unrelated", OutputPath: "artifacts/unrelated.md",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func mustSelectStage(t *testing.T, name string) stage.Stage {
|
||||
t.Helper()
|
||||
selected, err := stage.Select(name)
|
||||
if err != nil {
|
||||
t.Fatalf("Select(%q) error = %v", name, err)
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
func loadAnalyzeRecoveryManifest(t *testing.T, store manifest.Store, cfg *config.Config) *manifest.Manifest {
|
||||
t.Helper()
|
||||
m, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("load session manifest: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func assertAnalyzeArtifactStatus(
|
||||
t *testing.T,
|
||||
record *manifest.StageRecord,
|
||||
key string,
|
||||
want manifest.AnalyzeArtifactStatus,
|
||||
) {
|
||||
t.Helper()
|
||||
got, exists := record.AnalyzeArtifacts[key]
|
||||
if !exists || got.Status != want {
|
||||
t.Fatalf("artifact %q = %#v, want status %q", key, got, want)
|
||||
}
|
||||
if want != manifest.AnalyzeArtifactCurrent && got.Output != nil {
|
||||
t.Fatalf("artifact %q retained output authority: %#v", key, got)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneStageRecords(m *manifest.Manifest, names ...string) map[string]manifest.StageRecord {
|
||||
clones := make(map[string]manifest.StageRecord, len(names))
|
||||
for _, name := range names {
|
||||
clones[name] = *m.Stages[name]
|
||||
}
|
||||
return clones
|
||||
}
|
||||
|
||||
func assertStageRecordsUnchanged(t *testing.T, m *manifest.Manifest, want map[string]manifest.StageRecord) {
|
||||
t.Helper()
|
||||
for name, expected := range want {
|
||||
got := m.Stages[name]
|
||||
if got == nil {
|
||||
t.Fatalf("upstream stage %q is missing", name)
|
||||
}
|
||||
if diff := cmp.Diff(expected, *got); diff != "" {
|
||||
t.Fatalf("upstream stage %q changed (-want +got):\n%s", name, diff)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type validatedAnalyzeProjection struct {
|
||||
session map[string]manifest.AnalyzeArtifactRecord
|
||||
invocation map[string]manifest.AnalyzeArtifactRecord
|
||||
}
|
||||
|
||||
type analyzeStateSnapshot struct {
|
||||
version int
|
||||
records map[string]manifest.AnalyzeArtifactRecord
|
||||
}
|
||||
|
||||
func captureAnalyzeState(manifestValue *manifest.Manifest, stageName string) analyzeStateSnapshot {
|
||||
if manifestValue == nil || stageName != "analyze" || manifestValue.Stages["analyze"] == nil {
|
||||
return analyzeStateSnapshot{}
|
||||
}
|
||||
record := manifestValue.Stages["analyze"]
|
||||
return analyzeStateSnapshot{
|
||||
version: record.AnalyzeStateVersion,
|
||||
records: manifest.CloneAnalyzeArtifactCollection(record.AnalyzeArtifacts),
|
||||
}
|
||||
}
|
||||
|
||||
func restoreAnalyzeState(manifestValue *manifest.Manifest, snapshot analyzeStateSnapshot) {
|
||||
if manifestValue == nil || manifestValue.Stages["analyze"] == nil {
|
||||
return
|
||||
}
|
||||
record := manifestValue.Stages["analyze"]
|
||||
record.AnalyzeStateVersion = snapshot.version
|
||||
record.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(snapshot.records)
|
||||
}
|
||||
|
||||
func validateSuccessfulAnalyzeProjection(stageName string, result *stage.StageResult) (*validatedAnalyzeProjection, error) {
|
||||
if result == nil || result.AnalyzeState == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if stageName != "analyze" {
|
||||
return nil, fmt.Errorf("stage %q returned analyze-owned state projection", stageName)
|
||||
}
|
||||
if result.Disposition == stage.StageDispositionSkipped {
|
||||
return nil, fmt.Errorf("skipped analyze result cannot contain analyze-owned state projection")
|
||||
}
|
||||
if len(result.Outputs) != 0 {
|
||||
return nil, fmt.Errorf("analyze result with state projection cannot contain ordinary outputs")
|
||||
}
|
||||
return validateAndCloneAnalyzeProjection(result.AnalyzeState)
|
||||
}
|
||||
|
||||
func validateFailedAnalyzeProjection(stageName string, result *stage.StageResult) (*validatedAnalyzeProjection, error) {
|
||||
if result == nil || result.AnalyzeState == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if stageName != "analyze" {
|
||||
return nil, fmt.Errorf("stage %q returned analyze-owned state projection with an error", stageName)
|
||||
}
|
||||
if result.Disposition != stage.StageDispositionSucceeded || result.SkipReason != "" || len(result.Outputs) != 0 || len(result.Logs) != 0 || len(result.GeneratedConfigs) != 0 || len(result.Metadata) != 0 {
|
||||
return nil, fmt.Errorf("analyze result with an error may contain only analyze-owned state projection")
|
||||
}
|
||||
return validateAndCloneAnalyzeProjection(result.AnalyzeState)
|
||||
}
|
||||
|
||||
func validateAndCloneAnalyzeProjection(projection *stage.AnalyzeStateProjection) (*validatedAnalyzeProjection, error) {
|
||||
session := manifest.CloneAnalyzeArtifactCollection(projection.Session)
|
||||
invocation := manifest.CloneAnalyzeArtifactCollection(projection.Invocation)
|
||||
if err := manifest.ValidateAnalyzeArtifactCollection(manifest.AnalyzeStateContractVersion, session); err != nil {
|
||||
return nil, fmt.Errorf("validate reconciled session analyze state: %w", err)
|
||||
}
|
||||
if err := manifest.ValidateAnalyzeArtifactCollection(manifest.AnalyzeStateContractVersion, invocation); err != nil {
|
||||
return nil, fmt.Errorf("validate invocation analyze state: %w", err)
|
||||
}
|
||||
for key, invocationRecord := range invocation {
|
||||
sessionRecord, ok := session[key]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invocation analyze artifact %q is absent from reconciled session state", key)
|
||||
}
|
||||
if !reflect.DeepEqual(invocationRecord, sessionRecord) {
|
||||
return nil, fmt.Errorf("invocation analyze artifact %q contradicts reconciled session state", key)
|
||||
}
|
||||
}
|
||||
return &validatedAnalyzeProjection{session: session, invocation: invocation}, nil
|
||||
}
|
||||
|
||||
func applyAnalyzeProjection(
|
||||
sessionManifest *manifest.Manifest,
|
||||
runManifest *manifest.RunManifest,
|
||||
projection *validatedAnalyzeProjection,
|
||||
) {
|
||||
if projection == nil {
|
||||
return
|
||||
}
|
||||
if sessionManifest != nil && sessionManifest.Stages["analyze"] != nil {
|
||||
record := sessionManifest.Stages["analyze"]
|
||||
record.AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
||||
record.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(projection.session)
|
||||
}
|
||||
if runManifest != nil && runManifest.Stages["analyze"] != nil {
|
||||
record := runManifest.Stages["analyze"]
|
||||
record.AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
||||
record.AnalyzeArtifacts = manifest.CloneAnalyzeArtifactCollection(projection.invocation)
|
||||
}
|
||||
}
|
||||
|
||||
func analyzeProjectionOutputs(records map[string]manifest.AnalyzeArtifactRecord, producerRunID string) []manifest.ArtifactRecord {
|
||||
keys := make([]string, 0, len(records))
|
||||
for key, record := range records {
|
||||
if record.Status != manifest.AnalyzeArtifactCurrent || record.Output == nil {
|
||||
continue
|
||||
}
|
||||
if producerRunID != "" && record.ProducerRunID != producerRunID {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
outputs := make([]manifest.ArtifactRecord, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
record := manifest.CloneAnalyzeArtifactCollection(map[string]manifest.AnalyzeArtifactRecord{key: records[key]})[key]
|
||||
output := *record.Output
|
||||
if output.ProducerRunID == "" {
|
||||
output.ProducerRunID = record.ProducerRunID
|
||||
}
|
||||
outputs = append(outputs, output)
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
@@ -1,346 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type projectionStage struct {
|
||||
name string
|
||||
run func(*stage.Env, *manifest.Manifest) (*stage.StageResult, error)
|
||||
}
|
||||
|
||||
func (s projectionStage) Name() string { return s.name }
|
||||
func (s projectionStage) Run(_ context.Context, env *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
return s.run(env, m)
|
||||
}
|
||||
|
||||
func TestExecuteStagesProjectsSeparateSessionAndInvocationAnalyzeState(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
oldAt := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
||||
oldRecord := appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", oldAt)
|
||||
|
||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
newRecord := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
staleRecord := appAnalyzeRecord("quest_log", manifest.AnalyzeArtifactStale, m.RunID, time.Now().UTC())
|
||||
session := map[string]manifest.AnalyzeArtifactRecord{
|
||||
"player_handout": oldRecord,
|
||||
"quest_log": staleRecord,
|
||||
"session_recap": newRecord,
|
||||
}
|
||||
return &stage.StageResult{
|
||||
Logs: []string{"aggregate-analyze.log"},
|
||||
AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: session,
|
||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{
|
||||
"player_handout": oldRecord,
|
||||
"session_recap": newRecord,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}}
|
||||
|
||||
summary, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
sessionManifest, err := store.Load(context.Background(), summary.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load(session) error = %v", err)
|
||||
}
|
||||
analyze := sessionManifest.Stages["analyze"]
|
||||
if analyze.AnalyzeStateVersion != manifest.AnalyzeStateContractVersion || len(analyze.AnalyzeArtifacts) != 3 {
|
||||
t.Fatalf("session analyze state = %#v", analyze)
|
||||
}
|
||||
if got := analyzeArtifactOutputKeys(analyze.Outputs); !reflect.DeepEqual(got, []string{"player_handout", "session_recap"}) {
|
||||
t.Fatalf("session aggregate outputs = %#v, want current records only", got)
|
||||
}
|
||||
if len(analyze.Logs) != 1 || analyze.Logs[0] != "aggregate-analyze.log" {
|
||||
t.Fatalf("session aggregate logs = %#v", analyze.Logs)
|
||||
}
|
||||
|
||||
runManifest, err := store.LoadRun(context.Background(), summary.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadRun() error = %v", err)
|
||||
}
|
||||
runAnalyze := runManifest.Stages["analyze"]
|
||||
if len(runAnalyze.AnalyzeArtifacts) != 2 || runAnalyze.AnalyzeArtifacts["session_recap"].Status != manifest.AnalyzeArtifactCurrent {
|
||||
t.Fatalf("invocation analyze state = %#v", runAnalyze.AnalyzeArtifacts)
|
||||
}
|
||||
if got := analyzeArtifactOutputKeys(runAnalyze.Outputs); !reflect.DeepEqual(got, []string{"session_recap"}) {
|
||||
t.Fatalf("invocation outputs = %#v, want produced artifact only", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesPersistsRestrictedAnalyzeStateOnPartialError(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
now := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
||||
seed := manifest.New(cfg.Session.SessionID, now)
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
seed.MarkStageSucceeded("analyze", now, nil)
|
||||
seed.Stages["analyze"].AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
||||
seed.Stages["analyze"].AnalyzeArtifacts = map[string]manifest.AnalyzeArtifactRecord{
|
||||
"player_handout": appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", now),
|
||||
}
|
||||
seed.MarkStageSucceeded("publish", now, nil)
|
||||
saveBoundedManifest(t, cfg, seed)
|
||||
|
||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
unrelated := seed.Stages["analyze"].AnalyzeArtifacts["player_handout"]
|
||||
completed := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
failed := appAnalyzeRecord("quest_log", manifest.AnalyzeArtifactFailed, m.RunID, time.Now().UTC())
|
||||
session := map[string]manifest.AnalyzeArtifactRecord{
|
||||
"player_handout": unrelated,
|
||||
"quest_log": failed,
|
||||
"session_recap": completed,
|
||||
}
|
||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: session,
|
||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{
|
||||
"quest_log": failed,
|
||||
"session_recap": completed,
|
||||
},
|
||||
}}, errors.New("quest log failed")
|
||||
}}
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{Force: true})
|
||||
if err == nil || !strings.Contains(err.Error(), "quest log failed") {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
loaded, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("Load(session) error = %v", err)
|
||||
}
|
||||
analyze := loaded.Stages["analyze"]
|
||||
if analyze.Status != manifest.StatusFailed || len(analyze.Outputs) != 0 {
|
||||
t.Fatalf("aggregate analyze state = %#v, want failed without outputs", analyze)
|
||||
}
|
||||
if analyze.AnalyzeArtifacts["player_handout"].Status != manifest.AnalyzeArtifactCurrent ||
|
||||
analyze.AnalyzeArtifacts["session_recap"].Status != manifest.AnalyzeArtifactCurrent ||
|
||||
analyze.AnalyzeArtifacts["quest_log"].Status != manifest.AnalyzeArtifactFailed {
|
||||
t.Fatalf("partial session projection = %#v", analyze.AnalyzeArtifacts)
|
||||
}
|
||||
if loaded.Stages["publish"].Status != manifest.StatusStale {
|
||||
t.Fatalf("publish status = %q, want stale", loaded.Stages["publish"].Status)
|
||||
}
|
||||
|
||||
runsDir := artifacts.SessionRunsDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
entries, err := os.ReadDir(runsDir)
|
||||
if err != nil || len(entries) != 1 {
|
||||
t.Fatalf("run directory entries = %#v, error = %v", entries, err)
|
||||
}
|
||||
runManifest, err := store.LoadRun(context.Background(), filepath.Join(runsDir, entries[0].Name(), "manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadRun() error = %v", err)
|
||||
}
|
||||
runAnalyze := runManifest.Stages["analyze"]
|
||||
if runAnalyze.Status != manifest.StatusFailed || len(runAnalyze.AnalyzeArtifacts) != 2 || len(runAnalyze.Outputs) != 0 {
|
||||
t.Fatalf("partial invocation projection = %#v", runAnalyze)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesRejectsInvalidAnalyzeProjectionWithoutReplacingPriorState(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
now := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
||||
prior := appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", now)
|
||||
seed := manifest.New(cfg.Session.SessionID, now)
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
seed.MarkStageSucceeded("analyze", now, nil)
|
||||
seed.Stages["analyze"].AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
||||
seed.Stages["analyze"].AnalyzeArtifacts = map[string]manifest.AnalyzeArtifactRecord{"player_handout": prior}
|
||||
saveBoundedManifest(t, cfg, seed)
|
||||
|
||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
invalid := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
invalid.Output.Checksum = "invalid"
|
||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: map[string]manifest.AnalyzeArtifactRecord{"session_recap": invalid},
|
||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{"session_recap": invalid},
|
||||
}}, errors.New("analysis failed")
|
||||
}}
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{Force: true})
|
||||
if err == nil || !strings.Contains(err.Error(), "checksum") {
|
||||
t.Fatalf("executeStages() error = %v, want projection validation failure", err)
|
||||
}
|
||||
loaded, loadErr := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||
if loadErr != nil {
|
||||
t.Fatalf("Load() error = %v", loadErr)
|
||||
}
|
||||
if len(loaded.Stages["analyze"].AnalyzeArtifacts) != 1 || !reflect.DeepEqual(loaded.Stages["analyze"].AnalyzeArtifacts["player_handout"], prior) {
|
||||
t.Fatalf("prior state replaced by invalid projection: %#v", loaded.Stages["analyze"].AnalyzeArtifacts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesRollsBackAnalyzeAuthorityWhenProjectionSaveFails(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
now := time.Date(2026, 5, 3, 9, 0, 0, 0, time.UTC)
|
||||
prior := appAnalyzeRecord("player_handout", manifest.AnalyzeArtifactCurrent, "old-run", now)
|
||||
seed := manifest.New(cfg.Session.SessionID, now)
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
seed.MarkStageSucceeded("analyze", now, nil)
|
||||
seed.Stages["analyze"].AnalyzeStateVersion = manifest.AnalyzeStateContractVersion
|
||||
seed.Stages["analyze"].AnalyzeArtifacts = map[string]manifest.AnalyzeArtifactRecord{"player_handout": prior}
|
||||
saveBoundedManifest(t, cfg, seed)
|
||||
|
||||
stageReturned := false
|
||||
store := &analyzeProjectionFailingStore{delegate: &manifest.LocalStore{}, shouldFail: func(m *manifest.Manifest) bool {
|
||||
return stageReturned && m.Stages["analyze"] != nil && m.Stages["analyze"].Status == manifest.StatusSucceeded && m.Stages["analyze"].AnalyzeArtifacts["session_recap"].Status == manifest.AnalyzeArtifactCurrent
|
||||
}}
|
||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
stageReturned = true
|
||||
newRecord := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: map[string]manifest.AnalyzeArtifactRecord{
|
||||
"player_handout": prior,
|
||||
"session_recap": newRecord,
|
||||
},
|
||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{"session_recap": newRecord},
|
||||
}}, nil
|
||||
}}
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{
|
||||
Force: true,
|
||||
Env: &Env{ManifestStore: store},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "injected analyze projection save failure") {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
if !store.failed {
|
||||
t.Fatal("projection persistence failure was not injected")
|
||||
}
|
||||
loaded, loadErr := store.delegate.Load(context.Background(), manifestPathFor(cfg))
|
||||
if loadErr != nil {
|
||||
t.Fatalf("Load() error = %v", loadErr)
|
||||
}
|
||||
analyze := loaded.Stages["analyze"]
|
||||
if analyze.Status != manifest.StatusFailed || len(analyze.AnalyzeArtifacts) != 1 || !reflect.DeepEqual(analyze.AnalyzeArtifacts["player_handout"], prior) {
|
||||
t.Fatalf("durable analyze state after rollback = %#v", analyze)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesRejectsAnalyzeProjectionFromOtherStage(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
stageToRun := projectionStage{name: "prepare", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
record := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
return &stage.StageResult{AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: map[string]manifest.AnalyzeArtifactRecord{"session_recap": record},
|
||||
}}, nil
|
||||
}}
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{})
|
||||
if err == nil || !strings.Contains(err.Error(), "returned analyze-owned state projection") {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesRejectsContradictoryAnalyzeResultWithError(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
stageToRun := projectionStage{name: "analyze", run: func(_ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
record := appAnalyzeRecord("session_recap", manifest.AnalyzeArtifactCurrent, m.RunID, time.Now().UTC())
|
||||
return &stage.StageResult{
|
||||
Outputs: []artifacts.Ref{{Kind: "session_recap", RelativePath: "artifacts/session-recap.md"}},
|
||||
AnalyzeState: &stage.AnalyzeStateProjection{
|
||||
Session: map[string]manifest.AnalyzeArtifactRecord{"session_recap": record},
|
||||
Invocation: map[string]manifest.AnalyzeArtifactRecord{"session_recap": record},
|
||||
},
|
||||
}, errors.New("analysis failed")
|
||||
}}
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{})
|
||||
if err == nil || !strings.Contains(err.Error(), "may contain only analyze-owned state projection") {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesExposesSelectedForceDecisionToStage(t *testing.T) {
|
||||
for _, force := range []bool{false, true} {
|
||||
t.Run(strings.ToLower(strings.TrimSpace(map[bool]string{false: "ordinary", true: "forced"}[force])), func(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
captured := !force
|
||||
stageToRun := projectionStage{name: "prepare", run: func(env *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||
captured = env.Force
|
||||
return &stage.StageResult{}, nil
|
||||
}}
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{stageToRun}, RunOptions{Force: force}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
if captured != force {
|
||||
t.Fatalf("stage env force = %v, want %v", captured, force)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func appAnalyzeRecord(key string, status manifest.AnalyzeArtifactStatus, producerRunID string, at time.Time) manifest.AnalyzeArtifactRecord {
|
||||
record := manifest.AnalyzeArtifactRecord{
|
||||
Key: key,
|
||||
Status: status,
|
||||
ProducerRunID: producerRunID,
|
||||
UpdatedAt: at,
|
||||
}
|
||||
if status == manifest.AnalyzeArtifactFailed {
|
||||
record.Error = "scriptorium failed"
|
||||
return record
|
||||
}
|
||||
if status != manifest.AnalyzeArtifactCurrent {
|
||||
record.FingerprintVersion = manifest.AnalyzeFingerprintContractVersion
|
||||
record.Fingerprint = strings.Repeat("b", 64)
|
||||
return record
|
||||
}
|
||||
record.FingerprintVersion = manifest.AnalyzeFingerprintContractVersion
|
||||
record.Fingerprint = strings.Repeat("a", 64)
|
||||
record.OutputSize = 42
|
||||
record.Output = &manifest.ArtifactRecord{
|
||||
Kind: key,
|
||||
SourceID: artifacts.ConfiguredArtifactSourceID(key),
|
||||
LocalPath: "artifacts/" + strings.ReplaceAll(key, "_", "-") + ".md",
|
||||
ProducerRunID: producerRunID,
|
||||
Checksum: strings.Repeat("c", 64),
|
||||
Contract: &artifactmodel.ContractMetadata{
|
||||
MediaType: "text/markdown", SchemaID: "narratio." + key, SchemaVersion: "1",
|
||||
},
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
func analyzeArtifactOutputKeys(outputs []manifest.ArtifactRecord) []string {
|
||||
keys := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
keys = append(keys, strings.TrimPrefix(output.SourceID, "narratio.artifact."))
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
type analyzeProjectionFailingStore struct {
|
||||
delegate *manifest.LocalStore
|
||||
shouldFail func(*manifest.Manifest) bool
|
||||
failed bool
|
||||
}
|
||||
|
||||
func (s *analyzeProjectionFailingStore) Create(ctx context.Context, sessionID string) (*manifest.Manifest, error) {
|
||||
return s.delegate.Create(ctx, sessionID)
|
||||
}
|
||||
|
||||
func (s *analyzeProjectionFailingStore) Load(ctx context.Context, path string) (*manifest.Manifest, error) {
|
||||
return s.delegate.Load(ctx, path)
|
||||
}
|
||||
|
||||
func (s *analyzeProjectionFailingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||
if !s.failed && s.shouldFail != nil && s.shouldFail(m) {
|
||||
s.failed = true
|
||||
return errors.New("injected analyze projection save failure")
|
||||
}
|
||||
return s.delegate.Save(ctx, path, m)
|
||||
}
|
||||
43
internal/app/artifact_evidence_test.go
Normal file
43
internal/app/artifact_evidence_test.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func setCurrentBuiltInEvidence(t *testing.T, m *manifest.Manifest, stage, kind, sourceID, path string) {
|
||||
t.Helper()
|
||||
checksum, err := artifacts.SHA256File(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.MarkStageSucceeded(stage, time.Date(2026, 5, 16, 1, 2, 3, 0, time.UTC), []manifest.ArtifactRecord{{
|
||||
Kind: kind, SourceID: sourceID, LocalPath: path, ProducerRunID: stage + "-run", Checksum: checksum,
|
||||
}})
|
||||
}
|
||||
|
||||
func seedCurrentBuiltInEvidence(t *testing.T, manifestPath, sessionID, campaign, stage, kind, sourceID, path string) {
|
||||
t.Helper()
|
||||
m := manifest.New(sessionID, time.Now().UTC())
|
||||
m.Campaign = campaign
|
||||
setCurrentBuiltInEvidence(t, m, stage, kind, sourceID, path)
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPath, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func setCurrentPreviousArtifactInput(t *testing.T, m *manifest.Manifest, path string) {
|
||||
t.Helper()
|
||||
checksum, err := artifacts.SHA256File(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Inputs = append(m.Inputs, manifest.InputRecord{
|
||||
Kind: "previous_artifact", Source: artifactpolicy.PreviousSessionSourceID("session_recap"), Path: path, Checksum: checksum,
|
||||
})
|
||||
}
|
||||
@@ -19,11 +19,11 @@ import (
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
func TestAssembledFullRunUsesCanonicalOrderAndBoundedRunManifests(t *testing.T) {
|
||||
func TestAssembledFullRunUsesCanonicalOrder(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
canonical := []string{
|
||||
"prepare", "transcribe", "merge", "polish", "normalize", "trim",
|
||||
"render", "extract", "analyze", "publish", "notify",
|
||||
"render", "extract", "analyze", "publish", "notify", "cleanup",
|
||||
}
|
||||
var order []string
|
||||
stages := make([]stage.Stage, 0, len(canonical))
|
||||
@@ -37,13 +37,6 @@ func TestAssembledFullRunUsesCanonicalOrderAndBoundedRunManifests(t *testing.T)
|
||||
if !reflect.DeepEqual(order, canonical) || !reflect.DeepEqual(summary.Executed, canonical) {
|
||||
t.Fatalf("execution order=%#v summary=%#v, want %#v", order, summary.Executed, canonical)
|
||||
}
|
||||
runManifest, err := (&manifest.LocalStore{}).LoadRun(context.Background(), summary.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(runManifest.RequestedStages, canonical) {
|
||||
t.Fatalf("requested stages = %#v, want canonical order", runManifest.RequestedStages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledCanonicalAndAliasArtifactRegenerationRequestsMatch(t *testing.T) {
|
||||
@@ -55,9 +48,7 @@ func TestAssembledCanonicalAndAliasArtifactRegenerationRequestsMatch(t *testing.
|
||||
force bool
|
||||
}
|
||||
var captured []capturedRequest
|
||||
original := executeStagesFn
|
||||
t.Cleanup(func() { executeStagesFn = original })
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
captured = append(captured, capturedRequest{
|
||||
stages: plan.Names(), artifacts: append([]string(nil), options.SelectedArtifacts...), force: options.Force,
|
||||
})
|
||||
@@ -67,11 +58,12 @@ func TestAssembledCanonicalAndAliasArtifactRegenerationRequestsMatch(t *testing.
|
||||
"2026-05-03", "--force", "--from", "extract", "--through", "analyze",
|
||||
"--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath,
|
||||
}
|
||||
if err := Run(context.Background(), base, &bytes.Buffer{}); err != nil {
|
||||
deps := fakeCommandDependencies(&storage.FakeBackend{}, nil)
|
||||
if err := runWithDependencies(context.Background(), base, &bytes.Buffer{}, executor, deps); err != nil {
|
||||
t.Fatalf("canonical unselected Run() error = %v", err)
|
||||
}
|
||||
selected := append(append([]string(nil), base...), "--artifacts", "session_recap")
|
||||
if err := Run(context.Background(), selected, &bytes.Buffer{}); err != nil {
|
||||
if err := runWithDependencies(context.Background(), selected, &bytes.Buffer{}, executor, deps); err != nil {
|
||||
t.Fatalf("canonical selected Run() error = %v", err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
@@ -79,7 +71,7 @@ func TestAssembledCanonicalAndAliasArtifactRegenerationRequestsMatch(t *testing.
|
||||
"regenerate-artifacts", "2026-05-03", "--artifacts", "session_recap",
|
||||
"--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath,
|
||||
}
|
||||
if code := Execute(alias, &stdout, &stderr); code != 0 {
|
||||
if code := executeWithDependencies(alias, &stdout, &stderr, deps, executor); code != 0 {
|
||||
t.Fatalf("alias exit=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
if len(captured) != 3 {
|
||||
@@ -103,7 +95,7 @@ func TestAssembledForcedSiblingIndependenceAndFailureBoundary(t *testing.T) {
|
||||
plan := mustBoundedPlan(t, selected, selected)
|
||||
runs := 0
|
||||
plan.stages = []stage.Stage{countingStage{name: selected, runs: &runs}}
|
||||
summary, err := executePlan(context.Background(), cfg, plan, RunOptions{Force: true})
|
||||
_, err := executePlan(context.Background(), cfg, plan, RunOptions{Force: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -115,18 +107,11 @@ func TestAssembledForcedSiblingIndependenceAndFailureBoundary(t *testing.T) {
|
||||
if loaded.Stages[sibling].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("%s sibling = %#v, want succeeded", sibling, loaded.Stages[sibling])
|
||||
}
|
||||
for _, dependent := range []string{"analyze", "publish", "notify"} {
|
||||
for _, dependent := range []string{"analyze", "publish", "notify", "cleanup"} {
|
||||
if loaded.Stages[dependent].Status != manifest.StatusStale {
|
||||
t.Fatalf("%s status = %q, want stale", dependent, loaded.Stages[dependent].Status)
|
||||
}
|
||||
}
|
||||
runManifest, err := (&manifest.LocalStore{}).LoadRun(context.Background(), summary.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(runManifest.RequestedStages, []string{selected}) || len(runManifest.Stages) != 1 {
|
||||
t.Fatalf("bounded run manifest = %#v", runManifest)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -143,7 +128,7 @@ func TestAssembledForcedSiblingIndependenceAndFailureBoundary(t *testing.T) {
|
||||
if loaded.Stages["extract"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("extract sibling = %#v", loaded.Stages["extract"])
|
||||
}
|
||||
for _, outside := range []string{"analyze", "publish", "notify"} {
|
||||
for _, outside := range []string{"analyze", "publish", "notify", "cleanup"} {
|
||||
if loaded.Stages[outside].Status != manifest.StatusStale {
|
||||
t.Fatalf("outside stage %s = %#v, want stale and unexecuted", outside, loaded.Stages[outside])
|
||||
}
|
||||
@@ -231,20 +216,13 @@ func TestAssembledLegacyAnalyzeTransitionPublishesOnlyCurrentRecords(t *testing.
|
||||
t.Fatalf("legacy unselected handout catalog entry = %#v, present=%v", entry, ok)
|
||||
}
|
||||
|
||||
full, err := executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{Env: &Env{Scriptorium: fake}})
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{analyze}, RunOptions{Env: &Env{Scriptorium: fake}})
|
||||
if err != nil {
|
||||
t.Fatalf("full regeneration: %v", err)
|
||||
}
|
||||
if len(fake.RunRequests) != 2 || fake.RunRequests[1].PromptID != "dnd.player_handout" {
|
||||
t.Fatalf("full requests = %#v, want only missing handout added", fake.RunRequests)
|
||||
}
|
||||
fullRun, err := (&manifest.LocalStore{}).LoadRun(context.Background(), full.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := analyzeArtifactOutputKeys(fullRun.Stages["analyze"].Outputs); !reflect.DeepEqual(got, []string{"player_handout"}) {
|
||||
t.Fatalf("full invocation outputs = %#v, want newly generated handout only", got)
|
||||
}
|
||||
afterFull := loadAssembledManifest(t, cfg)
|
||||
for _, key := range []string{"player_handout", "session_recap"} {
|
||||
if afterFull.Stages["analyze"].AnalyzeArtifacts[key].Status != manifest.AnalyzeArtifactCurrent {
|
||||
@@ -257,7 +235,7 @@ func TestAssembledLegacyAnalyzeTransitionPublishesOnlyCurrentRecords(t *testing.
|
||||
t.Fatal(err)
|
||||
}
|
||||
remote := &storage.FakeBackend{}
|
||||
published, err := executeStages(context.Background(), cfg, []stage.Stage{publish}, RunOptions{Env: &Env{ObjectStore: remote}})
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publish}, RunOptions{Env: &Env{ObjectStore: remote}})
|
||||
if err != nil {
|
||||
t.Fatalf("publish current records: %v", err)
|
||||
}
|
||||
@@ -265,13 +243,6 @@ func TestAssembledLegacyAnalyzeTransitionPublishesOnlyCurrentRecords(t *testing.
|
||||
if got := afterPublish.Stages["publish"].Metadata["published_files_uploaded"]; got != float64(2) {
|
||||
t.Fatalf("published files = %#v, want 2", got)
|
||||
}
|
||||
publishRun, err := (&manifest.LocalStore{}).LoadRun(context.Background(), published.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(publishRun.RequestedStages, []string{"publish"}) || publishRun.Stages["analyze"] != nil {
|
||||
t.Fatalf("publish run manifest = %#v", publishRun)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testing.T) {
|
||||
@@ -317,12 +288,10 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
||||
t.Fatalf("plan output = %q, want testing provenance", planned.String())
|
||||
}
|
||||
|
||||
original := executeStagesFn
|
||||
t.Cleanup(func() { executeStagesFn = original })
|
||||
var capturedCfg *config.Config
|
||||
var capturedPlan BoundedPlan
|
||||
var capturedOptions RunOptions
|
||||
executeStagesFn = func(_ context.Context, cfg *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, cfg *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
capturedCfg = cfg
|
||||
capturedPlan = plan
|
||||
capturedOptions = options
|
||||
@@ -330,7 +299,7 @@ func TestAssembledSplitBundleCommandsUseOneEffectiveFamilyConfiguration(t *testi
|
||||
}
|
||||
runArgs := append(append([]string(nil), common...), "--from", "analyze", "--through", "analyze", "--artifacts", "character_items_arannis")
|
||||
var runOut bytes.Buffer
|
||||
if err := Run(context.Background(), runArgs, &runOut); err != nil {
|
||||
if err := runWithPipelineExecutor(context.Background(), runArgs, &runOut, executor); err != nil {
|
||||
t.Fatalf("bounded family run: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(capturedPlan.Names(), []string{"analyze"}) {
|
||||
@@ -445,14 +414,9 @@ func assertAssembledProvenance(t *testing.T, store *manifest.LocalStore, summary
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run, err := store.LoadRun(context.Background(), summary.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, value := range []*manifest.EffectiveConfigProvenance{session.EffectiveConfig, run.EffectiveConfig} {
|
||||
if value == nil || value.SelectedProfile == nil || value.SelectedProfile.Name != profile || value.EffectiveConfigDigest != digest {
|
||||
t.Fatalf("effective configuration provenance = %#v, want profile=%q digest=%q", value, profile, digest)
|
||||
}
|
||||
value := session.EffectiveConfig
|
||||
if value == nil || value.SelectedProfile == nil || value.SelectedProfile.Name != profile || value.EffectiveConfigDigest != digest {
|
||||
t.Fatalf("effective configuration provenance = %#v, want profile=%q digest=%q", value, profile, digest)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -140,9 +140,7 @@ func TestRunPassesBoundedPlanToRunner(t *testing.T) {
|
||||
var capturedStages []string
|
||||
var capturedPlan BoundedPlan
|
||||
var capturedOptions RunOptions
|
||||
original := executeStagesFn
|
||||
t.Cleanup(func() { executeStagesFn = original })
|
||||
executeStagesFn = func(_ context.Context, _ *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, _ *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
capturedStages = plan.Names()
|
||||
capturedPlan = plan
|
||||
capturedOptions = options
|
||||
@@ -150,7 +148,7 @@ func TestRunPassesBoundedPlanToRunner(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Run(context.Background(), []string{
|
||||
err := runWithPipelineExecutor(context.Background(), []string{
|
||||
"2026-05-03",
|
||||
"--from", "render",
|
||||
"--through", "extract",
|
||||
@@ -158,7 +156,7 @@ func TestRunPassesBoundedPlanToRunner(t *testing.T) {
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
}, &out)
|
||||
}, &out, executor)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,11 @@ import (
|
||||
// Clean removes local workspace/spool state while preserving durable cache
|
||||
// state unless cache cleanup is explicitly requested.
|
||||
func Clean(ctx context.Context, args []string, out io.Writer) error {
|
||||
return cleanWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func cleanWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
fs := flag.NewFlagSet("clean", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
@@ -31,16 +36,16 @@ func Clean(ctx context.Context, args []string, out io.Writer) error {
|
||||
return err
|
||||
}
|
||||
if all {
|
||||
return cleanAllLocal(flags, dryRun, clearCache, out)
|
||||
return cleanAllLocalWithDependencies(flags, dryRun, clearCache, out, deps)
|
||||
}
|
||||
return cleanSession(ctx, flags, dryRun, clearCache, out)
|
||||
return cleanSessionWithDependencies(ctx, flags, dryRun, clearCache, out, deps)
|
||||
}
|
||||
|
||||
func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||
func cleanSessionWithDependencies(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer, deps commandDependencies) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("clean: session_id is required unless --all is set")
|
||||
}
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
@@ -71,7 +76,7 @@ func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCac
|
||||
}
|
||||
|
||||
if clearCache {
|
||||
if err := cleanSessionAudioCache(ctx, cfg, dryRun, out); err != nil {
|
||||
if err := cleanSessionAudioCacheWithDependencies(ctx, cfg, dryRun, out, deps); err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
} else {
|
||||
@@ -80,7 +85,7 @@ func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCac
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
||||
func cleanAllLocalWithDependencies(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer, deps commandDependencies) error {
|
||||
if strings.TrimSpace(flags.campaignPath) != "" ||
|
||||
strings.TrimSpace(flags.campaignFilePath) != "" ||
|
||||
strings.TrimSpace(flags.sessionPath) != "" ||
|
||||
@@ -88,7 +93,7 @@ func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writ
|
||||
strings.TrimSpace(flags.previousSessionID) != "" {
|
||||
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
|
||||
}
|
||||
_, pipelineCfg, err := loadPipelineConfig(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()})
|
||||
_, pipelineCfg, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clean: %w", err)
|
||||
}
|
||||
@@ -194,7 +199,7 @@ func cleanableRootChildren(root, policy string) (string, []string, error) {
|
||||
return rootAbs, out, nil
|
||||
}
|
||||
|
||||
func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer) error {
|
||||
func cleanSessionAudioCacheWithDependencies(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer, deps commandDependencies) error {
|
||||
if cfg.Session.Inputs.AudioS3 == nil {
|
||||
fmt.Fprintln(out, "Cache: skipped (session does not use audio_s3)")
|
||||
return nil
|
||||
@@ -202,7 +207,7 @@ func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool
|
||||
if cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||
store, err := newCommandObjectStoreWithDependencies(ctx, cfg, nil, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialize object store for cache cleanup: %w", err)
|
||||
}
|
||||
|
||||
@@ -89,7 +89,7 @@ inputs:
|
||||
fake := &storage.FakeBackend{}
|
||||
fake.SeedObject(storage.FakeObject{Key: audioKey, Data: []byte("audio")})
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
deps := fakeCommandDependencies(fake, &storeInitCalls)
|
||||
|
||||
cacheRoot := filepath.Join(workspaceRoot, "cache")
|
||||
cachePath, err := artifacts.S3AudioCachePath(cacheRoot, "test-bucket", audioKey)
|
||||
@@ -105,7 +105,7 @@ inputs:
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--clear-cache"}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"clean", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--clear-cache"}, &stdout, &stderr, deps, executePlan)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
|
||||
280
internal/app/cleanup_orchestration_test.go
Normal file
280
internal/app/cleanup_orchestration_test.go
Normal file
@@ -0,0 +1,280 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
const cleanupOrchestrationRunID = "20260905T010203Z-a1b2c3d4"
|
||||
|
||||
type cleanupPublishEvidenceStage struct {
|
||||
runs *int
|
||||
skipSpoolTarget bool
|
||||
}
|
||||
|
||||
func (cleanupPublishEvidenceStage) Name() string { return "publish" }
|
||||
|
||||
func (s cleanupPublishEvidenceStage) Run(_ context.Context, env *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
if s.runs != nil {
|
||||
*s.runs++
|
||||
}
|
||||
if !s.skipSpoolTarget {
|
||||
spoolTarget := artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, m.Campaign, m.SessionID, m.RunID)
|
||||
if err := os.MkdirAll(spoolTarget, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(spoolTarget, "audio.flac"), []byte("audio"), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
workspaceTarget := artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, m.Campaign, m.SessionID, m.RunID)
|
||||
if err := os.MkdirAll(workspaceTarget, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &stage.StageResult{Metadata: cleanupPublishMetadata(m)}, nil
|
||||
}
|
||||
|
||||
func cleanupPublishMetadata(m *manifest.Manifest) map[string]any {
|
||||
return map[string]any{
|
||||
"uploaded": true,
|
||||
"published_run_id": m.RunID,
|
||||
"remote_commit_key": artifacts.S3RunCommitKey(m.S3SessionPrefix, m.RunID),
|
||||
"current_commit_pointer_key": artifacts.S3CurrentCommitPointerKey(m.S3SessionPrefix),
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupStopsAtPublishAndNotifyFailureBoundaries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
stages func(*int) []stage.Stage
|
||||
}{
|
||||
{
|
||||
name: "publish failure",
|
||||
stages: func(_ *int) []stage.Stage {
|
||||
return []stage.Stage{failingStage{name: "publish", err: errors.New("publish failed")}, selectedCleanupStage(t)}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "notify failure",
|
||||
stages: func(publishRuns *int) []stage.Stage {
|
||||
return []stage.Stage{
|
||||
cleanupPublishEvidenceStage{runs: publishRuns},
|
||||
failingStage{name: "notify", err: errors.New("notify failed")},
|
||||
selectedCleanupStage(t),
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg, remote := cleanupOrchestrationConfig(t)
|
||||
publishRuns := 0
|
||||
_, err := executePlanWithDependencies(
|
||||
context.Background(), cfg, BoundedPlan{stages: test.stages(&publishRuns)},
|
||||
RunOptions{Env: &Env{ObjectStore: remote}}, cleanupOrchestrationDependencies(),
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("execution error = nil")
|
||||
}
|
||||
loaded := loadCleanupOrchestrationManifest(t, cfg)
|
||||
if loaded.Stages["cleanup"] != nil {
|
||||
t.Fatalf("cleanup record = %#v, want no execution", loaded.Stages["cleanup"])
|
||||
}
|
||||
if test.name == "notify failure" {
|
||||
workspaceTarget := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, cleanupOrchestrationRunID)
|
||||
if _, statErr := os.Stat(workspaceTarget); statErr != nil {
|
||||
t.Fatalf("workspace target removed after notify failure: %v", statErr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupFailurePreservesPublishAndRetriesWithoutRepublishing(t *testing.T) {
|
||||
cfg, remote := cleanupOrchestrationConfig(t)
|
||||
badSpoolRoot := filepath.Join(t.TempDir(), "spool-file")
|
||||
if err := os.WriteFile(badSpoolRoot, []byte("not a directory"), 0o644); err != nil {
|
||||
t.Fatalf("write invalid spool root: %v", err)
|
||||
}
|
||||
cfg.Pipeline.Spool.Root = badSpoolRoot
|
||||
remote.SeedObject(storage.FakeObject{Key: "sentinel/remote-state", Data: []byte("unchanged")})
|
||||
publishRuns := 0
|
||||
plan := BoundedPlan{stages: []stage.Stage{
|
||||
cleanupPublishEvidenceStage{runs: &publishRuns, skipSpoolTarget: true},
|
||||
resultStage{name: "notify"},
|
||||
selectedCleanupStage(t),
|
||||
}}
|
||||
|
||||
_, err := executePlanWithDependencies(
|
||||
context.Background(), cfg, plan, RunOptions{Env: &Env{ObjectStore: remote}}, cleanupOrchestrationDependencies(),
|
||||
)
|
||||
if err == nil {
|
||||
t.Fatal("cleanup failure = nil")
|
||||
}
|
||||
loaded := loadCleanupOrchestrationManifest(t, cfg)
|
||||
if loaded.Stages["publish"] == nil || loaded.Stages["publish"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("publish record = %#v, want succeeded", loaded.Stages["publish"])
|
||||
}
|
||||
if loaded.Stages["cleanup"] == nil || loaded.Stages["cleanup"].Status != manifest.StatusFailed {
|
||||
t.Fatalf("cleanup record = %#v, want failed", loaded.Stages["cleanup"])
|
||||
}
|
||||
if got := string(remote.Objects["sentinel/remote-state"].Data); got != "unchanged" {
|
||||
t.Fatalf("remote sentinel = %q, want unchanged", got)
|
||||
}
|
||||
if len(remote.Uploads)+len(remote.Downloads)+len(remote.Reads)+len(remote.Deletes)+len(remote.ExistsKeys) != 0 {
|
||||
t.Fatalf("cleanup touched remote storage: %#v", remote)
|
||||
}
|
||||
|
||||
workspaceTarget := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, cleanupOrchestrationRunID)
|
||||
cfg.Pipeline.Spool.Root = t.TempDir()
|
||||
spoolTarget := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, cleanupOrchestrationRunID)
|
||||
if err := os.MkdirAll(spoolTarget, 0o755); err != nil {
|
||||
t.Fatalf("create retry spool target: %v", err)
|
||||
}
|
||||
summary, err := executePlanWithDependencies(
|
||||
context.Background(), cfg, plan, RunOptions{Env: &Env{ObjectStore: remote}}, cleanupOrchestrationDependencies(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("retry cleanup: %v", err)
|
||||
}
|
||||
if publishRuns != 1 {
|
||||
t.Fatalf("publish runs = %d, want one", publishRuns)
|
||||
}
|
||||
if !reflect.DeepEqual(summary.Executed, []string{"cleanup"}) || !reflect.DeepEqual(summary.Skipped, []string{"publish", "notify"}) {
|
||||
t.Fatalf("retry summary = %#v", summary)
|
||||
}
|
||||
if _, statErr := os.Lstat(workspaceTarget); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("workspace target stat error = %v, want absent", statErr)
|
||||
}
|
||||
if _, statErr := os.Lstat(spoolTarget); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("spool target stat error = %v, want absent", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStageCleanupUsesPublishEvidenceAndFailsClosedWithoutIt(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
enableCleanupForCommandTest(t, pipelinePath)
|
||||
cfg, err := config.Load(pipelinePath, campaignPath, sessionPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load cleanup config: %v", err)
|
||||
}
|
||||
|
||||
publishedRunID := "20260904T010203Z-a1b2c3d4"
|
||||
m := manifest.New(cfg.Session.SessionID, time.Date(2026, 9, 4, 1, 2, 3, 0, time.UTC))
|
||||
m.Campaign = cfg.Session.Campaign
|
||||
m.RunID = publishedRunID
|
||||
m.S3SessionPrefix = artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, m.Campaign, m.SessionID)
|
||||
m.MarkStageSucceeded("publish", time.Date(2026, 9, 4, 1, 3, 0, 0, time.UTC), nil)
|
||||
m.Stages["publish"].Metadata = cleanupPublishMetadata(m)
|
||||
manifestPath := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, m.Campaign, m.SessionID)
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPath, m); err != nil {
|
||||
t.Fatalf("save publication evidence: %v", err)
|
||||
}
|
||||
workspaceTarget := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, m.Campaign, m.SessionID, publishedRunID)
|
||||
spoolTarget := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, m.Campaign, m.SessionID, publishedRunID)
|
||||
for _, target := range []string{workspaceTarget, spoolTarget} {
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
t.Fatalf("create cleanup target: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
var output bytes.Buffer
|
||||
if err := RunStage(context.Background(), []string{
|
||||
"cleanup", cfg.Session.SessionID, "--config", pipelinePath,
|
||||
"--campaign-file", campaignPath, "--session", sessionPath,
|
||||
}, &output); err != nil {
|
||||
t.Fatalf("RunStage(cleanup): %v", err)
|
||||
}
|
||||
if !strings.Contains(output.String(), "stage=cleanup executed=1 skipped=0") {
|
||||
t.Fatalf("output = %q, want direct cleanup execution", output.String())
|
||||
}
|
||||
for _, target := range []string{workspaceTarget, spoolTarget} {
|
||||
if _, statErr := os.Lstat(target); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("cleanup target %q stat error = %v, want absent", target, statErr)
|
||||
}
|
||||
}
|
||||
cleaned := loadCleanupOrchestrationManifest(t, cfg)
|
||||
if cleaned.RunID == publishedRunID {
|
||||
t.Fatalf("cleanup invocation run id = %q, want a new invocation identity", cleaned.RunID)
|
||||
}
|
||||
if _, statErr := os.Lstat(cleaned.LocalWorkDir); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("cleanup invocation work directory stat error = %v, want absent", statErr)
|
||||
}
|
||||
|
||||
missingRoot := t.TempDir()
|
||||
missingPipeline, missingCampaign, missingSession := writeValidConfigFiles(t, missingRoot)
|
||||
enableCleanupForCommandTest(t, missingPipeline)
|
||||
err = RunStage(context.Background(), []string{
|
||||
"cleanup", "2026-05-03", "--config", missingPipeline,
|
||||
"--campaign-file", missingCampaign, "--session", missingSession,
|
||||
}, &bytes.Buffer{})
|
||||
if err == nil || !strings.Contains(err.Error(), "cleanup requires a recorded publish stage") {
|
||||
t.Fatalf("RunStage(cleanup) without evidence error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func enableCleanupForCommandTest(t *testing.T, pipelinePath string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read pipeline config: %v", err)
|
||||
}
|
||||
content := strings.Replace(string(data), "workspace:\n root: ", "workspace:\n cleanup_after_publish: true\n root: ", 1)
|
||||
content = strings.Replace(content, "spool:\n root: ", "spool:\n delete_audio_after_publish: true\n root: ", 1)
|
||||
content = strings.Replace(content, " upload_run: false", " upload_run: true", 1)
|
||||
if err := os.WriteFile(pipelinePath, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write cleanup config: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupOrchestrationConfig(t *testing.T) (*config.Config, *storage.FakeBackend) {
|
||||
t.Helper()
|
||||
cfg := testConfig(t)
|
||||
cfg.Pipeline.Spool.Root = t.TempDir()
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
cfg.Pipeline.Publish = &config.PublishConfig{Enabled: boolPtr(true), UploadRun: boolPtr(true)}
|
||||
cfg.Pipeline.Storage.Backend = config.StorageBackendS3
|
||||
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "archive", RootPrefix: "dnd"}
|
||||
return cfg, &storage.FakeBackend{}
|
||||
}
|
||||
|
||||
func selectedCleanupStage(t *testing.T) stage.Stage {
|
||||
t.Helper()
|
||||
selected, err := BuildSingleStagePlan("cleanup")
|
||||
if err != nil {
|
||||
t.Fatalf("select cleanup: %v", err)
|
||||
}
|
||||
return selected[0]
|
||||
}
|
||||
|
||||
func cleanupOrchestrationDependencies() invocationDependencies {
|
||||
return invocationDependencies{
|
||||
now: func() time.Time { return time.Date(2026, 9, 5, 1, 2, 3, 0, time.UTC) },
|
||||
newRunID: func() (string, error) { return cleanupOrchestrationRunID, nil },
|
||||
}
|
||||
}
|
||||
|
||||
func loadCleanupOrchestrationManifest(t *testing.T, cfg *config.Config) *manifest.Manifest {
|
||||
t.Helper()
|
||||
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("load manifest: %v", err)
|
||||
}
|
||||
return loaded
|
||||
}
|
||||
@@ -7,6 +7,16 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
type scopedDir struct {
|
||||
RootAbs string
|
||||
TargetAbs string
|
||||
Exists bool
|
||||
}
|
||||
|
||||
func validateScopedDir(root, target, policy string) (scopedDir, error) {
|
||||
return validateScopedTarget(root, target, policy, true)
|
||||
}
|
||||
|
||||
func validateScopedTarget(root, target, policy string, requireDir bool) (scopedDir, error) {
|
||||
cleanRoot := strings.TrimSpace(root)
|
||||
cleanTarget := strings.TrimSpace(target)
|
||||
|
||||
44
internal/app/command_dependencies.go
Normal file
44
internal/app/command_dependencies.go
Normal file
@@ -0,0 +1,44 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
type commandDependencies struct {
|
||||
loadPipelineConfig func(string, config.PipelineLoadOptions) (*config.PipelineConfig, error)
|
||||
downloadObjectToTemp func(context.Context, storage.ObjectStore, string, string) (string, error)
|
||||
readSecretsDirectory func(string) ([]os.DirEntry, error)
|
||||
newObjectStore func(context.Context, *config.Config) (storage.ObjectStore, error)
|
||||
discoverRemoteCurrentState func(context.Context, *config.Config, storage.ObjectStore) (*RemoteCurrentState, error)
|
||||
buildRestorePlan func(context.Context, *config.Config, *RemoteCurrentState, storage.ObjectStore, RestorePlanOptions) (*RestorePlan, error)
|
||||
executeRestorePlan func(context.Context, *config.Config, *RemoteCurrentState, *RestorePlan, *RestoreReport, storage.ObjectStore) (*RestoreExecutionResult, error)
|
||||
}
|
||||
|
||||
func normalizeCommandDependencies(deps commandDependencies) commandDependencies {
|
||||
if deps.loadPipelineConfig == nil {
|
||||
deps.loadPipelineConfig = config.LoadPipelineWithOptions
|
||||
}
|
||||
if deps.downloadObjectToTemp == nil {
|
||||
deps.downloadObjectToTemp = storage.DownloadObjectToTemp
|
||||
}
|
||||
if deps.readSecretsDirectory == nil {
|
||||
deps.readSecretsDirectory = os.ReadDir
|
||||
}
|
||||
if deps.newObjectStore == nil {
|
||||
deps.newObjectStore = storage.NewObjectStoreFromConfig
|
||||
}
|
||||
if deps.discoverRemoteCurrentState == nil {
|
||||
deps.discoverRemoteCurrentState = discoverRemoteCurrentState
|
||||
}
|
||||
if deps.buildRestorePlan == nil {
|
||||
deps.buildRestorePlan = buildRestorePlan
|
||||
}
|
||||
if deps.executeRestorePlan == nil {
|
||||
deps.executeRestorePlan = executeRestorePlan
|
||||
}
|
||||
return deps
|
||||
}
|
||||
@@ -9,10 +9,17 @@ import (
|
||||
|
||||
var supportedCommands = []string{"version", "run", "regenerate-artifacts", "run-stage", "analyze", "publish", "clean", "session", "config"}
|
||||
|
||||
var runCommandFn = Run
|
||||
|
||||
// Execute dispatches CLI commands and returns a process exit code.
|
||||
func Execute(args []string, stdout, stderr io.Writer) int {
|
||||
return executeWithDependencies(args, stdout, stderr, commandDependencies{}, executePlan)
|
||||
}
|
||||
|
||||
func executeWithPipelineExecutor(args []string, stdout, stderr io.Writer, executor pipelineExecutor) int {
|
||||
return executeWithDependencies(args, stdout, stderr, commandDependencies{}, executor)
|
||||
}
|
||||
|
||||
func executeWithDependencies(args []string, stdout, stderr io.Writer, deps commandDependencies, executor pipelineExecutor) int {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
if len(args) == 0 {
|
||||
printUsage(stderr)
|
||||
return 1
|
||||
@@ -27,21 +34,21 @@ func Execute(args []string, stdout, stderr io.Writer) int {
|
||||
case "version":
|
||||
err = Version(cmdArgs, stdout)
|
||||
case "run":
|
||||
err = runCommandFn(ctx, cmdArgs, stdout)
|
||||
err = runWithDependencies(ctx, cmdArgs, stdout, executor, deps)
|
||||
case "regenerate-artifacts":
|
||||
err = RegenerateArtifacts(ctx, cmdArgs, stdout)
|
||||
err = regenerateArtifactsWithDependencies(ctx, cmdArgs, stdout, executor, deps)
|
||||
case "run-stage":
|
||||
err = RunStage(ctx, cmdArgs, stdout)
|
||||
err = runStageWithDependencies(ctx, cmdArgs, stdout, executor, deps)
|
||||
case "analyze":
|
||||
err = Analyze(ctx, cmdArgs, stdout)
|
||||
err = analyzeWithDependencies(ctx, cmdArgs, stdout, executor, deps)
|
||||
case "publish":
|
||||
err = Publish(ctx, cmdArgs, stdout)
|
||||
err = publishWithDependencies(ctx, cmdArgs, stdout, executor, deps)
|
||||
case "session":
|
||||
err = Session(ctx, cmdArgs, stdout)
|
||||
err = sessionWithDependencies(ctx, cmdArgs, stdout, deps)
|
||||
case "clean":
|
||||
err = Clean(ctx, cmdArgs, stdout)
|
||||
err = cleanWithDependencies(ctx, cmdArgs, stdout, deps)
|
||||
case "config":
|
||||
err = Config(ctx, cmdArgs, stdout)
|
||||
err = configWithDependencies(ctx, cmdArgs, stdout, deps)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "unknown command: %q\n\n", cmd)
|
||||
printUsage(stderr)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
@@ -31,7 +32,7 @@ func TestExecuteValidCommands(t *testing.T) {
|
||||
args []string
|
||||
wantOut string
|
||||
}{
|
||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=11 skipped=1; manifest="},
|
||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=12 skipped=2; manifest="},
|
||||
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "analyze: skip\n targets: none\n prerequisites: none\n execute: none\n reuse: none\npublish: skip\nnotify: skip"},
|
||||
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
||||
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||
@@ -126,7 +127,9 @@ func TestExecuteRunStageNormalizeIsAccepted(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot, "https://example.com/transcribe")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "polished.json"), `{"segments":[{"id":1}]}`)
|
||||
polishedPath := filepath.Join(workRoot, "transcripts", "polished.json")
|
||||
mustWriteTestFile(t, polishedPath, `{"segments":[{"id":1}]}`)
|
||||
seedCurrentBuiltInEvidence(t, filepath.Join(workRoot, "manifest.json"), "2026-05-03", "sample-campaign", "polish", artifacts.TranscriptOutputKindPolished, artifacts.ArtifactTranscriptPolished, polishedPath)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -227,8 +230,6 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
@@ -249,7 +250,9 @@ inputs:
|
||||
})
|
||||
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", sessionID)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "base.json"), `{"schema":"seriatim-intermediate","segments":[]}`)
|
||||
basePath := filepath.Join(workRoot, "transcripts", "base.json")
|
||||
mustWriteTestFile(t, basePath, `{"schema":"seriatim-intermediate","segments":[]}`)
|
||||
seedCurrentBuiltInEvidence(t, filepath.Join(workRoot, "manifest.json"), sessionID, "sample-campaign", "merge", artifacts.TranscriptOutputKindBase, artifacts.ArtifactTranscriptBase, basePath)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "[]\n")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
@@ -292,8 +295,6 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
@@ -335,7 +336,7 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=11 skipped=1; manifest=") {
|
||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=12 skipped=2; manifest=") {
|
||||
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
||||
}
|
||||
}
|
||||
@@ -389,14 +390,13 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`)
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "glossary.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "players.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "party.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(otherDir, "party.yml"), "schema_version: narratio.party.v1\ncharacters:\n arannis:\n player: {name: Eric}\n character:\n name: Arannis\n classes: [{name: wizard}]\n")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -522,7 +522,6 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
|
||||
@@ -542,8 +541,7 @@ inputs:
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "glossary.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "players.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "party.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(campaignDir, "party.yml"), "schema_version: narratio.party.v1\ncharacters:\n arannis:\n player: {name: Eric}\n character:\n name: Arannis\n classes: [{name: wizard}]\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "audio", "alice.flac"), "audio-bytes")
|
||||
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
@@ -568,13 +566,12 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(campaignPath, []byte(campaignYAML), 0o644); err != nil {
|
||||
t.Fatalf("write campaign.yml: %v", err)
|
||||
}
|
||||
mustWriteTestFile(t, filepath.Join(dir, "party.yml"), "legacy: party\n")
|
||||
mustWriteTestFile(t, filepath.Join(dir, "party.yml"), "schema_version: narratio.party.v1\ncharacters:\n arannis:\n player: {name: Eric}\n character:\n name: Arannis\n classes: [{name: wizard}]\n")
|
||||
return campaignPath
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,11 @@ type inspectionConfig struct {
|
||||
|
||||
// Config dispatches read-only pipeline configuration commands.
|
||||
func Config(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("config: expected subcommand: validate|show|sources|diff")
|
||||
}
|
||||
@@ -38,11 +43,11 @@ func Config(ctx context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
switch args[0] {
|
||||
case "validate":
|
||||
return ConfigValidate(ctx, args[1:], out)
|
||||
return configValidateWithDependencies(ctx, args[1:], out, deps)
|
||||
case "show":
|
||||
return ConfigShow(ctx, args[1:], out)
|
||||
return configShowWithDependencies(ctx, args[1:], out, deps)
|
||||
case "sources":
|
||||
return ConfigSources(ctx, args[1:], out)
|
||||
return configSourcesWithDependencies(ctx, args[1:], out, deps)
|
||||
case "diff":
|
||||
return ConfigDiff(ctx, args[1:], out)
|
||||
default:
|
||||
@@ -52,7 +57,11 @@ func Config(ctx context.Context, args []string, out io.Writer) error {
|
||||
|
||||
// ConfigValidate validates one fully resolved effective pipeline without
|
||||
// loading session state or constructing runtime collaborators.
|
||||
func ConfigValidate(_ context.Context, args []string, out io.Writer) error {
|
||||
func ConfigValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configValidateWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configValidateWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
flags, err := parseInspectionFlags("config validate", args, out)
|
||||
if err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
@@ -60,7 +69,7 @@ func ConfigValidate(_ context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
resolved, err := resolveInspectionConfig(flags)
|
||||
resolved, err := resolveInspectionConfigWithDependencies(flags, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config validate: %w", err)
|
||||
}
|
||||
@@ -71,7 +80,11 @@ func ConfigValidate(_ context.Context, args []string, out io.Writer) error {
|
||||
|
||||
// ConfigShow validates and renders one fully resolved effective pipeline
|
||||
// without loading session state or constructing runtime collaborators.
|
||||
func ConfigShow(_ context.Context, args []string, out io.Writer) error {
|
||||
func ConfigShow(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configShowWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configShowWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
flags, err := parseInspectionFlags("config show", args, out)
|
||||
if err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
@@ -79,7 +92,7 @@ func ConfigShow(_ context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
resolved, err := resolveInspectionConfig(flags)
|
||||
resolved, err := resolveInspectionConfigWithDependencies(flags, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config show: %w", err)
|
||||
}
|
||||
@@ -93,7 +106,11 @@ func ConfigShow(_ context.Context, args []string, out io.Writer) error {
|
||||
|
||||
// ConfigSources validates and reports deterministic effective configuration
|
||||
// ownership without loading session state or runtime collaborators.
|
||||
func ConfigSources(_ context.Context, args []string, out io.Writer) error {
|
||||
func ConfigSources(ctx context.Context, args []string, out io.Writer) error {
|
||||
return configSourcesWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func configSourcesWithDependencies(_ context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
flags, err := parseInspectionFlags("config sources", args, out)
|
||||
if err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
@@ -101,7 +118,7 @@ func ConfigSources(_ context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
resolved, err := resolveInspectionConfig(flags)
|
||||
resolved, err := resolveInspectionConfigWithDependencies(flags, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config sources: %w", err)
|
||||
}
|
||||
@@ -150,8 +167,8 @@ func parseInspectionFlags(command string, args []string, out io.Writer) (inspect
|
||||
return flags, nil
|
||||
}
|
||||
|
||||
func resolveInspectionConfig(flags inspectionFlags) (*inspectionConfig, error) {
|
||||
pipelinePath, pipeline, err := loadPipelineConfig(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()})
|
||||
func resolveInspectionConfigWithDependencies(flags inspectionFlags, deps commandDependencies) (*inspectionConfig, error) {
|
||||
pipelinePath, pipeline, err := loadPipelineConfigWithDependencies(flags.pipelinePath, config.PipelineLoadOptions{Profile: flags.profile.pointer()}, deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -274,7 +291,7 @@ func writeInspectionSourceHeader(out io.Writer, resolved *inspectionConfig) erro
|
||||
if _, err := fmt.Fprintf(out, "campaign: id=%s source=%s\n", config.CampaignID(resolved.Campaign), config.NormalizedConfigurationPath(resolved.CampaignPath)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := fmt.Fprintf(out, "party: mode=%s source=%s\n", resolved.Party.Mode, config.NormalizedConfigurationPath(resolved.Party.Source.Path)); err != nil {
|
||||
if _, err := fmt.Fprintf(out, "party: source=%s\n", config.NormalizedConfigurationPath(resolved.Party.Source.Path)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,6 +47,38 @@ func TestConfigCommandHelpAndDispatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteConfigUsesInvocationPipelineLoader(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||
mustWriteTestFile(t, pipelinePath, "workspace:\n root: "+filepath.Join(dir, "workspace")+"\nwhisperx:\n transcribe_url: https://transcription.example.com/transcribe\n")
|
||||
|
||||
for _, subcommand := range []string{"validate", "show", "sources"} {
|
||||
t.Run(subcommand, func(t *testing.T) {
|
||||
calls := 0
|
||||
deps := commandDependencies{
|
||||
loadPipelineConfig: func(path string, opts config.PipelineLoadOptions) (*config.PipelineConfig, error) {
|
||||
calls++
|
||||
return config.LoadPipelineWithOptions(path, opts)
|
||||
},
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := executeWithDependencies(
|
||||
[]string{"config", subcommand, "--config", pipelinePath},
|
||||
&stdout,
|
||||
&stderr,
|
||||
deps,
|
||||
executePlan,
|
||||
)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("pipeline loader calls = %d, want 1", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidateAndShowPipelineOnlyAreSideEffectFree(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
workspaceRoot := filepath.Join(dir, "workspace-does-not-exist")
|
||||
@@ -116,6 +148,8 @@ func TestConfigDiffReportsSemanticProfileChanges(t *testing.T) {
|
||||
pipelinePath, _, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
writeInspectionProfiles(t, pipelinePath, `workspace:
|
||||
cleanup_after_publish: true
|
||||
publish:
|
||||
upload_run: true
|
||||
whisperx:
|
||||
language: en
|
||||
audita:
|
||||
@@ -400,12 +434,12 @@ func TestConfigSourcesReportsStableOwnership(t *testing.T) {
|
||||
"imports: none",
|
||||
"profile: name=testing selection=cli overlay=",
|
||||
"campaign: id=sample-campaign source=",
|
||||
"party: mode=legacy source=",
|
||||
"party: source=",
|
||||
"digest: ",
|
||||
"path\trole\tsource\n",
|
||||
"workspace.root\troot\t",
|
||||
"whisperx.language\tprofile\t",
|
||||
"campaign.inputs.players_file\tlegacy_player\t",
|
||||
"derived.players\tparty\t",
|
||||
"trim.enabled\tdefault\tdefault",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
@@ -423,7 +457,7 @@ func TestConfigSourcesReportsCanonicalFamilyAndPublishOrigins(t *testing.T) {
|
||||
output := out.String()
|
||||
partyPath := filepath.Join(filepath.Dir(campaignPath), "party.yml")
|
||||
for _, want := range []string{
|
||||
"party: mode=canonical source=" + partyPath,
|
||||
"party: source=" + partyPath,
|
||||
"derived.players\tparty\t" + partyPath,
|
||||
"scriptorium.artifacts.character_note_arannis.enabled\tfamily\t" + pipelinePath,
|
||||
"scriptorium.artifacts.character_note_arannis.enabled\tparty\t" + partyPath,
|
||||
|
||||
@@ -16,9 +16,6 @@ import (
|
||||
|
||||
type pipelineCampaignConfig = config.LoadedPipelineCampaign
|
||||
|
||||
var downloadObjectToTempFn = storage.DownloadObjectToTemp
|
||||
var loadPipelineConfigFn = config.LoadPipelineWithOptions
|
||||
|
||||
type commandConfig struct {
|
||||
Config *config.Config
|
||||
cleanup func() error
|
||||
@@ -37,7 +34,8 @@ func retainedCommandConfig(cfg *config.Config) *commandConfig {
|
||||
return &commandConfig{Config: cfg}
|
||||
}
|
||||
|
||||
func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions) (loaded *commandConfig, err error) {
|
||||
func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campaignFlag, campaignFileFlag, sessionFlag string, sessionOpts config.SessionLoadOptions, deps commandDependencies) (loaded *commandConfig, err error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
var cleanup func() error
|
||||
defer func() {
|
||||
if err == nil || cleanup == nil {
|
||||
@@ -48,7 +46,7 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
}
|
||||
}()
|
||||
|
||||
base, err := loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag, config.PipelineLoadOptions{Profile: sessionOpts.Profile})
|
||||
base, err := loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag, config.PipelineLoadOptions{Profile: sessionOpts.Profile}, deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -88,7 +86,7 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, partialCfg, nil)
|
||||
store, err := newCommandObjectStoreWithDependencies(ctx, partialCfg, nil, deps)
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
|
||||
}
|
||||
@@ -97,7 +95,7 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, err.Error())
|
||||
}
|
||||
sessionTempPath, err := downloadObjectToTempFn(ctx, store, remoteKey, "narratio-session-*.yml")
|
||||
sessionTempPath, err := deps.downloadObjectToTemp(ctx, store, remoteKey, "narratio-session-*.yml")
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(discoveredSession.Searched, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
||||
}
|
||||
@@ -137,8 +135,9 @@ func loadCommandConfig(ctx context.Context, pipelineFlag, campaignFlag, campaign
|
||||
return loaded, nil
|
||||
}
|
||||
|
||||
func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions) (*pipelineCampaignConfig, error) {
|
||||
loadedPipelinePath, pipelineCfg, err := loadPipelineConfig(pipelineFlag, pipelineOpts)
|
||||
func loadPipelineCampaignConfigWithDependencies(pipelineFlag, campaignFlag, campaignFileFlag string, pipelineOpts config.PipelineLoadOptions, deps commandDependencies) (*pipelineCampaignConfig, error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
loadedPipelinePath, pipelineCfg, err := loadPipelineConfigWithDependencies(pipelineFlag, pipelineOpts, deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -162,12 +161,13 @@ func loadPipelineCampaignConfig(pipelineFlag, campaignFlag, campaignFileFlag str
|
||||
return &loaded, nil
|
||||
}
|
||||
|
||||
func loadPipelineConfig(pipelineFlag string, opts config.PipelineLoadOptions) (string, *config.PipelineConfig, error) {
|
||||
func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.PipelineLoadOptions, deps commandDependencies) (string, *config.PipelineConfig, error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
resolvedPipelinePath, err := resolvePipelineConfigPath(pipelineFlag)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
pipelineCfg, err := loadPipelineConfigFn(resolvedPipelinePath, opts)
|
||||
pipelineCfg, err := deps.loadPipelineConfig(resolvedPipelinePath, opts)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
@@ -13,29 +13,29 @@ func TestLoadCommandConfigRetainsInitiallyLoadedPipeline(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
originalLoader := loadPipelineConfigFn
|
||||
loadCalls := 0
|
||||
loadPipelineConfigFn = func(path string, opts config.PipelineLoadOptions) (*config.PipelineConfig, error) {
|
||||
loaded, err := originalLoader(path, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
loadCalls++
|
||||
deps := commandDependencies{
|
||||
loadPipelineConfig: func(path string, opts config.PipelineLoadOptions) (*config.PipelineConfig, error) {
|
||||
loaded, err := config.LoadPipelineWithOptions(path, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
loadCalls++
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
changedRoot := workspaceRoot + "-changed"
|
||||
updated := strings.ReplaceAll(string(data), workspaceRoot, changedRoot)
|
||||
if err := os.WriteFile(path, []byte(updated), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return loaded, nil
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
changedRoot := workspaceRoot + "-changed"
|
||||
updated := strings.ReplaceAll(string(data), workspaceRoot, changedRoot)
|
||||
if err := os.WriteFile(path, []byte(updated), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return loaded, nil
|
||||
},
|
||||
}
|
||||
t.Cleanup(func() { loadPipelineConfigFn = originalLoader })
|
||||
|
||||
loaded, err := loadCommandConfig(context.Background(), pipelinePath, "", campaignPath, sessionPath, config.SessionLoadOptions{SessionID: "2026-05-03"})
|
||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, sessionPath, config.SessionLoadOptions{SessionID: "2026-05-03"}, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||
}
|
||||
|
||||
@@ -32,12 +32,6 @@ func applyEffectiveConfigProvenance(m *manifest.Manifest, cfg *config.Config) {
|
||||
}
|
||||
}
|
||||
|
||||
func applyEffectiveConfigProvenanceToRun(m *manifest.RunManifest, cfg *config.Config) {
|
||||
if m != nil {
|
||||
m.EffectiveConfig = effectiveConfigProvenance(cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func effectiveConfigSummary(cfg *config.Config) string {
|
||||
provenance := effectiveConfigProvenance(cfg)
|
||||
if provenance == nil {
|
||||
|
||||
@@ -156,7 +156,7 @@ func TestExtractLifecyclePrepareBindsVerifiedReferenceSnapshots(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
if len(summary.Executed) != 2 || len(runner.requests) != 1 {
|
||||
if len(summary.Executed) != 3 || len(runner.requests) != 1 {
|
||||
t.Fatalf("summary = %#v requests=%d", summary, len(runner.requests))
|
||||
}
|
||||
paths, err := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
@@ -210,7 +210,7 @@ func TestExtractLifecyclePrepareBindsVerifiedReferenceSnapshots(t *testing.T) {
|
||||
|
||||
func TestExtractLifecyclePreparedReferenceChangeRerunsExtractionAndInvalidatesDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
originalPaths := configureLifecycleReferences(t, cfg)
|
||||
configureLifecycleReferences(t, cfg)
|
||||
if _, err := executeStages(context.Background(), cfg, prepareExtractLifecyclePlan(t), RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("initial executeStages() error = %v", err)
|
||||
}
|
||||
@@ -222,9 +222,18 @@ func TestExtractLifecyclePreparedReferenceChangeRerunsExtractionAndInvalidatesDo
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPathFor(cfg), before); err != nil {
|
||||
t.Fatalf("Save(downstream success) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(originalPaths[artifactpolicy.SourceInputParty], []byte("changed party bytes\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(party source) error = %v", err)
|
||||
partyDocument, err := config.ParseParty([]byte(`schema_version: narratio.party.v1
|
||||
characters:
|
||||
changed:
|
||||
player: {name: Eric}
|
||||
character:
|
||||
name: Changed
|
||||
classes: [{name: wizard}]
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseParty(changed party) error = %v", err)
|
||||
}
|
||||
cfg.Party.Canonical = partyDocument
|
||||
|
||||
prepared := loadLifecycleManifest(t, cfg)
|
||||
prepare, err := stage.Select("prepare")
|
||||
@@ -266,17 +275,22 @@ func TestExtractLifecyclePreparedReferenceChangeRerunsExtractionAndInvalidatesDo
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleSessionOverrideBytesReachCanonicalReference(t *testing.T) {
|
||||
func TestExtractLifecycleCanonicalPartyBytesReachCanonicalReference(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
configureLifecycleReferences(t, cfg)
|
||||
overridePath := filepath.Join(filepath.Dir(cfg.SessionPath), "session-party.yml")
|
||||
if err := os.WriteFile(overridePath, []byte("session override party\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(session override) error = %v", err)
|
||||
partyBytes := []byte(`schema_version: narratio.party.v1
|
||||
characters:
|
||||
session:
|
||||
player: {name: Eric}
|
||||
character:
|
||||
name: Session
|
||||
classes: [{name: wizard}]
|
||||
`)
|
||||
partyDocument, err := config.ParseParty(partyBytes)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseParty(canonical party) error = %v", err)
|
||||
}
|
||||
cfg.StableInputs.PartyFile = config.ResolvedInputFile{
|
||||
Path: "./session-party.yml", ConfigPath: cfg.SessionPath, Source: "session_config",
|
||||
}
|
||||
cfg.Session.Inputs.PartyFile = "./session-party.yml"
|
||||
cfg.Party.Canonical = partyDocument
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, prepareExtractLifecyclePlan(t), RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
@@ -294,8 +308,8 @@ func TestExtractLifecycleSessionOverrideBytesReachCanonicalReference(t *testing.
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(prepared party) error = %v", err)
|
||||
}
|
||||
if string(contents) != "session override party\n" || partyPath == overridePath {
|
||||
t.Fatalf("prepared party path=%q contents=%q override=%q", partyPath, contents, overridePath)
|
||||
if string(contents) != string(partyBytes) || partyPath == cfg.Party.Source.Path {
|
||||
t.Fatalf("prepared party path=%q contents=%q source=%q", partyPath, contents, cfg.Party.Source.Path)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -400,7 +414,7 @@ func TestExtractLifecycleForcedFailureInvalidatesDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
plan, _ := BuildSingleStagePlan("extract")
|
||||
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
_, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("initial executeStages() error = %v", err)
|
||||
}
|
||||
@@ -426,14 +440,6 @@ func TestExtractLifecycleForcedFailureInvalidatesDownstream(t *testing.T) {
|
||||
if len(failed.Outputs) != 0 || len(failed.Logs) != 0 || len(failed.GeneratedConfigs) != 0 || len(failed.Metadata) != 0 {
|
||||
t.Fatalf("failed replacement inherited extraction result details: %#v", failed)
|
||||
}
|
||||
historical, err := (&manifest.LocalStore{}).LoadRun(context.Background(), first.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadRun(initial) error = %v", err)
|
||||
}
|
||||
historicalExtract := historical.Stages["extract"]
|
||||
if historicalExtract == nil || historicalExtract.Status != manifest.StatusSucceeded || len(historicalExtract.Outputs) == 0 || len(historicalExtract.Logs) == 0 || len(historicalExtract.Metadata) == 0 {
|
||||
t.Fatalf("historical extraction result = %#v, want preserved succeeded details", historicalExtract)
|
||||
}
|
||||
if _, err := os.Stat(succeeded.Outputs[0].LocalPath); err != nil {
|
||||
t.Fatalf("durable extraction output was not preserved: %v", err)
|
||||
}
|
||||
@@ -496,12 +502,12 @@ func TestExtractLifecycleRerunsAfterDirectTranscriptChange(t *testing.T) {
|
||||
t.Fatalf("Save(mutated) error = %v", err)
|
||||
}
|
||||
|
||||
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("rerun executeStages() error = %v", err)
|
||||
_, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err == nil || !strings.Contains(err.Error(), "checksum mismatch") {
|
||||
t.Fatalf("rerun executeStages() error = %v, want stale transcript evidence rejection", err)
|
||||
}
|
||||
if len(rerun.Executed) != 2 || len(rerun.Skipped) != 0 || len(runner.requests) != 2 || analyzeRuns != 2 {
|
||||
t.Fatalf("rerun summary = %#v requests=%d analyze=%d", rerun, len(runner.requests), analyzeRuns)
|
||||
if len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||
t.Fatalf("stale transcript should prevent extraction, requests=%d analyze=%d", len(runner.requests), analyzeRuns)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -631,7 +637,24 @@ func prepareExtractLifecyclePlan(t *testing.T) []stage.Stage {
|
||||
if err != nil {
|
||||
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
|
||||
}
|
||||
return append(prepare, extract...)
|
||||
return append(append(prepare, lifecycleTrimEvidenceStage{}), extract...)
|
||||
}
|
||||
|
||||
type lifecycleTrimEvidenceStage struct{}
|
||||
|
||||
func (lifecycleTrimEvidenceStage) Name() string { return "trim" }
|
||||
|
||||
func (lifecycleTrimEvidenceStage) Run(_ context.Context, env *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
paths := artifacts.NewLocalStore(env.Config.Pipeline.Workspace.Root).SessionPathsFor(m.Campaign, m.SessionID)
|
||||
path := filepath.Join(paths.ArtifactsDir, "trimmed.from-manifest.json")
|
||||
checksum, err := artifacts.SHA256File(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &stage.StageResult{Outputs: []artifacts.Ref{{
|
||||
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
|
||||
AbsolutePath: path, Checksum: checksum,
|
||||
}}}, nil
|
||||
}
|
||||
|
||||
func configureLifecycleReferences(t *testing.T, cfg *config.Config) map[string]string {
|
||||
@@ -724,9 +747,13 @@ func extractionLifecycleFixture(t *testing.T, enabled bool) (*config.Config, *st
|
||||
}
|
||||
m := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
m.Campaign = cfg.Session.Campaign
|
||||
inputChecksum, err := artifacts.SHA256File(inputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("SHA256File(input) error = %v", err)
|
||||
}
|
||||
m.MarkStageSucceeded("trim", time.Now().UTC(), []manifest.ArtifactRecord{{
|
||||
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
|
||||
LocalPath: inputPath,
|
||||
LocalPath: inputPath, ProducerRunID: "trim-run", Checksum: inputChecksum,
|
||||
}})
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), paths.ManifestPath, m); err != nil {
|
||||
t.Fatalf("Save(seed) error = %v", err)
|
||||
|
||||
@@ -9,11 +9,20 @@ import (
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func newCommandObjectStore(ctx context.Context, cfg *config.Config, logger *slog.Logger) (storage.ObjectStore, error) {
|
||||
if _, err := loadSecretsFromConfig(cfg, logger); err != nil {
|
||||
func newCommandObjectStoreWithDependencies(ctx context.Context, cfg *config.Config, logger *slog.Logger, deps commandDependencies) (storage.ObjectStore, error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
if _, err := loadSecretsFromConfigWithDependencies(cfg, logger, deps); err != nil {
|
||||
return nil, fmt.Errorf("load secrets from files: %w", err)
|
||||
}
|
||||
store, err := newObjectStoreFromConfigFn(ctx, cfg)
|
||||
return initializeObjectStoreBackend(ctx, cfg, deps.newObjectStore)
|
||||
}
|
||||
|
||||
func initializeObjectStoreBackend(
|
||||
ctx context.Context,
|
||||
cfg *config.Config,
|
||||
factory func(context.Context, *config.Config) (storage.ObjectStore, error),
|
||||
) (storage.ObjectStore, error) {
|
||||
store, err := factory(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("initialize object store backend: %w", err)
|
||||
}
|
||||
|
||||
@@ -24,8 +24,7 @@ func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
|
||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||
fake := &storage.FakeBackend{}
|
||||
called := false
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
called = true
|
||||
if got := os.Getenv(accessKeyEnv); got != "loaded-key-id" {
|
||||
return nil, errors.New("access key was not loaded before object store init")
|
||||
@@ -34,12 +33,9 @@ func TestNewCommandObjectStoreLoadsSecretsBeforeFactory(t *testing.T) {
|
||||
return nil, errors.New("secret key was not loaded before object store init")
|
||||
}
|
||||
return fake, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
}}
|
||||
|
||||
store, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
store, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||
}
|
||||
@@ -62,8 +58,7 @@ func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
|
||||
mustWriteSecretFile(t, filepath.Join(secretsDir, secretKeyEnv), "file-secret\n")
|
||||
|
||||
cfg := commandObjectStoreTestConfig(secretsDir)
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if got := os.Getenv(accessKeyEnv); got != "existing-key-id" {
|
||||
return nil, errors.New("existing access key was overwritten")
|
||||
}
|
||||
@@ -71,12 +66,9 @@ func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
|
||||
return nil, errors.New("existing secret key was overwritten")
|
||||
}
|
||||
return &storage.FakeBackend{}, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
}}
|
||||
|
||||
if _, err := newCommandObjectStore(context.Background(), cfg, nil); err != nil {
|
||||
if _, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps); err != nil {
|
||||
t.Fatalf("newCommandObjectStore() error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -84,16 +76,12 @@ func TestNewCommandObjectStorePreservesExistingEnv(t *testing.T) {
|
||||
func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
|
||||
cfg := commandObjectStoreTestConfig(filepath.Join(t.TempDir(), "missing"))
|
||||
called := false
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
called = true
|
||||
return &storage.FakeBackend{}, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
}}
|
||||
|
||||
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
_, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
@@ -107,15 +95,11 @@ func TestNewCommandObjectStoreSecretErrorStopsFactory(t *testing.T) {
|
||||
|
||||
func TestNewCommandObjectStoreFactoryErrorIsContextual(t *testing.T) {
|
||||
cfg := commandObjectStoreTestConfig("")
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
return nil, errors.New("factory boom")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
})
|
||||
}}
|
||||
|
||||
_, err := newCommandObjectStore(context.Background(), cfg, nil)
|
||||
_, err := newCommandObjectStoreWithDependencies(context.Background(), cfg, nil, deps)
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
|
||||
@@ -35,8 +35,8 @@ func buildHelperArtifactCatalog(cfg *config.Config, m *manifest.Manifest) (*arti
|
||||
return catalog, nil
|
||||
}
|
||||
|
||||
func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.ArtifactCatalog, locks *effectiveLocks, publishedRemoteState map[string]string) {
|
||||
lockSet := lockSourceSet(locks.All)
|
||||
func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.ArtifactCatalog, locks []artifacts.RemoteLock, publishedRemoteState map[string]string) {
|
||||
lockSet := remoteLockSourceSet(locks)
|
||||
fmt.Fprintln(out, "Built-in:")
|
||||
for _, transcript := range artifacts.RuntimeTranscriptArtifacts() {
|
||||
writeArtifactLine(out, transcript.SourceID, lockSet)
|
||||
@@ -71,7 +71,7 @@ func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.Art
|
||||
}
|
||||
}
|
||||
|
||||
func writeConfiguredArtifactLine(out io.Writer, source, state, provenance string, lockSet map[string]config.PublishLockRule, origins map[string]config.ArtifactFamilyMemberOrigin) {
|
||||
func writeConfiguredArtifactLine(out io.Writer, source, state, provenance string, lockSet map[string]struct{}, origins map[string]config.ArtifactFamilyMemberOrigin) {
|
||||
parts := []string{source, "planned", state}
|
||||
if key, ok := artifactpolicy.ParseConfiguredSource(source); ok {
|
||||
if origin, family := origins[key]; family {
|
||||
@@ -87,7 +87,7 @@ func writeConfiguredArtifactLine(out io.Writer, source, state, provenance string
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
func writeExtractionArtifactLine(out io.Writer, source, state, provenance string, lockSet map[string]config.PublishLockRule) {
|
||||
func writeExtractionArtifactLine(out io.Writer, source, state, provenance string, lockSet map[string]struct{}) {
|
||||
parts := []string{source, "planned", state}
|
||||
if strings.TrimSpace(provenance) != "" {
|
||||
parts = append(parts, "provenance="+strings.TrimSpace(provenance))
|
||||
@@ -98,7 +98,7 @@ func writeExtractionArtifactLine(out io.Writer, source, state, provenance string
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.PublishLockRule) {
|
||||
func writeArtifactLine(out io.Writer, source string, lockSet map[string]struct{}) {
|
||||
parts := []string{source}
|
||||
if _, ok := lockSet[source]; ok {
|
||||
parts = append(parts, "locked")
|
||||
@@ -106,7 +106,7 @@ func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.P
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
func writePublishedOutputLine(out io.Writer, rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog, lockSet map[string]config.PublishLockRule, remoteState map[string]string) {
|
||||
func writePublishedOutputLine(out io.Writer, rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog, lockSet map[string]struct{}, remoteState map[string]string) {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
parts := []string{source}
|
||||
if _, ok := lockSet[source]; ok {
|
||||
@@ -127,9 +127,16 @@ func writePublishedOutputLine(out io.Writer, rule config.PublishOutputRule, cata
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
func remotePublishedOutputAvailability(ctx context.Context, cfg *config.Config, store storage.ObjectStore, catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||
func remoteLockSourceSet(locks []artifacts.RemoteLock) map[string]struct{} {
|
||||
out := make(map[string]struct{}, len(locks))
|
||||
for _, lock := range locks {
|
||||
out[lock.Source] = struct{}{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func unresolvedRemotePublishedOutputAvailability(cfg *config.Config, catalog *artifacts.ArtifactCatalog, state string) map[string]string {
|
||||
out := map[string]string{}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
dest, _, err := helperPublishedOutputDest(rule, catalog)
|
||||
@@ -137,14 +144,7 @@ func remotePublishedOutputAvailability(ctx context.Context, cfg *config.Config,
|
||||
out[publishedOutputRemoteStateKey(source, "")] = "remote=error"
|
||||
continue
|
||||
}
|
||||
key := artifacts.S3PublishedOutputKey(sessionPrefix, dest)
|
||||
if exists, err := store.Exists(ctx, key); err == nil && exists {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = "remote=published"
|
||||
} else if err != nil {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = "remote=error"
|
||||
} else {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = "remote=missing"
|
||||
}
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = state
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -157,10 +157,10 @@ func remotePublishedOutputAvailabilityForCurrent(
|
||||
current *RemoteCurrentState,
|
||||
) map[string]string {
|
||||
if current == nil || current.Commit == nil {
|
||||
return remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
||||
return unresolvedRemotePublishedOutputAvailability(cfg, catalog, "remote=missing")
|
||||
}
|
||||
outcomes := committedPublishOutputOutcomes(current.Manifest)
|
||||
out := map[string]string{}
|
||||
runPrefix := artifacts.S3RunPrefix(current.SessionPrefix, current.RunID)
|
||||
for _, rule := range cfg.Pipeline.Publish.Outputs {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
dest, _, err := helperPublishedOutputDest(rule, catalog)
|
||||
@@ -168,26 +168,88 @@ func remotePublishedOutputAvailabilityForCurrent(
|
||||
out[publishedOutputRemoteStateKey(source, "")] = "remote=error"
|
||||
continue
|
||||
}
|
||||
key := artifacts.S3RunRelativeDestinationKey(runPrefix, dest)
|
||||
if committedPublishedOutput(current.Commit, key) {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = "remote=published"
|
||||
} else {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = "remote=missing"
|
||||
outcome := outcomes[source]
|
||||
state := ""
|
||||
if outcome != "" {
|
||||
state = "outcome=" + outcome + " "
|
||||
}
|
||||
declaration, ok := committedPublishedOutput(current.Commit, source)
|
||||
if !ok {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = state + "remote=missing"
|
||||
continue
|
||||
}
|
||||
identity, err := artifacts.ParsePublishedOutputObjectKey(current.SessionPrefix, declaration.DestinationKey)
|
||||
if err != nil {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = state + "remote=error"
|
||||
continue
|
||||
}
|
||||
exists, err := store.Exists(ctx, declaration.DestinationKey)
|
||||
if err != nil {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = state + "remote=error"
|
||||
continue
|
||||
}
|
||||
if !exists {
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = state + "remote=missing"
|
||||
continue
|
||||
}
|
||||
state += "remote=published"
|
||||
if outcome == "carried_locked" || identity.RunID != current.RunID {
|
||||
state += " declared_dest=" + identity.Destination
|
||||
state += " declared_key=" + declaration.DestinationKey
|
||||
state += " origin_run=" + identity.RunID
|
||||
}
|
||||
out[publishedOutputRemoteStateKey(source, dest)] = state
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func committedPublishOutputOutcomes(m *manifest.Manifest) map[string]string {
|
||||
out := map[string]string{}
|
||||
if m == nil || m.Stages["publish"] == nil {
|
||||
return out
|
||||
}
|
||||
raw := m.Stages["publish"].Metadata["publish_output_outcomes"]
|
||||
appendOutcome := func(item map[string]any) {
|
||||
source, sourceOK := item["source"].(string)
|
||||
status, statusOK := item["status"].(string)
|
||||
if sourceOK && statusOK && isPublishOutputOutcome(status) {
|
||||
out[source] = status
|
||||
}
|
||||
}
|
||||
switch items := raw.(type) {
|
||||
case []any:
|
||||
for _, rawItem := range items {
|
||||
if item, ok := rawItem.(map[string]any); ok {
|
||||
appendOutcome(item)
|
||||
}
|
||||
}
|
||||
case []map[string]any:
|
||||
for _, item := range items {
|
||||
appendOutcome(item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func committedPublishedOutput(commit *artifacts.RemoteCommitManifest, key string) bool {
|
||||
if commit == nil {
|
||||
func isPublishOutputOutcome(status string) bool {
|
||||
switch status {
|
||||
case "published", "carried_locked", "locked_unpublished", "optional_missing", "unselected":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func committedPublishedOutput(commit *artifacts.RemoteCommitManifest, source string) (artifacts.RemoteArtifact, bool) {
|
||||
if commit == nil {
|
||||
return artifacts.RemoteArtifact{}, false
|
||||
}
|
||||
for _, artifact := range commit.Artifacts {
|
||||
if artifact.Type == artifacts.RemoteArtifactTypePublishedOutput && artifact.DestinationKey == key {
|
||||
return true
|
||||
if artifact.Type == artifacts.RemoteArtifactTypePublishedOutput && artifact.Source == source {
|
||||
return artifact, true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return artifacts.RemoteArtifact{}, false
|
||||
}
|
||||
|
||||
func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
||||
|
||||
@@ -10,6 +10,11 @@ import (
|
||||
|
||||
// ArtifactsList lists effective artifact sources.
|
||||
func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
||||
return artifactsListWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func artifactsListWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
fs := flag.NewFlagSet("artifacts list", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
@@ -22,7 +27,7 @@ func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("artifacts list: session_id is required")
|
||||
}
|
||||
cfg, store, locks, m, cleanup, err := loadHelperContext(ctx, flags, remote)
|
||||
cfg, store, locks, m, cleanup, err := loadHelperContextWithDependencies(ctx, flags, remote, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("artifacts list: %w", err)
|
||||
}
|
||||
@@ -33,7 +38,12 @@ func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
publishedRemoteState := map[string]string{}
|
||||
if remote && store != nil {
|
||||
publishedRemoteState = remotePublishedOutputAvailability(ctx, cfg, store, catalog)
|
||||
current := inspectRemoteCurrentStateWithDependencies(ctx, cfg, store, deps)
|
||||
if current.Err != nil {
|
||||
publishedRemoteState = unresolvedRemotePublishedOutputAvailability(cfg, catalog, "remote=error")
|
||||
} else {
|
||||
publishedRemoteState = remotePublishedOutputAvailabilityForCurrent(ctx, cfg, store, catalog, current.State)
|
||||
}
|
||||
}
|
||||
writeArtifactList(out, cfg, catalog, locks, publishedRemoteState)
|
||||
return nil
|
||||
|
||||
@@ -5,11 +5,14 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
@@ -44,58 +47,72 @@ func (f commonConfigFlags) sessionOptions() config.SessionLoadOptions {
|
||||
|
||||
// Session dispatches session helper subcommands.
|
||||
func Session(ctx context.Context, args []string, out io.Writer) error {
|
||||
return sessionWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func sessionWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("session: expected subcommand: init|validate|status|plan|restore|artifacts|locks")
|
||||
}
|
||||
switch args[0] {
|
||||
case "init":
|
||||
return SessionInit(ctx, args[1:], out)
|
||||
return sessionInitWithDependencies(ctx, args[1:], out, deps)
|
||||
case "validate":
|
||||
return SessionValidate(ctx, args[1:], out)
|
||||
return sessionValidateWithDependencies(ctx, args[1:], out, deps)
|
||||
case "status":
|
||||
return Status(ctx, args[1:], out)
|
||||
return statusWithDependencies(ctx, args[1:], out, deps)
|
||||
case "plan":
|
||||
return Plan(ctx, args[1:], out)
|
||||
return planWithDependencies(ctx, args[1:], out, deps)
|
||||
case "restore":
|
||||
return Restore(ctx, args[1:], out)
|
||||
return restoreWithDependencies(ctx, args[1:], out, deps)
|
||||
case "artifacts":
|
||||
return ArtifactsList(ctx, args[1:], out)
|
||||
return artifactsListWithDependencies(ctx, args[1:], out, deps)
|
||||
case "locks":
|
||||
return SessionLocks(ctx, args[1:], out)
|
||||
return sessionLocksWithDependencies(ctx, args[1:], out, deps)
|
||||
default:
|
||||
return fmt.Errorf("session: unknown subcommand %q", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// SessionLocks dispatches session-oriented publish lock list and mutation
|
||||
// helpers while preserving the existing lock implementations.
|
||||
// helpers.
|
||||
func SessionLocks(ctx context.Context, args []string, out io.Writer) error {
|
||||
return sessionLocksWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func sessionLocksWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
if len(args) > 0 && !isCLIFlagToken(args[0]) {
|
||||
switch args[0] {
|
||||
case "add":
|
||||
return LocksAdd(ctx, args[1:], out)
|
||||
return locksAddWithDependencies(ctx, args[1:], out, deps)
|
||||
case "remove":
|
||||
return LocksRemove(ctx, args[1:], out)
|
||||
return locksRemoveWithDependencies(ctx, args[1:], out, deps)
|
||||
}
|
||||
}
|
||||
return LocksList(ctx, args, out)
|
||||
return locksListWithDependencies(ctx, args, out, deps)
|
||||
}
|
||||
|
||||
// Artifacts dispatches artifact helper subcommands.
|
||||
func Artifacts(ctx context.Context, args []string, out io.Writer) error {
|
||||
return artifactsWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func artifactsWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("artifacts: expected subcommand: list")
|
||||
}
|
||||
switch args[0] {
|
||||
case "list":
|
||||
return ArtifactsList(ctx, args[1:], out)
|
||||
return artifactsListWithDependencies(ctx, args[1:], out, deps)
|
||||
default:
|
||||
return fmt.Errorf("artifacts: unknown subcommand %q", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore bool) (*config.Config, storage.ObjectStore, *effectiveLocks, *manifest.Manifest, func(), error) {
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
func loadHelperContextWithDependencies(ctx context.Context, flags commonConfigFlags, needStore bool, deps commandDependencies) (*config.Config, storage.ObjectStore, []artifacts.RemoteLock, *manifest.Manifest, func(), error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
@@ -111,16 +128,23 @@ func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore b
|
||||
}
|
||||
var store storage.ObjectStore
|
||||
if needStore {
|
||||
store, err = newCommandObjectStore(ctx, cfg, nil)
|
||||
store, err = newCommandObjectStoreWithDependencies(ctx, cfg, nil, deps)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
} else {
|
||||
store, _ = objectStoreIfConfigured(ctx, cfg)
|
||||
store, _ = objectStoreIfConfiguredWithDependencies(ctx, cfg, deps)
|
||||
}
|
||||
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
var locks []artifacts.RemoteLock
|
||||
if store != nil {
|
||||
sessionPrefix, prefixErr := artifacts.ResolvePublishSessionPrefix(cfg, nil)
|
||||
if prefixErr != nil {
|
||||
return nil, nil, nil, nil, nil, prefixErr
|
||||
}
|
||||
locks, err = artifacts.ListRemoteLocks(ctx, store, sessionPrefix)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
}
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
m, err := loadLocalManifest(ctx, paths.ManifestPath)
|
||||
@@ -131,13 +155,34 @@ func loadHelperContext(ctx context.Context, flags commonConfigFlags, needStore b
|
||||
return cfg, store, locks, m, func() { _ = loaded.Close() }, nil
|
||||
}
|
||||
|
||||
func objectStoreIfConfigured(ctx context.Context, cfg *config.Config) (storage.ObjectStore, error) {
|
||||
func objectStoreIfConfiguredWithDependencies(ctx context.Context, cfg *config.Config, deps commandDependencies) (storage.ObjectStore, error) {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||
store, err := newCommandObjectStoreWithDependencies(ctx, cfg, nil, deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func writeLocalFile(path string, data []byte, force bool) error {
|
||||
cleaned := filepath.Clean(strings.TrimSpace(path))
|
||||
if cleaned == "" || cleaned == "." {
|
||||
return fmt.Errorf("output path is required")
|
||||
}
|
||||
if !force {
|
||||
if _, err := os.Stat(cleaned); err == nil {
|
||||
return fmt.Errorf("output file %q already exists; pass --force to overwrite", cleaned)
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("check output file %q: %w", cleaned, err)
|
||||
}
|
||||
}
|
||||
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(cleaned)); err != nil {
|
||||
return fmt.Errorf("create output directory: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(cleaned, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Chmod(cleaned, fileops.WorkspaceFileMode)
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -46,8 +46,8 @@ type remoteCurrentStateCheck struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
type effectiveLocksCheck struct {
|
||||
Locks *effectiveLocks
|
||||
type remoteLocksCheck struct {
|
||||
Locks []artifacts.RemoteLock
|
||||
Err error
|
||||
}
|
||||
|
||||
@@ -56,16 +56,21 @@ func inspectStableInputs(cfg *config.Config) []stableInputCheck {
|
||||
name string
|
||||
in config.ResolvedInputFile
|
||||
optional bool
|
||||
derived bool
|
||||
}{
|
||||
{name: "speakers", in: cfg.StableInputs.SpeakersFile},
|
||||
{name: "autocorrect", in: cfg.StableInputs.AutocorrectFile},
|
||||
{name: "glossary", in: cfg.StableInputs.GlossaryFile},
|
||||
{name: "players", in: cfg.StableInputs.PlayersFile},
|
||||
{name: "players", derived: true},
|
||||
{name: "party", in: cfg.StableInputs.PartyFile},
|
||||
{name: "spell_catalog", in: cfg.StableInputs.SpellCatalogFile, optional: true},
|
||||
}
|
||||
out := make([]stableInputCheck, 0, len(items))
|
||||
for _, item := range items {
|
||||
if item.derived {
|
||||
out = append(out, stableInputCheck{Name: item.name, Path: "derived from canonical party"})
|
||||
continue
|
||||
}
|
||||
if item.optional && strings.TrimSpace(item.in.Path) == "" {
|
||||
continue
|
||||
}
|
||||
@@ -166,16 +171,11 @@ func inspectPreviousArtifactReadiness(
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
plan, err := previouscache.Resolve(ctx, cfg, paths, requirements, store)
|
||||
if err != nil {
|
||||
var pointerMissing *artifacts.CurrentRunPointerMissingError
|
||||
var manifestMissing *artifacts.CurrentManifestMissingError
|
||||
var pointerMissing *artifacts.CurrentCommitPointerMissingError
|
||||
if errors.As(err, &pointerMissing) {
|
||||
out.Err = fmt.Errorf("remote %w", pointerMissing)
|
||||
return out
|
||||
}
|
||||
if errors.As(err, &manifestMissing) {
|
||||
out.Err = fmt.Errorf("remote %w", manifestMissing)
|
||||
return out
|
||||
}
|
||||
out.Err = fmt.Errorf("remote %w", err)
|
||||
return out
|
||||
}
|
||||
@@ -183,23 +183,31 @@ func inspectPreviousArtifactReadiness(
|
||||
return out
|
||||
}
|
||||
|
||||
func inspectRemoteCurrentState(ctx context.Context, cfg *config.Config, store storage.ObjectStore) remoteCurrentStateCheck {
|
||||
func inspectRemoteCurrentStateWithDependencies(ctx context.Context, cfg *config.Config, store storage.ObjectStore, deps commandDependencies) remoteCurrentStateCheck {
|
||||
if store == nil {
|
||||
return remoteCurrentStateCheck{}
|
||||
}
|
||||
current, err := discoverRemoteCurrentStateFn(ctx, cfg, store)
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
current, err := deps.discoverRemoteCurrentState(ctx, cfg, store)
|
||||
if err != nil {
|
||||
return remoteCurrentStateCheck{Err: err}
|
||||
}
|
||||
return remoteCurrentStateCheck{State: current}
|
||||
}
|
||||
|
||||
func inspectEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) effectiveLocksCheck {
|
||||
locks, err := loadEffectiveLocks(ctx, cfg, store)
|
||||
if err != nil {
|
||||
return effectiveLocksCheck{Err: err}
|
||||
func inspectRemoteLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) remoteLocksCheck {
|
||||
if store == nil {
|
||||
return remoteLocksCheck{}
|
||||
}
|
||||
return effectiveLocksCheck{Locks: locks}
|
||||
sessionPrefix, err := artifacts.ResolvePublishSessionPrefix(cfg, nil)
|
||||
if err != nil {
|
||||
return remoteLocksCheck{Err: err}
|
||||
}
|
||||
locks, err := artifacts.ListRemoteLocks(ctx, store, sessionPrefix)
|
||||
if err != nil {
|
||||
return remoteLocksCheck{Err: err}
|
||||
}
|
||||
return remoteLocksCheck{Locks: locks}
|
||||
}
|
||||
|
||||
func resolveLocalInspectionAudioPaths(sessionDir string, inputs config.SessionInputsConfig) ([]string, error) {
|
||||
|
||||
@@ -2,32 +2,24 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
// Locks dispatches publish lock list and mutation helpers.
|
||||
func Locks(ctx context.Context, args []string, out io.Writer) error {
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
switch args[0] {
|
||||
case "add":
|
||||
return LocksAdd(ctx, args[1:], out)
|
||||
case "remove":
|
||||
return LocksRemove(ctx, args[1:], out)
|
||||
default:
|
||||
return fmt.Errorf("locks: unknown subcommand %q", args[0])
|
||||
}
|
||||
}
|
||||
return LocksList(ctx, args, out)
|
||||
// LocksList lists independent publish locks.
|
||||
func LocksList(ctx context.Context, args []string, out io.Writer) error {
|
||||
return locksListWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
// LocksList lists effective publish locks.
|
||||
func LocksList(ctx context.Context, args []string, out io.Writer) error {
|
||||
func locksListWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
fs := flag.NewFlagSet("locks", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
@@ -38,17 +30,25 @@ func LocksList(ctx context.Context, args []string, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks: session_id is required")
|
||||
}
|
||||
cfg, _, locks, _, cleanup, err := loadHelperContext(ctx, flags, true)
|
||||
cfg, store, sessionPrefix, cleanup, err := loadLockCommandContextWithDependencies(ctx, flags, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks: %w", err)
|
||||
}
|
||||
defer cleanup()
|
||||
locks, err := artifacts.ListRemoteLocks(ctx, store, sessionPrefix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks: %w", err)
|
||||
}
|
||||
writeLocks(out, cfg, locks)
|
||||
return nil
|
||||
}
|
||||
|
||||
// LocksAdd adds or updates one remote lock.
|
||||
func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
||||
return locksAddWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func locksAddWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
fs := flag.NewFlagSet("locks add", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
@@ -64,31 +64,28 @@ func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks add: session_id is required")
|
||||
}
|
||||
cfg, store, locks, _, cleanup, err := loadHelperContext(ctx, flags, true)
|
||||
_, store, sessionPrefix, cleanup, err := loadLockCommandContextWithDependencies(ctx, flags, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
defer cleanup()
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks add"); err != nil {
|
||||
lock, err := artifacts.NewRemoteLock(source, strings.TrimSpace(reason), nowUTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
||||
return fmt.Errorf("locks add: source %q is locked by pipeline config and cannot be modified remotely", source)
|
||||
}
|
||||
if err := mutateRemoteLockStore(ctx, cfg, store, func(lockStore *config.PublishLockStore) error {
|
||||
remoteSet := lockSourceSet(lockStore.Locks)
|
||||
if _, exists := remoteSet[source]; exists && !force {
|
||||
return fmt.Errorf("remote lock for %q already exists; pass --force to update", source)
|
||||
}
|
||||
remoteSet[source] = config.PublishLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
||||
lockStore.Locks = lockMapValues(remoteSet)
|
||||
normalized, err := config.ValidatePublishLockRules(lockStore.Locks, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks")
|
||||
if !force {
|
||||
current, err := artifacts.LookupRemoteLock(ctx, store, sessionPrefix, source)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if current.Present {
|
||||
return fmt.Errorf("locks add: remote lock for %q already exists; pass --force to update", source)
|
||||
}
|
||||
}
|
||||
if err := artifacts.AddRemoteLock(ctx, store, sessionPrefix, lock, force); err != nil {
|
||||
if errors.Is(err, storage.ErrConditionNotMet) {
|
||||
return fmt.Errorf("locks add: remote lock for %q changed concurrently; retry: %w", source, err)
|
||||
}
|
||||
lockStore.Locks = normalized
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "narratio session locks add: locked %s\n", source)
|
||||
@@ -97,6 +94,10 @@ func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
||||
|
||||
// LocksRemove removes one remote lock.
|
||||
func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
||||
return locksRemoveWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func locksRemoveWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
fs := flag.NewFlagSet("locks remove", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
@@ -108,34 +109,56 @@ func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("locks remove: session_id is required")
|
||||
}
|
||||
cfg, store, locks, _, cleanup, err := loadHelperContext(ctx, flags, true)
|
||||
_, store, sessionPrefix, cleanup, err := loadLockCommandContextWithDependencies(ctx, flags, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
defer cleanup()
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks remove"); err != nil {
|
||||
current, err := artifacts.LookupRemoteLock(ctx, store, sessionPrefix, source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
if err := mutateRemoteLockStore(ctx, cfg, store, func(lockStore *config.PublishLockStore) error {
|
||||
remoteSet := lockSourceSet(lockStore.Locks)
|
||||
if _, ok := remoteSet[source]; !ok {
|
||||
if _, static := lockSourceSet(locks.Static)[source]; static {
|
||||
return fmt.Errorf("source %q is locked by pipeline config and cannot be unlocked remotely", source)
|
||||
}
|
||||
return fmt.Errorf("remote lock for %q does not exist", source)
|
||||
}
|
||||
delete(remoteSet, source)
|
||||
lockStore.Locks = lockMapValues(remoteSet)
|
||||
return nil
|
||||
}); err != nil {
|
||||
if err := artifacts.RemoveRemoteLock(ctx, store, sessionPrefix, source); err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
if !current.Present {
|
||||
_, err = fmt.Fprintf(out, "narratio session locks remove: %s was not locked\n", source)
|
||||
return err
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "narratio session locks remove: unlocked %s\n", source)
|
||||
return err
|
||||
}
|
||||
|
||||
func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
|
||||
if locks == nil || len(locks.All) == 0 {
|
||||
func loadLockCommandContextWithDependencies(ctx context.Context, flags commonConfigFlags, deps commandDependencies) (*config.Config, storage.ObjectStore, string, func(), error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
if err != nil {
|
||||
return nil, nil, "", nil, err
|
||||
}
|
||||
release := true
|
||||
defer func() {
|
||||
if release {
|
||||
_ = loaded.Close()
|
||||
}
|
||||
}()
|
||||
cfg := loaded.Config
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return nil, nil, "", nil, err
|
||||
}
|
||||
store, err := newCommandObjectStoreWithDependencies(ctx, cfg, nil, deps)
|
||||
if err != nil {
|
||||
return nil, nil, "", nil, err
|
||||
}
|
||||
sessionPrefix, err := artifacts.ResolvePublishSessionPrefix(cfg, nil)
|
||||
if err != nil {
|
||||
return nil, nil, "", nil, err
|
||||
}
|
||||
release = false
|
||||
return cfg, store, sessionPrefix, func() { _ = loaded.Close() }, nil
|
||||
}
|
||||
|
||||
func writeLocks(out io.Writer, cfg *config.Config, locks []artifacts.RemoteLock) {
|
||||
if len(locks) == 0 {
|
||||
fmt.Fprintln(out, "Publish locks: none")
|
||||
return
|
||||
}
|
||||
@@ -146,12 +169,7 @@ func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
|
||||
published[strings.TrimSpace(rule.Source)] = rule
|
||||
}
|
||||
}
|
||||
staticSet := lockSourceSet(locks.Static)
|
||||
for _, lock := range locks.All {
|
||||
origin := "remote"
|
||||
if _, ok := staticSet[lock.Source]; ok {
|
||||
origin = "pipeline"
|
||||
}
|
||||
for _, lock := range locks {
|
||||
promo := "not-published"
|
||||
if _, ok := published[lock.Source]; ok {
|
||||
promo = "published"
|
||||
@@ -160,22 +178,6 @@ func writeLocks(out io.Writer, cfg *config.Config, locks *effectiveLocks) {
|
||||
if reason == "" {
|
||||
reason = "(no reason)"
|
||||
}
|
||||
fmt.Fprintf(out, "- %s origin=%s %s reason=%s\n", lock.Source, origin, promo, reason)
|
||||
fmt.Fprintf(out, "- %s %s reason=%s created_at=%s\n", lock.Source, promo, reason, lock.CreatedAt.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
}
|
||||
|
||||
func lockMapValues(in map[string]config.PublishLockRule) []config.PublishLockRule {
|
||||
keys := make([]string, 0, len(in))
|
||||
for key := range in {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := make([]config.PublishLockRule, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
item := in[key]
|
||||
item.Source = key
|
||||
item.Reason = strings.TrimSpace(item.Reason)
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -19,6 +19,11 @@ import (
|
||||
|
||||
// SessionInit creates a local or remote session.yml skeleton.
|
||||
func SessionInit(ctx context.Context, args []string, out io.Writer) error {
|
||||
return sessionInitWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func sessionInitWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
fs := flag.NewFlagSet("session init", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var pipelinePath, campaignPath, campaignFilePath, sessionID, previousSessionID, date, title, output, audioS3Prefix, audioDir string
|
||||
@@ -51,7 +56,7 @@ func SessionInit(ctx context.Context, args []string, out io.Writer) error {
|
||||
return fmt.Errorf("session init: --audio-dir and --audio-s3-prefix are mutually exclusive")
|
||||
}
|
||||
|
||||
base, err := loadPipelineCampaignConfig(pipelinePath, campaignPath, campaignFilePath, config.PipelineLoadOptions{Profile: profile.pointer()})
|
||||
base, err := loadPipelineCampaignConfigWithDependencies(pipelinePath, campaignPath, campaignFilePath, config.PipelineLoadOptions{Profile: profile.pointer()}, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
@@ -98,7 +103,7 @@ func SessionInit(ctx context.Context, args []string, out io.Writer) error {
|
||||
return err
|
||||
}
|
||||
|
||||
store, err := newCommandObjectStore(ctx, cfg, nil)
|
||||
store, err := newCommandObjectStoreWithDependencies(ctx, cfg, nil, deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
|
||||
@@ -28,7 +28,6 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(campaignPath, []byte(campaign), 0o644); err != nil {
|
||||
|
||||
@@ -13,6 +13,11 @@ import (
|
||||
|
||||
// SessionValidate performs a read-only session preflight.
|
||||
func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||
return sessionValidateWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func sessionValidateWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
fs := flag.NewFlagSet("session validate", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
@@ -25,7 +30,7 @@ func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
|
||||
findings := []finding{}
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
if err != nil {
|
||||
findings = append(findings, errorFinding("config", err.Error()))
|
||||
return renderFindings(out, "", "", findings)
|
||||
@@ -43,7 +48,7 @@ func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||
findings = append(findings, validateStableInputFindings(cfg)...)
|
||||
findings = append(findings, validateLocalAudioFindings(cfg)...)
|
||||
|
||||
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||
store, storeErr := objectStoreIfConfiguredWithDependencies(ctx, cfg, deps)
|
||||
if storeErr != nil {
|
||||
findings = append(findings, errorFinding("storage", storeErr.Error()))
|
||||
}
|
||||
@@ -76,13 +81,13 @@ func SessionValidate(ctx context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
}
|
||||
|
||||
locks := inspectEffectiveLocks(ctx, cfg, store)
|
||||
locks := inspectRemoteLocks(ctx, cfg, store)
|
||||
if locks.Err != nil {
|
||||
findings = append(findings, errorFinding("locks", locks.Err.Error()))
|
||||
} else if len(locks.Locks.All) == 0 {
|
||||
findings = append(findings, okFinding("locks", "no effective publish locks"))
|
||||
} else if len(locks.Locks) == 0 {
|
||||
findings = append(findings, okFinding("locks", "no publish locks"))
|
||||
} else {
|
||||
for _, lock := range locks.Locks.All {
|
||||
for _, lock := range locks.Locks {
|
||||
findings = append(findings, warnFinding("locks", fmt.Sprintf("%s locked: %s", lock.Source, strings.TrimSpace(lock.Reason))))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,11 @@ import (
|
||||
|
||||
// Status reports effective local/remote session state.
|
||||
func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
return statusWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func statusWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
var flags commonConfigFlags
|
||||
@@ -26,7 +31,7 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("status: session_id is required")
|
||||
}
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("status: %w", err)
|
||||
}
|
||||
@@ -56,13 +61,15 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
writeStageStatuses(out, m)
|
||||
}
|
||||
|
||||
store, storeErr := objectStoreIfConfigured(ctx, cfg)
|
||||
store, storeErr := objectStoreIfConfiguredWithDependencies(ctx, cfg, deps)
|
||||
var remoteCurrent *RemoteCurrentState
|
||||
var remoteCurrentErr error
|
||||
if storeErr != nil {
|
||||
fmt.Fprintf(out, "Remote publish: unavailable: %v\n", storeErr)
|
||||
} else if store != nil {
|
||||
current := inspectRemoteCurrentState(ctx, cfg, store)
|
||||
current := inspectRemoteCurrentStateWithDependencies(ctx, cfg, store, deps)
|
||||
if current.Err != nil {
|
||||
remoteCurrentErr = current.Err
|
||||
fmt.Fprintf(out, "Remote publish: missing or unavailable: %v\n", current.Err)
|
||||
} else {
|
||||
remoteCurrent = current.State
|
||||
@@ -82,7 +89,7 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg), effective),
|
||||
))
|
||||
|
||||
lockChecks := inspectEffectiveLocks(ctx, cfg, store)
|
||||
lockChecks := inspectRemoteLocks(ctx, cfg, store)
|
||||
locks := lockChecks.Locks
|
||||
lockErr := lockChecks.Err
|
||||
if catalog, catalogErr := buildHelperArtifactCatalog(cfg, localManifest); catalogErr != nil {
|
||||
@@ -90,14 +97,15 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
} else if storeErr == nil {
|
||||
catalogLocks := locks
|
||||
if lockErr != nil {
|
||||
catalogLocks = &effectiveLocks{
|
||||
Static: staticPublishLocks(cfg),
|
||||
All: staticPublishLocks(cfg),
|
||||
}
|
||||
catalogLocks = nil
|
||||
}
|
||||
publishedRemoteState := map[string]string{}
|
||||
if store != nil {
|
||||
publishedRemoteState = remotePublishedOutputAvailabilityForCurrent(ctx, cfg, store, catalog, remoteCurrent)
|
||||
if remoteCurrentErr != nil {
|
||||
publishedRemoteState = unresolvedRemotePublishedOutputAvailability(cfg, catalog, "remote=error")
|
||||
} else {
|
||||
publishedRemoteState = remotePublishedOutputAvailabilityForCurrent(ctx, cfg, store, catalog, remoteCurrent)
|
||||
}
|
||||
}
|
||||
fmt.Fprintln(out, "Remote outputs:")
|
||||
writeArtifactList(out, cfg, catalog, catalogLocks, publishedRemoteState)
|
||||
|
||||
@@ -18,6 +18,11 @@ import (
|
||||
|
||||
// Plan validates configuration and prints a read-only execution preview.
|
||||
func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
return planWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func planWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) error {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
request, err := parseBoundedRunRequest("plan", args, out)
|
||||
if err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
@@ -26,7 +31,7 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
return err
|
||||
}
|
||||
flags := request.Config
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("plan: %w", err)
|
||||
}
|
||||
@@ -73,26 +78,20 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("plan: fingerprint semantic configuration for stage %q: %w", selectedStage.Name(), err)
|
||||
}
|
||||
action := decideStageAction(selectedStage, model, request.Force)
|
||||
decision := decideStageExecution(recordedStageStatus(model, selectedStage.Name()), request.Force, nil)
|
||||
var validation *stage.ResumeValidation
|
||||
if action == stageActionSkip {
|
||||
if decision.action == stageActionSkip {
|
||||
checked, validationErr := evaluateStageResume(ctx, selectedStage, stageEnv, model, semanticConfig)
|
||||
if validationErr != nil {
|
||||
return fmt.Errorf("plan: validate resume for stage %q: %w", selectedStage.Name(), validationErr)
|
||||
}
|
||||
validation = checked
|
||||
if checked != nil && !checked.Resumable {
|
||||
at := time.Now().UTC()
|
||||
model.MarkStageStale(selectedStage.Name(), at, checked.Reason)
|
||||
if _, invalidationErr := invalidateDependentSucceededStagesWithReason(
|
||||
model, selectedStage.Name(), at, staleReasonNotResumable,
|
||||
); invalidationErr != nil {
|
||||
return fmt.Errorf("plan: model resume invalidation for stage %q: %w", selectedStage.Name(), invalidationErr)
|
||||
}
|
||||
action = stageActionRun
|
||||
}
|
||||
decision = decideStageExecution(recordedStageStatus(model, selectedStage.Name()), request.Force, checked)
|
||||
}
|
||||
if action == stageActionRun && selectedStage.Name() == "analyze" {
|
||||
if _, err := applyPreExecutionDecision(model, selectedStage.Name(), decision, time.Now().UTC()); err != nil {
|
||||
return fmt.Errorf("plan: model resume invalidation for stage %q: %w", selectedStage.Name(), err)
|
||||
}
|
||||
if decision.action == stageActionRun && selectedStage.Name() == "analyze" {
|
||||
if validator, ok := selectedStage.(stage.ResumeValidator); ok {
|
||||
checked, validationErr := validator.ValidateResume(ctx, stageEnv, model)
|
||||
if validationErr != nil {
|
||||
@@ -102,12 +101,12 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
validation = &checked
|
||||
}
|
||||
}
|
||||
if action == stageActionRun {
|
||||
if decision.action == stageActionRun {
|
||||
runCount++
|
||||
} else {
|
||||
skipCount++
|
||||
}
|
||||
if _, err := fmt.Fprintf(out, "%s: %s\n", selectedStage.Name(), action); err != nil {
|
||||
if _, err := fmt.Fprintf(out, "%s: %s\n", selectedStage.Name(), decision.action); err != nil {
|
||||
return err
|
||||
}
|
||||
if validation != nil && validation.Analyze != nil {
|
||||
@@ -115,8 +114,8 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if action == stageActionRun {
|
||||
if err := modelPlannedStageRun(model, selectedStage, cfg, request.Force, semanticConfig); err != nil {
|
||||
if decision.action == stageActionRun {
|
||||
if err := modelPlannedStageRun(model, selectedStage, cfg, decision, semanticConfig); err != nil {
|
||||
return fmt.Errorf("plan: model stage %q: %w", selectedStage.Name(), err)
|
||||
}
|
||||
}
|
||||
@@ -149,40 +148,23 @@ func modelPlannedStageRun(
|
||||
model *manifest.Manifest,
|
||||
selectedStage stage.Stage,
|
||||
cfg *config.Config,
|
||||
force bool,
|
||||
decision preExecutionDecision,
|
||||
semanticConfig *manifest.SemanticConfigFingerprint,
|
||||
) error {
|
||||
prior := capturePriorStageOutcome(model, selectedStage.Name())
|
||||
at := time.Now().UTC()
|
||||
model.MarkStageRunning(selectedStage.Name(), at)
|
||||
if force {
|
||||
if _, err := invalidateDependentSucceededStagesWithReason(
|
||||
model, selectedStage.Name(), at, staleReasonForcedReplacement,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := beginStageAttempt(model, selectedStage.Name(), decision, at); err != nil {
|
||||
return err
|
||||
}
|
||||
if reason := plannedSelfSkipReason(selectedStage.Name(), cfg); reason != "" {
|
||||
model.MarkStageSkipped(selectedStage.Name(), at, reason)
|
||||
setSessionStageSemanticConfig(model, selectedStage.Name(), semanticConfig)
|
||||
if !prior.isSameSelfSkip(reason) {
|
||||
_, err := invalidateDependentSucceededStagesWithReason(
|
||||
model, selectedStage.Name(), at, staleReasonSelfSkip,
|
||||
)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
_, err := applyCompletedStageOutcome(model, selectedStage.Name(), &stage.StageResult{
|
||||
Disposition: stage.StageDispositionSkipped,
|
||||
SkipReason: reason,
|
||||
}, nil, semanticConfig, prior, at)
|
||||
return err
|
||||
}
|
||||
model.MarkStageSucceeded(selectedStage.Name(), at, nil)
|
||||
setSessionStageSemanticConfig(model, selectedStage.Name(), semanticConfig)
|
||||
if !prior.exists || prior.status != manifest.StatusSucceeded {
|
||||
if _, err := invalidateDependentSucceededStagesWithReason(
|
||||
model, selectedStage.Name(), at, staleReasonChangedResult,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
_, err := applyCompletedStageOutcome(model, selectedStage.Name(), &stage.StageResult{}, nil, semanticConfig, prior, at)
|
||||
return err
|
||||
}
|
||||
|
||||
func plannedSelfSkipReason(stageName string, cfg *config.Config) string {
|
||||
|
||||
@@ -32,12 +32,12 @@ func TestPlanDoesNotCreateWorkdir(t *testing.T) {
|
||||
if !strings.Contains(got, "narratio session plan: read-only workdir at") {
|
||||
t.Fatalf("first output = %q, want read-only workdir", got)
|
||||
}
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "extract", "analyze", "publish", "notify"} {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "extract", "analyze", "publish", "notify", "cleanup"} {
|
||||
if !strings.Contains(got, name+": run") {
|
||||
t.Fatalf("first output = %q, missing stage %q", got, name)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(got, "totals: run=11 skip=0") {
|
||||
if !strings.Contains(got, "totals: run=12 skip=0") {
|
||||
t.Fatalf("first output = %q, want totals", got)
|
||||
}
|
||||
|
||||
@@ -83,8 +83,8 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||
if !strings.Contains(got, "trim: run") {
|
||||
t.Fatalf("output = %q, want trim run", got)
|
||||
}
|
||||
if !strings.Contains(got, "totals: run=9 skip=2") {
|
||||
t.Fatalf("output = %q, want totals run=9 skip=2", got)
|
||||
if !strings.Contains(got, "totals: run=10 skip=2") {
|
||||
t.Fatalf("output = %q, want totals run=10 skip=2", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,8 +117,6 @@ inputs:
|
||||
speakers_file: ./speakers.yml
|
||||
autocorrect_file: ./autocorrect.yml
|
||||
glossary_file: ./glossary.yml
|
||||
players_file: ./players.yml
|
||||
party_file: ./party.yml
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline.yml: %v", err)
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
func TestBuildFullPlanOrder(t *testing.T) {
|
||||
got := BuildFullPlan()
|
||||
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "extract", "analyze", "publish", "notify"}
|
||||
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "extract", "analyze", "publish", "notify", "cleanup"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("len(plan) = %d, want %d", len(got), len(want))
|
||||
}
|
||||
@@ -22,15 +22,15 @@ func TestBuildFullPlanOrder(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBuildSingleStagePlan(t *testing.T) {
|
||||
stages, err := BuildSingleStagePlan("polish")
|
||||
stages, err := BuildSingleStagePlan("cleanup")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildSingleStagePlan() error = %v", err)
|
||||
}
|
||||
if len(stages) != 1 {
|
||||
t.Fatalf("len(plan) = %d, want 1", len(stages))
|
||||
}
|
||||
if stages[0].Name() != "polish" {
|
||||
t.Fatalf("stage name = %q, want %q", stages[0].Name(), "polish")
|
||||
if stages[0].Name() != "cleanup" {
|
||||
t.Fatalf("stage name = %q, want %q", stages[0].Name(), "cleanup")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,6 +92,19 @@ func TestBuildBoundedPlanDefaultsToFullCanonicalPlan(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBoundedPlanThroughNotifyExcludesCleanup(t *testing.T) {
|
||||
plan, err := BuildBoundedPlan("", "notify")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildBoundedPlan() error = %v", err)
|
||||
}
|
||||
if plan.Contains("cleanup") {
|
||||
t.Fatalf("plan = %#v, cleanup must be outside a notify-bounded run", plan.Names())
|
||||
}
|
||||
if got := plan.Through(); got != "notify" {
|
||||
t.Fatalf("through = %q, want notify", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBoundedPlanRejectsInvalidBounds(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -99,9 +112,9 @@ func TestBuildBoundedPlanRejectsInvalidBounds(t *testing.T) {
|
||||
through string
|
||||
want []string
|
||||
}{
|
||||
{name: "unknown from", from: "missing", through: "analyze", want: []string{"unknown from stage", "missing", "prepare", "notify"}},
|
||||
{name: "unknown through", from: "extract", through: "missing", want: []string{"unknown through stage", "missing", "prepare", "notify"}},
|
||||
{name: "reversed", from: "publish", through: "render", want: []string{"publish", "occurs after", "render", "prepare", "notify"}},
|
||||
{name: "unknown from", from: "missing", through: "analyze", want: []string{"unknown from stage", "missing", "prepare", "cleanup"}},
|
||||
{name: "unknown through", from: "extract", through: "missing", want: []string{"unknown through stage", "missing", "prepare", "cleanup"}},
|
||||
{name: "reversed", from: "publish", through: "render", want: []string{"publish", "occurs after", "render", "prepare", "cleanup"}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
var removeRunScopedDirFn = removeRunScopedDir
|
||||
|
||||
func runPostPublishCleanup(ctx context.Context, env *Env, manifestPath string, m *manifest.Manifest, executed []string) error {
|
||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil || m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
cleanup := m.PostPublishCleanup
|
||||
if cleanup == nil {
|
||||
var err error
|
||||
cleanup, err = createPostPublishCleanup(env.Config, m, executed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cleanup == nil {
|
||||
return nil
|
||||
}
|
||||
m.PostPublishCleanup = cleanup
|
||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||
return fmt.Errorf("persist post-publish cleanup obligation %q: %w", manifestPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
for index := range cleanup.Targets {
|
||||
target := &cleanup.Targets[index]
|
||||
if target.Completed {
|
||||
continue
|
||||
}
|
||||
if err := removeRunScopedDirFn(target.Root, target.Path, target.Policy); err != nil {
|
||||
return fmt.Errorf("complete post-publish cleanup for %q: %w", target.Path, err)
|
||||
}
|
||||
target.Completed = true
|
||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||
target.Completed = false
|
||||
return fmt.Errorf("persist post-publish cleanup completion %q: %w", manifestPath, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func createPostPublishCleanup(cfg *config.Config, m *manifest.Manifest, executed []string) (*manifest.PostPublishCleanup, error) {
|
||||
spoolRequested := cfg.Pipeline.Spool.DeleteAudioAfterPublish
|
||||
workRequested := cfg.Pipeline.Workspace.CleanupAfterPublish
|
||||
if !spoolRequested && !workRequested {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
sr := publishStageRecordForCleanup(m)
|
||||
publishedRunID, eligible, _ := publishCleanupEligible(cfg, m, sr, executed)
|
||||
if !eligible {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
spoolDir := strings.TrimSpace(m.LocalSpoolDir)
|
||||
if spoolDir == "" {
|
||||
spoolDir = artifacts.SessionSpoolAudioDir(
|
||||
cfg.Pipeline.Spool.Root,
|
||||
strings.TrimSpace(m.Campaign),
|
||||
strings.TrimSpace(m.SessionID),
|
||||
publishedRunID,
|
||||
)
|
||||
}
|
||||
workDir := strings.TrimSpace(m.LocalWorkDir)
|
||||
if workDir == "" {
|
||||
workDir = artifacts.SessionRunRootForCampaign(
|
||||
cfg.Pipeline.Workspace.Root,
|
||||
strings.TrimSpace(m.Campaign),
|
||||
strings.TrimSpace(m.SessionID),
|
||||
publishedRunID,
|
||||
)
|
||||
}
|
||||
|
||||
cleanup := &manifest.PostPublishCleanup{
|
||||
CommittedRunID: publishedRunID,
|
||||
RemoteCommitKey: strings.TrimSpace(asString(sr.Metadata["remote_commit_key"])),
|
||||
CurrentCommitPointerKey: strings.TrimSpace(asString(sr.Metadata["current_commit_pointer_key"])),
|
||||
}
|
||||
if spoolRequested {
|
||||
cleanup.Targets = append(cleanup.Targets, manifest.CleanupTarget{
|
||||
Policy: "pipeline.spool.delete_audio_after_publish",
|
||||
Root: strings.TrimSpace(cfg.Pipeline.Spool.Root),
|
||||
Path: filepath.Clean(spoolDir),
|
||||
})
|
||||
}
|
||||
if workRequested {
|
||||
cleanup.Targets = append(cleanup.Targets, manifest.CleanupTarget{
|
||||
Policy: "pipeline.workspace.cleanup_after_publish",
|
||||
Root: strings.TrimSpace(cfg.Pipeline.Workspace.Root),
|
||||
Path: filepath.Clean(workDir),
|
||||
})
|
||||
}
|
||||
return cleanup, nil
|
||||
}
|
||||
|
||||
func publishStageRecordForCleanup(m *manifest.Manifest) *manifest.StageRecord {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
sr := m.Stages["publish"]
|
||||
if sr == nil || sr.Status != manifest.StatusSucceeded {
|
||||
return nil
|
||||
}
|
||||
return sr
|
||||
}
|
||||
|
||||
func publishCleanupEligible(cfg *config.Config, m *manifest.Manifest, sr *manifest.StageRecord, executed []string) (string, bool, string) {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Publish == nil {
|
||||
return "", false, "publish configuration is missing"
|
||||
}
|
||||
enabled := true
|
||||
if cfg.Pipeline.Publish.Enabled != nil {
|
||||
enabled = *cfg.Pipeline.Publish.Enabled
|
||||
}
|
||||
if !enabled {
|
||||
return "", false, "publish.enabled is false"
|
||||
}
|
||||
uploadRun := true
|
||||
if cfg.Pipeline.Publish.UploadRun != nil {
|
||||
uploadRun = *cfg.Pipeline.Publish.UploadRun
|
||||
}
|
||||
if !uploadRun {
|
||||
return "", false, "publish.upload_run is false"
|
||||
}
|
||||
if sr == nil || sr.Metadata == nil {
|
||||
return "", false, "publish metadata is missing"
|
||||
}
|
||||
if skipped, _ := sr.Metadata["skipped"].(bool); skipped {
|
||||
return "", false, "publish stage was skipped"
|
||||
}
|
||||
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
|
||||
return "", false, "publish did not upload run record"
|
||||
}
|
||||
if strings.TrimSpace(asString(sr.Metadata["remote_commit_key"])) == "" {
|
||||
return "", false, "publish remote commit key is missing"
|
||||
}
|
||||
if strings.TrimSpace(asString(sr.Metadata["current_commit_pointer_key"])) == "" {
|
||||
return "", false, "publish current commit pointer key is missing"
|
||||
}
|
||||
publishedRunID := strings.TrimSpace(asString(sr.Metadata["published_run_id"]))
|
||||
if publishedRunID == "" && containsStage(executed, "publish") && m != nil {
|
||||
publishedRunID = strings.TrimSpace(m.RunID)
|
||||
}
|
||||
if publishedRunID == "" {
|
||||
return "", false, "publish run id is missing"
|
||||
}
|
||||
return publishedRunID, true, ""
|
||||
}
|
||||
|
||||
func containsStage(names []string, target string) bool {
|
||||
for _, name := range names {
|
||||
if name == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type scopedDir struct {
|
||||
RootAbs string
|
||||
TargetAbs string
|
||||
Exists bool
|
||||
}
|
||||
|
||||
func removeRunScopedDir(root, target, policy string) error {
|
||||
dir, err := validateScopedDir(root, target, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !dir.Exists {
|
||||
return nil
|
||||
}
|
||||
if err := fileops.RemoveAllUnderRoot(dir.RootAbs, dir.TargetAbs); err != nil {
|
||||
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateScopedDir(root, target, policy string) (scopedDir, error) {
|
||||
return validateScopedTarget(root, target, policy, true)
|
||||
}
|
||||
|
||||
func asString(v any) string {
|
||||
s, _ := v.(string)
|
||||
return s
|
||||
}
|
||||
@@ -1,662 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type publishSuccessStage struct {
|
||||
metadata map[string]any
|
||||
targets *cleanupSeed
|
||||
}
|
||||
|
||||
func (publishSuccessStage) Name() string { return "publish" }
|
||||
func (s publishSuccessStage) Run(_ context.Context, _ *stage.Env, m *manifest.Manifest) (*stage.StageResult, error) {
|
||||
if s.targets != nil {
|
||||
s.targets.runWorkDir = m.LocalWorkDir
|
||||
s.targets.spoolAudioDir = m.LocalSpoolDir
|
||||
if err := os.MkdirAll(filepath.Join(m.LocalWorkDir, "logs"), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(m.LocalWorkDir, "logs", "stage.log"), []byte("log\n"), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.MkdirAll(m.LocalSpoolDir, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(m.LocalSpoolDir, "speaker.flac"), []byte("flac\n"), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
md := map[string]any{
|
||||
"stage": "publish",
|
||||
"uploaded": true,
|
||||
"published_run_id": m.RunID,
|
||||
"remote_commit_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/20260519T010203Z-a1b2c3d4/commit.json",
|
||||
"current_commit_pointer_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/current/commit-pointer.json",
|
||||
}
|
||||
for k, v := range s.metadata {
|
||||
md[k] = v
|
||||
}
|
||||
return &stage.StageResult{Metadata: md}, nil
|
||||
}
|
||||
|
||||
type notifyFailStage struct{}
|
||||
|
||||
func (notifyFailStage) Name() string { return "notify" }
|
||||
func (notifyFailStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||
return nil, errors.New("notify failed")
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupDisabledKeepsLocalDirs(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = false
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
assertExists(t, seed.localSourceAudio)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupSpoolOnly(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertMissing(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
assertExists(t, seed.localSourceAudio)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupWorkdirOnly(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = false
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertExists(t, cfg.Pipeline.Workspace.Root)
|
||||
assertExists(t, seed.otherRunDir)
|
||||
assertExists(t, seed.previousCachePath)
|
||||
assertMissing(t, seed.runWorkDir)
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupBothPolicies(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertMissing(t, seed.spoolAudioDir)
|
||||
assertMissing(t, seed.runWorkDir)
|
||||
assertExists(t, seed.otherRunDir)
|
||||
assertExists(t, seed.previousCachePath)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupRetriesWhenInitialObligationSaveFails(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||
|
||||
failed := false
|
||||
store := &cleanupFailingManifestStore{
|
||||
delegate: &manifest.LocalStore{},
|
||||
fail: func(m *manifest.Manifest) error {
|
||||
if !failed && m.PostPublishCleanup != nil {
|
||||
failed = true
|
||||
return errors.New("injected obligation save failure")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "post-publish cleanup incomplete") {
|
||||
t.Fatalf("executeStages() error = %v, want incomplete cleanup", err)
|
||||
}
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertCleanupPending(t, cfg)
|
||||
|
||||
store.fail = nil
|
||||
if _, err := executeStages(context.Background(), cfg, nil, RunOptions{Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("non-publish executeStages() error = %v", err)
|
||||
}
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertCleanupPending(t, cfg)
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Force: true, Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("publish retry executeStages() error = %v", err)
|
||||
}
|
||||
assertMissing(t, seed.spoolAudioDir)
|
||||
assertCleanupComplete(t, cfg)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupRetriesFailedDeletionWithoutTouchingOtherRuns(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
originalRemove := removeRunScopedDirFn
|
||||
removeRunScopedDirFn = func(root, target, policy string) error {
|
||||
if policy == "pipeline.workspace.cleanup_after_publish" {
|
||||
return errors.New("injected deletion failure")
|
||||
}
|
||||
return originalRemove(root, target, policy)
|
||||
}
|
||||
t.Cleanup(func() { removeRunScopedDirFn = originalRemove })
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "post-publish cleanup incomplete") {
|
||||
t.Fatalf("executeStages() error = %v, want incomplete cleanup", err)
|
||||
}
|
||||
assertMissing(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
assertCleanupPending(t, cfg)
|
||||
assertExists(t, seed.otherRunDir)
|
||||
assertExists(t, seed.previousCachePath)
|
||||
|
||||
removeRunScopedDirFn = originalRemove
|
||||
if _, err := executeStages(context.Background(), cfg, nil, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("non-publish executeStages() error = %v", err)
|
||||
}
|
||||
assertExists(t, seed.runWorkDir)
|
||||
assertCleanupPending(t, cfg)
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Force: true, Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("publish retry executeStages() error = %v", err)
|
||||
}
|
||||
assertMissing(t, seed.runWorkDir)
|
||||
assertExists(t, seed.otherRunDir)
|
||||
assertExists(t, seed.previousCachePath)
|
||||
assertCleanupComplete(t, cfg)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupRetriesWhenCompletionEvidenceSaveFails(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||
|
||||
failed := false
|
||||
store := &cleanupFailingManifestStore{
|
||||
delegate: &manifest.LocalStore{},
|
||||
fail: func(m *manifest.Manifest) error {
|
||||
if m.PostPublishCleanup == nil {
|
||||
return nil
|
||||
}
|
||||
for _, target := range m.PostPublishCleanup.Targets {
|
||||
if !failed && target.Completed {
|
||||
failed = true
|
||||
return errors.New("injected completion evidence failure")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{
|
||||
Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "post-publish cleanup incomplete") {
|
||||
t.Fatalf("executeStages() error = %v, want incomplete cleanup", err)
|
||||
}
|
||||
assertMissing(t, seed.spoolAudioDir)
|
||||
assertCleanupPending(t, cfg)
|
||||
|
||||
store.fail = nil
|
||||
if _, err := executeStages(context.Background(), cfg, nil, RunOptions{Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("non-publish executeStages() error = %v", err)
|
||||
}
|
||||
assertCleanupPending(t, cfg)
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Force: true, Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("publish retry executeStages() error = %v", err)
|
||||
}
|
||||
assertCleanupComplete(t, cfg)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, nil, RunOptions{Env: &Env{ManifestStore: store, ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("idempotent non-publish executeStages() error = %v", err)
|
||||
}
|
||||
assertMissing(t, seed.spoolAudioDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenPublishFails(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{failingStage{name: "publish", err: errors.New("publish failed")}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "stage \"publish\" failed") {
|
||||
t.Fatalf("executeStages() error = %v, want publish failure", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenPublishSkipped(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{metadata: map[string]any{"skipped": true}, targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenCommitPointerMissing(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{metadata: map[string]any{"current_commit_pointer_key": ""}, targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenPublishUploadDisabled(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
cfg.Pipeline.Publish.UploadRun = boolPtr(false)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{targets: &seed}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupWaitsUntilAllStagesSucceed(t *testing.T) {
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}, notifyFailStage{}}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "stage \"notify\" failed") {
|
||||
t.Fatalf("executeStages() error = %v, want notify failure", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupFailsOnUnsafePath(t *testing.T) {
|
||||
cfg, _ := cleanupFixtureConfig(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = false
|
||||
|
||||
manifestPath := manifestPathFor(cfg)
|
||||
store := &manifest.LocalStore{}
|
||||
m, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
m.MarkStageSucceeded("publish", time.Now().UTC(), nil)
|
||||
m.Stages["publish"].Metadata = map[string]any{
|
||||
"uploaded": true,
|
||||
"published_run_id": m.RunID,
|
||||
"remote_commit_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/runs/20260516T010203Z-1a2b3c4d/commit.json",
|
||||
"current_commit_pointer_key": "dnd/campaigns/sample-campaign/sessions/2026-05-03/current/commit-pointer.json",
|
||||
}
|
||||
m.PostPublishCleanup = &manifest.PostPublishCleanup{
|
||||
CommittedRunID: m.RunID,
|
||||
RemoteCommitKey: m.Stages["publish"].Metadata["remote_commit_key"].(string),
|
||||
CurrentCommitPointerKey: m.Stages["publish"].Metadata["current_commit_pointer_key"].(string),
|
||||
Targets: []manifest.CleanupTarget{{
|
||||
Policy: "pipeline.spool.delete_audio_after_publish",
|
||||
Root: cfg.Pipeline.Spool.Root,
|
||||
Path: filepath.Join(filepath.Dir(cfg.Pipeline.Spool.Root), "outside-spool"),
|
||||
}},
|
||||
}
|
||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
|
||||
if _, err = executeStages(context.Background(), cfg, nil, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}}); err != nil {
|
||||
t.Fatalf("non-publish executeStages() error = %v", err)
|
||||
}
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishSuccessStage{}}, RunOptions{Force: true, Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "refusing to delete path outside root") {
|
||||
t.Fatalf("executeStages() error = %v, want safe-path failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenOutputIsMissing(t *testing.T) {
|
||||
cfg, seed, runID := publishStageCleanupFixture(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
cfg.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
{Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)},
|
||||
}
|
||||
|
||||
publishStageImpl, err := stage.Select("publish")
|
||||
if err != nil {
|
||||
t.Fatalf("Select(publish) error = %v", err)
|
||||
}
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishStageImpl}, RunOptions{Env: &Env{ObjectStore: &storage.FakeBackend{}}})
|
||||
if err == nil || !strings.Contains(err.Error(), "required output source unavailable") {
|
||||
t.Fatalf("executeStages() error = %v, want required output source unavailable failure", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
assertExists(t, filepath.Join(seed.runWorkDir, "manifest.json"))
|
||||
assertExists(t, artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID))
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenCommittedManifestUploadFails(t *testing.T) {
|
||||
cfg, seed, _ := publishStageCleanupFixture(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
|
||||
publishStageImpl, err := stage.Select("publish")
|
||||
if err != nil {
|
||||
t.Fatalf("Select(publish) error = %v", err)
|
||||
}
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishStageImpl}, RunOptions{
|
||||
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, fail: func(key string) bool {
|
||||
return strings.HasSuffix(key, "/session-manifest.json")
|
||||
}}},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "immutable object") {
|
||||
t.Fatalf("executeStages() error = %v, want committed-manifest failure", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
func TestPostPublishCleanupNotRunWhenCommitPointerUploadFails(t *testing.T) {
|
||||
cfg, seed, _ := publishStageCleanupFixture(t)
|
||||
cfg.Pipeline.Spool.DeleteAudioAfterPublish = true
|
||||
cfg.Pipeline.Workspace.CleanupAfterPublish = true
|
||||
failKey := artifacts.S3CurrentCommitPointerKey(seed.sessionPrefix)
|
||||
|
||||
publishStageImpl, err := stage.Select("publish")
|
||||
if err != nil {
|
||||
t.Fatalf("Select(publish) error = %v", err)
|
||||
}
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{publishStageImpl}, RunOptions{
|
||||
Env: &Env{ObjectStore: &failKeyStore{delegate: &storage.FakeBackend{}, failKey: failKey}},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "current commit pointer") {
|
||||
t.Fatalf("executeStages() error = %v, want current-commit-pointer failure", err)
|
||||
}
|
||||
|
||||
assertExists(t, seed.spoolAudioDir)
|
||||
assertExists(t, seed.runWorkDir)
|
||||
}
|
||||
|
||||
type cleanupSeed struct {
|
||||
runID string
|
||||
runWorkDir string
|
||||
otherRunDir string
|
||||
spoolAudioDir string
|
||||
localSourceAudio string
|
||||
previousCachePath string
|
||||
sessionPrefix string
|
||||
}
|
||||
|
||||
type cleanupFailingManifestStore struct {
|
||||
delegate manifest.Store
|
||||
fail func(*manifest.Manifest) error
|
||||
}
|
||||
|
||||
func (s *cleanupFailingManifestStore) Create(ctx context.Context, sessionID string) (*manifest.Manifest, error) {
|
||||
return s.delegate.Create(ctx, sessionID)
|
||||
}
|
||||
|
||||
func (s *cleanupFailingManifestStore) Load(ctx context.Context, path string) (*manifest.Manifest, error) {
|
||||
return s.delegate.Load(ctx, path)
|
||||
}
|
||||
|
||||
func (s *cleanupFailingManifestStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||
if s.fail != nil {
|
||||
if err := s.fail(m); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.delegate.Save(ctx, path, m)
|
||||
}
|
||||
|
||||
func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
|
||||
t.Helper()
|
||||
|
||||
cfg := testConfig(t)
|
||||
cfg.Pipeline.Publish = &config.PublishConfig{Enabled: boolPtr(true), UploadRun: boolPtr(true)}
|
||||
cfg.Pipeline.Spool.Root = filepath.Join(t.TempDir(), "spool")
|
||||
|
||||
runID := "20260516T010203Z-1a2b3c4d"
|
||||
runWorkDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||
otherRunDir := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, "20260516T010204Z-5e6f7a8b")
|
||||
spoolAudioDir := artifacts.SessionSpoolAudioDir(cfg.Pipeline.Spool.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID)
|
||||
previousCachePath := mustPreviousArtifactPathForCampaign(t,
|
||||
cfg.Pipeline.Workspace.Root,
|
||||
cfg.Session.Campaign,
|
||||
cfg.Session.SessionID,
|
||||
"session_recap.md",
|
||||
)
|
||||
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "stage.log"), "log\n")
|
||||
mustWriteFile(t, filepath.Join(otherRunDir, "logs", "stage.log"), "other\n")
|
||||
mustWriteFile(t, filepath.Join(spoolAudioDir, "speaker.flac"), "flac\n")
|
||||
mustWriteFile(t, previousCachePath, "# previous recap\n")
|
||||
|
||||
localSourceAudio := filepath.Join(filepath.Dir(cfg.SessionPath), "audio", "alice.flac")
|
||||
mustWriteFile(t, localSourceAudio, "source\n")
|
||||
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
seed.RunID = runID
|
||||
seed.LocalWorkDir = runWorkDir
|
||||
seed.LocalSpoolDir = spoolAudioDir
|
||||
seed.S3Bucket = "my-dnd-archive"
|
||||
seed.S3SessionPrefix = "dnd/campaigns/sample-campaign/sessions/2026-05-03/"
|
||||
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
if err := os.MkdirAll(filepath.Dir(manifestPathFor(cfg)), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||
t.Fatalf("seed manifest save error = %v", err)
|
||||
}
|
||||
|
||||
return cfg, cleanupSeed{
|
||||
runID: runID,
|
||||
runWorkDir: runWorkDir,
|
||||
otherRunDir: otherRunDir,
|
||||
spoolAudioDir: spoolAudioDir,
|
||||
localSourceAudio: localSourceAudio,
|
||||
previousCachePath: previousCachePath,
|
||||
sessionPrefix: seed.S3SessionPrefix,
|
||||
}
|
||||
}
|
||||
|
||||
func publishStageCleanupFixture(t *testing.T) (*config.Config, cleanupSeed, string) {
|
||||
t.Helper()
|
||||
|
||||
cfg, seed := cleanupFixtureConfig(t)
|
||||
runID := "20260516T010203Z-1a2b3c4d"
|
||||
cfg.Pipeline.Storage.S3 = &config.StorageS3Config{
|
||||
Bucket: "my-dnd-archive",
|
||||
RootPrefix: "dnd",
|
||||
}
|
||||
cfg.Pipeline.Publish = &config.PublishConfig{
|
||||
Enabled: boolPtr(true),
|
||||
UploadRun: boolPtr(true),
|
||||
Outputs: []config.PublishOutputRule{
|
||||
{Source: "narratio.transcript.final_trimmed", Dest: "transcripts/final.trimmed.json", Required: boolPtr(true)},
|
||||
{Source: "narratio.artifact.session_recap", Dest: "artifacts/session_recap.md", Required: boolPtr(true)},
|
||||
},
|
||||
}
|
||||
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{
|
||||
Artifacts: map[string]config.ScriptoriumArtifactConfig{
|
||||
"session_recap": {
|
||||
OutputPath: "artifacts/session_recap.md",
|
||||
},
|
||||
},
|
||||
}
|
||||
writePublishFixtureRunFiles(
|
||||
t,
|
||||
seed.runWorkDir,
|
||||
artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID),
|
||||
)
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
seedManifest, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||
seedManifest.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
}
|
||||
setAppAnalyzeEvidence(seedManifest, "session_recap", "artifacts/session_recap.md", []byte("# recap\n"))
|
||||
seedManifest.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
seedManifest.S3RunPrefix = artifacts.S3RunPrefix(seedManifest.S3SessionPrefix, runID)
|
||||
if err := store.Save(context.Background(), manifestPathFor(cfg), seedManifest); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
seed.sessionPrefix = seedManifest.S3SessionPrefix
|
||||
return cfg, seed, runID
|
||||
}
|
||||
|
||||
func writePublishFixtureRunFiles(t *testing.T, runWorkDir, sessionRoot string) {
|
||||
t.Helper()
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "prepare", "inputs", "session.yml"), "session_id: 2026-05-03\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "transcribe", "outputs", "transcripts", "raw", "speaker.json"), "{}\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "trim", "outputs", "transcripts", "final.trimmed.json"), "{\"segments\":[]}\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "analyze", "outputs", "artifacts", "session_recap.md"), "# recap\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "polish", "reports", "audita.report.json"), "{}\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "merge", "config", "seriatim.generated.yml"), "key: value\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "audita.stderr.log"), "stderr\n")
|
||||
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
|
||||
|
||||
mustWriteFile(t, filepath.Join(sessionRoot, "transcripts", "final.trimmed.json"), "{\"segments\":[]}\n")
|
||||
mustWriteFile(t, filepath.Join(sessionRoot, "artifacts", "session_recap.md"), "# recap\n")
|
||||
}
|
||||
|
||||
type failKeyStore struct {
|
||||
delegate *storage.FakeBackend
|
||||
failKey string
|
||||
fail func(string) bool
|
||||
}
|
||||
|
||||
func (s *failKeyStore) fails(key string) bool {
|
||||
return strings.TrimSpace(key) == strings.TrimSpace(s.failKey) || (s.fail != nil && s.fail(key))
|
||||
}
|
||||
|
||||
func (s *failKeyStore) List(ctx context.Context, prefix string) ([]storage.ObjectInfo, error) {
|
||||
return s.delegate.List(ctx, prefix)
|
||||
}
|
||||
|
||||
func (s *failKeyStore) Read(ctx context.Context, key string) (storage.ObjectInfo, io.ReadCloser, error) {
|
||||
return s.delegate.Read(ctx, key)
|
||||
}
|
||||
|
||||
func (s *failKeyStore) Download(ctx context.Context, key, localPath string) error {
|
||||
return s.delegate.Download(ctx, key, localPath)
|
||||
}
|
||||
|
||||
func (s *failKeyStore) Upload(ctx context.Context, localPath, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
|
||||
if s.fails(key) {
|
||||
return storage.ObjectInfo{}, errors.New("forced upload failure")
|
||||
}
|
||||
return s.delegate.Upload(ctx, localPath, key, opts)
|
||||
}
|
||||
|
||||
func (s *failKeyStore) UploadReader(ctx context.Context, source io.Reader, key string, opts storage.UploadOptions) (storage.ObjectInfo, error) {
|
||||
if s.fails(key) {
|
||||
return storage.ObjectInfo{}, errors.New("forced upload failure")
|
||||
}
|
||||
return s.delegate.UploadReader(ctx, source, key, opts)
|
||||
}
|
||||
|
||||
func (s *failKeyStore) UploadConditional(ctx context.Context, source io.Reader, key string, opts storage.UploadOptions, condition storage.WriteCondition) (storage.ObjectInfo, error) {
|
||||
if s.fails(key) {
|
||||
return storage.ObjectInfo{}, errors.New("forced upload failure")
|
||||
}
|
||||
return s.delegate.UploadConditional(ctx, source, key, opts, condition)
|
||||
}
|
||||
|
||||
func (s *failKeyStore) Exists(ctx context.Context, key string) (bool, error) {
|
||||
return s.delegate.Exists(ctx, key)
|
||||
}
|
||||
|
||||
func assertExists(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("expected path to exist %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertMissing(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected path to be removed %q, stat err=%v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertCleanupPending(t *testing.T, cfg *config.Config) {
|
||||
t.Helper()
|
||||
m, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if m.PostPublishCleanup == nil {
|
||||
t.Fatal("expected a persisted cleanup obligation")
|
||||
}
|
||||
for _, target := range m.PostPublishCleanup.Targets {
|
||||
if !target.Completed {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("expected at least one cleanup target to remain incomplete")
|
||||
}
|
||||
|
||||
func assertCleanupComplete(t *testing.T, cfg *config.Config) {
|
||||
t.Helper()
|
||||
m, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if m.PostPublishCleanup == nil {
|
||||
t.Fatal("expected a persisted cleanup obligation")
|
||||
}
|
||||
for _, target := range m.PostPublishCleanup.Targets {
|
||||
if !target.Completed {
|
||||
t.Fatalf("cleanup target remains incomplete: %#v", target)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,26 +16,24 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
enableCommandTestProfiles(t, pipelinePath)
|
||||
|
||||
original := executeStagesFn
|
||||
t.Cleanup(func() { executeStagesFn = original })
|
||||
var language string
|
||||
executeStagesFn = func(_ context.Context, cfg *config.Config, _ BoundedPlan, _ RunOptions) (*RunSummary, error) {
|
||||
executor := func(_ context.Context, cfg *config.Config, _ BoundedPlan, _ RunOptions) (*RunSummary, error) {
|
||||
language = cfg.Pipeline.WhisperX.Language
|
||||
return &RunSummary{SessionID: cfg.Session.SessionID, ManifestPath: "manifest.json"}, nil
|
||||
}
|
||||
|
||||
if err := Run(context.Background(), []string{
|
||||
if err := runWithPipelineExecutor(context.Background(), []string{
|
||||
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", "testing",
|
||||
}, io.Discard); err != nil {
|
||||
}, io.Discard, executor); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if language != "fr" {
|
||||
t.Fatalf("explicit profile language = %q, want fr", language)
|
||||
}
|
||||
|
||||
if err := Run(context.Background(), []string{
|
||||
if err := runWithPipelineExecutor(context.Background(), []string{
|
||||
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath,
|
||||
}, io.Discard); err != nil {
|
||||
}, io.Discard, executor); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if language != "en" {
|
||||
@@ -43,9 +41,9 @@ func TestRunProfileSelectionFlowsThroughSharedLoader(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, profile := range []string{"unknown", ""} {
|
||||
err := Run(context.Background(), []string{
|
||||
err := runWithPipelineExecutor(context.Background(), []string{
|
||||
"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--profile", profile,
|
||||
}, io.Discard)
|
||||
}, io.Discard, executor)
|
||||
if err == nil || !strings.Contains(err.Error(), "profile") {
|
||||
t.Fatalf("profile %q error = %v, want selection rejection", profile, err)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,14 @@ import (
|
||||
// RegenerateArtifacts expands the convenience command into its canonical run
|
||||
// invocation. The run command remains the sole owner of parsing and execution.
|
||||
func RegenerateArtifacts(ctx context.Context, args []string, out io.Writer) error {
|
||||
return regenerateArtifactsWithDependencies(ctx, args, out, executePlan, commandDependencies{})
|
||||
}
|
||||
|
||||
func regenerateArtifactsWithPipelineExecutor(ctx context.Context, args []string, out io.Writer, executor pipelineExecutor) error {
|
||||
return regenerateArtifactsWithDependencies(ctx, args, out, executor, commandDependencies{})
|
||||
}
|
||||
|
||||
func regenerateArtifactsWithDependencies(ctx context.Context, args []string, out io.Writer, executor pipelineExecutor, deps commandDependencies) error {
|
||||
if containsHelpOption(args) {
|
||||
printRegenerateArtifactsHelp(out)
|
||||
return nil
|
||||
@@ -21,7 +29,7 @@ func RegenerateArtifacts(ctx context.Context, args []string, out io.Writer) erro
|
||||
}
|
||||
expanded = append(expanded, "--force", "--from", "extract", "--through", "analyze")
|
||||
expanded = append(expanded, args...)
|
||||
return runCommandFn(ctx, expanded, out)
|
||||
return runWithDependencies(ctx, expanded, out, executor, deps)
|
||||
}
|
||||
|
||||
func containsHelpOption(args []string) bool {
|
||||
|
||||
@@ -7,82 +7,80 @@ import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
func TestRegenerateArtifactsForwardsExactCanonicalRunArguments(t *testing.T) {
|
||||
original := runCommandFn
|
||||
t.Cleanup(func() { runCommandFn = original })
|
||||
var captured []string
|
||||
runCommandFn = func(_ context.Context, args []string, _ io.Writer) error {
|
||||
captured = append([]string(nil), args...)
|
||||
return nil
|
||||
func TestRegenerateArtifactsForwardsCanonicalPipelineRequest(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
enableCommandTestProfiles(t, pipelinePath)
|
||||
var capturedPlan BoundedPlan
|
||||
var capturedOptions RunOptions
|
||||
executor := func(_ context.Context, cfg *config.Config, plan BoundedPlan, options RunOptions) (*RunSummary, error) {
|
||||
capturedPlan = plan
|
||||
capturedOptions = options
|
||||
return &RunSummary{SessionID: cfg.Session.SessionID, ManifestPath: "manifest.json"}, nil
|
||||
}
|
||||
|
||||
code := Execute([]string{
|
||||
code := executeWithPipelineExecutor([]string{
|
||||
"regenerate-artifacts", "2026-05-03",
|
||||
"--artifacts", "session_recap,player_handout",
|
||||
"--artifacts=player_handout",
|
||||
"--config", "pipeline.yml",
|
||||
"--campaign", "sample-campaign",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--profile", "testing",
|
||||
}, io.Discard, io.Discard)
|
||||
}, io.Discard, io.Discard, executor)
|
||||
if code != 0 {
|
||||
t.Fatalf("Execute() code = %d, want 0", code)
|
||||
t.Fatalf("executeWithPipelineExecutor() code = %d, want 0", code)
|
||||
}
|
||||
want := []string{
|
||||
"2026-05-03", "--force", "--from", "extract", "--through", "analyze",
|
||||
"--artifacts", "session_recap,player_handout",
|
||||
"--artifacts=player_handout",
|
||||
"--config", "pipeline.yml",
|
||||
"--campaign", "sample-campaign",
|
||||
"--profile", "testing",
|
||||
if got, want := capturedPlan.Names(), []string{"extract", "analyze"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("plan = %#v, want %#v", got, want)
|
||||
}
|
||||
if !reflect.DeepEqual(captured, want) {
|
||||
t.Fatalf("forwarded args = %#v, want %#v", captured, want)
|
||||
if !capturedOptions.Force {
|
||||
t.Fatal("force = false, want true")
|
||||
}
|
||||
if got, want := capturedOptions.SelectedArtifacts, []string{"player_handout", "session_recap"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("selected artifacts = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegenerateArtifactsForwardsSessionIDCompatibilityFlag(t *testing.T) {
|
||||
original := runCommandFn
|
||||
t.Cleanup(func() { runCommandFn = original })
|
||||
var captured []string
|
||||
runCommandFn = func(_ context.Context, args []string, _ io.Writer) error {
|
||||
captured = append([]string(nil), args...)
|
||||
return nil
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFilesWithScriptoriumArtifacts(t, workspaceRoot)
|
||||
var capturedSessionID string
|
||||
executor := func(_ context.Context, cfg *config.Config, _ BoundedPlan, _ RunOptions) (*RunSummary, error) {
|
||||
capturedSessionID = cfg.Session.SessionID
|
||||
return &RunSummary{SessionID: cfg.Session.SessionID, ManifestPath: "manifest.json"}, nil
|
||||
}
|
||||
|
||||
code := Execute([]string{
|
||||
code := executeWithPipelineExecutor([]string{
|
||||
"regenerate-artifacts",
|
||||
"--config", "pipeline.yml",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--session-id", "2026-05-03",
|
||||
"--artifacts", "session_recap",
|
||||
}, io.Discard, io.Discard)
|
||||
}, io.Discard, io.Discard, executor)
|
||||
if code != 0 {
|
||||
t.Fatalf("Execute() code = %d, want 0", code)
|
||||
t.Fatalf("executeWithPipelineExecutor() code = %d, want 0", code)
|
||||
}
|
||||
want := []string{
|
||||
"--force", "--from", "extract", "--through", "analyze",
|
||||
"--config", "pipeline.yml",
|
||||
"--session-id", "2026-05-03",
|
||||
"--artifacts", "session_recap",
|
||||
}
|
||||
if !reflect.DeepEqual(captured, want) {
|
||||
t.Fatalf("forwarded args = %#v, want %#v", captured, want)
|
||||
if capturedSessionID != "2026-05-03" {
|
||||
t.Fatalf("session id = %q, want compatibility flag value", capturedSessionID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegenerateArtifactsHelpDoesNotInvokeRun(t *testing.T) {
|
||||
original := runCommandFn
|
||||
t.Cleanup(func() { runCommandFn = original })
|
||||
called := false
|
||||
runCommandFn = func(_ context.Context, _ []string, _ io.Writer) error {
|
||||
executor := func(context.Context, *config.Config, BoundedPlan, RunOptions) (*RunSummary, error) {
|
||||
called = true
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
if code := Execute([]string{"regenerate-artifacts", "--help"}, &stdout, &stderr); code != 0 {
|
||||
if code := executeWithPipelineExecutor([]string{"regenerate-artifacts", "--help"}, &stdout, &stderr, executor); code != 0 {
|
||||
t.Fatalf("Execute() code = %d, stderr = %q", code, stderr.String())
|
||||
}
|
||||
if called {
|
||||
|
||||
@@ -1,180 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
)
|
||||
|
||||
type effectiveLocks struct {
|
||||
Static []config.PublishLockRule
|
||||
Remote []config.PublishLockRule
|
||||
All []config.PublishLockRule
|
||||
Key string
|
||||
}
|
||||
|
||||
const (
|
||||
remoteLockMutationAttempts = 4
|
||||
// MaxRemoteLockStoreBytes bounds the mutable remote publish-lock document.
|
||||
MaxRemoteLockStoreBytes int64 = 1 << 20
|
||||
)
|
||||
|
||||
func remoteLocksKey(cfg *config.Config) (string, error) {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return "", fmt.Errorf("resolved config is required")
|
||||
}
|
||||
if cfg.Pipeline.Storage.S3 == nil {
|
||||
return "", fmt.Errorf("pipeline.storage.s3 configuration is required")
|
||||
}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(
|
||||
cfg.Pipeline.Storage.S3.RootPrefix,
|
||||
cfg.Session.Campaign,
|
||||
cfg.Session.SessionID,
|
||||
)
|
||||
return artifacts.S3SessionLocksKey(sessionPrefix), nil
|
||||
}
|
||||
|
||||
func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*config.PublishLockStore, string, string, error) {
|
||||
key, err := remoteLocksKey(cfg)
|
||||
if err != nil {
|
||||
return nil, "", "", err
|
||||
}
|
||||
if store == nil {
|
||||
return nil, key, "", fmt.Errorf("remote lock store is required")
|
||||
}
|
||||
info, data, err := storage.ReadObjectBounded(ctx, store, key, MaxRemoteLockStoreBytes)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return &config.PublishLockStore{}, key, "", nil
|
||||
}
|
||||
var limitErr *storage.ReadLimitError
|
||||
if errors.As(err, &limitErr) {
|
||||
return nil, key, "", fmt.Errorf("read remote lock control object %q with %d-byte limit: %w", key, MaxRemoteLockStoreBytes, err)
|
||||
}
|
||||
return nil, key, "", fmt.Errorf("read remote locks %q: %w", key, err)
|
||||
}
|
||||
if strings.TrimSpace(info.ETag) == "" {
|
||||
return nil, key, "", fmt.Errorf("read remote locks %q: object has no generation", key)
|
||||
}
|
||||
lockStore, err := config.LoadPublishLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius)
|
||||
if err != nil {
|
||||
return nil, key, "", err
|
||||
}
|
||||
return lockStore, key, info.ETag, nil
|
||||
}
|
||||
|
||||
func loadEffectiveLocks(ctx context.Context, cfg *config.Config, store storage.ObjectStore) (*effectiveLocks, error) {
|
||||
staticLocks := staticPublishLocks(cfg)
|
||||
if store == nil {
|
||||
return &effectiveLocks{
|
||||
Static: staticLocks,
|
||||
All: append([]config.PublishLockRule(nil), staticLocks...),
|
||||
}, nil
|
||||
}
|
||||
lockStore, key, _, err := loadRemoteLockStore(ctx, cfg, store)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remoteLocks := append([]config.PublishLockRule(nil), lockStore.Locks...)
|
||||
return &effectiveLocks{
|
||||
Static: staticLocks,
|
||||
Remote: remoteLocks,
|
||||
All: config.MergePublishLockRules(staticLocks, remoteLocks),
|
||||
Key: key,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func staticPublishLocks(cfg *config.Config) []config.PublishLockRule {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Publish == nil {
|
||||
return nil
|
||||
}
|
||||
return append([]config.PublishLockRule(nil), cfg.Pipeline.Publish.Locks...)
|
||||
}
|
||||
|
||||
func applyEffectiveLocks(cfg *config.Config, locks []config.PublishLockRule) {
|
||||
if cfg == nil || cfg.Pipeline == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Pipeline.Publish == nil {
|
||||
cfg.Pipeline.Publish = &config.PublishConfig{}
|
||||
}
|
||||
cfg.Pipeline.Publish.Locks = append([]config.PublishLockRule(nil), locks...)
|
||||
}
|
||||
|
||||
func mutateRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.ObjectStore, mutate func(*config.PublishLockStore) error) error {
|
||||
for attempt := 0; attempt < remoteLockMutationAttempts; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
lockStore, key, generation, err := loadRemoteLockStore(ctx, cfg, store)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := mutate(lockStore); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := config.MarshalPublishLockStore(lockStore)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
condition := storage.WriteCondition{MatchETag: generation}
|
||||
if generation == "" {
|
||||
condition = storage.WriteCondition{RequireAbsent: true}
|
||||
}
|
||||
_, err = store.UploadConditional(ctx, bytes.NewReader(data), key, storage.UploadOptions{ContentType: "application/x-yaml; charset=utf-8"}, condition)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, storage.ErrConditionNotMet) {
|
||||
return fmt.Errorf("upload remote locks %q: %w", key, err)
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("update remote locks: concurrent updates prevented a conditional write after %d attempts", remoteLockMutationAttempts)
|
||||
}
|
||||
|
||||
func lockSourceSet(locks []config.PublishLockRule) map[string]config.PublishLockRule {
|
||||
out := make(map[string]config.PublishLockRule, len(locks))
|
||||
for _, lock := range locks {
|
||||
source := strings.TrimSpace(lock.Source)
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
lock.Source = source
|
||||
lock.Reason = strings.TrimSpace(lock.Reason)
|
||||
out[source] = lock
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func writeLocalFile(path string, data []byte, force bool) error {
|
||||
cleaned := filepath.Clean(strings.TrimSpace(path))
|
||||
if cleaned == "" || cleaned == "." {
|
||||
return fmt.Errorf("output path is required")
|
||||
}
|
||||
if !force {
|
||||
if _, err := os.Stat(cleaned); err == nil {
|
||||
return fmt.Errorf("output file %q already exists; pass --force to overwrite", cleaned)
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("check output file %q: %w", cleaned, err)
|
||||
}
|
||||
}
|
||||
if err := fileops.EnsureWorkspaceDirectory(filepath.Dir(cleaned)); err != nil {
|
||||
return fmt.Errorf("create output directory: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(cleaned, data, fileops.WorkspaceFileMode); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Chmod(cleaned, fileops.WorkspaceFileMode)
|
||||
}
|
||||
@@ -25,11 +25,12 @@ inputs:
|
||||
prefix: audio/
|
||||
`)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps := fakeCommandDependencies(fake, &storeInitCalls)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr, deps, executePlan)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -74,16 +75,19 @@ inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps, storeInitCalls := remoteSessionLoaderDependencies(fake)
|
||||
|
||||
loaded, err := loadCommandConfig(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"})
|
||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"}, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||
}
|
||||
defer func() { _ = loaded.Close() }()
|
||||
if loaded.Config.Party.Mode != config.PartyModeCanonical || loaded.Config.StableInputs.PlayersFile.Source != "derived_from_party" {
|
||||
t.Fatalf("remote config party = %#v, players = %#v", loaded.Config.Party, loaded.Config.StableInputs.PlayersFile)
|
||||
if loaded.Config.Party.Canonical == nil {
|
||||
t.Fatalf("remote config party = %#v", loaded.Config.Party)
|
||||
}
|
||||
if *storeInitCalls != 1 {
|
||||
t.Fatalf("object store init calls = %d, want 1", *storeInitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,7 +96,7 @@ func TestRemoteSessionConfigIsRemovedAfterEveryCommandExit(t *testing.T) {
|
||||
name string
|
||||
sessionYAML string
|
||||
command []string
|
||||
configureRun func()
|
||||
executor pipelineExecutor
|
||||
wantSuccessful bool
|
||||
}{
|
||||
{
|
||||
@@ -126,10 +130,8 @@ inputs:
|
||||
prefix: audio/
|
||||
`,
|
||||
command: []string{"run", "2026-05-03"},
|
||||
configureRun: func() {
|
||||
executeStagesFn = func(context.Context, *config.Config, BoundedPlan, RunOptions) (*RunSummary, error) {
|
||||
return nil, errors.New("adapter failed")
|
||||
}
|
||||
executor: func(context.Context, *config.Config, BoundedPlan, RunOptions) (*RunSummary, error) {
|
||||
return nil, errors.New("adapter failed")
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -140,10 +142,8 @@ inputs:
|
||||
prefix: audio/
|
||||
`,
|
||||
command: []string{"run", "2026-05-03"},
|
||||
configureRun: func() {
|
||||
executeStagesFn = func(context.Context, *config.Config, BoundedPlan, RunOptions) (*RunSummary, error) {
|
||||
return nil, context.Canceled
|
||||
}
|
||||
executor: func(context.Context, *config.Config, BoundedPlan, RunOptions) (*RunSummary, error) {
|
||||
return nil, context.Canceled
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -155,31 +155,32 @@ inputs:
|
||||
fake := &storage.FakeBackend{}
|
||||
seedRemoteSessionConfig(t, fake, "2026-05-03", tt.sessionYAML)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps := fakeCommandDependencies(fake, &storeInitCalls)
|
||||
|
||||
var downloadedPath string
|
||||
captureRemoteSessionTempPath(t, &downloadedPath)
|
||||
if tt.configureRun != nil {
|
||||
origExecuteStagesFn := executeStagesFn
|
||||
t.Cleanup(func() { executeStagesFn = origExecuteStagesFn })
|
||||
tt.configureRun()
|
||||
tempRoot := t.TempDir()
|
||||
t.Setenv("TMPDIR", tempRoot)
|
||||
executor := tt.executor
|
||||
if executor == nil {
|
||||
executor = executePlan
|
||||
}
|
||||
|
||||
args := append(append([]string(nil), tt.command...), "--config", pipelinePath, "--campaign-file", campaignPath)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(args, &stdout, &stderr)
|
||||
code := executeWithDependencies(args, &stdout, &stderr, deps, executor)
|
||||
if tt.wantSuccessful && code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !tt.wantSuccessful && code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
if downloadedPath == "" {
|
||||
t.Fatal("remote session download path was not captured")
|
||||
entries, err := os.ReadDir(tempRoot)
|
||||
if err != nil {
|
||||
t.Fatalf("read temp root: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(downloadedPath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("downloaded remote session path still exists or could not be inspected: %q, err=%v", downloadedPath, err)
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("temporary remote session files remain after command exit: %v", entries)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -194,12 +195,12 @@ inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps, _ := remoteSessionLoaderDependencies(fake)
|
||||
|
||||
var downloadedPath string
|
||||
captureRemoteSessionTempPath(t, &downloadedPath)
|
||||
loaded, err := loadCommandConfig(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"})
|
||||
deps.downloadObjectToTemp = captureRemoteSessionTempPath(&downloadedPath)
|
||||
loaded, err := loadCommandConfigWithDependencies(context.Background(), pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: "2026-05-03"}, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("loadCommandConfig() error = %v", err)
|
||||
}
|
||||
@@ -217,17 +218,14 @@ inputs:
|
||||
}
|
||||
}
|
||||
|
||||
func captureRemoteSessionTempPath(t *testing.T, destination *string) {
|
||||
t.Helper()
|
||||
original := downloadObjectToTempFn
|
||||
downloadObjectToTempFn = func(ctx context.Context, store storage.ObjectStore, key, pattern string) (string, error) {
|
||||
path, err := original(ctx, store, key, pattern)
|
||||
func captureRemoteSessionTempPath(destination *string) func(context.Context, storage.ObjectStore, string, string) (string, error) {
|
||||
return func(ctx context.Context, store storage.ObjectStore, key, pattern string) (string, error) {
|
||||
path, err := storage.DownloadObjectToTemp(ctx, store, key, pattern)
|
||||
if err == nil {
|
||||
*destination = path
|
||||
}
|
||||
return path, err
|
||||
}
|
||||
t.Cleanup(func() { downloadObjectToTempFn = original })
|
||||
}
|
||||
|
||||
func TestExecuteRemoteSessionFallbackLoadsSecretsBeforeObjectStoreInit(t *testing.T) {
|
||||
@@ -247,23 +245,17 @@ inputs:
|
||||
audio_s3:
|
||||
prefix: audio/
|
||||
`)
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if os.Getenv(accessKeyEnv) != "remote-session-key-id" || os.Getenv(secretKeyEnv) != "remote-session-secret" {
|
||||
return nil, fmt.Errorf("secrets were not loaded before remote session object store init")
|
||||
}
|
||||
return fake, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||
})
|
||||
}}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr, deps, executePlan)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
@@ -274,11 +266,12 @@ func TestExecuteExplicitLocalSessionPrecedenceSkipsRemote(t *testing.T) {
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps := fakeCommandDependencies(fake, &storeInitCalls)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr, deps, executePlan)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -292,11 +285,12 @@ func TestExecuteLocalSessionDiscoveryPrecedenceSkipsRemote(t *testing.T) {
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
setSessionConfigSearchPaths(t, []string{sessionPath})
|
||||
deps := fakeCommandDependencies(fake, &storeInitCalls)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr, deps, executePlan)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
@@ -311,11 +305,12 @@ func TestExecuteRemoteSessionMissingObjectFailsClearly(t *testing.T) {
|
||||
fake := &storage.FakeBackend{}
|
||||
var storeInitCalls int
|
||||
missingSessionPath := filepath.Join(t.TempDir(), "session.yml")
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{missingSessionPath})
|
||||
setSessionConfigSearchPaths(t, []string{missingSessionPath})
|
||||
deps := fakeCommandDependencies(fake, &storeInitCalls)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr, deps, executePlan)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -331,11 +326,12 @@ func TestExecuteRemoteSessionRequiresSessionID(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, &storage.FakeBackend{}, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps := fakeCommandDependencies(&storage.FakeBackend{}, &storeInitCalls)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"session", "plan", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr, deps, executePlan)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -350,20 +346,14 @@ func TestExecuteRemoteSessionRequiresSessionID(t *testing.T) {
|
||||
func TestExecuteRemoteSessionStorageInitErrorFailsClearly(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, _ := writeValidConfigFiles(t, workspaceRoot)
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||
config.DefaultSessionConfigSearchPaths = []string{filepath.Join(t.TempDir(), "session.yml")}
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps := commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
return nil, errors.New("storage unavailable")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||
})
|
||||
}}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr, deps, executePlan)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -378,11 +368,12 @@ func TestExecuteRemoteSessionMalformedYAMLFailsStrictDecode(t *testing.T) {
|
||||
fake := &storage.FakeBackend{}
|
||||
seedRemoteSessionConfig(t, fake, "2026-05-03", "session_id: 2026-05-03\nunknown: true\n")
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps := fakeCommandDependencies(fake, &storeInitCalls)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr, deps, executePlan)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -401,11 +392,12 @@ inputs:
|
||||
prefix: audio/
|
||||
`)
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps := fakeCommandDependencies(fake, &storeInitCalls)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr, deps, executePlan)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -420,11 +412,12 @@ func TestExecuteRemoteSessionMismatchFails(t *testing.T) {
|
||||
fake := &storage.FakeBackend{}
|
||||
seedRemoteSessionConfig(t, fake, "2026-05-03", "session_id: 2026-05-04\ninputs:\n audio_s3:\n prefix: audio/\n")
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
setSessionConfigSearchPaths(t, []string{filepath.Join(t.TempDir(), "session.yml")})
|
||||
deps := fakeCommandDependencies(fake, &storeInitCalls)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr)
|
||||
code := executeWithDependencies([]string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath}, &stdout, &stderr, deps, executePlan)
|
||||
if code == 0 {
|
||||
t.Fatal("exit code = 0, want non-zero")
|
||||
}
|
||||
@@ -433,21 +426,25 @@ func TestExecuteRemoteSessionMismatchFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func restoreAppConfigTestGlobals(t *testing.T, fake *storage.FakeBackend, storeInitCalls *int, sessionDefaults []string) {
|
||||
t.Helper()
|
||||
origStoreFn := newObjectStoreFromConfigFn
|
||||
origSessionDefaults := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||
config.DefaultSessionConfigSearchPaths = append([]string(nil), sessionDefaults...)
|
||||
newObjectStoreFromConfigFn = func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
func remoteSessionLoaderDependencies(fake *storage.FakeBackend) (commandDependencies, *int) {
|
||||
storeInitCalls := new(int)
|
||||
return fakeCommandDependencies(fake, storeInitCalls), storeInitCalls
|
||||
}
|
||||
|
||||
func fakeCommandDependencies(fake *storage.FakeBackend, storeInitCalls *int) commandDependencies {
|
||||
return commandDependencies{newObjectStore: func(context.Context, *config.Config) (storage.ObjectStore, error) {
|
||||
if storeInitCalls != nil {
|
||||
(*storeInitCalls)++
|
||||
*storeInitCalls++
|
||||
}
|
||||
return fake, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
newObjectStoreFromConfigFn = origStoreFn
|
||||
config.DefaultSessionConfigSearchPaths = origSessionDefaults
|
||||
})
|
||||
}}
|
||||
}
|
||||
|
||||
func setSessionConfigSearchPaths(t *testing.T, paths []string) {
|
||||
t.Helper()
|
||||
original := append([]string(nil), config.DefaultSessionConfigSearchPaths...)
|
||||
config.DefaultSessionConfigSearchPaths = append([]string(nil), paths...)
|
||||
t.Cleanup(func() { config.DefaultSessionConfigSearchPaths = original })
|
||||
}
|
||||
|
||||
func seedRemoteSessionConfig(t *testing.T, fake *storage.FakeBackend, sessionID, content string) string {
|
||||
|
||||
@@ -10,19 +10,18 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/logging"
|
||||
)
|
||||
|
||||
var newObjectStoreFromConfigFn = storage.NewObjectStoreFromConfig
|
||||
var discoverRemoteCurrentStateFn = discoverRemoteCurrentState
|
||||
var buildRestorePlanFn = buildRestorePlan
|
||||
var executeRestorePlanFn = executeRestorePlan
|
||||
|
||||
// Restore validates restore CLI/config inputs and storage preflight for future restore phases.
|
||||
func Restore(ctx context.Context, args []string, out io.Writer) (resultErr error) {
|
||||
return restoreWithDependencies(ctx, args, out, commandDependencies{})
|
||||
}
|
||||
|
||||
func restoreWithDependencies(ctx context.Context, args []string, out io.Writer, deps commandDependencies) (resultErr error) {
|
||||
deps = normalizeCommandDependencies(deps)
|
||||
positionalSessionID, args := pullLeadingSessionID(args)
|
||||
fs := flag.NewFlagSet("restore", flag.ContinueOnError)
|
||||
fs.SetOutput(out)
|
||||
@@ -54,7 +53,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) (resultErr error
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("restore: session_id is required")
|
||||
}
|
||||
loaded, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions(), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
@@ -63,16 +62,16 @@ func Restore(ctx context.Context, args []string, out io.Writer) (resultErr error
|
||||
if err := config.Validate(cfg); err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
objectStore, err := newCommandObjectStore(ctx, cfg, logging.NewLogger(os.Stderr, slog.LevelInfo))
|
||||
objectStore, err := newCommandObjectStoreWithDependencies(ctx, cfg, logging.NewLogger(os.Stderr, slog.LevelInfo), deps)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
current, err := discoverRemoteCurrentStateFn(ctx, cfg, objectStore)
|
||||
current, err := deps.discoverRemoteCurrentState(ctx, cfg, objectStore)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: %w", err)
|
||||
}
|
||||
if dryRun {
|
||||
plan, err := buildRestorePlanFn(ctx, cfg, current, objectStore, RestorePlanOptions{
|
||||
plan, err := deps.buildRestorePlan(ctx, cfg, current, objectStore, RestorePlanOptions{
|
||||
IncludeAudio: includeAudio,
|
||||
Force: force,
|
||||
DryRun: true,
|
||||
@@ -115,7 +114,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) (resultErr error
|
||||
// Classification and installation share the same transition lock as a
|
||||
// runner. This prevents a runner from making reuse decisions against state
|
||||
// that restore is about to replace.
|
||||
plan, err := buildRestorePlanFn(ctx, cfg, current, objectStore, RestorePlanOptions{
|
||||
plan, err := deps.buildRestorePlan(ctx, cfg, current, objectStore, RestorePlanOptions{
|
||||
IncludeAudio: includeAudio,
|
||||
Force: force,
|
||||
})
|
||||
@@ -147,7 +146,7 @@ func Restore(ctx context.Context, args []string, out io.Writer) (resultErr error
|
||||
return fmt.Errorf("restore: mark incomplete restore: %w", err)
|
||||
}
|
||||
|
||||
result, err := executeRestorePlanFn(ctx, cfg, current, plan, report, objectStore)
|
||||
result, err := deps.executeRestorePlan(ctx, cfg, current, plan, report, objectStore)
|
||||
if err != nil {
|
||||
report.setFailed(err)
|
||||
if _, reportErr := persistRestoreReport(artifactStore, cfg, report); reportErr != nil {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user