Compare commits
18 Commits
df58595d1e
...
v0.13.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 74e2d21de5 | |||
| 7cb18a1a40 | |||
| b556fc2f4f | |||
| b99bd38eb4 | |||
| 701b6726d7 | |||
| 665039f4dc | |||
| ef8dae776e | |||
| d01775b68a | |||
| 0d6f2dd0ce | |||
| df40cbec6e | |||
| 0341e0c7c0 | |||
| 39af7d4f3c | |||
| bba582b4ca | |||
| 1f16a85330 | |||
| f9482639d4 | |||
| dce721cdbd | |||
| 98734644d6 | |||
| 951383226c |
@@ -1,7 +1,8 @@
|
|||||||
# narratio
|
# narratio
|
||||||
|
|
||||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
||||||
polished transcripts and generated artifacts.
|
polished transcripts, validated Notarius extraction lanes, and generated
|
||||||
|
artifacts.
|
||||||
|
|
||||||
It runs a deterministic workflow with manifest-driven continuation, remote
|
It runs a deterministic workflow with manifest-driven continuation, remote
|
||||||
publish, and restore support.
|
publish, and restore support.
|
||||||
|
|||||||
15
docs/cli.md
15
docs/cli.md
@@ -75,7 +75,10 @@ narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common co
|
|||||||
Behavior:
|
Behavior:
|
||||||
|
|
||||||
- evaluates full stage order;
|
- evaluates full stage order;
|
||||||
- skips already-succeeded stages unless `--force` is set;
|
- runs `extract` between `trim` and `render`; an omitted or disabled Notarius
|
||||||
|
configuration records an explicit `notarius_disabled` self-skip;
|
||||||
|
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
||||||
|
resume check finds its durable result obsolete;
|
||||||
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||||
- writes session and run manifests.
|
- writes session and run manifests.
|
||||||
|
|
||||||
@@ -93,6 +96,7 @@ Valid stage names:
|
|||||||
- `polish`
|
- `polish`
|
||||||
- `normalize`
|
- `normalize`
|
||||||
- `trim`
|
- `trim`
|
||||||
|
- `extract`
|
||||||
- `render`
|
- `render`
|
||||||
- `analyze`
|
- `analyze`
|
||||||
- `publish`
|
- `publish`
|
||||||
@@ -218,7 +222,10 @@ default restore scope, report location, and conflict-handling workflow.
|
|||||||
narratio session artifacts <session_id> [--remote] [...common config flags]
|
narratio session artifacts <session_id> [--remote] [...common config flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Lists effective built-in and configured artifact sources, publish rules, lock state, and optional remote published-state availability.
|
Lists effective built-in, configured Scriptorium, and configured extraction
|
||||||
|
sources; reports planned, available, unavailable, and published state without
|
||||||
|
reading payload bodies; and includes publish rules, lock state, and optional
|
||||||
|
remote published-state availability.
|
||||||
|
|
||||||
### `session locks`
|
### `session locks`
|
||||||
|
|
||||||
@@ -248,7 +255,9 @@ Effects:
|
|||||||
|
|
||||||
- filters analyze execution to selected configured artifacts;
|
- filters analyze execution to selected configured artifacts;
|
||||||
- filters publish rules that source `narratio.artifact.<name>`;
|
- filters publish rules that source `narratio.artifact.<name>`;
|
||||||
- does not filter built-in transcript/bounds publish sources.
|
- does not filter built-in transcript/bounds or explicitly configured
|
||||||
|
`narratio.extraction.<name>` publish sources; and
|
||||||
|
- does not select or filter Notarius lanes.
|
||||||
|
|
||||||
## Common Workflows
|
## Common Workflows
|
||||||
|
|
||||||
|
|||||||
@@ -118,6 +118,9 @@ Rules:
|
|||||||
|
|
||||||
- `outputs[].source` is required.
|
- `outputs[].source` is required.
|
||||||
- `outputs[].dest` may be omitted when derivable from source.
|
- `outputs[].dest` may be omitted when derivable from source.
|
||||||
|
- extraction sources require an explicit `outputs[].dest` and publish only when
|
||||||
|
a rule names that source; the Notarius index and complete bundle are not
|
||||||
|
publish sources.
|
||||||
- `outputs[].required` defaults to `true`.
|
- `outputs[].required` defaults to `true`.
|
||||||
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
||||||
|
|
||||||
@@ -196,6 +199,13 @@ Rules:
|
|||||||
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
||||||
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
||||||
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||||
|
| `pipeline.notarius.enabled` | bool | No | `false` |
|
||||||
|
| `pipeline.notarius.binary` | string | No | `notarius` |
|
||||||
|
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
||||||
|
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
||||||
|
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
||||||
|
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
||||||
|
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
||||||
| `pipeline.render.enabled` | bool | No | `true` |
|
| `pipeline.render.enabled` | bool | No | `true` |
|
||||||
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
|
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
|
||||||
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
|
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
|
||||||
@@ -211,6 +221,26 @@ Rules:
|
|||||||
| `pipeline.notification.recipient` | string | No | empty |
|
| `pipeline.notification.recipient` | string | No | empty |
|
||||||
| `pipeline.notification.timeout` | duration | No | empty |
|
| `pipeline.notification.timeout` | duration | No | empty |
|
||||||
|
|
||||||
|
### Notarius Output Entries
|
||||||
|
|
||||||
|
For each `pipeline.notarius.outputs.<name>`:
|
||||||
|
|
||||||
|
| Field | Type | Required | Rule |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `lane_id` | string | Yes | unique Notarius lane ID |
|
||||||
|
| `media_type` | string | Yes | exact accepted descriptor media type |
|
||||||
|
| `schema_id` | string | Yes | exact accepted descriptor schema ID |
|
||||||
|
| `schema_version` | string | Yes | exact accepted descriptor schema version |
|
||||||
|
| `module_key` | string | No | exact accepted module key when set |
|
||||||
|
|
||||||
|
Output names must match `^[a-z][a-z0-9_]*$` and become selectable sources named
|
||||||
|
`narratio.extraction.<name>`. Lane IDs must be unique. Every declared output is
|
||||||
|
required from a successful Notarius result; a missing, rejected, duplicate, or
|
||||||
|
contract-incompatible lane fails extraction. See the
|
||||||
|
[complete maintained example](../examples/pipeline.full.annotated.yml) for the
|
||||||
|
current ten-lane D&D mapping and the [Notarius contract](./integrations/notarius.md)
|
||||||
|
for compatibility ownership.
|
||||||
|
|
||||||
### Scriptorium Artifact Entries
|
### Scriptorium Artifact Entries
|
||||||
|
|
||||||
For each `pipeline.scriptorium.artifacts.<name>`:
|
For each `pipeline.scriptorium.artifacts.<name>`:
|
||||||
@@ -233,7 +263,7 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
|
|||||||
|
|
||||||
| Field | Type | Required | Rule |
|
| Field | Type | Required | Rule |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.extraction.<name>`, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
||||||
| `artifact` | string | No | optional passthrough adapter field |
|
| `artifact` | string | No | optional passthrough adapter field |
|
||||||
| `path` | string | No | optional passthrough adapter field |
|
| `path` | string | No | optional passthrough adapter field |
|
||||||
| `required` | bool | No | optional input requirement |
|
| `required` | bool | No | optional input requirement |
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ focused stage documents.
|
|||||||
## Integration Contracts
|
## Integration Contracts
|
||||||
|
|
||||||
- [Audita](./audita.md): transcript polishing (`audita process`).
|
- [Audita](./audita.md): transcript polishing (`audita process`).
|
||||||
|
- [Notarius](./notarius.md): complete pipeline execution and safe JSON bundle
|
||||||
|
discovery (`notarius run`).
|
||||||
- [Seriatim](./seriatim.md): merge, normalize, trim, and render operations.
|
- [Seriatim](./seriatim.md): merge, normalize, trim, and render operations.
|
||||||
- [Scriptorium](./scriptorium.md): artifact generation and debug rendering
|
- [Scriptorium](./scriptorium.md): artifact generation and debug rendering
|
||||||
(`scriptorium run|render`).
|
(`scriptorium run|render`).
|
||||||
|
|||||||
83
docs/integrations/notarius.md
Normal file
83
docs/integrations/notarius.md
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
# Notarius Integration Contract
|
||||||
|
|
||||||
|
## Boundary
|
||||||
|
|
||||||
|
Narratio uses Notarius as a subprocess to extract configured structured JSON
|
||||||
|
lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
|
||||||
|
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
||||||
|
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
||||||
|
|
||||||
|
Canonical Notarius references:
|
||||||
|
|
||||||
|
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/subprocess.md)
|
||||||
|
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/dnd-pipeline.md)
|
||||||
|
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/run-result.md)
|
||||||
|
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/json-output.md)
|
||||||
|
|
||||||
|
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
||||||
|
records the exact current constraints for all ten D&D lanes. Treat the linked
|
||||||
|
Notarius documents as canonical when changing those values; Narratio does not
|
||||||
|
duplicate the complete schemas.
|
||||||
|
|
||||||
|
## Invocation
|
||||||
|
|
||||||
|
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
||||||
|
configuration path, input path, output directory, and working directory to
|
||||||
|
absolute paths and invokes:
|
||||||
|
|
||||||
|
```text
|
||||||
|
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> --json
|
||||||
|
```
|
||||||
|
|
||||||
|
Standard output is reserved for the JSON receipt. Standard error is captured
|
||||||
|
separately as diagnostic output. Narratio applies the configured timeout and
|
||||||
|
does not interpret stdout as a receipt unless the subprocess exits successfully.
|
||||||
|
It does not pass a Narratio session ID or run `notarius config validate`
|
||||||
|
automatically; the configured working directory and inherited environment
|
||||||
|
apply to the subprocess.
|
||||||
|
|
||||||
|
## Accepted Result
|
||||||
|
|
||||||
|
Narratio currently accepts receipt schema `notarius.run-result.v1`. The receipt
|
||||||
|
must identify the configured pipeline, and its `index_file` must be exactly
|
||||||
|
`index.json` beneath the reported bundle root. The production index must name
|
||||||
|
the management files exactly as `manifest.json`, `rejected.json`, and
|
||||||
|
`warnings.json`. All receipt, index, and lane paths must stay inside that
|
||||||
|
bundle; symlinks and non-regular lane payloads are rejected.
|
||||||
|
|
||||||
|
Supported receipt and index shapes tolerate unknown fields for forward
|
||||||
|
compatibility, while required identity, validation, count, manifest,
|
||||||
|
rejection, warning, and lane-list fields remain mandatory. Narratio applies
|
||||||
|
bounded reads to the receipt, index, rejection, and warning documents. Optional
|
||||||
|
chunk-map and evidence-context descriptors must carry their complete generic
|
||||||
|
contract metadata when present.
|
||||||
|
|
||||||
|
For every entry in `pipeline.notarius.outputs`, Narratio requires exactly one
|
||||||
|
index descriptor with the configured lane ID, media type, schema ID, schema
|
||||||
|
version, and, when configured, module key. Missing, duplicate, rejected, or
|
||||||
|
incompatible required lanes fail extraction even if Notarius exited zero.
|
||||||
|
Unconfigured lanes may remain in the preserved bundle but do not become
|
||||||
|
selectable Narratio sources.
|
||||||
|
|
||||||
|
Each accepted configured lane is registered as
|
||||||
|
`narratio.extraction.<output_key>`. The bundle index is retained for audit and
|
||||||
|
resume validation but is not selectable. Scriptorium and publish rules consume
|
||||||
|
only explicitly named lane sources; `--artifacts` never selects Notarius lanes.
|
||||||
|
|
||||||
|
## Failure And Compatibility Behavior
|
||||||
|
|
||||||
|
- Startup and nonzero-exit errors fail extraction and retain captured diagnostics.
|
||||||
|
- Invalid receipt JSON or an unsupported receipt schema fails before bundle use.
|
||||||
|
- Unsafe or incompatible index data and required-lane rejection fail before the
|
||||||
|
staged bundle is promoted to durable storage.
|
||||||
|
- Contract and external provenance metadata are preserved on lane artifact
|
||||||
|
records and through explicit publication.
|
||||||
|
|
||||||
|
Rejection and warning summaries retain structured stage, scope, lane, and
|
||||||
|
reason-code fields for diagnostics without exposing free-form external messages
|
||||||
|
or reading lane payload bodies.
|
||||||
|
|
||||||
|
Configuration fields and defaults are in [Configuration](../config.md).
|
||||||
|
Operator paths, rerun procedures, and bundle retention are in
|
||||||
|
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
||||||
|
for failure recovery.
|
||||||
@@ -16,6 +16,7 @@ Primary adapters:
|
|||||||
- `seriatim.Runner`
|
- `seriatim.Runner`
|
||||||
- `audita.Runner`
|
- `audita.Runner`
|
||||||
- `scriptorium.Runner`
|
- `scriptorium.Runner`
|
||||||
|
- `notarius.Runner`
|
||||||
- `storage.ObjectStore`
|
- `storage.ObjectStore`
|
||||||
- `notify.Sender`
|
- `notify.Sender`
|
||||||
|
|
||||||
@@ -40,9 +41,13 @@ Adapters do not own:
|
|||||||
- Seriatim subprocess runner.
|
- Seriatim subprocess runner.
|
||||||
- Audita subprocess runner.
|
- Audita subprocess runner.
|
||||||
- Scriptorium subprocess runner.
|
- Scriptorium subprocess runner.
|
||||||
|
- Notarius subprocess runner when extraction is enabled.
|
||||||
- Noop notifier (`notify.NoopSender`).
|
- Noop notifier (`notify.NoopSender`).
|
||||||
- Object store only when required by selected stages/config.
|
- Object store only when required by selected stages/config.
|
||||||
|
|
||||||
|
Notarius is composed only when extraction is enabled; the extract stage owns
|
||||||
|
receipt, bundle, and configured-lane policy rather than the adapter.
|
||||||
|
|
||||||
Object-store construction goes through `newCommandObjectStore`, which loads
|
Object-store construction goes through `newCommandObjectStore`, which loads
|
||||||
configured filesystem secrets before adapter initialization.
|
configured filesystem secrets before adapter initialization.
|
||||||
|
|
||||||
@@ -56,16 +61,18 @@ configured filesystem secrets before adapter initialization.
|
|||||||
|
|
||||||
- Composition: `internal/app/runner.go`, `internal/app/object_store.go`
|
- Composition: `internal/app/runner.go`, `internal/app/object_store.go`
|
||||||
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
||||||
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,storage,notify}`
|
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
|
||||||
- `internal/adapters/whisperx/http_test.go`
|
- `internal/adapters/whisperx/http_test.go`
|
||||||
- `internal/adapters/seriatim/subprocess_test.go`
|
- `internal/adapters/seriatim/subprocess_test.go`
|
||||||
- `internal/adapters/audita/subprocess_test.go`
|
- `internal/adapters/audita/subprocess_test.go`
|
||||||
- `internal/adapters/scriptorium/subprocess_test.go`
|
- `internal/adapters/scriptorium/subprocess_test.go`
|
||||||
|
- `internal/adapters/notarius/subprocess_test.go`
|
||||||
- `internal/adapters/storage/*_test.go`
|
- `internal/adapters/storage/*_test.go`
|
||||||
- `internal/app/runner_test.go`
|
- `internal/app/runner_test.go`
|
||||||
|
|
||||||
See the [WhisperX](../integrations/whisperx.md),
|
See the [WhisperX](../integrations/whisperx.md),
|
||||||
[Seriatim](../integrations/seriatim.md), [Audita](../integrations/audita.md),
|
[Seriatim](../integrations/seriatim.md), [Audita](../integrations/audita.md),
|
||||||
and [Scriptorium](../integrations/scriptorium.md) contracts before changing an
|
[Scriptorium](../integrations/scriptorium.md), and
|
||||||
|
[Notarius](../integrations/notarius.md) contracts before changing an
|
||||||
externally visible boundary. Operator-selected values belong in
|
externally visible boundary. Operator-selected values belong in
|
||||||
[Configuration](../config.md).
|
[Configuration](../config.md).
|
||||||
|
|||||||
@@ -24,12 +24,15 @@ Registry entries bind each ID to its producer, output kind, canonical fallback,
|
|||||||
and content validator. The focused stage documents own their input/output flow;
|
and content validator. The focused stage documents own their input/output flow;
|
||||||
[Configuration](../config.md) owns where operators may select these IDs.
|
[Configuration](../config.md) owns where operators may select these IDs.
|
||||||
|
|
||||||
## Configured and Previous-Session Sources
|
## Configured, Extraction, And Previous-Session Sources
|
||||||
|
|
||||||
- configured source ID format: `narratio.artifact.<artifact_key>`
|
- configured source ID format: `narratio.artifact.<artifact_key>`
|
||||||
|
- extraction source ID format: `narratio.extraction.<output_key>`
|
||||||
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
||||||
|
|
||||||
Both formats are validated by strict source-policy rules.
|
All formats are validated by strict source-policy rules. Extraction sources are
|
||||||
|
registered only from `pipeline.notarius.outputs`; the Notarius index has no
|
||||||
|
selectable source ID.
|
||||||
|
|
||||||
## Runtime Catalog
|
## Runtime Catalog
|
||||||
|
|
||||||
@@ -58,6 +61,15 @@ Configured sources (`narratio.artifact.*`):
|
|||||||
|
|
||||||
- resolve only through runtime catalog availability.
|
- resolve only through runtime catalog availability.
|
||||||
|
|
||||||
|
Extraction sources (`narratio.extraction.*`):
|
||||||
|
|
||||||
|
- use the shared registration and manifest hydration path in
|
||||||
|
`extraction_catalog.go`;
|
||||||
|
- require a current successful extract record with the exact configured source,
|
||||||
|
compatible contract and Notarius provenance, a confined regular durable
|
||||||
|
payload, and matching checksum; and
|
||||||
|
- are never inferred by scanning the Notarius bundle directory.
|
||||||
|
|
||||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||||
|
|
||||||
- resolve only from local `previous/` cache state;
|
- resolve only from local `previous/` cache state;
|
||||||
@@ -127,19 +139,23 @@ physical layout.
|
|||||||
|
|
||||||
- source ID formats are stable contracts;
|
- source ID formats are stable contracts;
|
||||||
- artifact resolution is deterministic and manifest-aware;
|
- artifact resolution is deterministic and manifest-aware;
|
||||||
|
- extraction sources are available only from a compatible successful manifest
|
||||||
|
record;
|
||||||
- previous-session source resolution in `analyze` is local-only;
|
- previous-session source resolution in `analyze` is local-only;
|
||||||
- remote current-state key construction remains centralized in artifacts helpers.
|
- remote current-state key construction remains centralized in artifacts helpers.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
|
|
||||||
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
||||||
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`
|
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`,
|
||||||
|
`internal/artifacts/extraction_catalog.go`
|
||||||
- Current state: `internal/artifacts/current_state.go`
|
- Current state: `internal/artifacts/current_state.go`
|
||||||
- Paths and keys: `internal/artifacts/paths.go`,
|
- Paths and keys: `internal/artifacts/paths.go`,
|
||||||
`internal/artifacts/s3_keys.go`
|
`internal/artifacts/s3_keys.go`
|
||||||
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
||||||
- Tests: `internal/artifacts/artifact_resolver_test.go`,
|
- Tests: `internal/artifacts/artifact_resolver_test.go`,
|
||||||
`internal/artifacts/catalog_test.go`,
|
`internal/artifacts/catalog_test.go`,
|
||||||
|
`internal/artifacts/extraction_catalog_test.go`,
|
||||||
`internal/artifacts/current_state_test.go`,
|
`internal/artifacts/current_state_test.go`,
|
||||||
`internal/artifacts/paths_model_test.go`,
|
`internal/artifacts/paths_model_test.go`,
|
||||||
`internal/artifacts/previous_requirements_test.go`
|
`internal/artifacts/previous_requirements_test.go`
|
||||||
|
|||||||
@@ -50,13 +50,33 @@ The model admits these stage states:
|
|||||||
|
|
||||||
The application runner marks an executing stage running and then succeeded or
|
The application runner marks an executing stage running and then succeeded or
|
||||||
failed in both manifests, persisting each transition. On success it records
|
failed in both manifests, persisting each transition. On success it records
|
||||||
outputs, logs, generated configuration references, and metadata. A successful
|
outputs, logs, generated configuration references, and metadata. Artifact
|
||||||
forced rerun marks only succeeded downstream session-stage records stale.
|
records may include optional contract and external provenance objects; old
|
||||||
|
manifests remain compatible when those fields are absent. A successful forced
|
||||||
|
rerun marks only succeeded downstream session-stage records stale.
|
||||||
|
|
||||||
|
Starting an execution clears the current session-stage record's prior outputs,
|
||||||
|
logs, generated configuration references, and metadata. Failed and skipped
|
||||||
|
transitions enforce the same clearing rule directly, while success repopulates
|
||||||
|
only fields returned by the new result. Marking a record stale does not clear
|
||||||
|
those details because resume validation and diagnosis may still require them
|
||||||
|
before execution begins. Invocation run manifests remain immutable audit
|
||||||
|
records of their own outcomes.
|
||||||
|
|
||||||
|
A stage may explicitly return a skipped disposition and stable reason. The
|
||||||
|
runner persists that outcome in both manifests, clears older outputs for the
|
||||||
|
session-stage record along with older logs, generated configuration references,
|
||||||
|
and metadata, then applies any bounded details from the current skip and
|
||||||
|
continues. This self-skip is distinct from deciding not to execute an
|
||||||
|
already-succeeded stage and is reconsidered on later runs. Skipped results
|
||||||
|
cannot contain outputs.
|
||||||
|
|
||||||
When an already-succeeded stage is skipped, the invocation run manifest records
|
When an already-succeeded stage is skipped, the invocation run manifest records
|
||||||
the `skip` action and reason. The session manifest deliberately retains its
|
the `skip` action and reason. The session manifest deliberately retains its
|
||||||
existing succeeded record because it remains the cross-invocation progress
|
existing succeeded record because it remains the cross-invocation progress
|
||||||
authority.
|
authority. Stages with a resume validator, currently extraction, may reject an
|
||||||
|
otherwise eligible skip when the recorded durable result is obsolete; the
|
||||||
|
runner marks it stale and executes it.
|
||||||
|
|
||||||
Session manifest is the authoritative stage-progress ledger across invocations.
|
Session manifest is the authoritative stage-progress ledger across invocations.
|
||||||
Run manifest is invocation-scoped audit state.
|
Run manifest is invocation-scoped audit state.
|
||||||
@@ -64,6 +84,9 @@ Run manifest is invocation-scoped audit state.
|
|||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- stage resume/skip decisions are session-manifest driven.
|
- stage resume/skip decisions are session-manifest driven.
|
||||||
|
- running, failed, and self-skipped stages do not retain result payloads from
|
||||||
|
an earlier success.
|
||||||
|
- stale stages retain prior details until replacement execution starts.
|
||||||
- force reruns stale downstream succeeded stages.
|
- force reruns stale downstream succeeded stages.
|
||||||
- run manifest does not replace session manifest as progress authority.
|
- run manifest does not replace session manifest as progress authority.
|
||||||
|
|
||||||
|
|||||||
@@ -53,10 +53,11 @@ The implemented canonical order is:
|
|||||||
4. [`polish`](stage-polish.md)
|
4. [`polish`](stage-polish.md)
|
||||||
5. [`normalize`](stage-normalize.md)
|
5. [`normalize`](stage-normalize.md)
|
||||||
6. [`trim`](stage-trim.md)
|
6. [`trim`](stage-trim.md)
|
||||||
7. [`render`](stage-render.md)
|
7. [`extract`](stage-extract.md)
|
||||||
8. [`analyze`](stage-analyze.md)
|
8. [`render`](stage-render.md)
|
||||||
9. [`publish`](stage-publish.md)
|
9. [`analyze`](stage-analyze.md)
|
||||||
10. `notify` (placeholder)
|
10. [`publish`](stage-publish.md)
|
||||||
|
11. `notify` (placeholder)
|
||||||
|
|
||||||
`notify` currently has optional notifier call behavior and no persisted pipeline
|
`notify` currently has optional notifier call behavior and no persisted pipeline
|
||||||
outputs; its default collaborator is a no-op sender. The focused stage
|
outputs; its default collaborator is a no-op sender. The focused stage
|
||||||
@@ -83,6 +84,7 @@ and execution semantics.
|
|||||||
- [`polish`](stage-polish.md)
|
- [`polish`](stage-polish.md)
|
||||||
- [`normalize`](stage-normalize.md)
|
- [`normalize`](stage-normalize.md)
|
||||||
- [`trim`](stage-trim.md)
|
- [`trim`](stage-trim.md)
|
||||||
|
- [`extract`](stage-extract.md)
|
||||||
- [`render`](stage-render.md)
|
- [`render`](stage-render.md)
|
||||||
- [`analyze`](stage-analyze.md)
|
- [`analyze`](stage-analyze.md)
|
||||||
- [`publish`](stage-publish.md)
|
- [`publish`](stage-publish.md)
|
||||||
|
|||||||
84
docs/internal/stage-extract.md
Normal file
84
docs/internal/stage-extract.md
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
# Internal: Extract Stage
|
||||||
|
|
||||||
|
## Responsibility
|
||||||
|
|
||||||
|
`extract` runs after `trim` and before `render`. It converts the canonical
|
||||||
|
`narratio.transcript.final_trimmed` JSON into configured Notarius lane artifacts.
|
||||||
|
An omitted or disabled Notarius section makes the stage explicitly self-skip
|
||||||
|
with reason `notarius_disabled`, no outputs, and no Notarius runner.
|
||||||
|
|
||||||
|
The external protocol is documented in the
|
||||||
|
[Notarius integration contract](../integrations/notarius.md). Configuration
|
||||||
|
fields belong in [Configuration](../config.md), and physical paths and force
|
||||||
|
procedures belong in [Operations](../operations.md).
|
||||||
|
|
||||||
|
## Lifecycle
|
||||||
|
|
||||||
|
`internal/stage/extract.go`:
|
||||||
|
|
||||||
|
1. resolves the final trimmed transcript from the shared artifact catalog;
|
||||||
|
2. resolves and fingerprints the Notarius invocation contract;
|
||||||
|
3. creates a run-local staging directory and invokes the injected
|
||||||
|
`notarius.Runner`;
|
||||||
|
4. validates the successful receipt, confined index, configured required lane
|
||||||
|
descriptors, and regular payload files;
|
||||||
|
5. atomically promotes the complete bundle to its immutable durable location;
|
||||||
|
6. records one non-selectable `notarius_index` output and one selectable
|
||||||
|
`notarius_lane` output per configured lane; and
|
||||||
|
7. registers each lane as `narratio.extraction.<output_key>` for downstream
|
||||||
|
Scriptorium and publish resolution.
|
||||||
|
|
||||||
|
Lane records retain checksum, contract, producer run ID, and Notarius system,
|
||||||
|
run, pipeline, and lane provenance. Stage metadata retains the durable bundle
|
||||||
|
root, receipt, diagnostic paths, rejection/warning summaries, producing
|
||||||
|
Narratio run ID, and invocation fingerprint. Validation completes before
|
||||||
|
promotion, so a rejected result cannot expose a partial durable bundle.
|
||||||
|
|
||||||
|
Any executed extraction outcome that replaces a different effective outcome
|
||||||
|
marks succeeded downstream stages stale. Repeating the same disabled self-skip
|
||||||
|
with no outputs is stable and does not repeatedly invalidate downstream stages.
|
||||||
|
|
||||||
|
## Resume Validation
|
||||||
|
|
||||||
|
`internal/stage/extract_resume.go` permits a skip only when the existing stage
|
||||||
|
record succeeded and still matches the current invocation fingerprint. The
|
||||||
|
fingerprint covers the resolved executable and config paths, pipeline ID,
|
||||||
|
timeout, working directory, and sorted configured output contracts.
|
||||||
|
|
||||||
|
The validator then checks the producing run identity, canonical immutable
|
||||||
|
bundle root, path confinement and absence of symlink components, receipt
|
||||||
|
identity, exactly one canonical index, the exact configured source set,
|
||||||
|
contracts and provenance, regular-file status, and stored checksums. Missing or
|
||||||
|
obsolete results are non-resumable and run again; unsafe filesystem conditions
|
||||||
|
return an error rather than silently accepting or replacing data.
|
||||||
|
|
||||||
|
The fingerprint cannot observe files imported by Notarius configuration,
|
||||||
|
profile contents, prompt/module definitions, or other transitive inputs.
|
||||||
|
Operators must force extraction after changing any such input.
|
||||||
|
|
||||||
|
## Failure Behavior
|
||||||
|
|
||||||
|
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
|
||||||
|
index compatibility, required-lane rejection, payload inspection, checksum, or
|
||||||
|
promotion errors fail the stage through ordinary manifest transition handling.
|
||||||
|
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
|
||||||
|
not given selectable extraction sources unless the complete configured result
|
||||||
|
has passed validation and promotion.
|
||||||
|
|
||||||
|
When a replacement attempt begins, the current session-stage record no longer
|
||||||
|
advertises payload from the previous success. A failed replacement therefore
|
||||||
|
has no current outputs, logs, generated configuration references, or metadata,
|
||||||
|
while the earlier invocation manifest and immutable promoted bundle remain
|
||||||
|
available for audit and recovery.
|
||||||
|
|
||||||
|
## Implementation And Focused Tests
|
||||||
|
|
||||||
|
- Stage execution, selection, and resume validation: `internal/stage/extract.go`,
|
||||||
|
`internal/stage/extract_resume.go`,
|
||||||
|
`internal/stage/extract_test.go`
|
||||||
|
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
||||||
|
`internal/adapters/notarius/subprocess_test.go`
|
||||||
|
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
||||||
|
`internal/artifacts/extraction_catalog_test.go`
|
||||||
|
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
||||||
|
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
||||||
@@ -26,8 +26,13 @@ Exact remote placement and the operator workflow belong in
|
|||||||
|
|
||||||
- stage can self-skip when publish disabled or run upload disabled.
|
- stage can self-skip when publish disabled or run upload disabled.
|
||||||
- validates prerequisite stage success and object-store availability.
|
- validates prerequisite stage success and object-store availability.
|
||||||
- collects deterministic run file list plus run `manifest.json`.
|
- collects a deterministic run file list plus run `manifest.json`, excluding
|
||||||
|
`audio/**` and the run-local `extract/notarius-output/**` staging bundle.
|
||||||
|
- keeps run-local Notarius receipt and stderr diagnostics eligible for the run
|
||||||
|
archive.
|
||||||
- resolves publish output sources through runtime artifact catalog and manifest-aware resolution.
|
- resolves publish output sources through runtime artifact catalog and manifest-aware resolution.
|
||||||
|
- publishes extraction lanes only through explicit configured output rules;
|
||||||
|
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
|
||||||
- selected artifact filter applies to configured artifact sources only.
|
- selected artifact filter applies to configured artifact sources only.
|
||||||
- locked outputs are skipped intentionally (including required ones).
|
- locked outputs are skipped intentionally (including required ones).
|
||||||
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
||||||
@@ -48,7 +53,9 @@ Includes counts/lists for:
|
|||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- `current/run_id.txt` is the remote commit marker and is written last.
|
- `current/run_id.txt` is the remote commit marker and is written last.
|
||||||
- run upload excludes `audio/**`.
|
- run upload excludes `audio/**` and `extract/notarius-output/**`.
|
||||||
|
- `extract/notarius.receipt.json` and `extract/notarius.stderr.log` remain
|
||||||
|
eligible run-record diagnostics.
|
||||||
- publish locks are not overridden by `--force`.
|
- publish locks are not overridden by `--force`.
|
||||||
|
|
||||||
The commit boundary and cleanup gate are normative architecture invariants; see
|
The commit boundary and cleanup gate are normative architecture invariants; see
|
||||||
|
|||||||
@@ -24,6 +24,16 @@ materialized into canonical session paths before stage success. Managed
|
|||||||
previous-session cache paths remain session-durable and are never redirected
|
previous-session cache paths remain session-durable and are never redirected
|
||||||
into run-local output space.
|
into run-local output space.
|
||||||
|
|
||||||
|
Extraction uses run-local receipt, stderr, and output-root helpers, then
|
||||||
|
promotes the validated external bundle to the unique immutable Notarius bundle
|
||||||
|
path supplied by `internal/artifacts`. `internal/fileops.PromoteDirectory`
|
||||||
|
copies only regular files and directories to a same-filesystem temporary
|
||||||
|
sibling. Source traversal uses confined directory handles and identity checks
|
||||||
|
so replacing an inspected root, directory, or file is rejected rather than
|
||||||
|
followed. The completed tree is atomically renamed without replacing an
|
||||||
|
existing destination. Exact physical paths belong in
|
||||||
|
[Operations](../operations.md#extraction-workflow).
|
||||||
|
|
||||||
## Locking
|
## Locking
|
||||||
|
|
||||||
`artifacts.LocalStore` enforces the single-writer session lock via `.lock`
|
`artifacts.LocalStore` enforces the single-writer session lock via `.lock`
|
||||||
@@ -54,9 +64,11 @@ deletion scope belong in [CLI](../cli.md#clean) and
|
|||||||
- Path model and local store: `internal/artifacts/paths.go`,
|
- Path model and local store: `internal/artifacts/paths.go`,
|
||||||
`internal/artifacts/local.go`
|
`internal/artifacts/local.go`
|
||||||
- Run-local materialization: `internal/stage/run_local.go`
|
- Run-local materialization: `internal/stage/run_local.go`
|
||||||
|
- Immutable bundle promotion: `internal/fileops/directory.go`
|
||||||
- Cleanup confinement: `internal/app/cleanup_targets.go`,
|
- Cleanup confinement: `internal/app/cleanup_targets.go`,
|
||||||
`internal/app/post_publish_cleanup.go`
|
`internal/app/post_publish_cleanup.go`
|
||||||
- Tests: `internal/artifacts/paths_model_test.go`,
|
- Tests: `internal/artifacts/paths_model_test.go`,
|
||||||
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
||||||
|
`internal/fileops/directory_test.go`,
|
||||||
`internal/app/cleanup_targets_test.go`,
|
`internal/app/cleanup_targets_test.go`,
|
||||||
`internal/app/post_publish_cleanup_test.go`
|
`internal/app/post_publish_cleanup_test.go`
|
||||||
|
|||||||
@@ -75,16 +75,22 @@ Canonical stage order:
|
|||||||
4. `polish`
|
4. `polish`
|
||||||
5. `normalize`
|
5. `normalize`
|
||||||
6. `trim`
|
6. `trim`
|
||||||
7. `render`
|
7. `extract`
|
||||||
8. `analyze`
|
8. `render`
|
||||||
9. `publish`
|
9. `analyze`
|
||||||
10. `notify`
|
10. `publish`
|
||||||
|
11. `notify`
|
||||||
|
|
||||||
Execution rules:
|
Execution rules:
|
||||||
|
|
||||||
- succeeded stages are skipped unless `--force` is set;
|
- succeeded stages are skipped unless `--force` is set;
|
||||||
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||||
- force rerunning a succeeded upstream stage marks succeeded downstream stages as `stale`.
|
- forcing an upstream stage marks succeeded downstream stages as `stale` before
|
||||||
|
the replacement runs; and
|
||||||
|
- an executed failure, changed self-skip, or success that replaces a different
|
||||||
|
effective upstream outcome also marks succeeded downstream stages stale. A
|
||||||
|
repeated self-skip with the same reason and no outputs is stable and does not
|
||||||
|
perpetually rerun downstream work.
|
||||||
|
|
||||||
Single-stage execution:
|
Single-stage execution:
|
||||||
|
|
||||||
@@ -101,7 +107,64 @@ Selection behavior:
|
|||||||
- validates names against `pipeline.scriptorium.artifacts`;
|
- validates names against `pipeline.scriptorium.artifacts`;
|
||||||
- filters analyze execution to selected configured artifacts;
|
- filters analyze execution to selected configured artifacts;
|
||||||
- filters publish rules for `narratio.artifact.<name>` sources only;
|
- filters publish rules for `narratio.artifact.<name>` sources only;
|
||||||
- does not suppress built-in transcript or bounds publish sources.
|
- does not suppress built-in transcript, bounds, or explicitly configured
|
||||||
|
`narratio.extraction.<name>` publish sources; and
|
||||||
|
- never partially selects Notarius lanes.
|
||||||
|
|
||||||
|
## Extraction Workflow
|
||||||
|
|
||||||
|
When Notarius is omitted or disabled, `extract` records an explicit skipped
|
||||||
|
outcome with reason `notarius_disabled` and no outputs. A later invocation
|
||||||
|
reconsiders the skipped stage, so enabling Notarius does not require force.
|
||||||
|
|
||||||
|
When Notarius extraction is enabled, the stage consumes the final trimmed JSON
|
||||||
|
and preserves the complete validated Notarius bundle at:
|
||||||
|
|
||||||
|
- `artifacts/notarius/{narratio_run_id}/`
|
||||||
|
|
||||||
|
The directory is immutable once promoted. Configured lanes become
|
||||||
|
`narratio.extraction.<name>` sources for Scriptorium and explicit publish rules;
|
||||||
|
the bundle and `index.json` are retained for audit and resume validation but
|
||||||
|
are not selectable or published implicitly.
|
||||||
|
|
||||||
|
Starting a replacement clears the previous extraction payload from the current
|
||||||
|
session-stage record. If that replacement fails or self-skips, the current
|
||||||
|
record does not fall back to the earlier outputs. The earlier run manifest and
|
||||||
|
immutable bundle remain available for inspection, but downstream resolution
|
||||||
|
requires a new current successful extraction record.
|
||||||
|
|
||||||
|
Atomic Notarius bundle promotion is supported on Linux, macOS, and Windows.
|
||||||
|
On other operating systems, extraction fails before copying the bundle into a
|
||||||
|
temporary promotion tree because Narratio has no verified atomic no-replace
|
||||||
|
directory primitive there. This is an extraction limitation, not a broader
|
||||||
|
platform-support guarantee for every Narratio workflow.
|
||||||
|
|
||||||
|
Run-local diagnostics are:
|
||||||
|
|
||||||
|
- `runs/{run_id}/extract/notarius.receipt.json`
|
||||||
|
- `runs/{run_id}/extract/notarius.stderr.log`
|
||||||
|
- `runs/{run_id}/extract/notarius-output/` before durable promotion
|
||||||
|
|
||||||
|
The run-record upload excludes the complete
|
||||||
|
`extract/notarius-output/**` subtree. The receipt and stderr files remain
|
||||||
|
eligible run-record diagnostics. The durable bundle is never scanned for
|
||||||
|
implicit publication; only lanes named by explicit `pipeline.publish.outputs`
|
||||||
|
rules are uploaded.
|
||||||
|
|
||||||
|
To intentionally replace the current extraction result, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage extract 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Narratio automatically reruns extraction when its recorded invocation contract
|
||||||
|
or durable output validation changes. It cannot fingerprint configuration
|
||||||
|
files, profiles, prompts, modules, or references loaded transitively by
|
||||||
|
Notarius. Force extraction after changing any of those inputs, even when the
|
||||||
|
top-level Narratio and Notarius config paths remain the same. A forced extract
|
||||||
|
marks successful downstream stages stale. Ordinary extraction failures or
|
||||||
|
outcome changes also stale affected downstream stages, while an identical
|
||||||
|
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
||||||
|
|
||||||
## Publish Workflow
|
## Publish Workflow
|
||||||
|
|
||||||
@@ -119,8 +182,10 @@ narratio run-stage publish 2026-04-04 --force
|
|||||||
|
|
||||||
Publish commit model:
|
Publish commit model:
|
||||||
|
|
||||||
- uploads run files under `{session_prefix}/runs/{run_id}/`;
|
- uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
|
||||||
- uploads configured published outputs;
|
audio and the run-local Notarius staging bundle;
|
||||||
|
- uploads configured published outputs, including only explicitly configured
|
||||||
|
extraction lanes;
|
||||||
- uploads `previous/**` cache files when present;
|
- uploads `previous/**` cache files when present;
|
||||||
- writes `current/manifest.json`;
|
- writes `current/manifest.json`;
|
||||||
- writes `current/run_id.txt` last.
|
- writes `current/run_id.txt` last.
|
||||||
@@ -198,6 +263,10 @@ Durable session paths:
|
|||||||
- `config/**`
|
- `config/**`
|
||||||
- `runs/**`
|
- `runs/**`
|
||||||
|
|
||||||
|
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
||||||
|
stderr, and pre-promotion output remain in the producing run's `extract`
|
||||||
|
directory as described in [Extraction Workflow](#extraction-workflow).
|
||||||
|
|
||||||
Run-local layout:
|
Run-local layout:
|
||||||
|
|
||||||
- `runs/{run_id}/{stage}/outputs`
|
- `runs/{run_id}/{stage}/outputs`
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ their domains:
|
|||||||
|
|
||||||
- WhisperX performs transcription;
|
- WhisperX performs transcription;
|
||||||
- Seriatim performs deterministic transcript processing and rendering;
|
- Seriatim performs deterministic transcript processing and rendering;
|
||||||
- Audita performs transcript correction and polishing; and
|
- Audita performs transcript correction and polishing;
|
||||||
|
- Notarius extracts validated structured artifact bundles; and
|
||||||
- Scriptorium executes prompts and produces configured artifacts.
|
- Scriptorium executes prompts and produces configured artifacts.
|
||||||
|
|
||||||
Narratio owns orchestration, configuration resolution, session and run state,
|
Narratio owns orchestration, configuration resolution, session and run state,
|
||||||
@@ -51,9 +52,9 @@ HTTP, subprocess, notification, and object-storage mechanics, including command
|
|||||||
construction, transport behavior, provider response handling, and external
|
construction, transport behavior, provider response handling, and external
|
||||||
error adaptation. External dependency types must remain inside the adapter that
|
error adaptation. External dependency types must remain inside the adapter that
|
||||||
owns them unless that dependency is the adapter's explicit public contract.
|
owns them unless that dependency is the adapter's explicit public contract.
|
||||||
WhisperX HTTP behavior, Seriatim, Audita, and Scriptorium command construction,
|
WhisperX HTTP behavior, Seriatim, Audita, Notarius, and Scriptorium command
|
||||||
notification transport, and object-storage SDK details remain behind these
|
construction, notification transport, and object-storage SDK details remain
|
||||||
boundaries.
|
behind these boundaries.
|
||||||
|
|
||||||
State and path services must not infer stage policy. Storage implementations
|
State and path services must not infer stage policy. Storage implementations
|
||||||
receive explicit bucket-relative keys and do not infer campaign, session, run,
|
receive explicit bucket-relative keys and do not infer campaign, session, run,
|
||||||
@@ -89,6 +90,13 @@ should preserve enough local state and diagnostics for inspection, recovery,
|
|||||||
and resume. Forcing an upstream stage invalidates succeeded downstream work
|
and resume. Forcing an upstream stage invalidates succeeded downstream work
|
||||||
according to the canonical stage order.
|
according to the canonical stage order.
|
||||||
|
|
||||||
|
A stage may explicitly self-skip with a stable reason and no outputs. That
|
||||||
|
outcome is persisted, clears older outputs owned by the stage, and is
|
||||||
|
reconsidered on a later invocation. A stage may also validate whether an
|
||||||
|
otherwise successful recorded result is still resumable; an obsolete result
|
||||||
|
is staled and rerun, while an unsafe condition that prevents a sound decision
|
||||||
|
stops execution.
|
||||||
|
|
||||||
Shared behavior should live behind a narrow service or helper with one clear
|
Shared behavior should live behind a narrow service or helper with one clear
|
||||||
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
|
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
|
||||||
artifact, or path policy ad hoc.
|
artifact, or path policy ad hoc.
|
||||||
@@ -141,6 +149,9 @@ not reconstruct canonical paths through scattered string concatenation.
|
|||||||
Artifact resolution is deterministic and manifest-aware. Producers materialize
|
Artifact resolution is deterministic and manifest-aware. Producers materialize
|
||||||
canonical outputs before reporting success, and consumers resolve declared
|
canonical outputs before reporting success, and consumers resolve declared
|
||||||
artifact identities rather than infer files from unrelated directory contents.
|
artifact identities rather than infer files from unrelated directory contents.
|
||||||
|
External artifact bundles become current only through validated immutable
|
||||||
|
promotion and manifest records; directory presence alone never establishes
|
||||||
|
availability.
|
||||||
|
|
||||||
Writes, moves, replacements, and deletions must use narrow, explicit,
|
Writes, moves, replacements, and deletions must use narrow, explicit,
|
||||||
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
||||||
|
|||||||
@@ -1,379 +0,0 @@
|
|||||||
# Notarius Extraction Stage
|
|
||||||
|
|
||||||
## Status
|
|
||||||
|
|
||||||
Proposed.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Add a first-class Narratio `extract` stage that runs Notarius against the
|
|
||||||
session's final trimmed transcript, validates and collects the resulting
|
|
||||||
structured D&D artifacts, and registers those artifacts for later use by the
|
|
||||||
`analyze` and `publish` stages.
|
|
||||||
|
|
||||||
This feature should integrate Notarius through Narratio's existing stage,
|
|
||||||
adapter, manifest, workspace, and artifact-catalog boundaries. It must not turn
|
|
||||||
Narratio into a generic workflow engine or a second configuration language for
|
|
||||||
Notarius pipelines.
|
|
||||||
|
|
||||||
## User Outcome
|
|
||||||
|
|
||||||
An operator can enable one configured Notarius pipeline for a Narratio
|
|
||||||
campaign. During a normal run, Narratio will:
|
|
||||||
|
|
||||||
1. finish producing the session transcript tiers;
|
|
||||||
2. invoke Notarius once with the final trimmed Seriatim JSON transcript;
|
|
||||||
3. collect and validate the configured structured artifact lanes;
|
|
||||||
4. record their exact files and provenance in the Narratio manifest; and
|
|
||||||
5. make those artifacts selectable as inputs to Scriptorium artifacts in the
|
|
||||||
later `analyze` stage.
|
|
||||||
|
|
||||||
The maintained D&D example should demonstrate all ten lanes emitted by
|
|
||||||
Notarius's complete `dnd-session` pipeline.
|
|
||||||
|
|
||||||
## Target Stage Architecture
|
|
||||||
|
|
||||||
### Canonical Order
|
|
||||||
|
|
||||||
The canonical stage order becomes:
|
|
||||||
|
|
||||||
```text
|
|
||||||
prepare -> transcribe -> merge -> polish -> normalize -> trim -> render
|
|
||||||
-> extract -> analyze -> publish -> notify
|
|
||||||
```
|
|
||||||
|
|
||||||
`extract` is deliberately after all transcript-producing stages and before
|
|
||||||
analysis. Its source document is the manifest-resolved
|
|
||||||
`narratio.transcript.final_trimmed` artifact, normally
|
|
||||||
`transcripts/final.trimmed.json`. It does not consume rendered Markdown.
|
|
||||||
|
|
||||||
Adding the stage must update full-plan construction, explicit stage selection,
|
|
||||||
downstream invalidation, prerequisite checks, resume behavior, run manifests,
|
|
||||||
CLI stage validation and help, and every canonical-stage inventory. Forcing an
|
|
||||||
upstream transcript stage must stale a previously successful `extract` stage
|
|
||||||
and its downstream stages. Forcing `extract` must stale `analyze`, `publish`,
|
|
||||||
and `notify` according to existing rules.
|
|
||||||
|
|
||||||
### Stage Boundary
|
|
||||||
|
|
||||||
The stage owns Narratio policy and state transitions:
|
|
||||||
|
|
||||||
- resolve the final trimmed transcript through the runtime artifact catalog;
|
|
||||||
- build a Narratio-level Notarius request from validated configuration and
|
|
||||||
run-local paths;
|
|
||||||
- call a narrow Notarius adapter;
|
|
||||||
- apply the configured required-output policy;
|
|
||||||
- materialize the validated bundle into its canonical session location;
|
|
||||||
- return manifest-ready artifact references and bounded metadata; and
|
|
||||||
- fail without marking the stage successful when any required contract or
|
|
||||||
materialization step fails.
|
|
||||||
|
|
||||||
The stage must not construct subprocess arguments, infer Notarius output
|
|
||||||
filenames, parse provider logs, or decode individual D&D payload bodies.
|
|
||||||
|
|
||||||
### Adapter Boundary
|
|
||||||
|
|
||||||
Add a dedicated Notarius adapter package with a small interface, production
|
|
||||||
subprocess implementation, and test fake. Its request should contain only the
|
|
||||||
resolved Notarius binary, configuration path, pipeline ID, transcript path,
|
|
||||||
output root, working directory, timeout, and process-log destinations needed
|
|
||||||
for one run.
|
|
||||||
|
|
||||||
The adapter owns:
|
|
||||||
|
|
||||||
- optional `notarius config validate` preflight for the configured pipeline;
|
|
||||||
- exact `notarius run ... --json` argument construction;
|
|
||||||
- stdout and stderr separation;
|
|
||||||
- context cancellation and timeout propagation through Narratio's shared
|
|
||||||
subprocess boundary;
|
|
||||||
- exit-status handling;
|
|
||||||
- decoding the `notarius.run-result.v1` success receipt;
|
|
||||||
- receipt and index path-confinement checks;
|
|
||||||
- decoding `index.json` and resolving descriptor paths safely beneath the
|
|
||||||
reported output directory; and
|
|
||||||
- returning a transport-neutral result containing the bundle location,
|
|
||||||
receipt summary, lane descriptors, pipeline-wide descriptors, warnings and
|
|
||||||
rejection locations, and diagnostic log paths.
|
|
||||||
|
|
||||||
Only exit status zero permits receipt decoding. Receipt, index, or descriptor
|
|
||||||
paths that are absolute where a logical relative path is required, or that
|
|
||||||
escape their owning root, are integration failures. Unknown fields in a
|
|
||||||
supported receipt or index schema should be tolerated. Unsupported schema
|
|
||||||
versions and incompatible descriptor metadata should fail clearly.
|
|
||||||
|
|
||||||
The adapter must not write Narratio manifests, choose required lanes, decide
|
|
||||||
analysis inputs, or contain D&D domain logic.
|
|
||||||
|
|
||||||
## Configuration Contract
|
|
||||||
|
|
||||||
Add a strict optional `pipeline.notarius` configuration section. Omission or
|
|
||||||
`enabled: false` keeps the current workflow usable and causes `extract` to
|
|
||||||
self-skip without outputs.
|
|
||||||
|
|
||||||
The section should provide:
|
|
||||||
|
|
||||||
- `enabled`: explicit opt-in;
|
|
||||||
- `binary`: Notarius executable, defaulting to `notarius`;
|
|
||||||
- `config_path`: required when enabled;
|
|
||||||
- `pipeline_id`: required when enabled;
|
|
||||||
- `timeout`: a positive stage timeout with a documented default;
|
|
||||||
- `working_directory`: optional explicit subprocess working directory,
|
|
||||||
defaulting to the directory containing `config_path`; and
|
|
||||||
- an `outputs` map defining the Notarius lane artifacts Narratio promises to
|
|
||||||
collect.
|
|
||||||
|
|
||||||
Each output-map key is a stable Narratio extraction key. Each value must define:
|
|
||||||
|
|
||||||
- the exact Notarius `lane_id`;
|
|
||||||
- the expected `media_type`;
|
|
||||||
- the expected `schema_id`;
|
|
||||||
- the expected `schema_version`; and
|
|
||||||
- optionally an expected `module_key` when the operator needs to constrain the
|
|
||||||
producing module as part of compatibility.
|
|
||||||
|
|
||||||
Narratio derives the downstream source ID
|
|
||||||
`narratio.extraction.<output-key>` from the map key. Keys and lane IDs must be
|
|
||||||
non-empty, unique after normalization, path-safe under the existing artifact
|
|
||||||
policy, and collision-free with built-in and configured artifact identities.
|
|
||||||
Every configured output is required: a successful Notarius process that omits
|
|
||||||
one, rejects it, or reports incompatible descriptor metadata fails the
|
|
||||||
`extract` stage.
|
|
||||||
|
|
||||||
This explicit map keeps Narratio's consumer contract stable when a Notarius
|
|
||||||
lane ID or schema changes and avoids hard-coding the current D&D family into a
|
|
||||||
generic adapter. It also replaces a separate `required_lanes` list, which would
|
|
||||||
duplicate configuration.
|
|
||||||
|
|
||||||
Narratio should not reproduce Notarius lane selection, references, LLM
|
|
||||||
profiles, model settings, retries, concurrency, or prompt configuration. Those
|
|
||||||
remain in the referenced Notarius configuration. Narratio should not expose a
|
|
||||||
runtime lane-selection flag for `extract`; one stage invocation runs the
|
|
||||||
configured Notarius pipeline as a unit.
|
|
||||||
|
|
||||||
All configured paths should become absolute during Narratio configuration
|
|
||||||
resolution. The deterministic default working directory allows a Notarius
|
|
||||||
profile path relative to that directory, but operator documentation should
|
|
||||||
still recommend absolute deployment paths where practical. Notarius reference
|
|
||||||
paths continue to follow Notarius's own configuration-relative rules.
|
|
||||||
|
|
||||||
## Output And Artifact Model
|
|
||||||
|
|
||||||
### Canonical Bundle
|
|
||||||
|
|
||||||
Run Notarius against a run-local output root. After all configured descriptors
|
|
||||||
are validated, materialize the contents of the exact run-specific Notarius
|
|
||||||
bundle into a fixed canonical session directory:
|
|
||||||
|
|
||||||
```text
|
|
||||||
artifacts/notarius/
|
|
||||||
```
|
|
||||||
|
|
||||||
Preserve its relative layout, including `index.json`, `manifest.json`,
|
|
||||||
`rejected.json`, `warnings.json`, `lanes/`, and any indexed `chunk-map.json` or
|
|
||||||
`evidence-context.json`. Materialize the complete directory as one narrow,
|
|
||||||
transactional replacement so a failed or interrupted rerun cannot mix files
|
|
||||||
from different Notarius runs.
|
|
||||||
|
|
||||||
The raw subprocess receipt and stderr log belong in the run-local `extract`
|
|
||||||
report and log directories. The raw receipt identifies the original run-local
|
|
||||||
Notarius bundle and must not be rewritten to pretend that the canonical copy
|
|
||||||
was its original `output_directory`. Narratio's manifest is the durable ledger
|
|
||||||
for the canonical materialized paths.
|
|
||||||
|
|
||||||
### Registered Artifact Sources
|
|
||||||
|
|
||||||
For each configured output, locate the lane through the canonical copy of
|
|
||||||
`index.json` and record a manifest artifact with:
|
|
||||||
|
|
||||||
- source ID `narratio.extraction.<output-key>`;
|
|
||||||
- canonical lane-file path discovered from the index;
|
|
||||||
- producer stage and Narratio run ID;
|
|
||||||
- checksum;
|
|
||||||
- Notarius lane ID; and
|
|
||||||
- descriptor media type, schema identity/version, and module key when present.
|
|
||||||
|
|
||||||
If the current manifest model cannot carry descriptor compatibility metadata,
|
|
||||||
extend its artifact metadata in a backward-tolerant way rather than encoding
|
|
||||||
that information in filenames or source IDs.
|
|
||||||
|
|
||||||
Also record the canonical Notarius index as a stage output or stage metadata so
|
|
||||||
operators can discover the complete bundle, including non-lane artifacts. The
|
|
||||||
configured lane sources are the stable interface for analysis; the index and
|
|
||||||
bundle remain the provenance and inspection interface.
|
|
||||||
|
|
||||||
## Analysis And Publish Integration
|
|
||||||
|
|
||||||
Extend the runtime artifact catalog and configured Scriptorium input validation
|
|
||||||
so an enabled analysis artifact can declare, for example:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
inputs:
|
|
||||||
npc_registry:
|
|
||||||
source: narratio.extraction.npc_registry
|
|
||||||
```
|
|
||||||
|
|
||||||
Resolution must remain manifest-first and verify that the recorded artifact
|
|
||||||
was produced by a successful current `extract` stage. A required extraction
|
|
||||||
source that is unavailable must fail analysis with guidance to configure or
|
|
||||||
rerun `extract`; an optional source may be omitted according to the existing
|
|
||||||
Scriptorium input contract.
|
|
||||||
|
|
||||||
Publish source resolution should accept configured
|
|
||||||
`narratio.extraction.<output-key>` sources through the same artifact catalog so
|
|
||||||
operators may publish selected structured artifacts without manually copying
|
|
||||||
paths. The existing `--artifacts` flag remains scoped to Scriptorium artifact
|
|
||||||
selection and must not partially execute the Notarius pipeline.
|
|
||||||
|
|
||||||
No current-session analysis artifact should consume an incidental file from a
|
|
||||||
failed, stale, skipped, or superseded extraction run.
|
|
||||||
|
|
||||||
## Failure, Skip, Resume, And Diagnostics
|
|
||||||
|
|
||||||
- Missing or invalid enabled Notarius configuration fails configuration
|
|
||||||
validation before stage execution where statically discoverable.
|
|
||||||
- A disabled or absent Notarius configuration makes `extract` skip with clear
|
|
||||||
stage metadata and no new outputs.
|
|
||||||
- A missing or invalid final trimmed transcript fails `extract` before starting
|
|
||||||
Notarius.
|
|
||||||
- Preflight failure, nonzero Notarius exit, cancellation, timeout, malformed or
|
|
||||||
unsupported receipt/index data, unsafe paths, incompatible descriptors,
|
|
||||||
rejected required outputs, or missing configured lanes fails the entire
|
|
||||||
stage.
|
|
||||||
- Process success does not override Narratio's required-output policy.
|
|
||||||
- A failed run retains bounded run-local receipt bytes, stderr, and the
|
|
||||||
unpublished Notarius bundle for diagnosis, subject to Narratio's existing
|
|
||||||
sensitive-data and cleanup policies.
|
|
||||||
- The canonical bundle and manifest artifacts are updated only after complete
|
|
||||||
validation and materialization.
|
|
||||||
- Resume skips a succeeded, non-stale `extract` stage only when its
|
|
||||||
manifest-recorded canonical index and configured lane outputs still validate.
|
|
||||||
- Force and staleness behavior follows the ordinary stage contract; it must not
|
|
||||||
depend on merely finding `artifacts/notarius/` on disk.
|
|
||||||
|
|
||||||
Transcripts, Notarius outputs, evidence context, manifests, receipts, and logs
|
|
||||||
are private campaign material. Subprocess arguments and manifest metadata must
|
|
||||||
not contain secrets. Credentials remain in the environment or in mechanisms
|
|
||||||
owned by Notarius and PromptKit.
|
|
||||||
|
|
||||||
## Maintained D&D Example
|
|
||||||
|
|
||||||
Add or update a Narratio example that enables Notarius's complete
|
|
||||||
`dnd-session` pipeline and maps these ten required lanes to stable extraction
|
|
||||||
keys:
|
|
||||||
|
|
||||||
| Output key | Notarius lane ID |
|
|
||||||
| --- | --- |
|
|
||||||
| `item_registry` | `item-registry` |
|
|
||||||
| `npc_registry` | `npc-registry` |
|
|
||||||
| `location_registry` | `location-registry` |
|
|
||||||
| `scene_descriptions` | `scene-descriptions` |
|
|
||||||
| `item_occurrences` | `item-occurrences` |
|
|
||||||
| `spells` | `spells` |
|
|
||||||
| `combat_turns` | `combat-turns` |
|
|
||||||
| `npc_occurrences` | `npc-occurrences` |
|
|
||||||
| `location_occurrences` | `location-occurrences` |
|
|
||||||
| `enemy_events` | `enemy-events` |
|
|
||||||
|
|
||||||
The example must include each lane's current media type and schema identity
|
|
||||||
from Notarius's published contracts. It should also demonstrate at least one
|
|
||||||
Scriptorium analysis artifact consuming one or more
|
|
||||||
`narratio.extraction.*` sources. The example must use placeholders and relative
|
|
||||||
paths suitable for the example tree, contain no credentials, and pass the
|
|
||||||
repository's configuration validation tests.
|
|
||||||
|
|
||||||
## Compatibility Policy
|
|
||||||
|
|
||||||
The initial integration baseline is the public subprocess contract available
|
|
||||||
in Notarius v0.3.0:
|
|
||||||
|
|
||||||
- successful JSON receipt schema `notarius.run-result.v1`;
|
|
||||||
- production JSON bundle discovery through `index.json`; and
|
|
||||||
- the schema IDs and versions explicitly configured for required lanes.
|
|
||||||
|
|
||||||
Runtime compatibility should be decided from those published contracts, not
|
|
||||||
from textual parsing of `notarius --version`. New optional receipt or index
|
|
||||||
fields must not break Narratio. An unsupported receipt version or lane schema
|
|
||||||
must fail before the artifact is registered for analysis.
|
|
||||||
|
|
||||||
## Documentation Deliverables When Implemented
|
|
||||||
|
|
||||||
Update current-behavior documentation in the same change that implements the
|
|
||||||
feature:
|
|
||||||
|
|
||||||
- add `docs/integrations/notarius.md` for the external CLI, receipt, bundle,
|
|
||||||
and adapter contract, linking to Notarius's canonical documentation;
|
|
||||||
- add `docs/internal/stage-extract.md` for stage inputs, outputs, collaborators,
|
|
||||||
state transitions, failures, and focused tests;
|
|
||||||
- update `docs/internal/adapters.md`, `docs/internal/artifacts.md`,
|
|
||||||
`docs/internal/manifest.md`, and the internal stage inventory;
|
|
||||||
- update `docs/policy/architecture.md` to list Notarius among isolated external
|
|
||||||
systems and preserve the adapter/stage boundary;
|
|
||||||
- update `docs/config.md`, `docs/cli.md`, `docs/operations.md`,
|
|
||||||
`docs/troubleshooting.md`, `README.md`, and maintained examples only to the
|
|
||||||
extent their canonical scopes require; and
|
|
||||||
- update `docs/development.md` only to the extent its canonical contributor
|
|
||||||
routing scope requires.
|
|
||||||
|
|
||||||
Outside this roadmap, do not describe `extract`, Notarius configuration, or
|
|
||||||
`narratio.extraction.*` sources as implemented until the code exists.
|
|
||||||
|
|
||||||
## Testing And Validation Expectations
|
|
||||||
|
|
||||||
Implementation should provide focused tests for:
|
|
||||||
|
|
||||||
- strict configuration decoding, defaults, required fields, path resolution,
|
|
||||||
output-map validation, normalized-key collisions, and example loading;
|
|
||||||
- exact stage order, selection, downstream staleness, resume, force, and
|
|
||||||
prerequisite behavior;
|
|
||||||
- adapter command construction, deterministic working directory, environment
|
|
||||||
inheritance, stdout/stderr separation, cancellation, timeout, and nonzero
|
|
||||||
exits;
|
|
||||||
- supported and unsupported receipt versions, unknown optional fields,
|
|
||||||
malformed receipts, index decoding, and path escapes at every boundary;
|
|
||||||
- descriptor lookup by lane ID rather than filename, expected metadata checks,
|
|
||||||
missing/rejected configured lanes, and tolerated unconfigured lanes;
|
|
||||||
- run-local execution, transactional canonical-bundle replacement, checksums,
|
|
||||||
failed-run preservation, and manifest recording;
|
|
||||||
- artifact-catalog resolution from `narratio.extraction.*` into analysis and
|
|
||||||
publish, including required, optional, missing, stale, and skipped cases; and
|
|
||||||
- end-to-end stage execution with a fake Notarius adapter, without live LLM or
|
|
||||||
external subprocess requirements in the ordinary test suite.
|
|
||||||
|
|
||||||
Run the repository-wide Go tests, vet, build, and maintained example validation
|
|
||||||
after focused tests pass.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
- `extract` is a first-class transactional stage between `render` and
|
|
||||||
`analyze` everywhere Narratio models stage order or state.
|
|
||||||
- Narratio invokes Notarius only through a narrow, tested adapter.
|
|
||||||
- The stage consumes the manifest-resolved final trimmed Seriatim transcript.
|
|
||||||
- The Notarius configuration remains owned by Notarius; Narratio configures
|
|
||||||
only invocation and its downstream consumer contract.
|
|
||||||
- Every configured output is discovered through the receipt and `index.json`,
|
|
||||||
contract-checked, materialized transactionally, and recorded with a stable
|
|
||||||
`narratio.extraction.*` source ID.
|
|
||||||
- The complete D&D example maps all ten current lanes and passes strict config
|
|
||||||
validation.
|
|
||||||
- Analysis can consume extraction sources through the existing artifact input
|
|
||||||
model, and publish can select them through the artifact catalog.
|
|
||||||
- Failed, partial, rejected, unsafe, stale, or incompatible output never becomes
|
|
||||||
a current analysis input.
|
|
||||||
- Resume and force behavior remains manifest-driven.
|
|
||||||
- Documentation accurately describes the implemented stage, adapter,
|
|
||||||
configuration, operations, and compatibility boundary without duplicating
|
|
||||||
Notarius's canonical schemas.
|
|
||||||
|
|
||||||
## Non-Goals
|
|
||||||
|
|
||||||
- Reimplementing Notarius extraction, prompts, schemas, references, retries,
|
|
||||||
profiles, or lane orchestration in Narratio.
|
|
||||||
- Allowing one Narratio run to invoke arbitrary extractor programs or multiple
|
|
||||||
Notarius pipelines.
|
|
||||||
- Making `extract` a configurable DAG or folding it into the Scriptorium
|
|
||||||
`analyze` stage.
|
|
||||||
- Partially selecting Notarius lanes through Narratio's `--artifacts` flag.
|
|
||||||
- Decoding D&D payload bodies in the generic Notarius adapter.
|
|
||||||
- Supporting previous-session extraction artifacts in the initial feature.
|
|
||||||
- Requiring live Notarius, PromptKit, an LLM provider, or external services in
|
|
||||||
the ordinary unit test suite.
|
|
||||||
@@ -117,6 +117,147 @@ Safe fix:
|
|||||||
|
|
||||||
Relevant reference: [CLI artifact selection](./cli.md).
|
Relevant reference: [CLI artifact selection](./cli.md).
|
||||||
|
|
||||||
|
## Notarius executable missing
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction fails while resolving or starting the Notarius executable.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- `pipeline.notarius.binary` is not installed, executable, or on `PATH`;
|
||||||
|
- a configured executable path is wrong.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- install a compatible Notarius release or correct the binary setting, then
|
||||||
|
rerun extraction.
|
||||||
|
|
||||||
|
Relevant references: [Notarius configuration](./config.md#notarius-output-entries)
|
||||||
|
and [Notarius integration](./integrations/notarius.md).
|
||||||
|
|
||||||
|
## Notarius exits nonzero
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction reports a Notarius exit error instead of a receipt.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
- inspect `runs/{run_id}/extract/notarius.stderr.log`; stdout is reserved for
|
||||||
|
the receipt and is not merged with diagnostics.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- correct the reported Notarius pipeline, input, provider, or configuration
|
||||||
|
failure and rerun extraction. Do not edit a staged output bundle into place.
|
||||||
|
|
||||||
|
After a failed replacement, an older immutable bundle may still exist even
|
||||||
|
though the current session manifest has no successful extraction payload. This
|
||||||
|
is expected audit state, not a signal to relink the old bundle manually.
|
||||||
|
|
||||||
|
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
|
## Atomic Notarius promotion unsupported
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction fails with `atomic no-replace directory promotion is unsupported`
|
||||||
|
before a durable bundle or temporary promotion tree is created.
|
||||||
|
|
||||||
|
Likely cause:
|
||||||
|
|
||||||
|
- Narratio is running on an operating system other than Linux, macOS, or
|
||||||
|
Windows, where the required atomic no-replace directory primitive has not
|
||||||
|
been implemented and verified.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- run extraction on Linux, macOS, or Windows. Do not replace the atomic commit
|
||||||
|
with a manual copy or move; the session manifest must never observe a partial
|
||||||
|
or overwritten bundle.
|
||||||
|
|
||||||
|
This is an extraction-specific platform boundary, not a support statement for
|
||||||
|
unrelated Narratio workflows. See
|
||||||
|
[Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
|
## Notarius receipt or index incompatible
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction rejects the receipt schema, pipeline identity, bundle/index path,
|
||||||
|
lane descriptor, or payload path even though Notarius exited successfully.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- Narratio and Notarius versions disagree on their consumer contract;
|
||||||
|
- the configured pipeline or lane constraints are stale;
|
||||||
|
- output paths escape the bundle or traverse symlinks.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- compare installed Notarius output with the canonical Notarius contracts,
|
||||||
|
including receipt `index_file: index.json` and index management names
|
||||||
|
`manifest.json`, `rejected.json`, and `warnings.json`; align
|
||||||
|
`pipeline.notarius` constraints and rerun. Do not bypass confinement or schema
|
||||||
|
checks.
|
||||||
|
|
||||||
|
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
||||||
|
|
||||||
|
## Required Notarius lane rejected or missing
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction fails because a configured lane is rejected, missing, duplicated,
|
||||||
|
or incompatible, including after a zero exit.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- inspect the Notarius diagnostic log and bundle rejection/warning information;
|
||||||
|
- correct the Notarius module or the exact declared lane contract;
|
||||||
|
- remove an output declaration only if downstream consumers genuinely no longer
|
||||||
|
require that source, then rerun extraction.
|
||||||
|
|
||||||
|
Every configured output is required. Narratio does not promote a partial result.
|
||||||
|
|
||||||
|
## Extraction resume invalidated
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- a previously successful extraction runs again during ordinary continuation.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- the executable/config path, pipeline ID, timeout, working directory, or
|
||||||
|
configured output contracts changed;
|
||||||
|
- the durable bundle, index, lane set, provenance, regular-file status, or
|
||||||
|
checksum no longer validates.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- allow the automatic rerun after verifying the current configuration. Treat
|
||||||
|
an unsafe path or symlink error as filesystem corruption or tampering and
|
||||||
|
investigate it rather than replacing files manually.
|
||||||
|
|
||||||
|
## Notarius transitive configuration changed
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- Notarius profiles, prompts, modules, imported files, or references changed,
|
||||||
|
but Narratio still considers the previous extraction resumable.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage extract 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Narratio fingerprints its invocation contract, not the contents of transitive
|
||||||
|
Notarius inputs. Always force extraction after changing them; downstream
|
||||||
|
successful stages are then marked stale normally.
|
||||||
|
|
||||||
|
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
## Previous-session artifact input missing
|
## Previous-session artifact input missing
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ in the [configuration reference](../docs/config.md).
|
|||||||
external tools, and configured Scriptorium artifacts.
|
external tools, and configured Scriptorium artifacts.
|
||||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||||
pipeline section with explanatory comments.
|
pipeline section with explanatory comments.
|
||||||
|
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||||
|
Scriptorium artifact consuming only three declared Notarius lanes.
|
||||||
|
|
||||||
The existing `internal/config` example test loads and validates each pipeline
|
The existing `internal/config` example test loads and validates each pipeline
|
||||||
with the sample campaign and a compatible local- or S3-audio session.
|
with the sample campaign and a compatible local- or S3-audio session.
|
||||||
|
|||||||
55
examples/pipeline.extraction-subset.yml
Normal file
55
examples/pipeline.extraction-subset.yml
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
# Purpose-specific extraction example: a Scriptorium session brief consumes
|
||||||
|
# only the three Notarius lanes it needs.
|
||||||
|
|
||||||
|
campaigns:
|
||||||
|
root: /usr/local/share/narratio/campaigns
|
||||||
|
default_campaign_id: sample-campaign
|
||||||
|
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: "https://transcription.example.com/transcribe"
|
||||||
|
|
||||||
|
notarius:
|
||||||
|
enabled: true
|
||||||
|
binary: notarius
|
||||||
|
config_path: /usr/local/etc/notarius/config.yml
|
||||||
|
pipeline_id: dnd-session
|
||||||
|
timeout: 3h
|
||||||
|
outputs:
|
||||||
|
npc_registry:
|
||||||
|
lane_id: npc-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/npc-registry
|
||||||
|
location_registry:
|
||||||
|
lane_id: location-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.location_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/location-registry
|
||||||
|
scene_descriptions:
|
||||||
|
lane_id: scene-descriptions
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.scene_descriptions
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/scene-descriptions
|
||||||
|
|
||||||
|
scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
config_path: /usr/local/etc/scriptorium/config.yml
|
||||||
|
artifacts:
|
||||||
|
session_brief:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_brief
|
||||||
|
output_path: artifacts/session_brief.md
|
||||||
|
inputs:
|
||||||
|
npcs:
|
||||||
|
source: narratio.extraction.npc_registry
|
||||||
|
required: true
|
||||||
|
locations:
|
||||||
|
source: narratio.extraction.location_registry
|
||||||
|
required: true
|
||||||
|
scenes:
|
||||||
|
source: narratio.extraction.scene_descriptions
|
||||||
|
required: true
|
||||||
|
|
||||||
@@ -60,6 +60,11 @@ publish:
|
|||||||
- source: narratio.artifact.player_handout
|
- source: narratio.artifact.player_handout
|
||||||
dest: artifacts/player_handout.md
|
dest: artifacts/player_handout.md
|
||||||
required: false
|
required: false
|
||||||
|
# Extraction lanes publish only when named explicitly; the bundle and index
|
||||||
|
# are never implicit publish sources.
|
||||||
|
- source: narratio.extraction.npc_registry
|
||||||
|
dest: artifacts/extraction/npc-registry.json
|
||||||
|
required: true
|
||||||
|
|
||||||
whisperx:
|
whisperx:
|
||||||
# Required.
|
# Required.
|
||||||
@@ -123,6 +128,78 @@ trim:
|
|||||||
seriatim:
|
seriatim:
|
||||||
report: false
|
report: false
|
||||||
|
|
||||||
|
notarius:
|
||||||
|
# Optional structured extraction between trim and render.
|
||||||
|
enabled: true
|
||||||
|
binary: notarius
|
||||||
|
config_path: /usr/local/etc/notarius/config.yml
|
||||||
|
pipeline_id: dnd-session
|
||||||
|
timeout: 3h
|
||||||
|
working_directory: /usr/local/etc/notarius
|
||||||
|
# Each key creates source narratio.extraction.<key>. These constraints match
|
||||||
|
# the current Notarius D&D lane contracts; update them with Notarius.
|
||||||
|
outputs:
|
||||||
|
item_registry:
|
||||||
|
lane_id: item-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.item_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/item-registry
|
||||||
|
npc_registry:
|
||||||
|
lane_id: npc-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/npc-registry
|
||||||
|
location_registry:
|
||||||
|
lane_id: location-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.location_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/location-registry
|
||||||
|
scene_descriptions:
|
||||||
|
lane_id: scene-descriptions
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.scene_descriptions
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/scene-descriptions
|
||||||
|
item_occurrences:
|
||||||
|
lane_id: item-occurrences
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.item_occurrences
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/item-occurrences
|
||||||
|
spells:
|
||||||
|
lane_id: spells
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.spells
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/spells
|
||||||
|
combat_turns:
|
||||||
|
lane_id: combat-turns
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.combat_turns
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/combat-turns
|
||||||
|
npc_occurrences:
|
||||||
|
lane_id: npc-occurrences
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_occurrences
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/npc-occurrences
|
||||||
|
location_occurrences:
|
||||||
|
lane_id: location-occurrences
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.location_occurrences
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/location-occurrences
|
||||||
|
enemy_events:
|
||||||
|
lane_id: enemy-events
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.enemy_events
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/enemy-events
|
||||||
|
|
||||||
scriptorium:
|
scriptorium:
|
||||||
binary: scriptorium
|
binary: scriptorium
|
||||||
config_path: /usr/local/etc/scriptorium/config.yml
|
config_path: /usr/local/etc/scriptorium/config.yml
|
||||||
|
|||||||
1
go.mod
1
go.mod
@@ -7,6 +7,7 @@ require (
|
|||||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||||
github.com/aws/smithy-go v1.25.1
|
github.com/aws/smithy-go v1.25.1
|
||||||
|
golang.org/x/sys v0.47.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -34,6 +34,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt
|
|||||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||||
|
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||||
|
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
|||||||
22
internal/adapters/notarius/fake.go
Normal file
22
internal/adapters/notarius/fake.go
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
package notarius
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// FakeRunner is a configurable in-memory runner for stage tests.
|
||||||
|
type FakeRunner struct {
|
||||||
|
Requests []RunRequest
|
||||||
|
Result RunResult
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run records the request and returns the configured result or error.
|
||||||
|
func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return RunResult{}, err
|
||||||
|
}
|
||||||
|
f.Requests = append(f.Requests, req)
|
||||||
|
if f.Err != nil {
|
||||||
|
return RunResult{}, f.Err
|
||||||
|
}
|
||||||
|
return f.Result, nil
|
||||||
|
}
|
||||||
108
internal/adapters/notarius/runner.go
Normal file
108
internal/adapters/notarius/runner.go
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
// Package notarius declares the adapter contract for Notarius CLI invocations.
|
||||||
|
package notarius
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const ReceiptSchemaVersion = "notarius.run-result.v1"
|
||||||
|
|
||||||
|
// Runner is the adapter boundary for a complete Notarius pipeline invocation.
|
||||||
|
type Runner interface {
|
||||||
|
Run(ctx context.Context, req RunRequest) (RunResult, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
|
||||||
|
type RunRequest struct {
|
||||||
|
Binary string
|
||||||
|
ConfigPath string
|
||||||
|
PipelineID string
|
||||||
|
InputPath string
|
||||||
|
OutputRoot string
|
||||||
|
WorkingDirectory string
|
||||||
|
ReceiptPath string
|
||||||
|
LogPath string
|
||||||
|
Timeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Receipt is the transport-neutral successful run receipt.
|
||||||
|
type Receipt struct {
|
||||||
|
SchemaVersion string
|
||||||
|
RunID string
|
||||||
|
PipelineID string
|
||||||
|
OutputDirectory string
|
||||||
|
IndexFile string
|
||||||
|
NormalizedOutputCount int
|
||||||
|
RejectedOutputCount int
|
||||||
|
WarningCount int
|
||||||
|
ValidationStatus string
|
||||||
|
DebugDirectory string
|
||||||
|
}
|
||||||
|
|
||||||
|
// LaneDescriptor identifies one normalized lane payload discovered through the index.
|
||||||
|
type LaneDescriptor struct {
|
||||||
|
LaneID string
|
||||||
|
File string
|
||||||
|
Path string
|
||||||
|
MediaType string
|
||||||
|
ModuleKey string
|
||||||
|
SchemaID string
|
||||||
|
SchemaName string
|
||||||
|
SchemaVersion string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PipelineDescriptor identifies a pipeline-wide artifact discovered through the index.
|
||||||
|
type PipelineDescriptor struct {
|
||||||
|
ArtifactKind string
|
||||||
|
File string
|
||||||
|
Path string
|
||||||
|
MediaType string
|
||||||
|
SchemaID string
|
||||||
|
SchemaName string
|
||||||
|
SchemaVersion string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Index describes the validated bundle-management and artifact paths.
|
||||||
|
type Index struct {
|
||||||
|
Path string
|
||||||
|
ManifestFile string
|
||||||
|
ManifestPath string
|
||||||
|
RejectedFile string
|
||||||
|
RejectedPath string
|
||||||
|
WarningsFile string
|
||||||
|
WarningsPath string
|
||||||
|
Lanes []LaneDescriptor
|
||||||
|
ChunkMap *PipelineDescriptor
|
||||||
|
EvidenceContext *PipelineDescriptor
|
||||||
|
}
|
||||||
|
|
||||||
|
// RejectionSummary retains structured rejection identity without free-form messages.
|
||||||
|
type RejectionSummary struct {
|
||||||
|
Stage string
|
||||||
|
StepID string
|
||||||
|
LaneID string
|
||||||
|
ModuleKey string
|
||||||
|
ChunkID string
|
||||||
|
ValidatorName string
|
||||||
|
ReasonCode string
|
||||||
|
}
|
||||||
|
|
||||||
|
// WarningSummary retains structured warning identity without free-form messages.
|
||||||
|
type WarningSummary struct {
|
||||||
|
Scope string
|
||||||
|
ReasonCode string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunResult describes a successfully decoded and validated Notarius bundle.
|
||||||
|
type RunResult struct {
|
||||||
|
Receipt Receipt
|
||||||
|
Index Index
|
||||||
|
BundleRoot string
|
||||||
|
ReceiptPath string
|
||||||
|
LogPath string
|
||||||
|
ExitCode int
|
||||||
|
Duration time.Duration
|
||||||
|
Rejections []RejectionSummary
|
||||||
|
Warnings []WarningSummary
|
||||||
|
}
|
||||||
524
internal/adapters/notarius/subprocess.go
Normal file
524
internal/adapters/notarius/subprocess.go
Normal file
@@ -0,0 +1,524 @@
|
|||||||
|
package notarius
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
maxReceiptBytes = 1 << 20
|
||||||
|
maxIndexBytes = 4 << 20
|
||||||
|
maxSummaryBytes = 4 << 20
|
||||||
|
canonicalIndexFile = "index.json"
|
||||||
|
canonicalManifestFile = "manifest.json"
|
||||||
|
canonicalRejectedFile = "rejected.json"
|
||||||
|
canonicalWarningsFile = "warnings.json"
|
||||||
|
)
|
||||||
|
|
||||||
|
type subprocessRun func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error)
|
||||||
|
|
||||||
|
// SubprocessRunner invokes Notarius through its public CLI.
|
||||||
|
type SubprocessRunner struct {
|
||||||
|
run subprocessRun
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSubprocessRunner constructs a production Notarius subprocess runner.
|
||||||
|
func NewSubprocessRunner() *SubprocessRunner {
|
||||||
|
return &SubprocessRunner{run: subprocess.Run}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run executes a complete Notarius pipeline and discovers its published bundle.
|
||||||
|
func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||||
|
if r == nil || r.run == nil {
|
||||||
|
return RunResult{}, fmt.Errorf("notarius subprocess runner is nil")
|
||||||
|
}
|
||||||
|
if err := validateRunRequest(req); err != nil {
|
||||||
|
return RunResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
args := []string{
|
||||||
|
"run", req.PipelineID,
|
||||||
|
"--config", req.ConfigPath,
|
||||||
|
"--input", req.InputPath,
|
||||||
|
"--output-dir", req.OutputRoot,
|
||||||
|
"--json",
|
||||||
|
}
|
||||||
|
processResult, err := r.run(ctx, subprocess.RunRequest{
|
||||||
|
Executable: req.Binary,
|
||||||
|
Args: args,
|
||||||
|
WorkingDir: req.WorkingDirectory,
|
||||||
|
Timeout: req.Timeout,
|
||||||
|
StdoutLogPath: req.ReceiptPath,
|
||||||
|
StderrLogPath: req.LogPath,
|
||||||
|
})
|
||||||
|
baseResult := RunResult{
|
||||||
|
ReceiptPath: req.ReceiptPath,
|
||||||
|
LogPath: req.LogPath,
|
||||||
|
ExitCode: processResult.ExitCode,
|
||||||
|
Duration: processResult.Duration,
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, fmt.Errorf("run notarius pipeline %q: %w", req.PipelineID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
receipt, err := loadReceipt(req.ReceiptPath, req.PipelineID)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, err
|
||||||
|
}
|
||||||
|
bundleRoot, err := validateBundleRoot(req.OutputRoot, receipt.OutputDirectory)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, err
|
||||||
|
}
|
||||||
|
indexPath, err := resolveRegularFile(bundleRoot, receipt.IndexFile)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, fmt.Errorf("resolve receipt index file: %w", err)
|
||||||
|
}
|
||||||
|
index, err := loadIndex(bundleRoot, indexPath)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, err
|
||||||
|
}
|
||||||
|
rejections, err := loadRejections(index.RejectedPath)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, err
|
||||||
|
}
|
||||||
|
warnings, err := loadWarnings(index.WarningsPath)
|
||||||
|
if err != nil {
|
||||||
|
return baseResult, err
|
||||||
|
}
|
||||||
|
|
||||||
|
baseResult.Receipt = receipt
|
||||||
|
baseResult.Index = index
|
||||||
|
baseResult.BundleRoot = bundleRoot
|
||||||
|
baseResult.Rejections = rejections
|
||||||
|
baseResult.Warnings = warnings
|
||||||
|
return baseResult, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRunRequest(req RunRequest) error {
|
||||||
|
if strings.TrimSpace(req.Binary) == "" {
|
||||||
|
return fmt.Errorf("notarius binary is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(req.PipelineID) == "" {
|
||||||
|
return fmt.Errorf("notarius pipeline id is required")
|
||||||
|
}
|
||||||
|
if req.Timeout <= 0 {
|
||||||
|
return fmt.Errorf("notarius timeout must be positive")
|
||||||
|
}
|
||||||
|
for label, path := range map[string]string{
|
||||||
|
"config": req.ConfigPath,
|
||||||
|
"input": req.InputPath,
|
||||||
|
"output root": req.OutputRoot,
|
||||||
|
"working directory": req.WorkingDirectory,
|
||||||
|
"receipt": req.ReceiptPath,
|
||||||
|
"log": req.LogPath,
|
||||||
|
} {
|
||||||
|
if strings.TrimSpace(path) == "" {
|
||||||
|
return fmt.Errorf("notarius %s path is required", label)
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(path) {
|
||||||
|
return fmt.Errorf("notarius %s path must be absolute", label)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if filepath.Clean(req.ReceiptPath) == filepath.Clean(req.LogPath) {
|
||||||
|
return fmt.Errorf("notarius receipt and log paths must be different")
|
||||||
|
}
|
||||||
|
if err := requireRegularFile(req.ConfigPath); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius config path: %w", err)
|
||||||
|
}
|
||||||
|
if err := requireRegularFile(req.InputPath); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius input path: %w", err)
|
||||||
|
}
|
||||||
|
if err := requireDirectory(req.OutputRoot); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius output root: %w", err)
|
||||||
|
}
|
||||||
|
if err := requireDirectory(req.WorkingDirectory); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius working directory: %w", err)
|
||||||
|
}
|
||||||
|
if err := validateLogDestination(req.ReceiptPath); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius receipt path: %w", err)
|
||||||
|
}
|
||||||
|
if err := validateLogDestination(req.LogPath); err != nil {
|
||||||
|
return fmt.Errorf("validate notarius log path: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type receiptDocument struct {
|
||||||
|
SchemaVersion string `json:"schema_version"`
|
||||||
|
RunID string `json:"run_id"`
|
||||||
|
PipelineID string `json:"pipeline_id"`
|
||||||
|
OutputDirectory string `json:"output_directory"`
|
||||||
|
IndexFile string `json:"index_file"`
|
||||||
|
NormalizedOutputCount *int `json:"normalized_output_count"`
|
||||||
|
RejectedOutputCount *int `json:"rejected_output_count"`
|
||||||
|
WarningCount *int `json:"warning_count"`
|
||||||
|
ValidationStatus string `json:"validation_status"`
|
||||||
|
DebugDirectory string `json:"debug_directory"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadReceipt(path, pipelineID string) (Receipt, error) {
|
||||||
|
var document receiptDocument
|
||||||
|
if err := decodeBoundedJSON(path, maxReceiptBytes, &document); err != nil {
|
||||||
|
return Receipt{}, fmt.Errorf("decode notarius receipt: %w", err)
|
||||||
|
}
|
||||||
|
if document.SchemaVersion != ReceiptSchemaVersion {
|
||||||
|
return Receipt{}, fmt.Errorf("unsupported notarius receipt schema version %q", document.SchemaVersion)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(document.RunID) == "" || strings.TrimSpace(document.PipelineID) == "" ||
|
||||||
|
strings.TrimSpace(document.OutputDirectory) == "" || strings.TrimSpace(document.ValidationStatus) == "" ||
|
||||||
|
document.NormalizedOutputCount == nil ||
|
||||||
|
document.RejectedOutputCount == nil || document.WarningCount == nil {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt is missing required fields")
|
||||||
|
}
|
||||||
|
if document.IndexFile != canonicalIndexFile {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt index_file %q is incompatible; want %q", document.IndexFile, canonicalIndexFile)
|
||||||
|
}
|
||||||
|
if document.PipelineID != pipelineID {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt pipeline id %q does not match requested pipeline %q", document.PipelineID, pipelineID)
|
||||||
|
}
|
||||||
|
if *document.NormalizedOutputCount < 0 || *document.RejectedOutputCount < 0 || *document.WarningCount < 0 {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt counts must be non-negative")
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(document.OutputDirectory) {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt output directory must be absolute")
|
||||||
|
}
|
||||||
|
if document.DebugDirectory != "" && !filepath.IsAbs(document.DebugDirectory) {
|
||||||
|
return Receipt{}, fmt.Errorf("notarius receipt debug directory must be absolute when present")
|
||||||
|
}
|
||||||
|
return Receipt{
|
||||||
|
SchemaVersion: document.SchemaVersion,
|
||||||
|
RunID: document.RunID,
|
||||||
|
PipelineID: document.PipelineID,
|
||||||
|
OutputDirectory: filepath.Clean(document.OutputDirectory),
|
||||||
|
IndexFile: document.IndexFile,
|
||||||
|
NormalizedOutputCount: *document.NormalizedOutputCount,
|
||||||
|
RejectedOutputCount: *document.RejectedOutputCount,
|
||||||
|
WarningCount: *document.WarningCount,
|
||||||
|
ValidationStatus: document.ValidationStatus,
|
||||||
|
DebugDirectory: document.DebugDirectory,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type indexDocument struct {
|
||||||
|
ManifestFile string `json:"manifest_file"`
|
||||||
|
OutputFiles *[]laneDocument `json:"output_files"`
|
||||||
|
RejectedFile string `json:"rejected_file"`
|
||||||
|
WarningsFile string `json:"warnings_file"`
|
||||||
|
ChunkMap *pipelineDocument `json:"chunk_map"`
|
||||||
|
EvidenceContext *pipelineDocument `json:"evidence_context"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type laneDocument struct {
|
||||||
|
LaneID string `json:"lane_id"`
|
||||||
|
File string `json:"file"`
|
||||||
|
MediaType string `json:"media_type"`
|
||||||
|
ModuleKey string `json:"module_key"`
|
||||||
|
SchemaID string `json:"schema_id"`
|
||||||
|
SchemaName string `json:"schema_name"`
|
||||||
|
SchemaVersion string `json:"schema_version"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type pipelineDocument struct {
|
||||||
|
ArtifactKind string `json:"artifact_kind"`
|
||||||
|
File string `json:"file"`
|
||||||
|
MediaType string `json:"media_type"`
|
||||||
|
SchemaID string `json:"schema_id"`
|
||||||
|
SchemaName string `json:"schema_name"`
|
||||||
|
SchemaVersion string `json:"schema_version"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadIndex(bundleRoot, indexPath string) (Index, error) {
|
||||||
|
var document indexDocument
|
||||||
|
if err := decodeBoundedJSON(indexPath, maxIndexBytes, &document); err != nil {
|
||||||
|
return Index{}, fmt.Errorf("decode notarius index: %w", err)
|
||||||
|
}
|
||||||
|
for _, field := range []struct {
|
||||||
|
name string
|
||||||
|
got string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "manifest_file", got: document.ManifestFile, want: canonicalManifestFile},
|
||||||
|
{name: "rejected_file", got: document.RejectedFile, want: canonicalRejectedFile},
|
||||||
|
{name: "warnings_file", got: document.WarningsFile, want: canonicalWarningsFile},
|
||||||
|
} {
|
||||||
|
if field.got != field.want {
|
||||||
|
return Index{}, fmt.Errorf("notarius index %s %q is incompatible; want %q", field.name, field.got, field.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if document.OutputFiles == nil {
|
||||||
|
return Index{}, fmt.Errorf("notarius index is missing required output_files")
|
||||||
|
}
|
||||||
|
|
||||||
|
index := Index{
|
||||||
|
Path: indexPath,
|
||||||
|
ManifestFile: document.ManifestFile,
|
||||||
|
RejectedFile: document.RejectedFile,
|
||||||
|
WarningsFile: document.WarningsFile,
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if index.ManifestPath, err = resolveRegularFile(bundleRoot, index.ManifestFile); err != nil {
|
||||||
|
return Index{}, fmt.Errorf("resolve notarius manifest file: %w", err)
|
||||||
|
}
|
||||||
|
if index.RejectedPath, err = resolveRegularFile(bundleRoot, index.RejectedFile); err != nil {
|
||||||
|
return Index{}, fmt.Errorf("resolve notarius rejection file: %w", err)
|
||||||
|
}
|
||||||
|
if index.WarningsPath, err = resolveRegularFile(bundleRoot, index.WarningsFile); err != nil {
|
||||||
|
return Index{}, fmt.Errorf("resolve notarius warning file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
seenLanes := make(map[string]struct{}, len(*document.OutputFiles))
|
||||||
|
for _, lane := range *document.OutputFiles {
|
||||||
|
if strings.TrimSpace(lane.LaneID) == "" || strings.TrimSpace(lane.File) == "" {
|
||||||
|
return Index{}, fmt.Errorf("notarius lane descriptors require lane_id and file")
|
||||||
|
}
|
||||||
|
if _, exists := seenLanes[lane.LaneID]; exists {
|
||||||
|
return Index{}, fmt.Errorf("notarius index contains duplicate lane id %q", lane.LaneID)
|
||||||
|
}
|
||||||
|
seenLanes[lane.LaneID] = struct{}{}
|
||||||
|
path, err := resolveRegularFile(bundleRoot, lane.File)
|
||||||
|
if err != nil {
|
||||||
|
return Index{}, fmt.Errorf("resolve notarius lane %q file: %w", lane.LaneID, err)
|
||||||
|
}
|
||||||
|
index.Lanes = append(index.Lanes, LaneDescriptor{
|
||||||
|
LaneID: lane.LaneID, File: lane.File, Path: path, MediaType: lane.MediaType,
|
||||||
|
ModuleKey: lane.ModuleKey, SchemaID: lane.SchemaID, SchemaName: lane.SchemaName,
|
||||||
|
SchemaVersion: lane.SchemaVersion,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if document.ChunkMap != nil {
|
||||||
|
index.ChunkMap, err = resolvePipelineDescriptor(bundleRoot, "chunk_map", *document.ChunkMap)
|
||||||
|
if err != nil {
|
||||||
|
return Index{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if document.EvidenceContext != nil {
|
||||||
|
index.EvidenceContext, err = resolvePipelineDescriptor(bundleRoot, "evidence_context", *document.EvidenceContext)
|
||||||
|
if err != nil {
|
||||||
|
return Index{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return index, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolvePipelineDescriptor(bundleRoot, label string, document pipelineDocument) (*PipelineDescriptor, error) {
|
||||||
|
if strings.TrimSpace(document.ArtifactKind) == "" || strings.TrimSpace(document.File) == "" ||
|
||||||
|
strings.TrimSpace(document.MediaType) == "" || strings.TrimSpace(document.SchemaID) == "" ||
|
||||||
|
strings.TrimSpace(document.SchemaName) == "" || strings.TrimSpace(document.SchemaVersion) == "" {
|
||||||
|
return nil, fmt.Errorf("notarius %s descriptor is missing required fields", label)
|
||||||
|
}
|
||||||
|
path, err := resolveRegularFile(bundleRoot, document.File)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve notarius %s file: %w", label, err)
|
||||||
|
}
|
||||||
|
return &PipelineDescriptor{
|
||||||
|
ArtifactKind: document.ArtifactKind, File: document.File, Path: path,
|
||||||
|
MediaType: document.MediaType, SchemaID: document.SchemaID,
|
||||||
|
SchemaName: document.SchemaName, SchemaVersion: document.SchemaVersion,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type rejectionDocument struct {
|
||||||
|
Rejected *[]struct {
|
||||||
|
Stage string `json:"stage"`
|
||||||
|
StepID string `json:"step_id"`
|
||||||
|
LaneID string `json:"lane_id"`
|
||||||
|
ModuleKey string `json:"module_key"`
|
||||||
|
ChunkID string `json:"chunk_id"`
|
||||||
|
ValidatorName string `json:"validator_name"`
|
||||||
|
ReasonCode string `json:"reason_code"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
} `json:"rejected"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadRejections(path string) ([]RejectionSummary, error) {
|
||||||
|
var document rejectionDocument
|
||||||
|
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode notarius rejections: %w", err)
|
||||||
|
}
|
||||||
|
if document.Rejected == nil {
|
||||||
|
return nil, fmt.Errorf("notarius rejection document is missing rejected array")
|
||||||
|
}
|
||||||
|
summaries := make([]RejectionSummary, 0, len(*document.Rejected))
|
||||||
|
for _, item := range *document.Rejected {
|
||||||
|
if strings.TrimSpace(item.Stage) == "" || strings.TrimSpace(item.Message) == "" {
|
||||||
|
return nil, fmt.Errorf("notarius rejection entries require stage and message")
|
||||||
|
}
|
||||||
|
summaries = append(summaries, RejectionSummary{
|
||||||
|
Stage: item.Stage, StepID: item.StepID, LaneID: item.LaneID,
|
||||||
|
ModuleKey: item.ModuleKey, ChunkID: item.ChunkID,
|
||||||
|
ValidatorName: item.ValidatorName, ReasonCode: item.ReasonCode,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return summaries, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type warningDocument struct {
|
||||||
|
Warnings *[]struct {
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
ReasonCode string `json:"reason_code"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
} `json:"warnings"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadWarnings(path string) ([]WarningSummary, error) {
|
||||||
|
var document warningDocument
|
||||||
|
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode notarius warnings: %w", err)
|
||||||
|
}
|
||||||
|
if document.Warnings == nil {
|
||||||
|
return nil, fmt.Errorf("notarius warning document is missing warnings array")
|
||||||
|
}
|
||||||
|
summaries := make([]WarningSummary, 0, len(*document.Warnings))
|
||||||
|
for _, item := range *document.Warnings {
|
||||||
|
if strings.TrimSpace(item.ReasonCode) == "" || strings.TrimSpace(item.Message) == "" {
|
||||||
|
return nil, fmt.Errorf("notarius warning entries require reason_code and message")
|
||||||
|
}
|
||||||
|
summaries = append(summaries, WarningSummary{Scope: item.Scope, ReasonCode: item.ReasonCode})
|
||||||
|
}
|
||||||
|
return summaries, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeBoundedJSON(path string, limit int64, destination any) error {
|
||||||
|
inspected, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if inspected.Mode()&os.ModeSymlink != 0 || !inspected.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("path %q must be a regular file without symlinks", path)
|
||||||
|
}
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = file.Close() }()
|
||||||
|
opened, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !opened.Mode().IsRegular() || !os.SameFile(inspected, opened) {
|
||||||
|
return fmt.Errorf("file %q changed before it could be read", path)
|
||||||
|
}
|
||||||
|
reader := io.LimitReader(file, limit+1)
|
||||||
|
data, err := io.ReadAll(reader)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if int64(len(data)) > limit {
|
||||||
|
return fmt.Errorf("file %q exceeds %d-byte limit", path, limit)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(data, destination); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateBundleRoot(outputRoot, bundleRoot string) (string, error) {
|
||||||
|
root := filepath.Clean(outputRoot)
|
||||||
|
bundle := filepath.Clean(bundleRoot)
|
||||||
|
relative, err := filepath.Rel(root, bundle)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("compare notarius output paths: %w", err)
|
||||||
|
}
|
||||||
|
if relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||||
|
return "", fmt.Errorf("notarius output directory %q is not beneath output root %q", bundleRoot, outputRoot)
|
||||||
|
}
|
||||||
|
if err := requireDirectoryTree(root, relative); err != nil {
|
||||||
|
return "", fmt.Errorf("validate notarius output directory: %w", err)
|
||||||
|
}
|
||||||
|
return bundle, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveRegularFile(root, logicalPath string) (string, error) {
|
||||||
|
resolved, err := pathsafe.JoinSlashRelativeUnderRoot(root, logicalPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
relative, err := filepath.Rel(root, resolved)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := requireRegularFileTree(root, relative); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return resolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireDirectoryTree(root, relative string) error {
|
||||||
|
if err := requireDirectory(root); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
current := root
|
||||||
|
for _, component := range strings.Split(relative, string(filepath.Separator)) {
|
||||||
|
current = filepath.Join(current, component)
|
||||||
|
if err := requireDirectory(current); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireRegularFileTree(root, relative string) error {
|
||||||
|
components := strings.Split(relative, string(filepath.Separator))
|
||||||
|
if len(components) == 0 {
|
||||||
|
return fmt.Errorf("regular file path is required")
|
||||||
|
}
|
||||||
|
if err := requireDirectory(root); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
current := root
|
||||||
|
for _, component := range components[:len(components)-1] {
|
||||||
|
current = filepath.Join(current, component)
|
||||||
|
if err := requireDirectory(current); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return requireRegularFile(filepath.Join(current, components[len(components)-1]))
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireDirectory(path string) error {
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||||
|
return fmt.Errorf("path %q must be a directory without symlinks", path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireRegularFile(path string) error {
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("path %q must be a regular file without symlinks", path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateLogDestination(path string) error {
|
||||||
|
if err := requireDirectory(filepath.Dir(path)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("path %q must be absent or a regular file without symlinks", path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
572
internal/adapters/notarius/subprocess_test.go
Normal file
572
internal/adapters/notarius/subprocess_test.go
Normal file
@@ -0,0 +1,572 @@
|
|||||||
|
package notarius
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
var captured sharedsubprocess.RunRequest
|
||||||
|
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
captured = processReq
|
||||||
|
writeValidBundleAndReceipt(t, req, true)
|
||||||
|
return sharedsubprocess.RunResult{ExitCode: 0, Duration: 2 * time.Second}, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
result, err := runner.Run(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
wantArgs := []string{
|
||||||
|
"run", "dnd-session", "--config", req.ConfigPath, "--input", req.InputPath,
|
||||||
|
"--output-dir", req.OutputRoot, "--json",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(captured.Args, wantArgs) {
|
||||||
|
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
|
||||||
|
}
|
||||||
|
if captured.Executable != req.Binary || captured.WorkingDir != req.WorkingDirectory || captured.Timeout != req.Timeout {
|
||||||
|
t.Fatalf("subprocess request = %#v", captured)
|
||||||
|
}
|
||||||
|
if captured.StdoutLogPath != req.ReceiptPath || captured.StderrLogPath != req.LogPath {
|
||||||
|
t.Fatalf("stream paths = stdout %q stderr %q", captured.StdoutLogPath, captured.StderrLogPath)
|
||||||
|
}
|
||||||
|
if captured.EnvOverrides != nil {
|
||||||
|
t.Fatalf("environment overrides = %#v, want inherited environment only", captured.EnvOverrides)
|
||||||
|
}
|
||||||
|
for _, arg := range captured.Args {
|
||||||
|
if arg == "--session-id" {
|
||||||
|
t.Fatal("subprocess args unexpectedly contain --session-id")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Receipt.SchemaVersion != ReceiptSchemaVersion || result.Receipt.RunID != "notarius-run-1" {
|
||||||
|
t.Fatalf("receipt = %#v", result.Receipt)
|
||||||
|
}
|
||||||
|
if len(result.Index.Lanes) != 1 || result.Index.Lanes[0].LaneID != "npc-registry" {
|
||||||
|
t.Fatalf("lanes = %#v", result.Index.Lanes)
|
||||||
|
}
|
||||||
|
if result.Index.ChunkMap == nil || result.Index.ChunkMap.ArtifactKind != "chunk_map" {
|
||||||
|
t.Fatalf("chunk map = %#v", result.Index.ChunkMap)
|
||||||
|
}
|
||||||
|
if result.Index.EvidenceContext == nil || result.Index.EvidenceContext.ArtifactKind != "evidence_context" {
|
||||||
|
t.Fatalf("evidence context = %#v", result.Index.EvidenceContext)
|
||||||
|
}
|
||||||
|
if len(result.Rejections) != 1 || result.Rejections[0].LaneID != "spells" || result.Rejections[0].ReasonCode != "invalid_spell" {
|
||||||
|
t.Fatalf("rejections = %#v", result.Rejections)
|
||||||
|
}
|
||||||
|
if len(result.Warnings) != 1 || result.Warnings[0].Scope != "lane:npc-registry" || result.Warnings[0].ReasonCode != "normalized_name" {
|
||||||
|
t.Fatalf("warnings = %#v", result.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerInheritsEnvironmentAndSeparatesStreams(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
writeValidBundleAndReceipt(t, req, false)
|
||||||
|
receiptFixture := req.ReceiptPath + ".fixture"
|
||||||
|
data, err := os.ReadFile(req.ReceiptPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(receiptFixture, data, 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(req.ReceiptPath); err != nil {
|
||||||
|
t.Fatalf("Remove(receipt) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
captureDir := filepath.Join(filepath.Dir(req.ReceiptPath), "capture")
|
||||||
|
if err := os.Mkdir(captureDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(capture) error = %v", err)
|
||||||
|
}
|
||||||
|
script := writeShellScript(t, `#!/bin/sh
|
||||||
|
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
|
||||||
|
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
|
||||||
|
printf 'diagnostic stream\n' >&2
|
||||||
|
cat "$NOTARIUS_RECEIPT_FIXTURE"
|
||||||
|
`)
|
||||||
|
req.Binary = script
|
||||||
|
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
|
||||||
|
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
|
||||||
|
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
||||||
|
|
||||||
|
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
|
||||||
|
assertTextFile(t, filepath.Join(captureDir, "environment"), "inherited-value")
|
||||||
|
assertTextFile(t, req.LogPath, "diagnostic stream\n")
|
||||||
|
receiptBytes, err := os.ReadFile(req.ReceiptPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(receiptBytes), "diagnostic stream") {
|
||||||
|
t.Fatal("receipt contains stderr output")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
scriptBody string
|
||||||
|
timeout time.Duration
|
||||||
|
cancel bool
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "nonzero", scriptBody: "printf '{malformed receipt'; printf 'failed\\n' >&2; exit 7\n", timeout: time.Second, want: "exit code 7"},
|
||||||
|
{name: "timeout", scriptBody: "sleep 5\n", timeout: 20 * time.Millisecond, want: "timed out"},
|
||||||
|
{name: "cancellation", scriptBody: "sleep 5\n", timeout: time.Second, cancel: true, want: "canceled"},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
|
||||||
|
req.Timeout = test.timeout
|
||||||
|
ctx := context.Background()
|
||||||
|
if test.cancel {
|
||||||
|
cancelCtx, cancel := context.WithCancel(ctx)
|
||||||
|
ctx = cancelCtx
|
||||||
|
time.AfterFunc(20*time.Millisecond, cancel)
|
||||||
|
}
|
||||||
|
_, err := NewSubprocessRunner().Run(ctx, req)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||||
|
t.Fatalf("Run() error = %v, want fragment %q", err, test.want)
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), "decode notarius receipt") {
|
||||||
|
t.Fatalf("Run() parsed stdout after process failure: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRunnerReturnsSharedSubprocessErrorWithoutReadingReceipt(t *testing.T) {
|
||||||
|
req := validRunRequest(t)
|
||||||
|
if err := os.WriteFile(req.ReceiptPath, []byte("not json"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(receipt) error = %v", err)
|
||||||
|
}
|
||||||
|
wantErr := errors.New("process failed")
|
||||||
|
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||||
|
return sharedsubprocess.RunResult{ExitCode: 9}, wantErr
|
||||||
|
}}
|
||||||
|
_, err := runner.Run(context.Background(), req)
|
||||||
|
if !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("Run() error = %v, want wrapped process error", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), "decode") {
|
||||||
|
t.Fatalf("Run() parsed receipt after failure: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadReceiptValidation(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
valid := map[string]any{
|
||||||
|
"schema_version": ReceiptSchemaVersion, "run_id": "run-1", "pipeline_id": "pipeline-1",
|
||||||
|
"output_directory": filepath.Join(root, "outputs", "run-1"), "index_file": "index.json",
|
||||||
|
"normalized_output_count": 1, "rejected_output_count": 0, "warning_count": 0,
|
||||||
|
"validation_status": "approved", "future_field": true,
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(map[string]any)
|
||||||
|
raw []byte
|
||||||
|
wantOK bool
|
||||||
|
wantError string
|
||||||
|
}{
|
||||||
|
{name: "unknown fields tolerated", wantOK: true},
|
||||||
|
{name: "malformed", raw: []byte("{")},
|
||||||
|
{name: "unsupported version", mutate: func(v map[string]any) { v["schema_version"] = "notarius.run-result.v2" }},
|
||||||
|
{name: "missing field", mutate: func(v map[string]any) { delete(v, "run_id") }},
|
||||||
|
{name: "pipeline mismatch", mutate: func(v map[string]any) { v["pipeline_id"] = "other" }},
|
||||||
|
{name: "relative output", mutate: func(v map[string]any) { v["output_directory"] = "run-1" }},
|
||||||
|
{name: "negative count", mutate: func(v map[string]any) { v["warning_count"] = -1 }},
|
||||||
|
{
|
||||||
|
name: "nested index", mutate: func(v map[string]any) { v["index_file"] = "nested/index.json" },
|
||||||
|
wantError: `index_file "nested/index.json"`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cleanable index", mutate: func(v map[string]any) { v["index_file"] = "./index.json" },
|
||||||
|
wantError: `index_file "./index.json"`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(root, strings.ReplaceAll(test.name, " ", "-")+".json")
|
||||||
|
values := cloneMap(valid)
|
||||||
|
if test.mutate != nil {
|
||||||
|
test.mutate(values)
|
||||||
|
}
|
||||||
|
if test.raw != nil {
|
||||||
|
if err := os.WriteFile(path, test.raw, 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
writeJSONFile(t, path, values)
|
||||||
|
}
|
||||||
|
_, err := loadReceipt(path, "pipeline-1")
|
||||||
|
if test.wantOK && err != nil {
|
||||||
|
t.Fatalf("loadReceipt() error = %v", err)
|
||||||
|
}
|
||||||
|
if !test.wantOK && err == nil {
|
||||||
|
t.Fatal("loadReceipt() error = nil, want validation failure")
|
||||||
|
}
|
||||||
|
if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
|
||||||
|
t.Fatalf("loadReceipt() error = %v, want fragment %q", err, test.wantError)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
oversized := filepath.Join(root, "oversized.json")
|
||||||
|
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxReceiptBytes+1)), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := loadReceipt(oversized, "pipeline-1"); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("loadReceipt(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateBundleRootRejectsEscapesAndSymlinks(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
outputRoot := filepath.Join(root, "output")
|
||||||
|
if err := os.Mkdir(outputRoot, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(output root) error = %v", err)
|
||||||
|
}
|
||||||
|
validBundle := filepath.Join(outputRoot, "run-1")
|
||||||
|
if err := os.Mkdir(validBundle, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(bundle) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := validateBundleRoot(outputRoot, validBundle); err != nil {
|
||||||
|
t.Fatalf("validateBundleRoot(valid) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
outside := filepath.Join(root, "output-other")
|
||||||
|
if err := os.Mkdir(outside, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(outside) error = %v", err)
|
||||||
|
}
|
||||||
|
for name, candidate := range map[string]string{"equal root": outputRoot, "escape": root, "prefix confusion": outside} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
if _, err := validateBundleRoot(outputRoot, candidate); err == nil {
|
||||||
|
t.Fatalf("validateBundleRoot(%q) error = nil", candidate)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
symlink := filepath.Join(outputRoot, "linked")
|
||||||
|
if err := os.Symlink(outside, symlink); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := validateBundleRoot(outputRoot, symlink); err == nil {
|
||||||
|
t.Fatal("validateBundleRoot(symlink) error = nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadIndexRejectsMalformedUnsafeAndUnsupportedDocuments(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
indexValue any
|
||||||
|
prepare func(*testing.T, string)
|
||||||
|
wantError string
|
||||||
|
}{
|
||||||
|
{name: "malformed", indexValue: json.RawMessage(`{"manifest_file":`)},
|
||||||
|
{name: "unsupported output shape", indexValue: map[string]any{"manifest_file": "manifest.json", "output_files": map[string]any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
|
||||||
|
{name: "missing management path", indexValue: map[string]any{"output_files": []any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
|
||||||
|
{name: "renamed manifest", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["manifest_file"] = "metadata.json"
|
||||||
|
return value
|
||||||
|
}(), wantError: `manifest_file "metadata.json"`},
|
||||||
|
{name: "cleanable manifest", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["manifest_file"] = "./manifest.json"
|
||||||
|
return value
|
||||||
|
}(), wantError: `manifest_file "./manifest.json"`},
|
||||||
|
{name: "renamed rejections", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["rejected_file"] = "rejections.json"
|
||||||
|
return value
|
||||||
|
}(), wantError: `rejected_file "rejections.json"`},
|
||||||
|
{name: "renamed warnings", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["warnings_file"] = "diagnostics/warnings.json"
|
||||||
|
return value
|
||||||
|
}(), wantError: `warnings_file "diagnostics/warnings.json"`},
|
||||||
|
{name: "duplicate lane", indexValue: validIndexValue([]any{
|
||||||
|
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
|
||||||
|
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
|
||||||
|
})},
|
||||||
|
{name: "absolute logical path", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "/tmp/npc.json"}})},
|
||||||
|
{name: "lexical traversal", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../outside.json"}})},
|
||||||
|
{name: "root prefix confusion", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../bundle-other/npc.json"}})},
|
||||||
|
{name: "file symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "lanes/npc.json"}}), prepare: func(t *testing.T, bundle string) {
|
||||||
|
if err := os.Symlink(filepath.Join(bundle, "manifest.json"), filepath.Join(bundle, "lanes", "npc.json")); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "directory symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "linked/npc.json"}}), prepare: func(t *testing.T, bundle string) {
|
||||||
|
if err := os.Symlink(filepath.Join(bundle, "lanes"), filepath.Join(bundle, "linked")); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "missing management file", indexValue: validIndexValue([]any{}), prepare: func(t *testing.T, bundle string) {
|
||||||
|
if err := os.Remove(filepath.Join(bundle, "manifest.json")); err != nil {
|
||||||
|
t.Fatalf("Remove(manifest) error = %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "incomplete pipeline descriptor", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["chunk_map"] = map[string]any{"artifact_kind": "chunk_map", "file": "chunk-map.json"}
|
||||||
|
return value
|
||||||
|
}()},
|
||||||
|
{name: "pipeline descriptor escape", indexValue: func() any {
|
||||||
|
value := validIndexValue([]any{})
|
||||||
|
value["evidence_context"] = map[string]any{
|
||||||
|
"artifact_kind": "evidence_context", "file": "../evidence.json", "media_type": "application/json",
|
||||||
|
"schema_id": "evidence", "schema_name": "Evidence", "schema_version": "v1",
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}()},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
bundle := createBundleSkeleton(t)
|
||||||
|
indexPath := filepath.Join(bundle, "index.json")
|
||||||
|
if raw, ok := test.indexValue.(json.RawMessage); ok {
|
||||||
|
if err := os.WriteFile(indexPath, raw, 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(index) error = %v", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
writeJSONFile(t, indexPath, test.indexValue)
|
||||||
|
}
|
||||||
|
if test.prepare != nil {
|
||||||
|
test.prepare(t, bundle)
|
||||||
|
}
|
||||||
|
if _, err := loadIndex(bundle, indexPath); err == nil {
|
||||||
|
t.Fatal("loadIndex() error = nil, want failure")
|
||||||
|
} else if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
|
||||||
|
t.Fatalf("loadIndex() error = %v, want fragment %q", err, test.wantError)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
bundle := createBundleSkeleton(t)
|
||||||
|
oversizedIndex := filepath.Join(bundle, "index.json")
|
||||||
|
if err := os.WriteFile(oversizedIndex, []byte(strings.Repeat("x", maxIndexBytes+1)), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(oversized index) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := loadIndex(bundle, oversizedIndex); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("loadIndex(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
rejectedPath := filepath.Join(root, "rejected.json")
|
||||||
|
warningsPath := filepath.Join(root, "warnings.json")
|
||||||
|
writeJSONFile(t, rejectedPath, map[string]any{"rejected": []any{map[string]any{
|
||||||
|
"stage": "validate", "lane_id": "spells", "reason_code": "invalid", "message": "do not retain this", "future": true,
|
||||||
|
}}, "future": true})
|
||||||
|
writeJSONFile(t, warningsPath, map[string]any{"warnings": []any{map[string]any{
|
||||||
|
"scope": "lane:spells", "reason_code": "bounded", "message": "do not retain this", "future": true,
|
||||||
|
}}, "future": true})
|
||||||
|
rejections, err := loadRejections(rejectedPath)
|
||||||
|
if err != nil || len(rejections) != 1 || rejections[0].ReasonCode != "invalid" {
|
||||||
|
t.Fatalf("loadRejections() = %#v, %v", rejections, err)
|
||||||
|
}
|
||||||
|
warnings, err := loadWarnings(warningsPath)
|
||||||
|
if err != nil || len(warnings) != 1 || warnings[0].Scope != "lane:spells" {
|
||||||
|
t.Fatalf("loadWarnings() = %#v, %v", warnings, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, path := range map[string]string{"rejections": rejectedPath, "warnings": warningsPath} {
|
||||||
|
t.Run("malformed "+name, func(t *testing.T) {
|
||||||
|
if err := os.WriteFile(path, []byte("{"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if name == "rejections" {
|
||||||
|
_, err = loadRejections(path)
|
||||||
|
} else {
|
||||||
|
_, err = loadWarnings(path)
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("summary decoder error = nil")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
oversized := filepath.Join(root, "oversized.json")
|
||||||
|
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxSummaryBytes+1)), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := loadWarnings(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("loadWarnings(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := loadRejections(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("loadRejections(oversized) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
|
||||||
|
req := RunRequest{PipelineID: "pipeline"}
|
||||||
|
want := RunResult{BundleRoot: "/bundle"}
|
||||||
|
fake := &FakeRunner{Result: want}
|
||||||
|
got, err := fake.Run(context.Background(), req)
|
||||||
|
if err != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(fake.Requests, []RunRequest{req}) {
|
||||||
|
t.Fatalf("Run() = %#v, %v; requests = %#v", got, err, fake.Requests)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantErr := errors.New("configured failure")
|
||||||
|
fake.Err = wantErr
|
||||||
|
if _, err := fake.Run(context.Background(), req); !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("Run(configured error) = %v", err)
|
||||||
|
}
|
||||||
|
canceled, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
before := len(fake.Requests)
|
||||||
|
if _, err := fake.Run(canceled, req); !errors.Is(err, context.Canceled) || len(fake.Requests) != before {
|
||||||
|
t.Fatalf("Run(canceled) error = %v; requests = %d", err, len(fake.Requests))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validRunRequest(t *testing.T) RunRequest {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
configPath := filepath.Join(root, "notarius.yml")
|
||||||
|
inputPath := filepath.Join(root, "input.json")
|
||||||
|
outputRoot := filepath.Join(root, "outputs")
|
||||||
|
workingDirectory := filepath.Join(root, "work")
|
||||||
|
diagnostics := filepath.Join(root, "diagnostics")
|
||||||
|
for _, directory := range []string{outputRoot, workingDirectory, diagnostics} {
|
||||||
|
if err := os.Mkdir(directory, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(%q) error = %v", directory, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(config) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(inputPath, []byte("{}\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(input) error = %v", err)
|
||||||
|
}
|
||||||
|
return RunRequest{
|
||||||
|
Binary: "notarius", ConfigPath: configPath, PipelineID: "dnd-session", InputPath: inputPath,
|
||||||
|
OutputRoot: outputRoot, WorkingDirectory: workingDirectory,
|
||||||
|
ReceiptPath: filepath.Join(diagnostics, "receipt.json"), LogPath: filepath.Join(diagnostics, "stderr.log"),
|
||||||
|
Timeout: time.Second,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
|
||||||
|
t.Helper()
|
||||||
|
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
|
||||||
|
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(bundle) error = %v", err)
|
||||||
|
}
|
||||||
|
for path, data := range map[string]string{
|
||||||
|
"manifest.json": `{}`,
|
||||||
|
"lanes/npc.json": `{}`,
|
||||||
|
"chunk-map.json": `{}`,
|
||||||
|
"evidence-context.json": `{}`,
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(path)), []byte(data), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rejection := map[string]any{"stage": "validate", "lane_id": "spells", "reason_code": "invalid_spell", "message": strings.Repeat("external detail", 20)}
|
||||||
|
warning := map[string]any{"scope": "lane:npc-registry", "reason_code": "normalized_name", "message": strings.Repeat("external warning", 20)}
|
||||||
|
if includeUnknown {
|
||||||
|
rejection["future"] = true
|
||||||
|
warning["future"] = true
|
||||||
|
}
|
||||||
|
writeJSONFile(t, filepath.Join(bundle, "rejected.json"), map[string]any{"rejected": []any{rejection}, "future": true})
|
||||||
|
writeJSONFile(t, filepath.Join(bundle, "warnings.json"), map[string]any{"warnings": []any{warning}, "future": true})
|
||||||
|
index := validIndexValue([]any{map[string]any{
|
||||||
|
"lane_id": "npc-registry", "file": "lanes/npc.json", "media_type": "application/json",
|
||||||
|
"module_key": "dnd/npc-registry", "schema_id": "notarius.dnd.npc_registry",
|
||||||
|
"schema_name": "NPCRegistry", "schema_version": "v1", "future": true,
|
||||||
|
}})
|
||||||
|
index["chunk_map"] = map[string]any{
|
||||||
|
"artifact_kind": "chunk_map", "file": "chunk-map.json", "media_type": "application/json",
|
||||||
|
"schema_id": "notarius.chunk_map", "schema_name": "ChunkMap", "schema_version": "v1", "future": true,
|
||||||
|
}
|
||||||
|
index["evidence_context"] = map[string]any{
|
||||||
|
"artifact_kind": "evidence_context", "file": "evidence-context.json", "media_type": "application/json",
|
||||||
|
"schema_id": "notarius.evidence_context", "schema_name": "EvidenceContext", "schema_version": "v1", "future": true,
|
||||||
|
}
|
||||||
|
index["future"] = true
|
||||||
|
writeJSONFile(t, filepath.Join(bundle, "index.json"), index)
|
||||||
|
receipt := map[string]any{
|
||||||
|
"schema_version": ReceiptSchemaVersion, "run_id": "notarius-run-1", "pipeline_id": req.PipelineID,
|
||||||
|
"output_directory": bundle, "index_file": "index.json", "normalized_output_count": 1,
|
||||||
|
"rejected_output_count": 1, "warning_count": 1, "validation_status": "rejected",
|
||||||
|
}
|
||||||
|
if includeUnknown {
|
||||||
|
receipt["future"] = true
|
||||||
|
}
|
||||||
|
writeJSONFile(t, req.ReceiptPath, receipt)
|
||||||
|
}
|
||||||
|
|
||||||
|
func createBundleSkeleton(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
bundle := filepath.Join(t.TempDir(), "bundle")
|
||||||
|
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(bundle) error = %v", err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"manifest.json", "rejected.json", "warnings.json", "lanes/npc.json", "chunk-map.json"} {
|
||||||
|
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(name)), []byte("{}"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q) error = %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return bundle
|
||||||
|
}
|
||||||
|
|
||||||
|
func validIndexValue(lanes []any) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"manifest_file": "manifest.json", "output_files": lanes,
|
||||||
|
"rejected_file": "rejected.json", "warnings_file": "warnings.json",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeJSONFile(t *testing.T, path string, value any) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeShellScript(t *testing.T, body string) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(t.TempDir(), "notarius-helper")
|
||||||
|
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||||
|
t.Fatalf("WriteFile(script) error = %v", err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertTextFile(t *testing.T, path, want string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
if string(data) != want {
|
||||||
|
t.Fatalf("ReadFile(%q) = %q, want %q", path, string(data), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneMap(source map[string]any) map[string]any {
|
||||||
|
result := make(map[string]any, len(source))
|
||||||
|
for key, value := range source {
|
||||||
|
result[key] = value
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
@@ -21,7 +21,7 @@ func TestExecuteRunStageArtifactsUnsupportedStageFails(t *testing.T) {
|
|||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := Execute(
|
code := Execute(
|
||||||
[]string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
[]string{"run-stage", "extract", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||||
&stdout,
|
&stdout,
|
||||||
&stderr,
|
&stderr,
|
||||||
)
|
)
|
||||||
@@ -130,6 +130,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
|||||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||||
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
|
seed.MarkStageSkipped("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), "notarius_disabled")
|
||||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||||
t.Fatalf("save manifest: %v", err)
|
t.Fatalf("save manifest: %v", err)
|
||||||
}
|
}
|
||||||
@@ -143,7 +144,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out.String(), "executed=0 skipped=10") {
|
if !strings.Contains(out.String(), "executed=1 skipped=11") {
|
||||||
t.Fatalf("output = %q, want all stages skipped", out.String())
|
t.Fatalf("output = %q, want all stages skipped", out.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,8 +31,8 @@ func TestExecuteValidCommands(t *testing.T) {
|
|||||||
args []string
|
args []string
|
||||||
wantOut string
|
wantOut string
|
||||||
}{
|
}{
|
||||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=10 skipped=0; manifest="},
|
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=11 skipped=1; manifest="},
|
||||||
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nrender: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nextract: run\nrender: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
||||||
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
||||||
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||||
}
|
}
|
||||||
@@ -335,7 +335,7 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
|||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=10 skipped=0; manifest=") {
|
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=11 skipped=1; manifest=") {
|
||||||
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
416
internal/app/extract_lifecycle_test.go
Normal file
416
internal/app/extract_lifecycle_test.go
Normal file
@@ -0,0 +1,416 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||||
|
)
|
||||||
|
|
||||||
|
type materializingNotariusRunner struct {
|
||||||
|
cfg *config.NotariusConfig
|
||||||
|
requests []notarius.RunRequest
|
||||||
|
failuresRemaining int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *materializingNotariusRunner) Run(_ context.Context, req notarius.RunRequest) (notarius.RunResult, error) {
|
||||||
|
r.requests = append(r.requests, req)
|
||||||
|
if r.failuresRemaining > 0 {
|
||||||
|
r.failuresRemaining--
|
||||||
|
return notarius.RunResult{}, errors.New("notarius execution failed")
|
||||||
|
}
|
||||||
|
externalRunID := fmt.Sprintf("notarius-run-%d", len(r.requests))
|
||||||
|
bundle := filepath.Join(req.OutputRoot, externalRunID)
|
||||||
|
lanesDir := filepath.Join(bundle, "lanes")
|
||||||
|
if err := os.MkdirAll(lanesDir, 0o755); err != nil {
|
||||||
|
return notarius.RunResult{}, err
|
||||||
|
}
|
||||||
|
for path, content := range map[string]string{
|
||||||
|
filepath.Join(bundle, "index.json"): `{"manifest_file":"manifest.json"}`,
|
||||||
|
filepath.Join(bundle, "manifest.json"): `{}`,
|
||||||
|
filepath.Join(bundle, "rejected.json"): `{"rejected":[]}`,
|
||||||
|
filepath.Join(bundle, "warnings.json"): `{"warnings":[]}`,
|
||||||
|
filepath.Join(lanesDir, "npcs.json"): `{"npcs":[]}`,
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||||
|
return notarius.RunResult{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
output := r.cfg.Outputs["npc_registry"]
|
||||||
|
return notarius.RunResult{
|
||||||
|
Receipt: notarius.Receipt{
|
||||||
|
SchemaVersion: notarius.ReceiptSchemaVersion, RunID: externalRunID,
|
||||||
|
PipelineID: req.PipelineID, OutputDirectory: bundle, IndexFile: "index.json",
|
||||||
|
NormalizedOutputCount: 1, ValidationStatus: "valid",
|
||||||
|
},
|
||||||
|
BundleRoot: bundle,
|
||||||
|
Index: notarius.Index{
|
||||||
|
Path: filepath.Join(bundle, "index.json"), RejectedPath: filepath.Join(bundle, "rejected.json"),
|
||||||
|
WarningsPath: filepath.Join(bundle, "warnings.json"),
|
||||||
|
Lanes: []notarius.LaneDescriptor{{
|
||||||
|
LaneID: output.LaneID, File: "lanes/npcs.json", Path: filepath.Join(lanesDir, "npcs.json"),
|
||||||
|
MediaType: output.MediaType, SchemaID: output.SchemaID,
|
||||||
|
SchemaVersion: output.SchemaVersion, ModuleKey: output.ModuleKey,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleDisabledThenEnabled(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||||
|
plan, err := BuildSingleStagePlan("extract")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("disabled executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(first.Executed) != 1 || len(first.Skipped) != 1 || first.Skipped[0] != "extract" || len(runner.requests) != 0 {
|
||||||
|
t.Fatalf("disabled summary = %#v requests=%d", first, len(runner.requests))
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Pipeline.Notarius.Enabled = true
|
||||||
|
second, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("enabled executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(second.Executed) != 1 || len(second.Skipped) != 0 || len(runner.requests) != 1 {
|
||||||
|
t.Fatalf("enabled summary = %#v requests=%d", second, len(runner.requests))
|
||||||
|
}
|
||||||
|
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), second.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if loaded.Stages["extract"] == nil || loaded.Stages["extract"].Status != manifest.StatusSucceeded || len(loaded.Stages["extract"].Outputs) != 2 {
|
||||||
|
t.Fatalf("extract record = %#v, want succeeded manifest-ready outputs", loaded.Stages["extract"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleChangedOutcomeRerunsSucceededDownstream(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("disabled executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if analyzeRuns != 1 || len(runner.requests) != 0 {
|
||||||
|
t.Fatalf("disabled run analyze=%d Notarius=%d, want 1 and 0", analyzeRuns, len(runner.requests))
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Pipeline.Notarius.Enabled = true
|
||||||
|
summary, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("enabled executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if analyzeRuns != 2 || len(runner.requests) != 1 {
|
||||||
|
t.Fatalf("enabled run analyze=%d Notarius=%d, want 2 and 1", analyzeRuns, len(runner.requests))
|
||||||
|
}
|
||||||
|
if len(summary.Executed) != 2 || len(summary.Skipped) != 0 {
|
||||||
|
t.Fatalf("enabled summary = %#v, want extract and analyze executed", summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleFailureInvalidatesAndOrdinaryRetryRerunsDownstream(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
runner.failuresRemaining = 1
|
||||||
|
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil || !strings.Contains(err.Error(), "notarius execution failed") {
|
||||||
|
t.Fatalf("failed executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
failed := loadLifecycleManifest(t, cfg)
|
||||||
|
if failed.Stages["extract"].Status != manifest.StatusFailed || failed.Stages["analyze"].Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("failed lifecycle extract=%#v analyze=%#v", failed.Stages["extract"], failed.Stages["analyze"])
|
||||||
|
}
|
||||||
|
if failed.Stages["analyze"].Error == nil || failed.Stages["analyze"].Error.Message != staleReasonFailure {
|
||||||
|
t.Fatalf("analyze stale reason = %#v, want %q", failed.Stages["analyze"].Error, staleReasonFailure)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("retry executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if analyzeRuns != 1 || len(runner.requests) != 2 || len(summary.Executed) != 2 {
|
||||||
|
t.Fatalf("retry analyze=%d Notarius=%d summary=%#v", analyzeRuns, len(runner.requests), summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleForcedSelfSkipInvalidatesDownstream(t *testing.T) {
|
||||||
|
cfg, env, _ := extractionLifecycleFixture(t, false)
|
||||||
|
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||||
|
plan, _ := BuildSingleStagePlan("extract")
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true}); err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
loaded := loadLifecycleManifest(t, cfg)
|
||||||
|
if loaded.Stages["extract"].Status != manifest.StatusSkipped || loaded.Stages["analyze"].Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("forced self-skip extract=%#v analyze=%#v", loaded.Stages["extract"], loaded.Stages["analyze"])
|
||||||
|
}
|
||||||
|
if loaded.Stages["analyze"].Error == nil || loaded.Stages["analyze"].Error.Message != staleReasonForcedReplacement {
|
||||||
|
t.Fatalf("analyze stale reason = %#v, want %q", loaded.Stages["analyze"].Error, staleReasonForcedReplacement)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleForcedFailureInvalidatesDownstream(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
plan, _ := BuildSingleStagePlan("extract")
|
||||||
|
|
||||||
|
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("initial executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
succeeded := loadLifecycleManifest(t, cfg).Stages["extract"]
|
||||||
|
if succeeded == nil || succeeded.Status != manifest.StatusSucceeded || len(succeeded.Outputs) == 0 || len(succeeded.Logs) == 0 || len(succeeded.Metadata) == 0 {
|
||||||
|
t.Fatalf("initial extraction result = %#v, want succeeded result details", succeeded)
|
||||||
|
}
|
||||||
|
|
||||||
|
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||||
|
runner.failuresRemaining = 1
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true}); err == nil {
|
||||||
|
t.Fatal("executeStages() error = nil, want forced extraction failure")
|
||||||
|
}
|
||||||
|
loaded := loadLifecycleManifest(t, cfg)
|
||||||
|
if loaded.Stages["extract"].Status != manifest.StatusFailed || loaded.Stages["analyze"].Status != manifest.StatusStale {
|
||||||
|
t.Fatalf("forced failure extract=%#v analyze=%#v", loaded.Stages["extract"], loaded.Stages["analyze"])
|
||||||
|
}
|
||||||
|
if loaded.Stages["analyze"].Error == nil || loaded.Stages["analyze"].Error.Message != staleReasonForcedReplacement {
|
||||||
|
t.Fatalf("analyze stale reason = %#v, want %q", loaded.Stages["analyze"].Error, staleReasonForcedReplacement)
|
||||||
|
}
|
||||||
|
failed := loaded.Stages["extract"]
|
||||||
|
if len(failed.Outputs) != 0 || len(failed.Logs) != 0 || len(failed.GeneratedConfigs) != 0 || len(failed.Metadata) != 0 {
|
||||||
|
t.Fatalf("failed replacement inherited extraction result details: %#v", failed)
|
||||||
|
}
|
||||||
|
historical, err := (&manifest.LocalStore{}).LoadRun(context.Background(), first.RunManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadRun(initial) error = %v", err)
|
||||||
|
}
|
||||||
|
historicalExtract := historical.Stages["extract"]
|
||||||
|
if historicalExtract == nil || historicalExtract.Status != manifest.StatusSucceeded || len(historicalExtract.Outputs) == 0 || len(historicalExtract.Logs) == 0 || len(historicalExtract.Metadata) == 0 {
|
||||||
|
t.Fatalf("historical extraction result = %#v, want preserved succeeded details", historicalExtract)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(succeeded.Outputs[0].LocalPath); err != nil {
|
||||||
|
t.Fatalf("durable extraction output was not preserved: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleRepeatedSelfSkipPreservesSucceededDownstream(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("first executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
second, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if analyzeRuns != 1 || len(runner.requests) != 0 {
|
||||||
|
t.Fatalf("repeated disabled run analyze=%d Notarius=%d, want 1 and 0", analyzeRuns, len(runner.requests))
|
||||||
|
}
|
||||||
|
if len(second.Executed) != 1 || len(second.Skipped) != 2 {
|
||||||
|
t.Fatalf("second summary = %#v, want executed self-skip and skipped analyze", second)
|
||||||
|
}
|
||||||
|
loaded := loadLifecycleManifest(t, cfg)
|
||||||
|
if loaded.Stages["analyze"].Status != manifest.StatusSucceeded {
|
||||||
|
t.Fatalf("analyze = %#v, want succeeded", loaded.Stages["analyze"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleSkipsCurrentResumableResult(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||||
|
t.Fatalf("first executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
resumed, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resume executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(resumed.Executed) != 0 || len(resumed.Skipped) != 2 || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||||
|
t.Fatalf("resume summary = %#v requests=%d analyze=%d", resumed, len(runner.requests), analyzeRuns)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleResumesAndRerunsObsoleteResults(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*testing.T, *config.Config, *manifest.Manifest)
|
||||||
|
}{
|
||||||
|
{name: "configuration changed", mutate: func(_ *testing.T, cfg *config.Config, _ *manifest.Manifest) {
|
||||||
|
output := cfg.Pipeline.Notarius.Outputs["npc_registry"]
|
||||||
|
output.SchemaVersion = "v2"
|
||||||
|
cfg.Pipeline.Notarius.Outputs["npc_registry"] = output
|
||||||
|
}},
|
||||||
|
{name: "payload missing", mutate: func(t *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||||
|
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
||||||
|
t.Fatalf("Remove() error = %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "payload tampered", mutate: func(t *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||||
|
if err := os.WriteFile(m.Stages["extract"].Outputs[0].LocalPath, []byte(`{"npcs":["tampered"]}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "record incompatible", mutate: func(_ *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs[0].Contract.SchemaID = "incompatible"
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
plan, _ := BuildSingleStagePlan("extract")
|
||||||
|
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
persisted, err := (&manifest.LocalStore{}).Load(context.Background(), first.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
test.mutate(t, cfg, persisted)
|
||||||
|
if err := (&manifest.LocalStore{}).Save(context.Background(), first.ManifestPath, persisted); err != nil {
|
||||||
|
t.Fatalf("Save(mutated) error = %v", err)
|
||||||
|
}
|
||||||
|
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("rerun executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(rerun.Executed) != 1 || len(rerun.Skipped) != 0 || len(runner.requests) != 2 {
|
||||||
|
t.Fatalf("rerun summary = %#v requests=%d", rerun, len(runner.requests))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractLifecycleUnsafeResumeErrorPreservesSuccess(t *testing.T) {
|
||||||
|
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||||
|
analyzeRuns := 0
|
||||||
|
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||||
|
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
persisted, err := store.Load(context.Background(), first.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
persisted.Stages["extract"].Outputs[0].LocalPath = filepath.Join(cfg.Pipeline.Workspace.Root, "outside.json")
|
||||||
|
if err := store.Save(context.Background(), first.ManifestPath, persisted); err != nil {
|
||||||
|
t.Fatalf("Save() error = %v", err)
|
||||||
|
}
|
||||||
|
before, _ := json.Marshal(map[string]*manifest.StageRecord{
|
||||||
|
"extract": persisted.Stages["extract"],
|
||||||
|
"analyze": persisted.Stages["analyze"],
|
||||||
|
})
|
||||||
|
|
||||||
|
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil || !strings.Contains(err.Error(), "unsafe") {
|
||||||
|
t.Fatalf("executeStages() error = %v, want unsafe resume failure", err)
|
||||||
|
}
|
||||||
|
afterManifest, err := store.Load(context.Background(), first.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load(after) error = %v", err)
|
||||||
|
}
|
||||||
|
after, _ := json.Marshal(map[string]*manifest.StageRecord{
|
||||||
|
"extract": afterManifest.Stages["extract"],
|
||||||
|
"analyze": afterManifest.Stages["analyze"],
|
||||||
|
})
|
||||||
|
if string(before) != string(after) || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||||
|
t.Fatalf("successful records changed: before=%s after=%s requests=%d analyze=%d", before, after, len(runner.requests), analyzeRuns)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractionLifecyclePlan(t *testing.T, analyzeRuns *int) []stage.Stage {
|
||||||
|
t.Helper()
|
||||||
|
plan, err := BuildSingleStagePlan("extract")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
|
||||||
|
}
|
||||||
|
return append(plan, countingStage{name: "analyze", runs: analyzeRuns})
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadLifecycleManifest(t *testing.T, cfg *config.Config) *manifest.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
return loaded
|
||||||
|
}
|
||||||
|
|
||||||
|
func markLifecycleStageSucceeded(t *testing.T, cfg *config.Config, name string) {
|
||||||
|
t.Helper()
|
||||||
|
loaded := loadLifecycleManifest(t, cfg)
|
||||||
|
loaded.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||||
|
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPathFor(cfg), loaded); err != nil {
|
||||||
|
t.Fatalf("Save() error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractionLifecycleFixture(t *testing.T, enabled bool) (*config.Config, *stage.Env, *materializingNotariusRunner) {
|
||||||
|
t.Helper()
|
||||||
|
cfg := testConfig(t)
|
||||||
|
root := cfg.Pipeline.Workspace.Root
|
||||||
|
binary := filepath.Join(root, "notarius")
|
||||||
|
configPath := filepath.Join(root, "notarius.yml")
|
||||||
|
workingDirectory := filepath.Join(root, "notarius-work")
|
||||||
|
if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||||
|
t.Fatalf("WriteFile(binary) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(config) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Mkdir(workingDirectory, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(working directory) error = %v", err)
|
||||||
|
}
|
||||||
|
cfg.Pipeline.Notarius = &config.NotariusConfig{
|
||||||
|
Enabled: enabled, Binary: binary, ConfigPath: configPath, PipelineID: "dnd-session",
|
||||||
|
Timeout: "45m", WorkingDirectory: workingDirectory,
|
||||||
|
Outputs: map[string]config.NotariusOutputConfig{
|
||||||
|
"npc_registry": {
|
||||||
|
LaneID: "npc-registry", MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry",
|
||||||
|
SchemaVersion: "v1", ModuleKey: "dnd/npc-registry",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
paths, err := artifacts.NewLocalStore(root).EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("EnsureLayoutFor() error = %v", err)
|
||||||
|
}
|
||||||
|
inputPath := filepath.Join(paths.ArtifactsDir, "trimmed.from-manifest.json")
|
||||||
|
if err := os.WriteFile(inputPath, []byte(`{"segments":[]}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(input) error = %v", err)
|
||||||
|
}
|
||||||
|
m := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
|
m.Campaign = cfg.Session.Campaign
|
||||||
|
m.MarkStageSucceeded("trim", time.Now().UTC(), []manifest.ArtifactRecord{{
|
||||||
|
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
|
||||||
|
LocalPath: inputPath,
|
||||||
|
}})
|
||||||
|
if err := (&manifest.LocalStore{}).Save(context.Background(), paths.ManifestPath, m); err != nil {
|
||||||
|
t.Fatalf("Save(seed) error = %v", err)
|
||||||
|
}
|
||||||
|
runner := &materializingNotariusRunner{cfg: cfg.Pipeline.Notarius}
|
||||||
|
return cfg, &stage.Env{Notarius: runner}, runner
|
||||||
|
}
|
||||||
@@ -10,9 +10,10 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
|
|
||||||
func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog, error) {
|
func buildHelperArtifactCatalog(cfg *config.Config, m *manifest.Manifest) (*artifacts.ArtifactCatalog, error) {
|
||||||
catalog := artifacts.NewArtifactCatalog()
|
catalog := artifacts.NewArtifactCatalog()
|
||||||
if err := catalog.RegisterBuiltIns(); err != nil {
|
if err := catalog.RegisterBuiltIns(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -26,6 +27,14 @@ func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog,
|
|||||||
if err := catalog.RegisterConfiguredArtifacts(configured, nil); err != nil {
|
if err := catalog.RegisterConfiguredArtifacts(configured, nil); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
extractionDefinitions := artifacts.ExtractionDefinitionsFromConfig(cfg.Pipeline.Notarius)
|
||||||
|
if err := catalog.RegisterExtractionArtifacts(extractionDefinitions); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
if cfg.Pipeline.Notarius != nil && cfg.Pipeline.Notarius.Enabled {
|
||||||
|
catalog.HydrateExtractionArtifacts(paths, m, extractionDefinitions)
|
||||||
|
}
|
||||||
return catalog, nil
|
return catalog, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -40,6 +49,14 @@ func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.Art
|
|||||||
for _, entry := range catalog.ListConfigured() {
|
for _, entry := range catalog.ListConfigured() {
|
||||||
writeArtifactLine(out, entry.SourceID, lockSet)
|
writeArtifactLine(out, entry.SourceID, lockSet)
|
||||||
}
|
}
|
||||||
|
fmt.Fprintln(out, "Extraction:")
|
||||||
|
for _, entry := range catalog.ListExtraction() {
|
||||||
|
state := "unavailable"
|
||||||
|
if entry.Available {
|
||||||
|
state = "available"
|
||||||
|
}
|
||||||
|
writeExtractionArtifactLine(out, entry.SourceID, state, entry.Provenance, lockSet)
|
||||||
|
}
|
||||||
fmt.Fprintln(out, "Previous-session:")
|
fmt.Fprintln(out, "Previous-session:")
|
||||||
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
|
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
|
||||||
fmt.Fprintf(out, "- %s required=%t\n", artifactpolicy.PreviousSessionSourceID(req.Name), req.Required)
|
fmt.Fprintf(out, "- %s required=%t\n", artifactpolicy.PreviousSessionSourceID(req.Name), req.Required)
|
||||||
@@ -50,6 +67,17 @@ func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.Art
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func writeExtractionArtifactLine(out io.Writer, source, state, provenance string, lockSet map[string]config.PublishLockRule) {
|
||||||
|
parts := []string{source, "planned", state}
|
||||||
|
if strings.TrimSpace(provenance) != "" {
|
||||||
|
parts = append(parts, "provenance="+strings.TrimSpace(provenance))
|
||||||
|
}
|
||||||
|
if _, ok := lockSet[source]; ok {
|
||||||
|
parts = append(parts, "locked")
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||||
|
}
|
||||||
|
|
||||||
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.PublishLockRule) {
|
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.PublishLockRule) {
|
||||||
parts := []string{source}
|
parts := []string{source}
|
||||||
if _, ok := lockSet[source]; ok {
|
if _, ok := lockSet[source]; ok {
|
||||||
@@ -103,7 +131,12 @@ func remotePublishedOutputAvailability(ctx context.Context, cfg *config.Config,
|
|||||||
|
|
||||||
func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
||||||
source := strings.TrimSpace(rule.Source)
|
source := strings.TrimSpace(rule.Source)
|
||||||
normalized, err := artifactpolicy.ResolvePublishedDestination(source, rule.Dest, helperConfiguredOutputPathMap(catalog))
|
normalized, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(
|
||||||
|
source,
|
||||||
|
rule.Dest,
|
||||||
|
helperConfiguredOutputPathMap(catalog),
|
||||||
|
helperExtractionOutputSet(catalog),
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", false, err
|
return "", false, err
|
||||||
}
|
}
|
||||||
@@ -112,6 +145,19 @@ func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts
|
|||||||
return normalized, showDest, nil
|
return normalized, showDest, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func helperExtractionOutputSet(catalog *artifacts.ArtifactCatalog) map[string]struct{} {
|
||||||
|
out := map[string]struct{}{}
|
||||||
|
if catalog == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for _, entry := range catalog.ListExtraction() {
|
||||||
|
if strings.TrimSpace(entry.ExtractionKey) != "" {
|
||||||
|
out[entry.ExtractionKey] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func helperConfiguredOutputPathMap(catalog *artifacts.ArtifactCatalog) map[string]string {
|
func helperConfiguredOutputPathMap(catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||||
out := map[string]string{}
|
out := map[string]string{}
|
||||||
if catalog == nil {
|
if catalog == nil {
|
||||||
|
|||||||
@@ -22,11 +22,11 @@ func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if strings.TrimSpace(flags.sessionID) == "" {
|
if strings.TrimSpace(flags.sessionID) == "" {
|
||||||
return fmt.Errorf("artifacts list: session_id is required")
|
return fmt.Errorf("artifacts list: session_id is required")
|
||||||
}
|
}
|
||||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, remote)
|
cfg, store, locks, m, err := loadHelperContext(ctx, flags, remote)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("artifacts list: %w", err)
|
return fmt.Errorf("artifacts list: %w", err)
|
||||||
}
|
}
|
||||||
catalog, err := buildHelperArtifactCatalog(cfg)
|
catalog, err := buildHelperArtifactCatalog(cfg, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("artifacts list: %w", err)
|
return fmt.Errorf("artifacts list: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,8 +8,10 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
@@ -508,7 +510,7 @@ func TestExecuteLocksAddListAndRemoveUseRemoteLockStore(t *testing.T) {
|
|||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -772,6 +774,71 @@ func TestExecuteArtifactsListRemoteReportsPublishedAvailability(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteArtifactsListReportsExtractionLifecycleWithoutPayload(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
addExtractionOutputToPipeline(t, pipelinePath)
|
||||||
|
addPublishOutputsToPipeline(t, pipelinePath, `
|
||||||
|
outputs:
|
||||||
|
- source: narratio.extraction.encounters
|
||||||
|
dest: artifacts/encounters.json
|
||||||
|
required: true
|
||||||
|
`)
|
||||||
|
lanePath := writeOperatorExtractionManifest(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
publishedKey := artifacts.S3PublishedOutputKey(
|
||||||
|
artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"),
|
||||||
|
"artifacts/encounters.json",
|
||||||
|
)
|
||||||
|
fake.SeedObject(storage.FakeObject{Key: publishedKey, Data: []byte(`{"secret":"DO_NOT_PRINT"}`)})
|
||||||
|
var storeInitCalls int
|
||||||
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{
|
||||||
|
"session", "artifacts", "2026-05-03",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign-file", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
"--remote",
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
out := stdout.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"Extraction:",
|
||||||
|
"narratio.extraction.encounters planned available provenance=manifest.current_extract_run",
|
||||||
|
"narratio.extraction.encounters dest=artifacts/encounters.json remote=published",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(out, want) {
|
||||||
|
t.Fatalf("stdout = %q, want %q", out, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(out, "DO_NOT_PRINT") {
|
||||||
|
t.Fatalf("operator output exposed extraction payload: %q", out)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Remove(lanePath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
code = Execute([]string{
|
||||||
|
"session", "artifacts", "2026-05-03",
|
||||||
|
"--config", pipelinePath,
|
||||||
|
"--campaign-file", campaignPath,
|
||||||
|
"--session", sessionPath,
|
||||||
|
}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("unavailable exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "narratio.extraction.encounters planned unavailable") {
|
||||||
|
t.Fatalf("stdout = %q, want unavailable extraction state", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteArtifactsListRemoteUsesPublishOutputDestinations(t *testing.T) {
|
func TestExecuteArtifactsListRemoteUsesPublishOutputDestinations(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
@@ -1006,7 +1073,7 @@ func TestExecutePublishLoadsRemoteLocks(t *testing.T) {
|
|||||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||||
|
|
||||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||||
// The publish stage only checks the manifest statuses and source files.
|
// The publish stage only checks the manifest statuses and source files.
|
||||||
_ = stageName
|
_ = stageName
|
||||||
}
|
}
|
||||||
@@ -1041,6 +1108,72 @@ func addPublishOutputsToPipeline(t *testing.T, pipelinePath, publishYAML string)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func addExtractionOutputToPipeline(t *testing.T, pipelinePath string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data = append(data, []byte(`notarius:
|
||||||
|
enabled: true
|
||||||
|
config_path: notarius.yml
|
||||||
|
pipeline_id: campaign.extract
|
||||||
|
outputs:
|
||||||
|
encounters:
|
||||||
|
lane_id: encounters
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: encounters
|
||||||
|
schema_version: "1"
|
||||||
|
module_key: encounters
|
||||||
|
`)...)
|
||||||
|
if err := os.WriteFile(pipelinePath, data, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(filepath.Dir(pipelinePath), "notarius.yml"), []byte("{}\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeOperatorExtractionManifest(t *testing.T, workspaceRoot string) string {
|
||||||
|
t.Helper()
|
||||||
|
paths := artifacts.NewLocalStore(workspaceRoot).SessionPathsFor("sample-campaign", "2026-05-03")
|
||||||
|
bundleRoot := filepath.Join(paths.ArtifactsDir, "notarius", "extract-run-1")
|
||||||
|
lanePath := filepath.Join(bundleRoot, "lanes", "encounters.json")
|
||||||
|
indexPath := filepath.Join(bundleRoot, "index.json")
|
||||||
|
mustWriteTestFile(t, lanePath, `{"secret":"DO_NOT_PRINT"}`)
|
||||||
|
mustWriteTestFile(t, indexPath, `{"lanes":[]}`)
|
||||||
|
laneChecksum, err := artifacts.SHA256File(lanePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
indexChecksum, err := artifacts.SHA256File(indexPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := manifest.New("2026-05-03", time.Now().UTC())
|
||||||
|
m.Campaign = "sample-campaign"
|
||||||
|
m.Stages["extract"] = &manifest.StageRecord{
|
||||||
|
Name: "extract", Status: manifest.StatusSucceeded,
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"narratio_run_id": "extract-run-1", "bundle_root": bundleRoot,
|
||||||
|
"receipt": map[string]any{"run_id": "notarius-run-1", "pipeline_id": "campaign.extract"},
|
||||||
|
},
|
||||||
|
Outputs: []manifest.ArtifactRecord{
|
||||||
|
{
|
||||||
|
Kind: "notarius_lane", SourceID: artifacts.ExtractionArtifactSourceID("encounters"), LocalPath: lanePath,
|
||||||
|
ProducerRunID: "extract-run-1", Checksum: laneChecksum,
|
||||||
|
Contract: &artifactmodel.ContractMetadata{MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters"},
|
||||||
|
ExternalProvenance: &artifactmodel.ExternalProvenance{System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: "encounters"},
|
||||||
|
},
|
||||||
|
{Kind: "notarius_index", LocalPath: indexPath, ProducerRunID: "extract-run-1", Checksum: indexChecksum},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := (&manifest.LocalStore{}).Save(context.Background(), paths.ManifestPath, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return lanePath
|
||||||
|
}
|
||||||
|
|
||||||
func replaceInFileOrFatal(t *testing.T, path, old, new string) {
|
func replaceInFileOrFatal(t *testing.T, path, old, new string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
@@ -1076,7 +1209,7 @@ func writeValidPublishRunConfigFiles(t *testing.T, workspaceRoot string) (string
|
|||||||
m := manifest.New("2026-05-03", nowUTC())
|
m := manifest.New("2026-05-03", nowUTC())
|
||||||
m.Campaign = "sample-campaign"
|
m.Campaign = "sample-campaign"
|
||||||
m.RunID = "20260521T160000Z-test"
|
m.RunID = "20260521T160000Z-test"
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||||
m.MarkStageSucceeded(name, nowUTC(), nil)
|
m.MarkStageSucceeded(name, nowUTC(), nil)
|
||||||
}
|
}
|
||||||
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("locks add: %w", err)
|
return fmt.Errorf("locks add: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
|
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks add"); err != nil {
|
||||||
return fmt.Errorf("locks add: %w", err)
|
return fmt.Errorf("locks add: %w", err)
|
||||||
}
|
}
|
||||||
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
||||||
@@ -79,7 +79,7 @@ func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
}
|
}
|
||||||
remoteSet[source] = config.PublishLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
remoteSet[source] = config.PublishLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
||||||
remoteLocks := lockMapValues(remoteSet)
|
remoteLocks := lockMapValues(remoteSet)
|
||||||
if _, err := config.ValidatePublishLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
|
if _, err := config.ValidatePublishLockRules(remoteLocks, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks"); err != nil {
|
||||||
return fmt.Errorf("locks add: %w", err)
|
return fmt.Errorf("locks add: %w", err)
|
||||||
}
|
}
|
||||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
||||||
@@ -106,7 +106,7 @@ func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("locks remove: %w", err)
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
|
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks remove"); err != nil {
|
||||||
return fmt.Errorf("locks remove: %w", err)
|
return fmt.Errorf("locks remove: %w", err)
|
||||||
}
|
}
|
||||||
remoteSet := lockSourceSet(locks.Remote)
|
remoteSet := lockSourceSet(locks.Remote)
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Status reports effective local/remote session state.
|
// Status reports effective local/remote session state.
|
||||||
@@ -40,11 +41,13 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
writeStatusStableInputs(out, inspectStableInputs(cfg))
|
writeStatusStableInputs(out, inspectStableInputs(cfg))
|
||||||
writeStatusLocalAudio(out, inspectLocalAudioPresence(cfg))
|
writeStatusLocalAudio(out, inspectLocalAudioPresence(cfg))
|
||||||
|
|
||||||
|
var localManifest *manifest.Manifest
|
||||||
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
|
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
|
||||||
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
|
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
|
||||||
} else if m == nil {
|
} else if m == nil {
|
||||||
fmt.Fprintln(out, "Local manifest: missing")
|
fmt.Fprintln(out, "Local manifest: missing")
|
||||||
} else {
|
} else {
|
||||||
|
localManifest = m
|
||||||
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
|
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
|
||||||
writeStageStatuses(out, m)
|
writeStageStatuses(out, m)
|
||||||
}
|
}
|
||||||
@@ -72,7 +75,7 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
lockChecks := inspectEffectiveLocks(ctx, cfg, store)
|
lockChecks := inspectEffectiveLocks(ctx, cfg, store)
|
||||||
locks := lockChecks.Locks
|
locks := lockChecks.Locks
|
||||||
lockErr := lockChecks.Err
|
lockErr := lockChecks.Err
|
||||||
if catalog, catalogErr := buildHelperArtifactCatalog(cfg); catalogErr != nil {
|
if catalog, catalogErr := buildHelperArtifactCatalog(cfg, localManifest); catalogErr != nil {
|
||||||
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
|
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
|
||||||
} else if storeErr == nil {
|
} else if storeErr == nil {
|
||||||
catalogLocks := locks
|
catalogLocks := locks
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
var flags commonConfigFlags
|
var flags commonConfigFlags
|
||||||
var force bool
|
var force bool
|
||||||
addCommonConfigFlags(fs, &flags)
|
addCommonConfigFlags(fs, &flags)
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
fs.BoolVar(&force, "force", false, "show all stages as scheduled to rerun")
|
||||||
|
|
||||||
if err := parseSessionAwareFlags("plan", fs, args, &flags.sessionID); err != nil {
|
if err := parseSessionAwareFlags("plan", fs, args, &flags.sessionID); err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -27,12 +27,12 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
|||||||
if !strings.Contains(got, "narratio session plan: workdir prepared at") {
|
if !strings.Contains(got, "narratio session plan: workdir prepared at") {
|
||||||
t.Fatalf("first output = %q, want workdir prepared", got)
|
t.Fatalf("first output = %q, want workdir prepared", got)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||||
if !strings.Contains(got, name+": run") {
|
if !strings.Contains(got, name+": run") {
|
||||||
t.Fatalf("first output = %q, missing stage %q", got, name)
|
t.Fatalf("first output = %q, missing stage %q", got, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !strings.Contains(got, "totals: run=10 skip=0") {
|
if !strings.Contains(got, "totals: run=11 skip=0") {
|
||||||
t.Fatalf("first output = %q, want totals", got)
|
t.Fatalf("first output = %q, want totals", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,8 +84,8 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
|||||||
if !strings.Contains(got, "trim: run") {
|
if !strings.Contains(got, "trim: run") {
|
||||||
t.Fatalf("output = %q, want trim run", got)
|
t.Fatalf("output = %q, want trim run", got)
|
||||||
}
|
}
|
||||||
if !strings.Contains(got, "totals: run=8 skip=2") {
|
if !strings.Contains(got, "totals: run=9 skip=2") {
|
||||||
t.Fatalf("output = %q, want totals run=8 skip=2", got)
|
t.Fatalf("output = %q, want totals run=9 skip=2", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import "testing"
|
|||||||
|
|
||||||
func TestBuildFullPlanOrder(t *testing.T) {
|
func TestBuildFullPlanOrder(t *testing.T) {
|
||||||
got := BuildFullPlan()
|
got := BuildFullPlan()
|
||||||
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"}
|
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"}
|
||||||
if len(got) != len(want) {
|
if len(got) != len(want) {
|
||||||
t.Fatalf("len(plan) = %d, want %d", len(got), len(want))
|
t.Fatalf("len(plan) = %d, want %d", len(got), len(want))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -355,7 +355,7 @@ func publishStageCleanupFixture(t *testing.T) (*config.Config, cleanupSeed, stri
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Load() error = %v", err)
|
t.Fatalf("Load() error = %v", err)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||||
seedManifest.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
seedManifest.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||||
}
|
}
|
||||||
seedManifest.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
|
seedManifest.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
|
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
|
||||||
}
|
}
|
||||||
lockStore, err := config.LoadPublishLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium)
|
lockStore, err := config.LoadPublishLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, key, err
|
return nil, key, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,8 +9,10 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
@@ -62,6 +64,55 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteRestoreRoundTripsPublishedExtractionAndManifestMetadata(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
fake := &storage.FakeBackend{}
|
||||||
|
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||||
|
seedRestoreObject(fake, sessionPrefix+"artifacts/encounters.json", []byte(`{"encounters":[]}`))
|
||||||
|
seedRestoreObject(fake, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
|
||||||
|
|
||||||
|
remoteManifest := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
|
remoteManifest.Campaign = cfg.Session.Campaign
|
||||||
|
remoteManifest.RunID = "20260519T010203Z-a1b2c3d4"
|
||||||
|
remoteManifest.Stages["extract"] = &manifest.StageRecord{
|
||||||
|
Name: "extract", Status: manifest.StatusSucceeded,
|
||||||
|
Outputs: []manifest.ArtifactRecord{{
|
||||||
|
Kind: "notarius_lane", SourceID: artifacts.ExtractionArtifactSourceID("encounters"),
|
||||||
|
LocalPath: "/prior/workspace/artifacts/notarius/extract-run-1/lanes/encounters.json",
|
||||||
|
Contract: &artifactmodel.ContractMetadata{
|
||||||
|
MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1",
|
||||||
|
},
|
||||||
|
ExternalProvenance: &artifactmodel.ExternalProvenance{
|
||||||
|
System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: "encounters",
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
manifestBody, err := json.Marshal(remoteManifest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
seedRestoreObject(fake, manifestKey, manifestBody)
|
||||||
|
restoreWithStoreAndRealPhases(t, fake)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Execute([]string{"session", "restore", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
mustReadEquals(t, filepath.Join(sessionRoot, "artifacts", "encounters.json"), `{"encounters":[]}`)
|
||||||
|
restored, err := (&manifest.LocalStore{}).Load(context.Background(), filepath.Join(sessionRoot, "manifest.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load restored manifest: %v", err)
|
||||||
|
}
|
||||||
|
lane := restored.Stages["extract"].Outputs[0]
|
||||||
|
if lane.Contract == nil || lane.Contract.SchemaID != "encounters" || lane.ExternalProvenance == nil || lane.ExternalProvenance.RunID != "notarius-run-1" {
|
||||||
|
t.Fatalf("restored extraction metadata = %#v", lane)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
var force bool
|
var force bool
|
||||||
var selectedArtifacts artifactSelectionFlag
|
var selectedArtifacts artifactSelectionFlag
|
||||||
addCommonConfigFlags(fs, &flags)
|
addCommonConfigFlags(fs, &flags)
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
fs.BoolVar(&force, "force", false, "rerun stages even when already succeeded")
|
||||||
fs.Var(&selectedArtifacts, "artifacts", "configured artifact names to execute and publish (comma-separated or repeatable)")
|
fs.Var(&selectedArtifacts, "artifacts", "configured artifact names to execute and publish (comma-separated or repeatable)")
|
||||||
|
|
||||||
if err := parseSessionAwareFlags("run", fs, args, &flags.sessionID); err != nil {
|
if err := parseSessionAwareFlags("run", fs, args, &flags.sessionID); err != nil {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package app
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
@@ -23,22 +24,40 @@ type stageDecision struct {
|
|||||||
Action stageAction
|
Action stageAction
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
staleReasonForcedReplacement = "upstream stage was force-run"
|
||||||
|
staleReasonChangedResult = "upstream stage result changed"
|
||||||
|
staleReasonFailure = "upstream stage failed"
|
||||||
|
staleReasonSelfSkip = "upstream stage self-skipped"
|
||||||
|
staleReasonNotResumable = "upstream stage result was not resumable"
|
||||||
|
)
|
||||||
|
|
||||||
|
type priorStageOutcome struct {
|
||||||
|
exists bool
|
||||||
|
status manifest.StageStatus
|
||||||
|
skipReason string
|
||||||
|
outputs int
|
||||||
|
}
|
||||||
|
|
||||||
func decideStageActions(stages []stage.Stage, m *manifest.Manifest, force bool) []stageDecision {
|
func decideStageActions(stages []stage.Stage, m *manifest.Manifest, force bool) []stageDecision {
|
||||||
out := make([]stageDecision, 0, len(stages))
|
out := make([]stageDecision, 0, len(stages))
|
||||||
for _, s := range stages {
|
for _, s := range stages {
|
||||||
action := stageActionRun
|
|
||||||
// TODO: incorporate stale detection once checksum/input change tracking is implemented.
|
|
||||||
if !force && stageSucceeded(m, s.Name()) {
|
|
||||||
action = stageActionSkip
|
|
||||||
}
|
|
||||||
out = append(out, stageDecision{
|
out = append(out, stageDecision{
|
||||||
Stage: s,
|
Stage: s,
|
||||||
Action: action,
|
Action: decideStageAction(s, m, force),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func decideStageAction(s stage.Stage, m *manifest.Manifest, force bool) stageAction {
|
||||||
|
// TODO: incorporate stale detection once checksum/input change tracking is implemented.
|
||||||
|
if !force && stageSucceeded(m, s.Name()) {
|
||||||
|
return stageActionSkip
|
||||||
|
}
|
||||||
|
return stageActionRun
|
||||||
|
}
|
||||||
|
|
||||||
func stageSucceeded(m *manifest.Manifest, name string) bool {
|
func stageSucceeded(m *manifest.Manifest, name string) bool {
|
||||||
if m == nil || m.Stages == nil {
|
if m == nil || m.Stages == nil {
|
||||||
return false
|
return false
|
||||||
@@ -47,6 +66,29 @@ func stageSucceeded(m *manifest.Manifest, name string) bool {
|
|||||||
return sr != nil && sr.Status == manifest.StatusSucceeded
|
return sr != nil && sr.Status == manifest.StatusSucceeded
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func capturePriorStageOutcome(m *manifest.Manifest, name string) priorStageOutcome {
|
||||||
|
if m == nil || m.Stages == nil || m.Stages[name] == nil {
|
||||||
|
return priorStageOutcome{}
|
||||||
|
}
|
||||||
|
record := m.Stages[name]
|
||||||
|
outcome := priorStageOutcome{
|
||||||
|
exists: true,
|
||||||
|
status: record.Status,
|
||||||
|
outputs: len(record.Outputs),
|
||||||
|
}
|
||||||
|
if record.Error != nil && record.Error.Code == "skipped" {
|
||||||
|
outcome.skipReason = record.Error.Message
|
||||||
|
}
|
||||||
|
return outcome
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o priorStageOutcome) isSameSelfSkip(reason string) bool {
|
||||||
|
return o.exists &&
|
||||||
|
o.status == manifest.StatusSkipped &&
|
||||||
|
o.outputs == 0 &&
|
||||||
|
o.skipReason == strings.TrimSpace(reason)
|
||||||
|
}
|
||||||
|
|
||||||
func loadManifestIfPresent(ctx context.Context, cfg *config.Config) (*manifest.Manifest, error) {
|
func loadManifestIfPresent(ctx context.Context, cfg *config.Config) (*manifest.Manifest, error) {
|
||||||
path := artifacts.SessionManifestPathForCampaign(
|
path := artifacts.SessionManifestPathForCampaign(
|
||||||
cfg.Pipeline.Workspace.Root,
|
cfg.Pipeline.Workspace.Root,
|
||||||
@@ -91,7 +133,7 @@ func downstreamStageNames(stageName string) []string {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func invalidateDownstreamSucceededStages(m *manifest.Manifest, upstreamStage string, at time.Time) []string {
|
func invalidateDownstreamSucceededStagesWithReason(m *manifest.Manifest, upstreamStage string, at time.Time, reason string) []string {
|
||||||
if m == nil || m.Stages == nil {
|
if m == nil || m.Stages == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -102,7 +144,7 @@ func invalidateDownstreamSucceededStages(m *manifest.Manifest, upstreamStage str
|
|||||||
if sr == nil || sr.Status != manifest.StatusSucceeded {
|
if sr == nil || sr.Status != manifest.StatusSucceeded {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
m.MarkStageStale(downstream, at, "upstream stage rerun with force")
|
m.MarkStageStale(downstream, at, reason)
|
||||||
invalidated = append(invalidated, downstream)
|
invalidated = append(invalidated, downstream)
|
||||||
}
|
}
|
||||||
return invalidated
|
return invalidated
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ func TestDecideStageActions(t *testing.T) {
|
|||||||
|
|
||||||
func TestDownstreamStageNames(t *testing.T) {
|
func TestDownstreamStageNames(t *testing.T) {
|
||||||
got := downstreamStageNames("polish")
|
got := downstreamStageNames("polish")
|
||||||
want := []string{"normalize", "trim", "render", "analyze", "publish", "notify"}
|
want := []string{"normalize", "trim", "extract", "render", "analyze", "publish", "notify"}
|
||||||
if !reflect.DeepEqual(got, want) {
|
if !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("downstreamStageNames(polish) = %#v, want %#v", got, want)
|
t.Fatalf("downstreamStageNames(polish) = %#v, want %#v", got, want)
|
||||||
}
|
}
|
||||||
@@ -43,7 +43,7 @@ func TestDownstreamStageNames(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInvalidateDownstreamSucceededStages(t *testing.T) {
|
func TestInvalidateDownstreamSucceededStagesWithReason(t *testing.T) {
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
m := manifest.New("2026-05-03", now)
|
m := manifest.New("2026-05-03", now)
|
||||||
m.MarkStageSucceeded("prepare", now, nil)
|
m.MarkStageSucceeded("prepare", now, nil)
|
||||||
@@ -52,15 +52,16 @@ func TestInvalidateDownstreamSucceededStages(t *testing.T) {
|
|||||||
m.MarkStageSucceeded("polish", now, nil)
|
m.MarkStageSucceeded("polish", now, nil)
|
||||||
m.MarkStageSucceeded("normalize", now, nil)
|
m.MarkStageSucceeded("normalize", now, nil)
|
||||||
m.MarkStageSucceeded("trim", now, nil)
|
m.MarkStageSucceeded("trim", now, nil)
|
||||||
|
m.MarkStageSucceeded("extract", now, nil)
|
||||||
m.MarkStageSucceeded("render", now, nil)
|
m.MarkStageSucceeded("render", now, nil)
|
||||||
m.MarkStageFailed("analyze", now, "analysis failed")
|
m.MarkStageFailed("analyze", now, "analysis failed")
|
||||||
m.MarkStageSucceeded("publish", now, nil)
|
m.MarkStageSucceeded("publish", now, nil)
|
||||||
m.MarkStageSucceeded("notify", now, nil)
|
m.MarkStageSucceeded("notify", now, nil)
|
||||||
|
|
||||||
got := invalidateDownstreamSucceededStages(m, "polish", now.Add(1*time.Second))
|
got := invalidateDownstreamSucceededStagesWithReason(m, "polish", now.Add(1*time.Second), staleReasonChangedResult)
|
||||||
want := []string{"normalize", "trim", "render", "publish", "notify"}
|
want := []string{"normalize", "trim", "extract", "render", "publish", "notify"}
|
||||||
if !reflect.DeepEqual(got, want) {
|
if !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("invalidateDownstreamSucceededStages() = %#v, want %#v", got, want)
|
t.Fatalf("invalidateDownstreamSucceededStagesWithReason() = %#v, want %#v", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, stageName := range want {
|
for _, stageName := range want {
|
||||||
@@ -75,3 +76,30 @@ func TestInvalidateDownstreamSucceededStages(t *testing.T) {
|
|||||||
t.Fatalf("prepare status = %q, want succeeded", m.Stages["prepare"].Status)
|
t.Fatalf("prepare status = %q, want succeeded", m.Stages["prepare"].Status)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtractionPositionControlsForceInvalidation(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
tests := []struct {
|
||||||
|
upstream string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{upstream: "trim", want: []string{"extract", "render", "analyze", "publish", "notify"}},
|
||||||
|
{upstream: "extract", want: []string{"render", "analyze", "publish", "notify"}},
|
||||||
|
{upstream: "render", want: []string{"analyze", "publish", "notify"}},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.upstream, func(t *testing.T) {
|
||||||
|
m := manifest.New("2026-05-03", now)
|
||||||
|
for _, name := range canonicalStageNames() {
|
||||||
|
m.MarkStageSucceeded(name, now, nil)
|
||||||
|
}
|
||||||
|
got := invalidateDownstreamSucceededStagesWithReason(m, test.upstream, now.Add(time.Second), staleReasonForcedReplacement)
|
||||||
|
if !reflect.DeepEqual(got, test.want) {
|
||||||
|
t.Fatalf("invalidated = %#v, want %#v", got, test.want)
|
||||||
|
}
|
||||||
|
if test.upstream == "render" && m.Stages["extract"].Status != manifest.StatusSucceeded {
|
||||||
|
t.Fatalf("forcing render changed extract: %#v", m.Stages["extract"])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
|||||||
var force bool
|
var force bool
|
||||||
var selectedArtifacts artifactSelectionFlag
|
var selectedArtifacts artifactSelectionFlag
|
||||||
addCommonConfigFlags(fs, &flags)
|
addCommonConfigFlags(fs, &flags)
|
||||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
fs.BoolVar(&force, "force", false, "rerun the stage even when already succeeded")
|
||||||
fs.Var(&selectedArtifacts, "artifacts", "configured artifact names to execute or publish (comma-separated or repeatable)")
|
fs.Var(&selectedArtifacts, "artifacts", "configured artifact names to execute or publish (comma-separated or repeatable)")
|
||||||
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
|
|||||||
@@ -36,8 +36,8 @@ func TestRunContinuesAfterCompletedStages(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out.String(), "executed=8 skipped=2") {
|
if !strings.Contains(out.String(), "executed=9 skipped=3") {
|
||||||
t.Fatalf("output = %q, want executed=8 skipped=2", out.String())
|
t.Fatalf("output = %q, want executed=9 skipped=3", out.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
loaded, err := store.Load(context.Background(), manifestPath)
|
loaded, err := store.Load(context.Background(), manifestPath)
|
||||||
@@ -59,6 +59,7 @@ func TestRunNoRemainingStagesRecordsSkippedStages(t *testing.T) {
|
|||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||||
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
|
m.MarkStageSkipped("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), "notarius_disabled")
|
||||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||||
t.Fatalf("save manifest: %v", err)
|
t.Fatalf("save manifest: %v", err)
|
||||||
}
|
}
|
||||||
@@ -68,8 +69,15 @@ func TestRunNoRemainingStagesRecordsSkippedStages(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out.String(), "executed=0 skipped=10") {
|
if !strings.Contains(out.String(), "executed=1 skipped=11") {
|
||||||
t.Fatalf("output = %q, want executed=0 skipped=10", out.String())
|
t.Fatalf("output = %q, want disabled extraction to self-skip", out.String())
|
||||||
|
}
|
||||||
|
loaded, err := store.Load(context.Background(), manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load migrated manifest: %v", err)
|
||||||
|
}
|
||||||
|
if loaded.Stages["extract"] == nil || loaded.Stages["extract"].Status != manifest.StatusSkipped {
|
||||||
|
t.Fatalf("extract record = %#v, want stable disabled skip", loaded.Stages["extract"])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,7 +93,7 @@ func TestRunForceRerunsSucceeded(t *testing.T) {
|
|||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||||
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||||
@@ -97,7 +105,7 @@ func TestRunForceRerunsSucceeded(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out.String(), "executed=10 skipped=0") {
|
if !strings.Contains(out.String(), "executed=11 skipped=1") {
|
||||||
t.Fatalf("output = %q, want forced full rerun", out.String())
|
t.Fatalf("output = %q, want forced full rerun", out.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -132,6 +140,27 @@ func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunStageExtractIsAcceptedAndSelfSkipsWhenDisabled(t *testing.T) {
|
||||||
|
workspaceRoot := t.TempDir()
|
||||||
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := RunStage(context.Background(), []string{"extract", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("RunStage(extract) error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "stage=extract executed=1 skipped=1 force=false") {
|
||||||
|
t.Fatalf("output = %q, want disabled extraction self-skip", out.String())
|
||||||
|
}
|
||||||
|
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if loaded.Stages["extract"] == nil || loaded.Stages["extract"].Status != manifest.StatusSkipped {
|
||||||
|
t.Fatalf("extract record = %#v, want skipped", loaded.Stages["extract"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunStageSkipAndForce(t *testing.T) {
|
func TestRunStageSkipAndForce(t *testing.T) {
|
||||||
workspaceRoot := t.TempDir()
|
workspaceRoot := t.TempDir()
|
||||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||||
@@ -176,7 +205,7 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
|
|||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||||
seed.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
seed.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||||
@@ -196,7 +225,7 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("load manifest after force: %v", err)
|
t.Fatalf("load manifest after force: %v", err)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"normalize", "trim", "render", "analyze", "publish", "notify"} {
|
for _, name := range []string{"normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||||
if afterForce.Stages[name] == nil || afterForce.Stages[name].Status != manifest.StatusStale {
|
if afterForce.Stages[name] == nil || afterForce.Stages[name].Status != manifest.StatusStale {
|
||||||
t.Fatalf("stage %q = %#v, want stale", name, afterForce.Stages[name])
|
t.Fatalf("stage %q = %#v, want stale", name, afterForce.Stages[name])
|
||||||
}
|
}
|
||||||
@@ -207,7 +236,7 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Run() error = %v", err)
|
t.Fatalf("Run() error = %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(out.String(), "executed=6 skipped=4") {
|
if !strings.Contains(out.String(), "executed=7 skipped=5") {
|
||||||
t.Fatalf("output = %q, want run to execute stale downstream stages", out.String())
|
t.Fatalf("output = %q, want run to execute stale downstream stages", out.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/audita"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/audita"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notify"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notify"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
|
||||||
@@ -80,6 +81,9 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
}
|
}
|
||||||
env.Audita = runner
|
env.Audita = runner
|
||||||
}
|
}
|
||||||
|
if env.Notarius == nil && needsNotariusForRun(env.Config, stages) {
|
||||||
|
env.Notarius = notarius.NewSubprocessRunner()
|
||||||
|
}
|
||||||
if env.Scriptorium == nil {
|
if env.Scriptorium == nil {
|
||||||
env.Scriptorium = scriptorium.NewSubprocessRunner()
|
env.Scriptorium = scriptorium.NewSubprocessRunner()
|
||||||
}
|
}
|
||||||
@@ -167,6 +171,29 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
for _, d := range decisions {
|
for _, d := range decisions {
|
||||||
s := d.Stage
|
s := d.Stage
|
||||||
runNames = append(runNames, s.Name())
|
runNames = append(runNames, s.Name())
|
||||||
|
d.Action = decideStageAction(s, m, opts.Force)
|
||||||
|
|
||||||
|
if d.Action == stageActionSkip {
|
||||||
|
if validator, ok := s.(stage.ResumeValidator); ok {
|
||||||
|
validation, err := validator.ValidateResume(ctx, stageEnv, m)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("validate resume for stage %q: %w", s.Name(), err)
|
||||||
|
}
|
||||||
|
validation = validation.Normalized()
|
||||||
|
if !validation.Resumable {
|
||||||
|
staleAt := nowUTC()
|
||||||
|
m.MarkStageStale(s.Name(), staleAt, validation.Reason)
|
||||||
|
invalidateDownstreamSucceededStagesWithReason(
|
||||||
|
m, s.Name(), staleAt, staleReasonNotResumable,
|
||||||
|
)
|
||||||
|
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||||
|
return nil, fmt.Errorf("save manifest after resume validation for stage %q: %w", s.Name(), err)
|
||||||
|
}
|
||||||
|
env.Logger.Info("stage result is not resumable", "stage", s.Name(), "reason", validation.Reason)
|
||||||
|
d.Action = stageActionRun
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if d.Action == stageActionSkip {
|
if d.Action == stageActionSkip {
|
||||||
skipped = append(skipped, s.Name())
|
skipped = append(skipped, s.Name())
|
||||||
@@ -180,6 +207,7 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
executed = append(executed, s.Name())
|
executed = append(executed, s.Name())
|
||||||
|
priorOutcome := capturePriorStageOutcome(m, s.Name())
|
||||||
|
|
||||||
now := nowUTC()
|
now := nowUTC()
|
||||||
runManifest.SetStageAction(s.Name(), manifest.RunStageActionRun, now)
|
runManifest.SetStageAction(s.Name(), manifest.RunStageActionRun, now)
|
||||||
@@ -188,6 +216,9 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
return nil, fmt.Errorf("save run manifest before stage %q: %w", s.Name(), err)
|
return nil, fmt.Errorf("save run manifest before stage %q: %w", s.Name(), err)
|
||||||
}
|
}
|
||||||
m.MarkStageRunning(s.Name(), now)
|
m.MarkStageRunning(s.Name(), now)
|
||||||
|
if opts.Force {
|
||||||
|
invalidateDownstreamSucceededStagesWithReason(m, s.Name(), now, staleReasonForcedReplacement)
|
||||||
|
}
|
||||||
env.Logger.Info("starting stage", "stage", s.Name())
|
env.Logger.Info("starting stage", "stage", s.Name())
|
||||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||||
return nil, fmt.Errorf("save manifest before stage %q: %w", s.Name(), err)
|
return nil, fmt.Errorf("save manifest before stage %q: %w", s.Name(), err)
|
||||||
@@ -195,9 +226,13 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "running", "path", manifestPath)
|
env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "running", "path", manifestPath)
|
||||||
|
|
||||||
result, err := s.Run(ctx, stageEnv, m)
|
result, err := s.Run(ctx, stageEnv, m)
|
||||||
|
if err == nil {
|
||||||
|
err = validateStageResult(result)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
failedAt := nowUTC()
|
failedAt := nowUTC()
|
||||||
m.MarkStageFailed(s.Name(), failedAt, err.Error())
|
m.MarkStageFailed(s.Name(), failedAt, err.Error())
|
||||||
|
invalidateDownstreamSucceededStagesWithReason(m, s.Name(), failedAt, staleReasonFailure)
|
||||||
if saveErr := env.ManifestStore.Save(ctx, manifestPath, m); saveErr != nil {
|
if saveErr := env.ManifestStore.Save(ctx, manifestPath, m); saveErr != nil {
|
||||||
return nil, fmt.Errorf("stage %q failed (%v) and manifest save failed (%v)", s.Name(), err, saveErr)
|
return nil, fmt.Errorf("stage %q failed (%v) and manifest save failed (%v)", s.Name(), err, saveErr)
|
||||||
}
|
}
|
||||||
@@ -209,13 +244,34 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
|||||||
env.Logger.Info("stage failed", "stage", s.Name(), "error", err)
|
env.Logger.Info("stage failed", "stage", s.Name(), "error", err)
|
||||||
return nil, fmt.Errorf("stage %q failed: %w", s.Name(), err)
|
return nil, fmt.Errorf("stage %q failed: %w", s.Name(), err)
|
||||||
}
|
}
|
||||||
|
if result != nil && result.Disposition == stage.StageDispositionSkipped {
|
||||||
|
skipped = append(skipped, s.Name())
|
||||||
|
skippedAt := nowUTC()
|
||||||
|
m.MarkStageSkipped(s.Name(), skippedAt, result.SkipReason)
|
||||||
|
applyStageResultToManifest(m, s.Name(), result)
|
||||||
|
if !priorOutcome.isSameSelfSkip(result.SkipReason) {
|
||||||
|
invalidateDownstreamSucceededStagesWithReason(m, s.Name(), skippedAt, staleReasonSelfSkip)
|
||||||
|
}
|
||||||
|
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||||
|
return nil, fmt.Errorf("save manifest after self-skip %q: %w", s.Name(), err)
|
||||||
|
}
|
||||||
|
runManifest.MarkStageSkipped(s.Name(), skippedAt, result.SkipReason)
|
||||||
|
applyStageResultToRunManifest(runManifest, s.Name(), result)
|
||||||
|
syncRunManifestIdentityFromSession(m, runManifest)
|
||||||
|
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
|
||||||
|
return nil, fmt.Errorf("save run manifest after self-skip %q: %w", s.Name(), err)
|
||||||
|
}
|
||||||
|
env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "skipped", "path", manifestPath)
|
||||||
|
env.Logger.Info("stage skipped", "stage", s.Name(), "reason", result.SkipReason)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
outputs := mapResultOutputs(s.Name(), result, runID)
|
outputs := mapResultOutputs(s.Name(), result, runID)
|
||||||
succeededAt := nowUTC()
|
succeededAt := nowUTC()
|
||||||
m.MarkStageSucceeded(s.Name(), succeededAt, outputs)
|
m.MarkStageSucceeded(s.Name(), succeededAt, outputs)
|
||||||
applyStageResultToManifest(m, s.Name(), result)
|
applyStageResultToManifest(m, s.Name(), result)
|
||||||
if opts.Force {
|
if !priorOutcome.exists || priorOutcome.status != manifest.StatusSucceeded {
|
||||||
invalidateDownstreamSucceededStages(m, s.Name(), succeededAt)
|
invalidateDownstreamSucceededStagesWithReason(m, s.Name(), succeededAt, staleReasonChangedResult)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||||
@@ -407,26 +463,69 @@ func mapResultOutputs(stageName string, result *stage.StageResult, runID string)
|
|||||||
localPath = ref.RelativePath
|
localPath = ref.RelativePath
|
||||||
}
|
}
|
||||||
kind := ref.Kind
|
kind := ref.Kind
|
||||||
sourceID := ""
|
sourceID := strings.TrimSpace(ref.SourceID)
|
||||||
if stageName == "analyze" {
|
if sourceID == "" {
|
||||||
sourceID = artifacts.ConfiguredArtifactSourceID(ref.Kind)
|
if stageName == "analyze" {
|
||||||
kind = "scriptorium_artifact"
|
sourceID = artifacts.ConfiguredArtifactSourceID(ref.Kind)
|
||||||
} else {
|
kind = "scriptorium_artifact"
|
||||||
sourceID = sourceIDForOutputKind(kind)
|
} else {
|
||||||
|
sourceID = sourceIDForOutputKind(kind)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
out = append(out, manifest.ArtifactRecord{
|
out = append(out, manifest.ArtifactRecord{
|
||||||
Kind: kind,
|
Kind: kind,
|
||||||
SourceID: sourceID,
|
SourceID: sourceID,
|
||||||
LocalPath: localPath,
|
LocalPath: localPath,
|
||||||
ProducerRunID: runID,
|
Contract: cloneContractMetadata(ref.Contract),
|
||||||
RemoteKey: ref.RemoteKey,
|
ExternalProvenance: cloneExternalProvenance(ref.ExternalProvenance),
|
||||||
Checksum: ref.Checksum,
|
ProducerRunID: runID,
|
||||||
|
RemoteKey: ref.RemoteKey,
|
||||||
|
Checksum: ref.Checksum,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateStageResult(result *stage.StageResult) error {
|
||||||
|
if result == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch result.Disposition {
|
||||||
|
case stage.StageDispositionSucceeded:
|
||||||
|
if strings.TrimSpace(result.SkipReason) != "" {
|
||||||
|
return fmt.Errorf("successful result contains a skip reason")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
case stage.StageDispositionSkipped:
|
||||||
|
if strings.TrimSpace(result.SkipReason) == "" {
|
||||||
|
return fmt.Errorf("skipped result requires a skip reason")
|
||||||
|
}
|
||||||
|
if len(result.Outputs) != 0 {
|
||||||
|
return fmt.Errorf("skipped result contains %d output(s)", len(result.Outputs))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("unsupported stage result disposition %q", result.Disposition)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneContractMetadata(value *artifactmodel.ContractMetadata) *artifactmodel.ContractMetadata {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := *value
|
||||||
|
return &cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneExternalProvenance(value *artifactmodel.ExternalProvenance) *artifactmodel.ExternalProvenance {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
cloned := *value
|
||||||
|
return &cloned
|
||||||
|
}
|
||||||
|
|
||||||
func sourceIDForOutputKind(kind string) string {
|
func sourceIDForOutputKind(kind string) string {
|
||||||
trimmed := strings.TrimSpace(kind)
|
trimmed := strings.TrimSpace(kind)
|
||||||
if trimmed == "" {
|
if trimmed == "" {
|
||||||
@@ -598,6 +697,18 @@ func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func needsNotariusForRun(cfg *config.Config, stages []stage.Stage) bool {
|
||||||
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Notarius == nil || !cfg.Pipeline.Notarius.Enabled {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, candidate := range stages {
|
||||||
|
if candidate != nil && candidate.Name() == "extract" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func needsRemoteLocksForRun(cfg *config.Config, stages []stage.Stage) bool {
|
func needsRemoteLocksForRun(cfg *config.Config, stages []stage.Stage) bool {
|
||||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||||
return false
|
return false
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package app
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
@@ -16,6 +17,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
@@ -38,6 +40,25 @@ type countingStage struct {
|
|||||||
runs *int
|
runs *int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type resultStage struct {
|
||||||
|
name string
|
||||||
|
result *stage.StageResult
|
||||||
|
runs *int
|
||||||
|
order *[]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s resultStage) Name() string { return s.name }
|
||||||
|
func (s resultStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||||
|
func (s resultStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||||
|
if s.runs != nil {
|
||||||
|
*s.runs = *s.runs + 1
|
||||||
|
}
|
||||||
|
if s.order != nil {
|
||||||
|
*s.order = append(*s.order, s.name)
|
||||||
|
}
|
||||||
|
return s.result, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s countingStage) Name() string { return s.name }
|
func (s countingStage) Name() string { return s.name }
|
||||||
func (s countingStage) Declares() stage.IODecl { return stage.IODecl{} }
|
func (s countingStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||||
func (s countingStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
func (s countingStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||||
@@ -50,6 +71,34 @@ type captureSelectedArtifactsStage struct {
|
|||||||
captured *[]string
|
captured *[]string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type captureNotariusStage struct {
|
||||||
|
captured *bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type resumeCheckingStage struct {
|
||||||
|
name string
|
||||||
|
validation stage.ResumeValidation
|
||||||
|
validateErr error
|
||||||
|
runs *int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s resumeCheckingStage) Name() string { return s.name }
|
||||||
|
func (s resumeCheckingStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||||
|
func (s resumeCheckingStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||||
|
*s.runs++
|
||||||
|
return &stage.StageResult{}, nil
|
||||||
|
}
|
||||||
|
func (s resumeCheckingStage) ValidateResume(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (stage.ResumeValidation, error) {
|
||||||
|
return s.validation, s.validateErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s captureNotariusStage) Name() string { return "extract" }
|
||||||
|
func (s captureNotariusStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||||
|
func (s captureNotariusStage) Run(_ context.Context, env *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||||
|
*s.captured = env.Notarius != nil
|
||||||
|
return &stage.StageResult{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s captureSelectedArtifactsStage) Name() string { return s.name }
|
func (s captureSelectedArtifactsStage) Name() string { return s.name }
|
||||||
func (s captureSelectedArtifactsStage) Declares() stage.IODecl { return stage.IODecl{} }
|
func (s captureSelectedArtifactsStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||||
func (s captureSelectedArtifactsStage) Run(_ context.Context, env *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
func (s captureSelectedArtifactsStage) Run(_ context.Context, env *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||||
@@ -134,6 +183,28 @@ func TestExecuteStagesPropagatesSelectedArtifactsToEnv(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesComposesNotariusOnlyForEnabledExtraction(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
cfg.Pipeline.Notarius = &config.NotariusConfig{Enabled: true}
|
||||||
|
captured := false
|
||||||
|
|
||||||
|
_, err := executeStages(context.Background(), cfg, []stage.Stage{captureNotariusStage{captured: &captured}}, RunOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if !captured {
|
||||||
|
t.Fatal("extract stage did not receive the default Notarius runner")
|
||||||
|
}
|
||||||
|
|
||||||
|
if needsNotariusForRun(cfg, []stage.Stage{countingStage{name: "analyze", runs: new(int)}}) {
|
||||||
|
t.Fatal("Notarius runner requested without extract in the selected plan")
|
||||||
|
}
|
||||||
|
cfg.Pipeline.Notarius.Enabled = false
|
||||||
|
if needsNotariusForRun(cfg, []stage.Stage{captureNotariusStage{captured: new(bool)}}) {
|
||||||
|
t.Fatal("Notarius runner requested while extraction is disabled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteStagesAnalyzeOutputsPersistAsScriptoriumArtifacts(t *testing.T) {
|
func TestExecuteStagesAnalyzeOutputsPersistAsScriptoriumArtifacts(t *testing.T) {
|
||||||
cfg := testConfig(t)
|
cfg := testConfig(t)
|
||||||
storeForPaths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
|
storeForPaths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
|
||||||
@@ -199,6 +270,61 @@ func TestExecuteStagesAnalyzeOutputsPersistAsScriptoriumArtifacts(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapResultOutputsPrefersExplicitSourceAndCopiesMetadata(t *testing.T) {
|
||||||
|
contract := &artifactmodel.ContractMetadata{
|
||||||
|
MediaType: "application/json",
|
||||||
|
SchemaID: "notarius.dnd.npc_registry",
|
||||||
|
SchemaVersion: "v1",
|
||||||
|
}
|
||||||
|
provenance := &artifactmodel.ExternalProvenance{
|
||||||
|
System: "notarius",
|
||||||
|
RunID: "external-run",
|
||||||
|
PipelineID: "dnd-session",
|
||||||
|
ArtifactID: "npc-registry",
|
||||||
|
}
|
||||||
|
result := &stage.StageResult{Outputs: []artifacts.Ref{{
|
||||||
|
Kind: "structured_data",
|
||||||
|
SourceID: "narratio.example.npcs",
|
||||||
|
RelativePath: "artifacts/npcs.json",
|
||||||
|
Contract: contract,
|
||||||
|
ExternalProvenance: provenance,
|
||||||
|
}}}
|
||||||
|
|
||||||
|
got := mapResultOutputs("analyze", result, "narratio-run")
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("outputs len = %d, want 1", len(got))
|
||||||
|
}
|
||||||
|
if got[0].SourceID != "narratio.example.npcs" {
|
||||||
|
t.Fatalf("source_id = %q, want explicit source", got[0].SourceID)
|
||||||
|
}
|
||||||
|
if got[0].Kind != "structured_data" {
|
||||||
|
t.Fatalf("kind = %q, want explicit output kind preserved", got[0].Kind)
|
||||||
|
}
|
||||||
|
if got[0].Contract == nil || *got[0].Contract != *contract {
|
||||||
|
t.Fatalf("contract = %#v, want %#v", got[0].Contract, contract)
|
||||||
|
}
|
||||||
|
if got[0].ExternalProvenance == nil || *got[0].ExternalProvenance != *provenance {
|
||||||
|
t.Fatalf("external provenance = %#v, want %#v", got[0].ExternalProvenance, provenance)
|
||||||
|
}
|
||||||
|
if got[0].Contract == contract || got[0].ExternalProvenance == provenance {
|
||||||
|
t.Fatal("mapped metadata should not alias the stage result")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapResultOutputsRetainsFallbackInference(t *testing.T) {
|
||||||
|
transcript := mapResultOutputs("trim", &stage.StageResult{Outputs: []artifacts.Ref{{
|
||||||
|
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed,
|
||||||
|
}}}, "run-id")
|
||||||
|
if len(transcript) != 1 || transcript[0].SourceID != artifacts.ArtifactTranscriptFinalTrimmed {
|
||||||
|
t.Fatalf("transcript fallback = %#v, want final-trimmed source", transcript)
|
||||||
|
}
|
||||||
|
|
||||||
|
analyze := mapResultOutputs("analyze", &stage.StageResult{Outputs: []artifacts.Ref{{Kind: "session_recap"}}}, "run-id")
|
||||||
|
if len(analyze) != 1 || analyze[0].SourceID != "narratio.artifact.session_recap" || analyze[0].Kind != "scriptorium_artifact" {
|
||||||
|
t.Fatalf("analyze fallback = %#v, want configured artifact inference", analyze)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestNeedsObjectStoreForRunPrepareWithPreviousRequirements(t *testing.T) {
|
func TestNeedsObjectStoreForRunPrepareWithPreviousRequirements(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -273,7 +399,7 @@ func TestExecuteStagesPublishSkipsRequiredUnselectedConfiguredOutput(t *testing.
|
|||||||
manifestPath := manifestPathFor(cfg)
|
manifestPath := manifestPathFor(cfg)
|
||||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
seed.Campaign = cfg.Session.Campaign
|
seed.Campaign = cfg.Session.Campaign
|
||||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render"} {
|
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render"} {
|
||||||
seed.MarkStageSucceeded(stageName, time.Now().UTC(), nil)
|
seed.MarkStageSucceeded(stageName, time.Now().UTC(), nil)
|
||||||
}
|
}
|
||||||
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||||
@@ -332,8 +458,8 @@ func TestExecuteStagesPlaceholderSuccessUpdatesManifest(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("executeStages() error = %v", err)
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
}
|
}
|
||||||
if len(summary.StageNames) != 10 || len(summary.Executed) != 10 || len(summary.Skipped) != 0 {
|
if len(summary.StageNames) != 11 || len(summary.Executed) != 11 || len(summary.Skipped) != 1 || summary.Skipped[0] != "extract" {
|
||||||
t.Fatalf("summary = %#v, want all 10 executed", summary)
|
t.Fatalf("summary = %#v, want full plan with disabled extraction self-skip", summary)
|
||||||
}
|
}
|
||||||
|
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
@@ -342,11 +468,17 @@ func TestExecuteStagesPlaceholderSuccessUpdatesManifest(t *testing.T) {
|
|||||||
t.Fatalf("Load manifest error = %v", err)
|
t.Fatalf("Load manifest error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||||
sr := m.Stages[name]
|
sr := m.Stages[name]
|
||||||
if sr == nil {
|
if sr == nil {
|
||||||
t.Fatalf("missing stage record %q", name)
|
t.Fatalf("missing stage record %q", name)
|
||||||
}
|
}
|
||||||
|
if name == "extract" {
|
||||||
|
if sr.Status != manifest.StatusSkipped || sr.Error == nil || sr.Error.Message != "notarius_disabled" {
|
||||||
|
t.Fatalf("extract stage = %#v, want disabled skip", sr)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
if sr.Status != manifest.StatusSucceeded {
|
if sr.Status != manifest.StatusSucceeded {
|
||||||
t.Fatalf("stage %q status = %q, want %q", name, sr.Status, manifest.StatusSucceeded)
|
t.Fatalf("stage %q status = %q, want %q", name, sr.Status, manifest.StatusSucceeded)
|
||||||
}
|
}
|
||||||
@@ -497,6 +629,91 @@ func TestExecuteStagesSkipSucceededWhenNotForced(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesUsesOptionalResumeValidation(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
validation stage.ResumeValidation
|
||||||
|
wantRuns int
|
||||||
|
wantSkipped int
|
||||||
|
}{
|
||||||
|
{name: "resumable", validation: stage.Resumable(), wantSkipped: 1},
|
||||||
|
{name: "rerun", validation: stage.NonResumable("durable output changed"), wantRuns: 1},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
|
seed.MarkStageSucceeded("checked", time.Now().UTC(), nil)
|
||||||
|
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||||
|
t.Fatalf("Save() error = %v", err)
|
||||||
|
}
|
||||||
|
runs := 0
|
||||||
|
candidate := resumeCheckingStage{name: "checked", validation: test.validation, runs: &runs}
|
||||||
|
summary, err := executeStages(context.Background(), cfg, []stage.Stage{candidate}, RunOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if runs != test.wantRuns || len(summary.Skipped) != test.wantSkipped {
|
||||||
|
t.Fatalf("runs = %d summary = %#v", runs, summary)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesNonResumableResultRerunsSucceededDownstream(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
|
seed.MarkStageSucceeded("extract", time.Now().UTC(), nil)
|
||||||
|
seed.MarkStageSucceeded("render", time.Now().UTC(), nil)
|
||||||
|
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||||
|
t.Fatalf("Save() error = %v", err)
|
||||||
|
}
|
||||||
|
extractRuns, renderRuns := 0, 0
|
||||||
|
stages := []stage.Stage{
|
||||||
|
resumeCheckingStage{name: "extract", validation: stage.NonResumable("checksum changed"), runs: &extractRuns},
|
||||||
|
countingStage{name: "render", runs: &renderRuns},
|
||||||
|
}
|
||||||
|
summary, err := executeStages(context.Background(), cfg, stages, RunOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if extractRuns != 1 || renderRuns != 1 || len(summary.Executed) != 2 || len(summary.Skipped) != 0 {
|
||||||
|
t.Fatalf("extract runs=%d render runs=%d summary=%#v", extractRuns, renderRuns, summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesResumeValidationErrorPreservesSucceededRecord(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||||
|
seed.MarkStageSucceeded("checked", time.Now().UTC(), []manifest.ArtifactRecord{{Kind: "kept", LocalPath: "kept.json"}})
|
||||||
|
seed.Stages["checked"].Metadata = map[string]any{"kept": true}
|
||||||
|
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||||
|
t.Fatalf("Save() error = %v", err)
|
||||||
|
}
|
||||||
|
before, err := json.Marshal(seed.Stages["checked"])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal(before) error = %v", err)
|
||||||
|
}
|
||||||
|
runs := 0
|
||||||
|
candidate := resumeCheckingStage{name: "checked", validateErr: errors.New("inspection unavailable"), runs: &runs}
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{candidate}, RunOptions{}); err == nil || !strings.Contains(err.Error(), "inspection unavailable") {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
loaded, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
after, err := json.Marshal(loaded.Stages["checked"])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal(after) error = %v", err)
|
||||||
|
}
|
||||||
|
if string(after) != string(before) || runs != 0 {
|
||||||
|
t.Fatalf("succeeded record changed: before=%s after=%s runs=%d", before, after, runs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteStagesForceRerunsSucceeded(t *testing.T) {
|
func TestExecuteStagesForceRerunsSucceeded(t *testing.T) {
|
||||||
cfg := testConfig(t)
|
cfg := testConfig(t)
|
||||||
manifestPath := manifestPathFor(cfg)
|
manifestPath := manifestPathFor(cfg)
|
||||||
@@ -525,13 +742,48 @@ func TestExecuteStagesForceRerunsSucceeded(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesSuccessfulReplacementDoesNotInheritResultDetails(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
manifestPath := manifestPathFor(cfg)
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
|
||||||
|
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
||||||
|
existing.MarkStageSucceeded("transcribe", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), []manifest.ArtifactRecord{{
|
||||||
|
Kind: "transcript_raw", LocalPath: "transcripts/old.json",
|
||||||
|
}})
|
||||||
|
existingStage := existing.Stages["transcribe"]
|
||||||
|
existingStage.Logs = []string{"logs/old.log"}
|
||||||
|
existingStage.GeneratedConfigs = []string{"generated/old.yaml"}
|
||||||
|
existingStage.Metadata = map[string]any{"old_result": true}
|
||||||
|
if err := store.Save(context.Background(), manifestPath, existing); err != nil {
|
||||||
|
t.Fatalf("Save manifest error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
runs := 0
|
||||||
|
replacement := resultStage{name: "transcribe", result: &stage.StageResult{}, runs: &runs}
|
||||||
|
if _, err := executeStages(context.Background(), cfg, []stage.Stage{replacement}, RunOptions{Force: true}); err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
loaded, err := store.Load(context.Background(), manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load manifest error = %v", err)
|
||||||
|
}
|
||||||
|
record := loaded.Stages["transcribe"]
|
||||||
|
if runs != 1 || record == nil || record.Status != manifest.StatusSucceeded {
|
||||||
|
t.Fatalf("runs = %d, record = %#v, want one successful replacement", runs, record)
|
||||||
|
}
|
||||||
|
if len(record.Outputs) != 0 || len(record.Logs) != 0 || len(record.GeneratedConfigs) != 0 || len(record.Metadata) != 0 {
|
||||||
|
t.Fatalf("replacement inherited result details: %#v", record)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteStagesForceSuccessInvalidatesDownstreamSucceededStages(t *testing.T) {
|
func TestExecuteStagesForceSuccessInvalidatesDownstreamSucceededStages(t *testing.T) {
|
||||||
cfg := testConfig(t)
|
cfg := testConfig(t)
|
||||||
manifestPath := manifestPathFor(cfg)
|
manifestPath := manifestPathFor(cfg)
|
||||||
store := &manifest.LocalStore{}
|
store := &manifest.LocalStore{}
|
||||||
|
|
||||||
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
||||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "publish", "notify"} {
|
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "publish", "notify"} {
|
||||||
existing.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
|
existing.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
|
||||||
}
|
}
|
||||||
existing.MarkStageFailed("analyze", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), "previous analyze failure")
|
existing.MarkStageFailed("analyze", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), "previous analyze failure")
|
||||||
@@ -562,7 +814,7 @@ func TestExecuteStagesForceSuccessInvalidatesDownstreamSucceededStages(t *testin
|
|||||||
if loaded.Stages["polish"] == nil || loaded.Stages["polish"].Status != manifest.StatusSucceeded {
|
if loaded.Stages["polish"] == nil || loaded.Stages["polish"].Status != manifest.StatusSucceeded {
|
||||||
t.Fatalf("polish status = %#v, want succeeded", loaded.Stages["polish"])
|
t.Fatalf("polish status = %#v, want succeeded", loaded.Stages["polish"])
|
||||||
}
|
}
|
||||||
for _, stageName := range []string{"normalize", "trim", "render", "publish", "notify"} {
|
for _, stageName := range []string{"normalize", "trim", "extract", "render", "publish", "notify"} {
|
||||||
if loaded.Stages[stageName] == nil || loaded.Stages[stageName].Status != manifest.StatusStale {
|
if loaded.Stages[stageName] == nil || loaded.Stages[stageName].Status != manifest.StatusStale {
|
||||||
t.Fatalf("%s status = %#v, want stale", stageName, loaded.Stages[stageName])
|
t.Fatalf("%s status = %#v, want stale", stageName, loaded.Stages[stageName])
|
||||||
}
|
}
|
||||||
@@ -738,6 +990,126 @@ func TestExecuteStagesRunManifestRecordsSkippedStage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesPersistsSelfSkipAndContinues(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
store := &manifest.LocalStore{}
|
||||||
|
manifestPath := manifestPathFor(cfg)
|
||||||
|
seed := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
||||||
|
seed.MarkStageSucceeded("optional", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), []manifest.ArtifactRecord{{
|
||||||
|
Kind: "old_output",
|
||||||
|
SourceID: "narratio.example.old",
|
||||||
|
LocalPath: "artifacts/old.json",
|
||||||
|
}})
|
||||||
|
seed.Stages["optional"].Logs = []string{"old.log"}
|
||||||
|
seed.Stages["optional"].GeneratedConfigs = []string{"old.yml"}
|
||||||
|
seed.Stages["optional"].Metadata = map[string]any{"old": true}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||||
|
t.Fatalf("Save() seed manifest error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
order := []string{}
|
||||||
|
optionalRuns := 0
|
||||||
|
stages := []stage.Stage{
|
||||||
|
resultStage{
|
||||||
|
name: "optional",
|
||||||
|
runs: &optionalRuns,
|
||||||
|
order: &order,
|
||||||
|
result: &stage.StageResult{
|
||||||
|
Disposition: stage.StageDispositionSkipped,
|
||||||
|
SkipReason: "integration_disabled",
|
||||||
|
Logs: []string{"runs/current/optional.log"},
|
||||||
|
GeneratedConfigs: []string{"runs/current/optional.yml"},
|
||||||
|
Metadata: map[string]any{"enabled": false},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
resultStage{name: "later", order: &order, result: &stage.StageResult{}},
|
||||||
|
}
|
||||||
|
summary, err := executeStages(context.Background(), cfg, stages, RunOptions{Force: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if strings.Join(order, ",") != "optional,later" {
|
||||||
|
t.Fatalf("execution order = %v, want optional then later", order)
|
||||||
|
}
|
||||||
|
if len(summary.Executed) != 2 || len(summary.Skipped) != 1 || summary.Skipped[0] != "optional" {
|
||||||
|
t.Fatalf("summary = %#v, want optional executed and self-skipped before later", summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionManifest, err := store.Load(context.Background(), summary.ManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() session manifest error = %v", err)
|
||||||
|
}
|
||||||
|
selfSkipped := sessionManifest.Stages["optional"]
|
||||||
|
if selfSkipped == nil || selfSkipped.Status != manifest.StatusSkipped {
|
||||||
|
t.Fatalf("optional stage = %#v, want skipped", selfSkipped)
|
||||||
|
}
|
||||||
|
if len(selfSkipped.Outputs) != 0 {
|
||||||
|
t.Fatalf("optional outputs = %#v, want old outputs cleared", selfSkipped.Outputs)
|
||||||
|
}
|
||||||
|
if selfSkipped.Error == nil || selfSkipped.Error.Message != "integration_disabled" {
|
||||||
|
t.Fatalf("optional skip reason = %#v, want integration_disabled", selfSkipped.Error)
|
||||||
|
}
|
||||||
|
if len(selfSkipped.Logs) != 1 || selfSkipped.Logs[0] != "runs/current/optional.log" ||
|
||||||
|
len(selfSkipped.GeneratedConfigs) != 1 || selfSkipped.GeneratedConfigs[0] != "runs/current/optional.yml" ||
|
||||||
|
selfSkipped.Metadata["enabled"] != false || selfSkipped.Metadata["old"] != nil {
|
||||||
|
t.Fatalf("optional result details = %#v, want current bounded diagnostics and metadata", selfSkipped)
|
||||||
|
}
|
||||||
|
if later := sessionManifest.Stages["later"]; later == nil || later.Status != manifest.StatusSucceeded {
|
||||||
|
t.Fatalf("later stage = %#v, want succeeded", later)
|
||||||
|
}
|
||||||
|
|
||||||
|
runManifest, err := store.LoadRun(context.Background(), summary.RunManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadRun() error = %v", err)
|
||||||
|
}
|
||||||
|
runStage := runManifest.Stages["optional"]
|
||||||
|
if runStage == nil || runStage.Action != manifest.RunStageActionRun || runStage.Status != manifest.StatusSkipped {
|
||||||
|
t.Fatalf("run optional stage = %#v, want run action with skipped status", runStage)
|
||||||
|
}
|
||||||
|
if len(runStage.Logs) != 1 || runStage.Metadata["enabled"] != false {
|
||||||
|
t.Fatalf("run optional stage details = %#v, want result diagnostics and metadata", runStage)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = executeStages(context.Background(), cfg, []stage.Stage{stages[0]}, RunOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second executeStages() error = %v", err)
|
||||||
|
}
|
||||||
|
if optionalRuns != 2 {
|
||||||
|
t.Fatalf("optional runs = %d, want self-skipped stage reconsidered", optionalRuns)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteStagesRejectsSkippedResultWithOutputs(t *testing.T) {
|
||||||
|
cfg := testConfig(t)
|
||||||
|
invalid := resultStage{name: "optional", result: &stage.StageResult{
|
||||||
|
Disposition: stage.StageDispositionSkipped,
|
||||||
|
SkipReason: "integration_disabled",
|
||||||
|
Outputs: []artifacts.Ref{{Kind: "unexpected"}},
|
||||||
|
}}
|
||||||
|
|
||||||
|
summary, err := executeStages(context.Background(), cfg, []stage.Stage{invalid}, RunOptions{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("executeStages() error = nil, want invalid skipped result failure")
|
||||||
|
}
|
||||||
|
if summary != nil {
|
||||||
|
t.Fatalf("summary = %#v, want nil", summary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "skipped result contains 1 output") {
|
||||||
|
t.Fatalf("error = %q, want skipped-output validation", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded, loadErr := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||||
|
if loadErr != nil {
|
||||||
|
t.Fatalf("Load() session manifest error = %v", loadErr)
|
||||||
|
}
|
||||||
|
if got := loaded.Stages["optional"]; got == nil || got.Status != manifest.StatusFailed {
|
||||||
|
t.Fatalf("optional stage = %#v, want failed", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExecuteStagesRunLocalArtifactsAndCanonicalSync(t *testing.T) {
|
func TestExecuteStagesRunLocalArtifactsAndCanonicalSync(t *testing.T) {
|
||||||
cfg := testConfig(t)
|
cfg := testConfig(t)
|
||||||
stages := []stage.Stage{
|
stages := []stage.Stage{
|
||||||
@@ -986,7 +1358,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
|||||||
seed.S3Bucket = "my-dnd-archive"
|
seed.S3Bucket = "my-dnd-archive"
|
||||||
seed.S3SessionPrefix = "dnd/campaigns/" + cfg.Session.Campaign + "/sessions/" + cfg.Session.SessionID + "/"
|
seed.S3SessionPrefix = "dnd/campaigns/" + cfg.Session.Campaign + "/sessions/" + cfg.Session.SessionID + "/"
|
||||||
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
|
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||||
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||||
}
|
}
|
||||||
if err := tc.env.ManifestStore.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
if err := tc.env.ManifestStore.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||||
|
|||||||
@@ -346,7 +346,7 @@ func TestExecuteSessionLocksMutationAcceptsPositionalSessionID(t *testing.T) {
|
|||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
17
internal/artifactmodel/metadata.go
Normal file
17
internal/artifactmodel/metadata.go
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
package artifactmodel
|
||||||
|
|
||||||
|
// ContractMetadata identifies the data contract implemented by an artifact.
|
||||||
|
type ContractMetadata struct {
|
||||||
|
MediaType string `json:"media_type"`
|
||||||
|
SchemaID string `json:"schema_id"`
|
||||||
|
SchemaVersion string `json:"schema_version"`
|
||||||
|
ModuleKey string `json:"module_key,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExternalProvenance identifies an artifact produced by an external system.
|
||||||
|
type ExternalProvenance struct {
|
||||||
|
System string `json:"system"`
|
||||||
|
RunID string `json:"run_id"`
|
||||||
|
PipelineID string `json:"pipeline_id"`
|
||||||
|
ArtifactID string `json:"artifact_id"`
|
||||||
|
}
|
||||||
56
internal/artifactmodel/metadata_test.go
Normal file
56
internal/artifactmodel/metadata_test.go
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
package artifactmodel
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestArtifactMetadataJSON(t *testing.T) {
|
||||||
|
type envelope struct {
|
||||||
|
Contract *ContractMetadata `json:"contract,omitempty"`
|
||||||
|
ExternalProvenance *ExternalProvenance `json:"external_provenance,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
complete, err := json.Marshal(envelope{
|
||||||
|
Contract: &ContractMetadata{
|
||||||
|
MediaType: "application/json",
|
||||||
|
SchemaID: "notarius.dnd.npc_registry",
|
||||||
|
SchemaVersion: "v1",
|
||||||
|
ModuleKey: "dnd/npc-registry",
|
||||||
|
},
|
||||||
|
ExternalProvenance: &ExternalProvenance{
|
||||||
|
System: "notarius",
|
||||||
|
RunID: "run-123",
|
||||||
|
PipelineID: "dnd-session",
|
||||||
|
ArtifactID: "npc-registry",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal() complete metadata error = %v", err)
|
||||||
|
}
|
||||||
|
wantComplete := `{"contract":{"media_type":"application/json","schema_id":"notarius.dnd.npc_registry","schema_version":"v1","module_key":"dnd/npc-registry"},"external_provenance":{"system":"notarius","run_id":"run-123","pipeline_id":"dnd-session","artifact_id":"npc-registry"}}`
|
||||||
|
if string(complete) != wantComplete {
|
||||||
|
t.Fatalf("complete metadata JSON = %s, want %s", complete, wantComplete)
|
||||||
|
}
|
||||||
|
|
||||||
|
omitted, err := json.Marshal(envelope{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal() omitted metadata error = %v", err)
|
||||||
|
}
|
||||||
|
if string(omitted) != `{}` {
|
||||||
|
t.Fatalf("omitted metadata JSON = %s, want {}", omitted)
|
||||||
|
}
|
||||||
|
|
||||||
|
withoutModule, err := json.Marshal(envelope{Contract: &ContractMetadata{
|
||||||
|
MediaType: "application/json",
|
||||||
|
SchemaID: "notarius.dnd.npc_registry",
|
||||||
|
SchemaVersion: "v1",
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal() contract without module key error = %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(withoutModule), "module_key") {
|
||||||
|
t.Fatalf("contract JSON unexpectedly contains omitted module_key: %s", withoutModule)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,11 +18,14 @@ const (
|
|||||||
SourceInputGlossary = "narratio.input.glossary"
|
SourceInputGlossary = "narratio.input.glossary"
|
||||||
|
|
||||||
configuredSourcePrefix = "narratio.artifact."
|
configuredSourcePrefix = "narratio.artifact."
|
||||||
|
extractionSourcePrefix = "narratio.extraction."
|
||||||
previousConfiguredSrcPrefix = "narratio.previous_session.artifact."
|
previousConfiguredSrcPrefix = "narratio.previous_session.artifact."
|
||||||
)
|
)
|
||||||
|
|
||||||
var configuredSourceRE = regexp.MustCompile(`^narratio\.artifact\.([a-z][a-z0-9_]*)$`)
|
var configuredSourceRE = regexp.MustCompile(`^narratio\.artifact\.([a-z][a-z0-9_]*)$`)
|
||||||
|
var extractionSourceRE = regexp.MustCompile(`^narratio\.extraction\.([a-z][a-z0-9_]*)$`)
|
||||||
var previousSourceRE = regexp.MustCompile(`^narratio\.previous_session\.artifact\.([a-z][a-z0-9_]*)$`)
|
var previousSourceRE = regexp.MustCompile(`^narratio\.previous_session\.artifact\.([a-z][a-z0-9_]*)$`)
|
||||||
|
var configuredKeyRE = regexp.MustCompile(`^[a-z][a-z0-9_]*$`)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrUnsupportedScriptoriumInputSource = errors.New("unsupported scriptorium input source")
|
ErrUnsupportedScriptoriumInputSource = errors.New("unsupported scriptorium input source")
|
||||||
@@ -34,6 +37,7 @@ type SourceKind string
|
|||||||
const (
|
const (
|
||||||
SourceKindBuiltIn SourceKind = "built_in"
|
SourceKindBuiltIn SourceKind = "built_in"
|
||||||
SourceKindConfiguredArtifact SourceKind = "configured_artifact"
|
SourceKindConfiguredArtifact SourceKind = "configured_artifact"
|
||||||
|
SourceKindExtraction SourceKind = "extraction"
|
||||||
SourceKindPreviousArtifact SourceKind = "previous_session_configured_artifact"
|
SourceKindPreviousArtifact SourceKind = "previous_session_configured_artifact"
|
||||||
SourceKindStableInput SourceKind = "stable_input"
|
SourceKindStableInput SourceKind = "stable_input"
|
||||||
)
|
)
|
||||||
@@ -67,11 +71,30 @@ func (e *UnknownConfiguredArtifactError) Error() string {
|
|||||||
return fmt.Sprintf("references unknown artifact %q", e.ConfiguredKey)
|
return fmt.Sprintf("references unknown artifact %q", e.ConfiguredKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UnknownExtractionArtifactError reports a source that references an undefined extraction key.
|
||||||
|
type UnknownExtractionArtifactError struct {
|
||||||
|
ConfiguredKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *UnknownExtractionArtifactError) Error() string {
|
||||||
|
return fmt.Sprintf("references unknown extraction output %q", e.ConfiguredKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsConfiguredKey reports whether a key follows the configured-artifact key grammar.
|
||||||
|
func IsConfiguredKey(key string) bool {
|
||||||
|
return configuredKeyRE.MatchString(strings.TrimSpace(key))
|
||||||
|
}
|
||||||
|
|
||||||
// ConfiguredSourceID converts a configured artifact key into source id form.
|
// ConfiguredSourceID converts a configured artifact key into source id form.
|
||||||
func ConfiguredSourceID(key string) string {
|
func ConfiguredSourceID(key string) string {
|
||||||
return configuredSourcePrefix + strings.TrimSpace(key)
|
return configuredSourcePrefix + strings.TrimSpace(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExtractionSourceID converts an extraction output key into source id form.
|
||||||
|
func ExtractionSourceID(key string) string {
|
||||||
|
return extractionSourcePrefix + strings.TrimSpace(key)
|
||||||
|
}
|
||||||
|
|
||||||
// PreviousSessionSourceID converts a configured artifact key into previous-session source id form.
|
// PreviousSessionSourceID converts a configured artifact key into previous-session source id form.
|
||||||
func PreviousSessionSourceID(key string) string {
|
func PreviousSessionSourceID(key string) string {
|
||||||
return previousConfiguredSrcPrefix + strings.TrimSpace(key)
|
return previousConfiguredSrcPrefix + strings.TrimSpace(key)
|
||||||
@@ -86,6 +109,15 @@ func ParseConfiguredSource(source string) (string, bool) {
|
|||||||
return matches[1], true
|
return matches[1], true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParseExtractionSource extracts configured key from narratio.extraction.<key>.
|
||||||
|
func ParseExtractionSource(source string) (string, bool) {
|
||||||
|
matches := extractionSourceRE.FindStringSubmatch(strings.TrimSpace(source))
|
||||||
|
if len(matches) != 2 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return matches[1], true
|
||||||
|
}
|
||||||
|
|
||||||
// ParsePreviousSessionSource extracts configured key from narratio.previous_session.artifact.<key>.
|
// ParsePreviousSessionSource extracts configured key from narratio.previous_session.artifact.<key>.
|
||||||
func ParsePreviousSessionSource(source string) (string, bool) {
|
func ParsePreviousSessionSource(source string) (string, bool) {
|
||||||
matches := previousSourceRE.FindStringSubmatch(strings.TrimSpace(source))
|
matches := previousSourceRE.FindStringSubmatch(strings.TrimSpace(source))
|
||||||
@@ -110,6 +142,9 @@ func ClassifySource(source string) (Source, error) {
|
|||||||
if key, ok := ParseConfiguredSource(trimmed); ok {
|
if key, ok := ParseConfiguredSource(trimmed); ok {
|
||||||
return Source{ID: trimmed, Kind: SourceKindConfiguredArtifact, ConfiguredKey: key}, nil
|
return Source{ID: trimmed, Kind: SourceKindConfiguredArtifact, ConfiguredKey: key}, nil
|
||||||
}
|
}
|
||||||
|
if key, ok := ParseExtractionSource(trimmed); ok {
|
||||||
|
return Source{ID: trimmed, Kind: SourceKindExtraction, ConfiguredKey: key}, nil
|
||||||
|
}
|
||||||
if key, ok := ParsePreviousSessionSource(trimmed); ok {
|
if key, ok := ParsePreviousSessionSource(trimmed); ok {
|
||||||
return Source{ID: trimmed, Kind: SourceKindPreviousArtifact, ConfiguredKey: key}, nil
|
return Source{ID: trimmed, Kind: SourceKindPreviousArtifact, ConfiguredKey: key}, nil
|
||||||
}
|
}
|
||||||
@@ -186,24 +221,48 @@ func PreviousSessionSourceDescriptorForConfiguredKey(configuredKey string) (Prev
|
|||||||
func ValidateInputConfiguredReference(
|
func ValidateInputConfiguredReference(
|
||||||
descriptor ScriptoriumInputSourceDescriptor,
|
descriptor ScriptoriumInputSourceDescriptor,
|
||||||
configured map[string]struct{},
|
configured map[string]struct{},
|
||||||
|
) error {
|
||||||
|
return ValidateInputReference(descriptor, configured, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateInputReference checks that configured and extraction sources are declared
|
||||||
|
// by the effective pipeline configuration.
|
||||||
|
func ValidateInputReference(
|
||||||
|
descriptor ScriptoriumInputSourceDescriptor,
|
||||||
|
configured map[string]struct{},
|
||||||
|
extractions map[string]struct{},
|
||||||
) error {
|
) error {
|
||||||
switch descriptor.Source.Kind {
|
switch descriptor.Source.Kind {
|
||||||
case SourceKindConfiguredArtifact, SourceKindPreviousArtifact:
|
case SourceKindConfiguredArtifact, SourceKindPreviousArtifact:
|
||||||
if _, ok := configured[descriptor.Source.ConfiguredKey]; !ok {
|
if _, ok := configured[descriptor.Source.ConfiguredKey]; !ok {
|
||||||
return &UnknownConfiguredArtifactError{ConfiguredKey: descriptor.Source.ConfiguredKey}
|
return &UnknownConfiguredArtifactError{ConfiguredKey: descriptor.Source.ConfiguredKey}
|
||||||
}
|
}
|
||||||
|
case SourceKindExtraction:
|
||||||
|
if _, ok := extractions[descriptor.Source.ConfiguredKey]; !ok {
|
||||||
|
return &UnknownExtractionArtifactError{ConfiguredKey: descriptor.Source.ConfiguredKey}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidatePublishSource validates that a source is publish-compatible and references a known configured artifact.
|
// ValidatePublishSource validates that a source is publish-compatible and references a known configured artifact.
|
||||||
func ValidatePublishSource(source string, configured map[string]string) (Source, error) {
|
func ValidatePublishSource(source string, configured map[string]string) (Source, error) {
|
||||||
|
return ValidatePublishSourceWithExtractions(source, configured, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidatePublishSourceWithExtractions validates publish sources against the
|
||||||
|
// configured Scriptorium artifacts and extraction outputs.
|
||||||
|
func ValidatePublishSourceWithExtractions(
|
||||||
|
source string,
|
||||||
|
configured map[string]string,
|
||||||
|
extractions map[string]struct{},
|
||||||
|
) (Source, error) {
|
||||||
classified, err := ClassifySource(source)
|
classified, err := ClassifySource(source)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Source{}, fmt.Errorf("must be a built-in source id or narratio.artifact.<name>")
|
return Source{}, fmt.Errorf("must be a built-in source id, narratio.artifact.<name>, or configured narratio.extraction.<name>")
|
||||||
}
|
}
|
||||||
if classified.Kind == SourceKindPreviousArtifact {
|
if classified.Kind == SourceKindPreviousArtifact {
|
||||||
return Source{}, fmt.Errorf("must be a built-in source id or narratio.artifact.<name>")
|
return Source{}, fmt.Errorf("must be a built-in source id, narratio.artifact.<name>, or configured narratio.extraction.<name>")
|
||||||
}
|
}
|
||||||
if classified.Kind == SourceKindConfiguredArtifact {
|
if classified.Kind == SourceKindConfiguredArtifact {
|
||||||
if configured == nil {
|
if configured == nil {
|
||||||
@@ -213,6 +272,11 @@ func ValidatePublishSource(source string, configured map[string]string) (Source,
|
|||||||
return Source{}, fmt.Errorf("configured artifact %q is not defined in pipeline.scriptorium.artifacts", classified.ConfiguredKey)
|
return Source{}, fmt.Errorf("configured artifact %q is not defined in pipeline.scriptorium.artifacts", classified.ConfiguredKey)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if classified.Kind == SourceKindExtraction {
|
||||||
|
if _, ok := extractions[classified.ConfiguredKey]; !ok {
|
||||||
|
return Source{}, fmt.Errorf("extraction output %q is not defined in pipeline.notarius.outputs", classified.ConfiguredKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
return classified, nil
|
return classified, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -247,7 +311,17 @@ func DeriveDefaultPublishedDestination(source Source, configured map[string]stri
|
|||||||
// ResolvePublishedDestination validates and normalizes an explicit destination,
|
// ResolvePublishedDestination validates and normalizes an explicit destination,
|
||||||
// or derives one when omitted.
|
// or derives one when omitted.
|
||||||
func ResolvePublishedDestination(sourceID, explicitDest string, configured map[string]string) (string, error) {
|
func ResolvePublishedDestination(sourceID, explicitDest string, configured map[string]string) (string, error) {
|
||||||
source, err := ValidatePublishSource(sourceID, configured)
|
return ResolvePublishedDestinationWithExtractions(sourceID, explicitDest, configured, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolvePublishedDestinationWithExtractions validates and normalizes a destination
|
||||||
|
// while accepting extraction sources declared by the effective Notarius configuration.
|
||||||
|
func ResolvePublishedDestinationWithExtractions(
|
||||||
|
sourceID, explicitDest string,
|
||||||
|
configured map[string]string,
|
||||||
|
extractions map[string]struct{},
|
||||||
|
) (string, error) {
|
||||||
|
source, err := ValidatePublishSourceWithExtractions(sourceID, configured, extractions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ func TestClassifySource(t *testing.T) {
|
|||||||
{name: "built in transcript", source: "narratio.transcript.final_trimmed", wantKind: SourceKindBuiltIn},
|
{name: "built in transcript", source: "narratio.transcript.final_trimmed", wantKind: SourceKindBuiltIn},
|
||||||
{name: "built in bounds", source: "narratio.bounds.session", wantKind: SourceKindBuiltIn},
|
{name: "built in bounds", source: "narratio.bounds.session", wantKind: SourceKindBuiltIn},
|
||||||
{name: "configured artifact", source: "narratio.artifact.session_recap", wantKind: SourceKindConfiguredArtifact, wantKey: "session_recap"},
|
{name: "configured artifact", source: "narratio.artifact.session_recap", wantKind: SourceKindConfiguredArtifact, wantKey: "session_recap"},
|
||||||
|
{name: "extraction", source: "narratio.extraction.npc_registry", wantKind: SourceKindExtraction, wantKey: "npc_registry"},
|
||||||
{name: "previous session configured", source: "narratio.previous_session.artifact.session_recap", wantKind: SourceKindPreviousArtifact, wantKey: "session_recap"},
|
{name: "previous session configured", source: "narratio.previous_session.artifact.session_recap", wantKind: SourceKindPreviousArtifact, wantKey: "session_recap"},
|
||||||
{name: "unsupported", source: "narratio.unknown", wantErrLike: "unsupported artifact source"},
|
{name: "unsupported", source: "narratio.unknown", wantErrLike: "unsupported artifact source"},
|
||||||
}
|
}
|
||||||
@@ -56,6 +57,60 @@ func TestValidatePublishSource(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtractionSourcePolicy(t *testing.T) {
|
||||||
|
if got := ExtractionSourceID(" npc_registry "); got != "narratio.extraction.npc_registry" {
|
||||||
|
t.Fatalf("ExtractionSourceID() = %q, want narratio.extraction.npc_registry", got)
|
||||||
|
}
|
||||||
|
if key, ok := ParseExtractionSource(" narratio.extraction.npc_registry "); !ok || key != "npc_registry" {
|
||||||
|
t.Fatalf("ParseExtractionSource() = %q, %t; want npc_registry, true", key, ok)
|
||||||
|
}
|
||||||
|
for _, source := range []string{
|
||||||
|
"narratio.extraction.",
|
||||||
|
"narratio.extraction.NPC",
|
||||||
|
"narratio.extraction.npc-registry",
|
||||||
|
"narratio.extraction.npc_registry.extra",
|
||||||
|
} {
|
||||||
|
if _, ok := ParseExtractionSource(source); ok {
|
||||||
|
t.Fatalf("ParseExtractionSource(%q) unexpectedly matched", source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
descriptor, err := DescribeScriptoriumInputSource("narratio.extraction.npc_registry")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DescribeScriptoriumInputSource(extraction) error = %v", err)
|
||||||
|
}
|
||||||
|
if descriptor.Source.Kind != SourceKindExtraction || descriptor.Source.ConfiguredKey != "npc_registry" {
|
||||||
|
t.Fatalf("extraction descriptor = %#v", descriptor)
|
||||||
|
}
|
||||||
|
declared := map[string]struct{}{"npc_registry": {}}
|
||||||
|
if err := ValidateInputReference(descriptor, nil, declared); err != nil {
|
||||||
|
t.Fatalf("ValidateInputReference(declared extraction) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := ValidateInputReference(descriptor, nil, nil); err == nil {
|
||||||
|
t.Fatal("ValidateInputReference(unknown extraction) error = nil, want error")
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := ValidatePublishSourceWithExtractions("narratio.extraction.npc_registry", nil, declared); err != nil {
|
||||||
|
t.Fatalf("ValidatePublishSourceWithExtractions(declared) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := ValidatePublishSourceWithExtractions("narratio.extraction.unknown", nil, declared); err == nil {
|
||||||
|
t.Fatal("ValidatePublishSourceWithExtractions(unknown) error = nil, want error")
|
||||||
|
}
|
||||||
|
|
||||||
|
identities := map[string]struct{}{}
|
||||||
|
for _, sourceID := range []string{
|
||||||
|
ExtractionSourceID("npc_registry"),
|
||||||
|
ConfiguredSourceID("npc_registry"),
|
||||||
|
PreviousSessionSourceID("npc_registry"),
|
||||||
|
SourceBoundsSession,
|
||||||
|
} {
|
||||||
|
if _, exists := identities[sourceID]; exists {
|
||||||
|
t.Fatalf("source identity collision at %q", sourceID)
|
||||||
|
}
|
||||||
|
identities[sourceID] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvePublishedDestination(t *testing.T) {
|
func TestResolvePublishedDestination(t *testing.T) {
|
||||||
configured := map[string]string{"session_recap": "artifacts/session_recap.md"}
|
configured := map[string]string{"session_recap": "artifacts/session_recap.md"}
|
||||||
|
|
||||||
|
|||||||
@@ -144,6 +144,22 @@ func ConfiguredArtifactName(source string) (string, bool) {
|
|||||||
return artifactpolicy.ParseConfiguredSource(source)
|
return artifactpolicy.ParseConfiguredSource(source)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExtractionArtifactSourceID returns narratio.extraction.<name> for a configured key.
|
||||||
|
func ExtractionArtifactSourceID(key string) string {
|
||||||
|
return artifactpolicy.ExtractionSourceID(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsExtractionArtifactSource reports whether source is narratio.extraction.<name>.
|
||||||
|
func IsExtractionArtifactSource(source string) bool {
|
||||||
|
_, ok := artifactpolicy.ParseExtractionSource(source)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractionArtifactName extracts <name> from narratio.extraction.<name>.
|
||||||
|
func ExtractionArtifactName(source string) (string, bool) {
|
||||||
|
return artifactpolicy.ParseExtractionSource(source)
|
||||||
|
}
|
||||||
|
|
||||||
// IsPreviousSessionArtifactSource returns true when source is narratio.previous_session.artifact.<name>.
|
// IsPreviousSessionArtifactSource returns true when source is narratio.previous_session.artifact.<name>.
|
||||||
func IsPreviousSessionArtifactSource(source string) bool {
|
func IsPreviousSessionArtifactSource(source string) bool {
|
||||||
_, ok := artifactpolicy.ParsePreviousSessionSource(source)
|
_, ok := artifactpolicy.ParsePreviousSessionSource(source)
|
||||||
@@ -204,17 +220,19 @@ func ResolveSessionArtifact(paths SessionPaths, m *manifest.Manifest, source str
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ResolveSessionArtifactWithCatalog resolves built-in sources using existing rules and resolves
|
// ResolveSessionArtifactWithCatalog resolves built-in sources using existing rules and resolves
|
||||||
// configured narratio.artifact.<name> sources through runtime catalog availability.
|
// configured artifact and extraction sources through runtime catalog availability.
|
||||||
func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest, source string, catalog *ArtifactCatalog) (ResolvedSessionArtifact, error) {
|
func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest, source string, catalog *ArtifactCatalog) (ResolvedSessionArtifact, error) {
|
||||||
normalized := strings.TrimSpace(source)
|
normalized := strings.TrimSpace(source)
|
||||||
if IsPreviousSessionArtifactSource(normalized) {
|
if IsPreviousSessionArtifactSource(normalized) {
|
||||||
return ResolvePreviousSessionArtifactWithCatalog(paths, m, normalized, catalog)
|
return ResolvePreviousSessionArtifactWithCatalog(paths, m, normalized, catalog)
|
||||||
}
|
}
|
||||||
if !IsConfiguredArtifactSource(normalized) {
|
configuredSource := IsConfiguredArtifactSource(normalized)
|
||||||
|
extractionSource := IsExtractionArtifactSource(normalized)
|
||||||
|
if !configuredSource && !extractionSource {
|
||||||
return ResolveSessionArtifact(paths, m, normalized)
|
return ResolveSessionArtifact(paths, m, normalized)
|
||||||
}
|
}
|
||||||
if catalog == nil {
|
if catalog == nil {
|
||||||
return ResolvedSessionArtifact{}, fmt.Errorf("configured artifact source %q requires runtime artifact catalog", source)
|
return ResolvedSessionArtifact{}, fmt.Errorf("catalog-backed artifact source %q requires runtime artifact catalog", source)
|
||||||
}
|
}
|
||||||
entry, ok := catalog.Lookup(normalized)
|
entry, ok := catalog.Lookup(normalized)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -223,7 +241,11 @@ func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest,
|
|||||||
if !entry.Available {
|
if !entry.Available {
|
||||||
return ResolvedSessionArtifact{}, &SessionArtifactNotFoundError{ArtifactID: normalized}
|
return ResolvedSessionArtifact{}, &SessionArtifactNotFoundError{ArtifactID: normalized}
|
||||||
}
|
}
|
||||||
if err := validateResolvedContent(entry.Path, contentText); err != nil {
|
contentKind := contentText
|
||||||
|
if extractionSource {
|
||||||
|
contentKind = contentJSON
|
||||||
|
}
|
||||||
|
if err := validateResolvedContent(entry.Path, contentKind); err != nil {
|
||||||
return ResolvedSessionArtifact{}, fmt.Errorf("validate %q: %w", normalized, err)
|
return ResolvedSessionArtifact{}, fmt.Errorf("validate %q: %w", normalized, err)
|
||||||
}
|
}
|
||||||
return ResolvedSessionArtifact{
|
return ResolvedSessionArtifact{
|
||||||
@@ -231,6 +253,7 @@ func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest,
|
|||||||
Path: filepath.Clean(entry.Path),
|
Path: filepath.Clean(entry.Path),
|
||||||
ProducerStage: entry.ProducerStage,
|
ProducerStage: entry.ProducerStage,
|
||||||
OutputKind: entry.OutputKind,
|
OutputKind: entry.OutputKind,
|
||||||
|
ProducerRunID: entry.ProducerRunID,
|
||||||
Provenance: entry.Provenance,
|
Provenance: entry.Provenance,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -150,6 +150,23 @@ func TestPreviousSessionArtifactSourceHelpers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtractionArtifactSourceHelpers(t *testing.T) {
|
||||||
|
if got := ExtractionArtifactSourceID("npc_registry"); got != "narratio.extraction.npc_registry" {
|
||||||
|
t.Fatalf("ExtractionArtifactSourceID() = %q", got)
|
||||||
|
}
|
||||||
|
if !IsExtractionArtifactSource(" narratio.extraction.npc_registry ") {
|
||||||
|
t.Fatal("IsExtractionArtifactSource(valid) = false")
|
||||||
|
}
|
||||||
|
if key, ok := ExtractionArtifactName("narratio.extraction.npc_registry"); !ok || key != "npc_registry" {
|
||||||
|
t.Fatalf("ExtractionArtifactName() = %q, %t", key, ok)
|
||||||
|
}
|
||||||
|
for _, source := range []string{"narratio.extraction.", "narratio.extraction.npc-registry", "narratio.artifact.npc_registry"} {
|
||||||
|
if IsExtractionArtifactSource(source) {
|
||||||
|
t.Fatalf("IsExtractionArtifactSource(%q) = true, want false", source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolveSessionArtifactPrefersManifestOutput(t *testing.T) {
|
func TestResolveSessionArtifactPrefersManifestOutput(t *testing.T) {
|
||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
paths := buildSessionPaths(workspace, "campaign", "session")
|
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||||
|
|||||||
@@ -6,11 +6,13 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
ArtifactProvenanceGeneratedCurrentAnalyzeRun = "generated.current_analyze_run"
|
ArtifactProvenanceGeneratedCurrentAnalyzeRun = "generated.current_analyze_run"
|
||||||
ArtifactProvenanceDisabledFromDisk = "filesystem.disabled_artifact_output"
|
ArtifactProvenanceDisabledFromDisk = "filesystem.disabled_artifact_output"
|
||||||
|
ArtifactProvenanceCurrentExtractManifest = "manifest.current_extract_run"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ConfiguredArtifactDefinition describes one configured analyze artifact.
|
// ConfiguredArtifactDefinition describes one configured analyze artifact.
|
||||||
@@ -19,10 +21,40 @@ type ConfiguredArtifactDefinition struct {
|
|||||||
OutputPath string
|
OutputPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExtractionArtifactDefinition describes one configured Notarius output lane.
|
||||||
|
type ExtractionArtifactDefinition struct {
|
||||||
|
LaneID string
|
||||||
|
PipelineID string
|
||||||
|
MediaType string
|
||||||
|
SchemaID string
|
||||||
|
SchemaVersion string
|
||||||
|
ModuleKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractionDefinitionsFromConfig converts the effective Notarius output map into catalog definitions.
|
||||||
|
func ExtractionDefinitionsFromConfig(cfg *config.NotariusConfig) map[string]ExtractionArtifactDefinition {
|
||||||
|
if cfg == nil || len(cfg.Outputs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
definitions := make(map[string]ExtractionArtifactDefinition, len(cfg.Outputs))
|
||||||
|
for key, output := range cfg.Outputs {
|
||||||
|
definitions[key] = ExtractionArtifactDefinition{
|
||||||
|
LaneID: output.LaneID,
|
||||||
|
PipelineID: cfg.PipelineID,
|
||||||
|
MediaType: output.MediaType,
|
||||||
|
SchemaID: output.SchemaID,
|
||||||
|
SchemaVersion: output.SchemaVersion,
|
||||||
|
ModuleKey: output.ModuleKey,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return definitions
|
||||||
|
}
|
||||||
|
|
||||||
// CatalogEntry is one runtime catalog entry resolved by source ID.
|
// CatalogEntry is one runtime catalog entry resolved by source ID.
|
||||||
type CatalogEntry struct {
|
type CatalogEntry struct {
|
||||||
SourceID string
|
SourceID string
|
||||||
ConfiguredKey string
|
ConfiguredKey string
|
||||||
|
ExtractionKey string
|
||||||
CanonicalRelPath string
|
CanonicalRelPath string
|
||||||
ProducerStage string
|
ProducerStage string
|
||||||
OutputKind string
|
OutputKind string
|
||||||
@@ -31,12 +63,14 @@ type CatalogEntry struct {
|
|||||||
Available bool
|
Available bool
|
||||||
Path string
|
Path string
|
||||||
Provenance string
|
Provenance string
|
||||||
|
ProducerRunID string
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArtifactCatalog tracks built-in and configured artifact definitions and runtime state.
|
// ArtifactCatalog tracks built-in, configured, and extraction artifact definitions and runtime state.
|
||||||
type ArtifactCatalog struct {
|
type ArtifactCatalog struct {
|
||||||
entries map[string]CatalogEntry
|
entries map[string]CatalogEntry
|
||||||
configuredIndex map[string]string
|
configuredIndex map[string]string
|
||||||
|
extractionIndex map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewArtifactCatalog returns an empty runtime artifact catalog.
|
// NewArtifactCatalog returns an empty runtime artifact catalog.
|
||||||
@@ -44,9 +78,41 @@ func NewArtifactCatalog() *ArtifactCatalog {
|
|||||||
return &ArtifactCatalog{
|
return &ArtifactCatalog{
|
||||||
entries: map[string]CatalogEntry{},
|
entries: map[string]CatalogEntry{},
|
||||||
configuredIndex: map[string]string{},
|
configuredIndex: map[string]string{},
|
||||||
|
extractionIndex: map[string]string{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RegisterExtractionArtifacts registers the configured Notarius output lanes.
|
||||||
|
func (c *ArtifactCatalog) RegisterExtractionArtifacts(configured map[string]ExtractionArtifactDefinition) error {
|
||||||
|
keys := make([]string, 0, len(configured))
|
||||||
|
for key := range configured {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
|
||||||
|
for _, key := range keys {
|
||||||
|
trimmed := strings.TrimSpace(key)
|
||||||
|
if trimmed == "" {
|
||||||
|
return fmt.Errorf("extraction artifact keys must be non-empty")
|
||||||
|
}
|
||||||
|
if _, exists := c.extractionIndex[trimmed]; exists {
|
||||||
|
return fmt.Errorf("duplicate extraction artifact key %q", trimmed)
|
||||||
|
}
|
||||||
|
sourceID := ExtractionArtifactSourceID(trimmed)
|
||||||
|
if err := c.addEntry(CatalogEntry{
|
||||||
|
SourceID: sourceID,
|
||||||
|
ExtractionKey: trimmed,
|
||||||
|
ProducerStage: "extract",
|
||||||
|
OutputKind: "notarius_lane",
|
||||||
|
Planned: true,
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("register extraction artifact %q: %w", trimmed, err)
|
||||||
|
}
|
||||||
|
c.extractionIndex[trimmed] = sourceID
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// ConfiguredArtifactSourceID converts a configured artifact key into canonical source ID.
|
// ConfiguredArtifactSourceID converts a configured artifact key into canonical source ID.
|
||||||
func ConfiguredArtifactSourceID(key string) string {
|
func ConfiguredArtifactSourceID(key string) string {
|
||||||
return artifactpolicy.ConfiguredSourceID(key)
|
return artifactpolicy.ConfiguredSourceID(key)
|
||||||
@@ -151,6 +217,15 @@ func (c *ArtifactCatalog) SourceIDForConfiguredKey(key string) (string, bool) {
|
|||||||
return sourceID, ok
|
return sourceID, ok
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SourceIDForExtractionKey returns the canonical source ID for one Notarius output key.
|
||||||
|
func (c *ArtifactCatalog) SourceIDForExtractionKey(key string) (string, bool) {
|
||||||
|
if c == nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
sourceID, ok := c.extractionIndex[strings.TrimSpace(key)]
|
||||||
|
return sourceID, ok
|
||||||
|
}
|
||||||
|
|
||||||
// ListConfigured returns configured entries sorted by configured key.
|
// ListConfigured returns configured entries sorted by configured key.
|
||||||
func (c *ArtifactCatalog) ListConfigured() []CatalogEntry {
|
func (c *ArtifactCatalog) ListConfigured() []CatalogEntry {
|
||||||
if c == nil || len(c.configuredIndex) == 0 {
|
if c == nil || len(c.configuredIndex) == 0 {
|
||||||
@@ -169,6 +244,23 @@ func (c *ArtifactCatalog) ListConfigured() []CatalogEntry {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListExtraction returns extraction entries sorted by configured output key.
|
||||||
|
func (c *ArtifactCatalog) ListExtraction() []CatalogEntry {
|
||||||
|
if c == nil || len(c.extractionIndex) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
keys := make([]string, 0, len(c.extractionIndex))
|
||||||
|
for key := range c.extractionIndex {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
out := make([]CatalogEntry, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
out = append(out, c.entries[c.extractionIndex[key]])
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// MarkAvailableGenerated marks one source as available in current analyze execution.
|
// MarkAvailableGenerated marks one source as available in current analyze execution.
|
||||||
func (c *ArtifactCatalog) MarkAvailableGenerated(sourceID, path string) error {
|
func (c *ArtifactCatalog) MarkAvailableGenerated(sourceID, path string) error {
|
||||||
return c.markAvailable(sourceID, path, ArtifactProvenanceGeneratedCurrentAnalyzeRun)
|
return c.markAvailable(sourceID, path, ArtifactProvenanceGeneratedCurrentAnalyzeRun)
|
||||||
@@ -179,6 +271,16 @@ func (c *ArtifactCatalog) MarkAvailableFromDisk(sourceID, path string) error {
|
|||||||
return c.markAvailable(sourceID, path, ArtifactProvenanceDisabledFromDisk)
|
return c.markAvailable(sourceID, path, ArtifactProvenanceDisabledFromDisk)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *ArtifactCatalog) markAvailableFromExtractManifest(sourceID, path, producerRunID string) error {
|
||||||
|
if err := c.markAvailable(sourceID, path, ArtifactProvenanceCurrentExtractManifest); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
entry := c.entries[strings.TrimSpace(sourceID)]
|
||||||
|
entry.ProducerRunID = strings.TrimSpace(producerRunID)
|
||||||
|
c.entries[entry.SourceID] = entry
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *ArtifactCatalog) markAvailable(sourceID, path, provenance string) error {
|
func (c *ArtifactCatalog) markAvailable(sourceID, path, provenance string) error {
|
||||||
if c == nil {
|
if c == nil {
|
||||||
return fmt.Errorf("artifact catalog is nil")
|
return fmt.Errorf("artifact catalog is nil")
|
||||||
|
|||||||
180
internal/artifacts/extraction_catalog.go
Normal file
180
internal/artifacts/extraction_catalog.go
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
extractStageName = "extract"
|
||||||
|
extractionLaneKind = "notarius_lane"
|
||||||
|
extractionIndexKind = "notarius_index"
|
||||||
|
extractionMetadataRun = "narratio_run_id"
|
||||||
|
extractionMetadataRoot = "bundle_root"
|
||||||
|
)
|
||||||
|
|
||||||
|
type hydratedExtraction struct {
|
||||||
|
sourceID string
|
||||||
|
path string
|
||||||
|
}
|
||||||
|
|
||||||
|
// HydrateExtractionArtifacts marks extraction sources available only when the current
|
||||||
|
// manifest contains one complete, internally consistent, succeeded extraction bundle.
|
||||||
|
// Invalid, stale, incomplete, or unsafe records leave every extraction source unavailable.
|
||||||
|
func (c *ArtifactCatalog) HydrateExtractionArtifacts(
|
||||||
|
paths SessionPaths,
|
||||||
|
m *manifest.Manifest,
|
||||||
|
configured map[string]ExtractionArtifactDefinition,
|
||||||
|
) {
|
||||||
|
if c == nil || m == nil || len(configured) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
record := m.Stages[extractStageName]
|
||||||
|
if record == nil || record.Name != extractStageName || record.Status != manifest.StatusSucceeded {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
producerRunID := extractionMetadataString(record.Metadata, extractionMetadataRun)
|
||||||
|
if !safeExtractionPathSegment(producerRunID) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
bundleRoot := filepath.Clean(filepath.Join(paths.ArtifactsDir, "notarius", producerRunID))
|
||||||
|
if !filepath.IsAbs(bundleRoot) || extractionMetadataString(record.Metadata, extractionMetadataRoot) != bundleRoot {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !safeExistingExtractionDirectory(paths.Root, bundleRoot) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
receiptRunID, receiptPipelineID := extractionReceiptIdentity(record.Metadata)
|
||||||
|
if receiptRunID == "" || receiptPipelineID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
expected := make(map[string]ExtractionArtifactDefinition, len(configured))
|
||||||
|
for key, definition := range configured {
|
||||||
|
sourceID, ok := c.SourceIDForExtractionKey(key)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
expected[sourceID] = definition
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := make(map[string]struct{}, len(expected))
|
||||||
|
hydrated := make([]hydratedExtraction, 0, len(expected))
|
||||||
|
indexSeen := false
|
||||||
|
for _, output := range record.Outputs {
|
||||||
|
if strings.TrimSpace(output.ProducerRunID) != producerRunID {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if output.SourceID == "" {
|
||||||
|
if indexSeen || output.Kind != extractionIndexKind || filepath.Clean(output.LocalPath) != filepath.Join(bundleRoot, "index.json") ||
|
||||||
|
!validExtractionPayload(bundleRoot, output.LocalPath, output.Checksum) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
indexSeen = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
definition, ok := expected[output.SourceID]
|
||||||
|
if !ok || output.Kind != extractionLaneKind {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, duplicate := seen[output.SourceID]; duplicate {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !compatibleCatalogExtractionContract(output.Contract, definition) ||
|
||||||
|
!compatibleCatalogExtractionProvenance(output.ExternalProvenance, receiptRunID, receiptPipelineID, definition) ||
|
||||||
|
!validExtractionPayload(bundleRoot, output.LocalPath, output.Checksum) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seen[output.SourceID] = struct{}{}
|
||||||
|
hydrated = append(hydrated, hydratedExtraction{sourceID: output.SourceID, path: output.LocalPath})
|
||||||
|
}
|
||||||
|
if !indexSeen || len(seen) != len(expected) || len(record.Outputs) != len(expected)+1 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, item := range hydrated {
|
||||||
|
_ = c.markAvailableFromExtractManifest(item.sourceID, item.path, producerRunID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func compatibleCatalogExtractionContract(got *artifactmodel.ContractMetadata, want ExtractionArtifactDefinition) bool {
|
||||||
|
return got != nil && got.MediaType == want.MediaType && got.SchemaID == want.SchemaID &&
|
||||||
|
got.SchemaVersion == want.SchemaVersion && (want.ModuleKey == "" || got.ModuleKey == want.ModuleKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func compatibleCatalogExtractionProvenance(
|
||||||
|
got *artifactmodel.ExternalProvenance,
|
||||||
|
runID, pipelineID string,
|
||||||
|
want ExtractionArtifactDefinition,
|
||||||
|
) bool {
|
||||||
|
return got != nil && got.System == "notarius" && got.RunID == runID && got.PipelineID == pipelineID &&
|
||||||
|
pipelineID == strings.TrimSpace(want.PipelineID) && got.ArtifactID == want.LaneID
|
||||||
|
}
|
||||||
|
|
||||||
|
func validExtractionPayload(bundleRoot, path, checksum string) bool {
|
||||||
|
if !filepath.IsAbs(path) || !pathWithinExtractionRoot(bundleRoot, path) || strings.TrimSpace(checksum) == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !safeExtractionComponents(bundleRoot, path) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
actual, err := SHA256File(path)
|
||||||
|
if err != nil || actual != checksum {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(path)
|
||||||
|
return err == nil && json.Valid(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeExistingExtractionDirectory(sessionRoot, bundleRoot string) bool {
|
||||||
|
if !pathWithinExtractionRoot(sessionRoot, bundleRoot) || !safeExtractionComponents(sessionRoot, bundleRoot) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(bundleRoot)
|
||||||
|
return err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeExtractionComponents(root, target string) bool {
|
||||||
|
relative, err := filepath.Rel(filepath.Clean(root), filepath.Clean(target))
|
||||||
|
if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
current := filepath.Clean(root)
|
||||||
|
for _, part := range strings.Split(relative, string(filepath.Separator)) {
|
||||||
|
current = filepath.Join(current, part)
|
||||||
|
info, err := os.Lstat(current)
|
||||||
|
if err != nil || info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathWithinExtractionRoot(root, target string) bool {
|
||||||
|
relative, err := filepath.Rel(filepath.Clean(root), filepath.Clean(target))
|
||||||
|
return err == nil && relative != "." && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeExtractionPathSegment(value string) bool {
|
||||||
|
return value != "" && value != "." && value != ".." && filepath.Base(value) == value &&
|
||||||
|
!strings.ContainsAny(value, `/\\`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractionMetadataString(metadata map[string]any, key string) string {
|
||||||
|
value, _ := metadata[key].(string)
|
||||||
|
return strings.TrimSpace(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractionReceiptIdentity(metadata map[string]any) (string, string) {
|
||||||
|
receipt, _ := metadata["receipt"].(map[string]any)
|
||||||
|
return extractionMetadataString(receipt, "run_id"), extractionMetadataString(receipt, "pipeline_id")
|
||||||
|
}
|
||||||
222
internal/artifacts/extraction_catalog_test.go
Normal file
222
internal/artifacts/extraction_catalog_test.go
Normal file
@@ -0,0 +1,222 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestArtifactCatalogRegistersExtractionArtifactsDeterministically(t *testing.T) {
|
||||||
|
catalog := NewArtifactCatalog()
|
||||||
|
if err := catalog.RegisterConfiguredArtifacts(map[string]ConfiguredArtifactDefinition{
|
||||||
|
"summary": {Enabled: true},
|
||||||
|
}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := catalog.RegisterExtractionArtifacts(map[string]ExtractionArtifactDefinition{
|
||||||
|
"zeta": {LaneID: "zeta"},
|
||||||
|
"summary": {LaneID: "summary"},
|
||||||
|
"alpha": {LaneID: "alpha"},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("RegisterExtractionArtifacts() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
entries := catalog.ListExtraction()
|
||||||
|
if len(entries) != 3 || entries[0].ExtractionKey != "alpha" || entries[1].ExtractionKey != "summary" || entries[2].ExtractionKey != "zeta" {
|
||||||
|
t.Fatalf("ListExtraction() = %#v, want alpha, summary, zeta", entries)
|
||||||
|
}
|
||||||
|
extractionID, ok := catalog.SourceIDForExtractionKey("summary")
|
||||||
|
if !ok || extractionID != "narratio.extraction.summary" {
|
||||||
|
t.Fatalf("SourceIDForExtractionKey(summary) = %q, %v", extractionID, ok)
|
||||||
|
}
|
||||||
|
configuredID, _ := catalog.SourceIDForConfiguredKey("summary")
|
||||||
|
if configuredID == extractionID {
|
||||||
|
t.Fatalf("configured and extraction source families collided at %q", extractionID)
|
||||||
|
}
|
||||||
|
entry, ok := catalog.Lookup(extractionID)
|
||||||
|
if !ok || entry.ProducerStage != "extract" || entry.OutputKind != "notarius_lane" || !entry.Planned {
|
||||||
|
t.Fatalf("Lookup(%q) = %#v, %v", extractionID, entry, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArtifactCatalogRejectsDuplicateExtractionRegistration(t *testing.T) {
|
||||||
|
catalog := NewArtifactCatalog()
|
||||||
|
definitions := map[string]ExtractionArtifactDefinition{"summary": {LaneID: "summary"}}
|
||||||
|
if err := catalog.RegisterExtractionArtifacts(definitions); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := catalog.RegisterExtractionArtifacts(definitions); err == nil {
|
||||||
|
t.Fatal("second RegisterExtractionArtifacts() error = nil, want collision error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHydrateExtractionArtifactsAcceptsOnlyCompleteCurrentBundle(t *testing.T) {
|
||||||
|
paths, currentManifest, definitions := validExtractionCatalogFixture(t)
|
||||||
|
catalog := registeredExtractionCatalog(t, definitions)
|
||||||
|
catalog.HydrateExtractionArtifacts(paths, currentManifest, definitions)
|
||||||
|
|
||||||
|
entry, ok := catalog.Lookup(ExtractionArtifactSourceID("encounters"))
|
||||||
|
if !ok || !entry.Available {
|
||||||
|
t.Fatalf("hydrated entry = %#v, %v; want available", entry, ok)
|
||||||
|
}
|
||||||
|
if entry.Provenance != ArtifactProvenanceCurrentExtractManifest || entry.ProducerRunID != "extract-run-1" {
|
||||||
|
t.Fatalf("hydrated provenance = %#v", entry)
|
||||||
|
}
|
||||||
|
resolved, err := ResolveSessionArtifactWithCatalog(paths, currentManifest, entry.SourceID, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolveSessionArtifactWithCatalog() error = %v", err)
|
||||||
|
}
|
||||||
|
if resolved.Path != entry.Path || resolved.ProducerRunID != "extract-run-1" {
|
||||||
|
t.Fatalf("resolved = %#v", resolved)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHydrateExtractionArtifactsRejectsUntrustedManifestState(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(t *testing.T, paths SessionPaths, m *manifest.Manifest)
|
||||||
|
}{
|
||||||
|
{name: "missing stage", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) { delete(m.Stages, "extract") }},
|
||||||
|
{name: "skipped", mutate: setExtractionStatus(manifest.StatusSkipped)},
|
||||||
|
{name: "failed", mutate: setExtractionStatus(manifest.StatusFailed)},
|
||||||
|
{name: "stale", mutate: setExtractionStatus(manifest.StatusStale)},
|
||||||
|
{name: "interrupted", mutate: setExtractionStatus(manifest.StatusInterrupted)},
|
||||||
|
{name: "missing source", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs = m.Stages["extract"].Outputs[1:]
|
||||||
|
}},
|
||||||
|
{name: "inconsistent producer identity", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs[0].ProducerRunID = "another-run"
|
||||||
|
}},
|
||||||
|
{name: "incompatible contract", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs[0].Contract.SchemaVersion = "99"
|
||||||
|
}},
|
||||||
|
{name: "incompatible provenance", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs[0].ExternalProvenance.RunID = "another-notarius-run"
|
||||||
|
}},
|
||||||
|
{name: "missing file", mutate: func(t *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||||
|
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "tampered checksum", mutate: func(t *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||||
|
if err := os.WriteFile(m.Stages["extract"].Outputs[0].LocalPath, []byte(`{"tampered":true}`), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "unsafe path", mutate: func(t *testing.T, paths SessionPaths, m *manifest.Manifest) {
|
||||||
|
outside := filepath.Join(paths.Root, "incidental.json")
|
||||||
|
writeExtractionFixtureFile(t, outside, `{"incidental":true}`)
|
||||||
|
m.Stages["extract"].Outputs[0].LocalPath = outside
|
||||||
|
m.Stages["extract"].Outputs[0].Checksum = extractionFixtureChecksum(t, outside)
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
paths, currentManifest, definitions := validExtractionCatalogFixture(t)
|
||||||
|
test.mutate(t, paths, currentManifest)
|
||||||
|
catalog := registeredExtractionCatalog(t, definitions)
|
||||||
|
catalog.HydrateExtractionArtifacts(paths, currentManifest, definitions)
|
||||||
|
entry, _ := catalog.Lookup(ExtractionArtifactSourceID("encounters"))
|
||||||
|
if entry.Available {
|
||||||
|
t.Fatalf("entry became available from %s manifest", test.name)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveExtractionArtifactNeverDiscoversIncidentalBundleFile(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
paths := SessionPaths{Root: root, ArtifactsDir: filepath.Join(root, "artifacts")}
|
||||||
|
incidental := filepath.Join(paths.ArtifactsDir, "notarius", "incidental", "lanes", "encounters.json")
|
||||||
|
writeExtractionFixtureFile(t, incidental, `{"encounters":[]}`)
|
||||||
|
definitions := extractionFixtureDefinitions()
|
||||||
|
catalog := registeredExtractionCatalog(t, definitions)
|
||||||
|
|
||||||
|
_, err := ResolveSessionArtifactWithCatalog(paths, manifest.New("session", fixtureTime), ExtractionArtifactSourceID("encounters"), catalog)
|
||||||
|
if err == nil || !errors.Is(err, ErrSessionArtifactNotFound) {
|
||||||
|
t.Fatalf("ResolveSessionArtifactWithCatalog() error = %v, want not found", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var fixtureTime = mustFixtureTime()
|
||||||
|
|
||||||
|
func mustFixtureTime() (value time.Time) {
|
||||||
|
return time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setExtractionStatus(status manifest.StageStatus) func(*testing.T, SessionPaths, *manifest.Manifest) {
|
||||||
|
return func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) { m.Stages["extract"].Status = status }
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractionFixtureDefinitions() map[string]ExtractionArtifactDefinition {
|
||||||
|
return map[string]ExtractionArtifactDefinition{
|
||||||
|
"encounters": {
|
||||||
|
LaneID: "encounters", PipelineID: "campaign.extract", MediaType: "application/json",
|
||||||
|
SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func registeredExtractionCatalog(t *testing.T, definitions map[string]ExtractionArtifactDefinition) *ArtifactCatalog {
|
||||||
|
t.Helper()
|
||||||
|
catalog := NewArtifactCatalog()
|
||||||
|
if err := catalog.RegisterExtractionArtifacts(definitions); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return catalog
|
||||||
|
}
|
||||||
|
|
||||||
|
func validExtractionCatalogFixture(t *testing.T) (SessionPaths, *manifest.Manifest, map[string]ExtractionArtifactDefinition) {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
paths := SessionPaths{Root: root, ArtifactsDir: filepath.Join(root, "artifacts")}
|
||||||
|
bundleRoot := filepath.Join(paths.ArtifactsDir, "notarius", "extract-run-1")
|
||||||
|
lanePath := filepath.Join(bundleRoot, "lanes", "encounters.json")
|
||||||
|
indexPath := filepath.Join(bundleRoot, "index.json")
|
||||||
|
writeExtractionFixtureFile(t, lanePath, `{"encounters":[]}`)
|
||||||
|
writeExtractionFixtureFile(t, indexPath, `{"lanes":[]}`)
|
||||||
|
definitions := extractionFixtureDefinitions()
|
||||||
|
m := manifest.New("session", fixtureTime)
|
||||||
|
m.Stages["extract"] = &manifest.StageRecord{
|
||||||
|
Name: "extract", Status: manifest.StatusSucceeded,
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"narratio_run_id": "extract-run-1", "bundle_root": bundleRoot,
|
||||||
|
"receipt": map[string]any{"run_id": "notarius-run-1", "pipeline_id": "campaign.extract"},
|
||||||
|
},
|
||||||
|
Outputs: []manifest.ArtifactRecord{
|
||||||
|
{
|
||||||
|
Kind: "notarius_lane", SourceID: ExtractionArtifactSourceID("encounters"), LocalPath: lanePath,
|
||||||
|
ProducerRunID: "extract-run-1", Checksum: extractionFixtureChecksum(t, lanePath),
|
||||||
|
Contract: &artifactmodel.ContractMetadata{MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters"},
|
||||||
|
ExternalProvenance: &artifactmodel.ExternalProvenance{System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: "encounters"},
|
||||||
|
},
|
||||||
|
{Kind: "notarius_index", LocalPath: indexPath, ProducerRunID: "extract-run-1", Checksum: extractionFixtureChecksum(t, indexPath)},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return paths, m, definitions
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeExtractionFixtureFile(t *testing.T, path, body string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractionFixtureChecksum(t *testing.T, path string) string {
|
||||||
|
t.Helper()
|
||||||
|
checksum, err := SHA256File(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return checksum
|
||||||
|
}
|
||||||
@@ -86,6 +86,36 @@ func SessionRunStageDirForCampaign(rootDir, campaign, sessionID, runID, stageNam
|
|||||||
return filepath.Join(SessionRunRootForCampaign(rootDir, campaign, sessionID, runID), stageName)
|
return filepath.Join(SessionRunRootForCampaign(rootDir, campaign, sessionID, runID), stageName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SessionRunExtractDirForCampaign returns the invocation-local extraction directory.
|
||||||
|
func SessionRunExtractDirForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||||
|
return SessionRunStageDirForCampaign(rootDir, campaign, sessionID, runID, "extract")
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionRunNotariusReceiptPathForCampaign returns the invocation-local receipt path.
|
||||||
|
func SessionRunNotariusReceiptPathForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||||
|
return filepath.Join(SessionRunExtractDirForCampaign(rootDir, campaign, sessionID, runID), "notarius.receipt.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionRunNotariusLogPathForCampaign returns the invocation-local stderr log path.
|
||||||
|
func SessionRunNotariusLogPathForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||||
|
return filepath.Join(SessionRunExtractDirForCampaign(rootDir, campaign, sessionID, runID), "notarius.stderr.log")
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionRunNotariusOutputRootForCampaign returns the invocation-local Notarius output root.
|
||||||
|
func SessionRunNotariusOutputRootForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||||
|
return filepath.Join(SessionRunExtractDirForCampaign(rootDir, campaign, sessionID, runID), "notarius-output")
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionNotariusBundleDirForCampaign returns one immutable durable bundle destination.
|
||||||
|
func SessionNotariusBundleDirForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||||
|
return filepath.Join(
|
||||||
|
SessionWorkDirForCampaign(rootDir, campaign, sessionID),
|
||||||
|
config.PathArtifactsDirSegment,
|
||||||
|
"notarius",
|
||||||
|
runID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// SessionSpoolAudioDir returns the campaign/session/run scoped local spool audio path.
|
// SessionSpoolAudioDir returns the campaign/session/run scoped local spool audio path.
|
||||||
func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
|
func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
|
||||||
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
|
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
|
||||||
|
|||||||
@@ -49,6 +49,33 @@ func TestSessionRunManifestPathForCampaign(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSessionNotariusPathsForCampaign(t *testing.T) {
|
||||||
|
root := "/tmp/workspace"
|
||||||
|
campaign := "forsaken"
|
||||||
|
sessionID := "2026-04-19"
|
||||||
|
runID := "20260515T031522Z-a1b2c3d4"
|
||||||
|
extractDir := filepath.Join(root, "work", campaign, sessionID, "runs", runID, "extract")
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
got string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "extract directory", got: SessionRunExtractDirForCampaign(root, campaign, sessionID, runID), want: extractDir},
|
||||||
|
{name: "receipt", got: SessionRunNotariusReceiptPathForCampaign(root, campaign, sessionID, runID), want: filepath.Join(extractDir, "notarius.receipt.json")},
|
||||||
|
{name: "stderr", got: SessionRunNotariusLogPathForCampaign(root, campaign, sessionID, runID), want: filepath.Join(extractDir, "notarius.stderr.log")},
|
||||||
|
{name: "output root", got: SessionRunNotariusOutputRootForCampaign(root, campaign, sessionID, runID), want: filepath.Join(extractDir, "notarius-output")},
|
||||||
|
{name: "durable bundle", got: SessionNotariusBundleDirForCampaign(root, campaign, sessionID, runID), want: filepath.Join(root, "work", campaign, sessionID, "artifacts", "notarius", runID)},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if test.got != test.want {
|
||||||
|
t.Fatalf("path = %q, want %q", test.got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSessionPreviousPathsForCampaign(t *testing.T) {
|
func TestSessionPreviousPathsForCampaign(t *testing.T) {
|
||||||
root := "/tmp/workspace"
|
root := "/tmp/workspace"
|
||||||
previousDir := SessionPreviousDirForCampaign(root, "forsaken", "2026-04-19")
|
previousDir := SessionPreviousDirForCampaign(root, "forsaken", "2026-04-19")
|
||||||
|
|||||||
@@ -1,16 +1,23 @@
|
|||||||
package artifacts
|
package artifacts
|
||||||
|
|
||||||
import "os"
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
|
)
|
||||||
|
|
||||||
// Ref identifies a pipeline artifact and its local/remote coordinates.
|
// Ref identifies a pipeline artifact and its local/remote coordinates.
|
||||||
type Ref struct {
|
type Ref struct {
|
||||||
Kind string
|
Kind string
|
||||||
Category string
|
SourceID string
|
||||||
SessionID string
|
Category string
|
||||||
RelativePath string
|
SessionID string
|
||||||
AbsolutePath string
|
RelativePath string
|
||||||
RemoteKey string
|
AbsolutePath string
|
||||||
Checksum string
|
RemoteKey string
|
||||||
|
Checksum string
|
||||||
|
Contract *artifactmodel.ContractMetadata
|
||||||
|
ExternalProvenance *artifactmodel.ExternalProvenance
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store is the local artifact/workdir abstraction used by orchestration code.
|
// Store is the local artifact/workdir abstraction used by orchestration code.
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ type PipelineConfig struct {
|
|||||||
Trim *TrimConfig `yaml:"trim"`
|
Trim *TrimConfig `yaml:"trim"`
|
||||||
Render *RenderConfig `yaml:"render"`
|
Render *RenderConfig `yaml:"render"`
|
||||||
Scriptorium *ScriptoriumConfig `yaml:"scriptorium"`
|
Scriptorium *ScriptoriumConfig `yaml:"scriptorium"`
|
||||||
|
Notarius *NotariusConfig `yaml:"notarius"`
|
||||||
Notification NotificationConfig `yaml:"notification"`
|
Notification NotificationConfig `yaml:"notification"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -252,6 +253,26 @@ type ScriptoriumInputConfig struct {
|
|||||||
Required bool `yaml:"required"`
|
Required bool `yaml:"required"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NotariusConfig configures structured artifact extraction by Notarius.
|
||||||
|
type NotariusConfig struct {
|
||||||
|
Enabled bool `yaml:"enabled"`
|
||||||
|
Binary string `yaml:"binary"`
|
||||||
|
ConfigPath string `yaml:"config_path"`
|
||||||
|
PipelineID string `yaml:"pipeline_id"`
|
||||||
|
Timeout string `yaml:"timeout"`
|
||||||
|
WorkingDirectory string `yaml:"working_directory"`
|
||||||
|
Outputs map[string]NotariusOutputConfig `yaml:"outputs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NotariusOutputConfig declares one required extraction lane contract.
|
||||||
|
type NotariusOutputConfig struct {
|
||||||
|
LaneID string `yaml:"lane_id"`
|
||||||
|
MediaType string `yaml:"media_type"`
|
||||||
|
SchemaID string `yaml:"schema_id"`
|
||||||
|
SchemaVersion string `yaml:"schema_version"`
|
||||||
|
ModuleKey string `yaml:"module_key"`
|
||||||
|
}
|
||||||
|
|
||||||
// NotificationConfig configures notification backend settings.
|
// NotificationConfig configures notification backend settings.
|
||||||
type NotificationConfig struct {
|
type NotificationConfig struct {
|
||||||
Backend string `yaml:"backend"`
|
Backend string `yaml:"backend"`
|
||||||
|
|||||||
@@ -30,9 +30,11 @@ const (
|
|||||||
DefaultSeriatimCoalesceGap = 3.0
|
DefaultSeriatimCoalesceGap = 3.0
|
||||||
DefaultSeriatimReport = true
|
DefaultSeriatimReport = true
|
||||||
|
|
||||||
DefaultAuditaBinary = "audita"
|
DefaultAuditaBinary = "audita"
|
||||||
DefaultAuditaTimeout = "3h"
|
DefaultAuditaTimeout = "3h"
|
||||||
DefaultAuditaReport = true
|
DefaultAuditaReport = true
|
||||||
|
DefaultNotariusBinary = "notarius"
|
||||||
|
DefaultNotariusTimeout = "3h"
|
||||||
|
|
||||||
DefaultScriptoriumBinary = "scriptorium"
|
DefaultScriptoriumBinary = "scriptorium"
|
||||||
DefaultScriptoriumTimeout = "10m"
|
DefaultScriptoriumTimeout = "10m"
|
||||||
|
|||||||
@@ -19,6 +19,9 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
|
|||||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||||
}
|
}
|
||||||
applyPipelineDefaults(&cfg)
|
applyPipelineDefaults(&cfg)
|
||||||
|
if err := resolveNotariusPaths(&cfg, path); err != nil {
|
||||||
|
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||||
|
}
|
||||||
return &cfg, nil
|
return &cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -86,12 +89,17 @@ func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOpti
|
|||||||
|
|
||||||
// LoadPublishLockStoreBytes loads a mutable session lock store with strict
|
// LoadPublishLockStoreBytes loads a mutable session lock store with strict
|
||||||
// field checking and source validation.
|
// field checking and source validation.
|
||||||
func LoadPublishLockStoreBytes(label string, data []byte, scriptorium *ScriptoriumConfig) (*PublishLockStore, error) {
|
func LoadPublishLockStoreBytes(
|
||||||
|
label string,
|
||||||
|
data []byte,
|
||||||
|
scriptorium *ScriptoriumConfig,
|
||||||
|
notarius *NotariusConfig,
|
||||||
|
) (*PublishLockStore, error) {
|
||||||
var store PublishLockStore
|
var store PublishLockStore
|
||||||
if err := decodeStrictYAMLFromReader("publish lock store", label, strings.NewReader(string(data)), &store); err != nil {
|
if err := decodeStrictYAMLFromReader("publish lock store", label, strings.NewReader(string(data)), &store); err != nil {
|
||||||
return nil, fmt.Errorf("load publish lock store: %w", err)
|
return nil, fmt.Errorf("load publish lock store: %w", err)
|
||||||
}
|
}
|
||||||
locks, err := ValidatePublishLockRules(store.Locks, scriptorium, "locks")
|
locks, err := ValidatePublishLockRules(store.Locks, scriptorium, notarius, "locks")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("load publish lock store: %w", err)
|
return nil, fmt.Errorf("load publish lock store: %w", err)
|
||||||
}
|
}
|
||||||
@@ -359,6 +367,7 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
|
|||||||
}
|
}
|
||||||
applyRenderDefaults(&cfg.Render)
|
applyRenderDefaults(&cfg.Render)
|
||||||
applyScriptoriumDefaults(cfg.Scriptorium)
|
applyScriptoriumDefaults(cfg.Scriptorium)
|
||||||
|
applyNotariusDefaults(cfg.Notarius)
|
||||||
}
|
}
|
||||||
|
|
||||||
func applyCampaignsDefaults(cfg *CampaignsConfig) {
|
func applyCampaignsDefaults(cfg *CampaignsConfig) {
|
||||||
@@ -516,6 +525,57 @@ func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func applyNotariusDefaults(cfg *NotariusConfig) {
|
||||||
|
if cfg == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cfg.Binary == "" {
|
||||||
|
cfg.Binary = DefaultNotariusBinary
|
||||||
|
}
|
||||||
|
if cfg.Timeout == "" {
|
||||||
|
cfg.Timeout = DefaultNotariusTimeout
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
|
||||||
|
if cfg == nil || cfg.Notarius == nil || !cfg.Notarius.Enabled {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
pipelineAbs, err := filepath.Abs(pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("resolve pipeline config path %q: %w", pipelinePath, err)
|
||||||
|
}
|
||||||
|
baseDir := filepath.Dir(pipelineAbs)
|
||||||
|
resolve := func(value string) (string, error) {
|
||||||
|
trimmed := strings.TrimSpace(value)
|
||||||
|
if trimmed == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(trimmed) {
|
||||||
|
trimmed = filepath.Join(baseDir, trimmed)
|
||||||
|
}
|
||||||
|
return filepath.Abs(trimmed)
|
||||||
|
}
|
||||||
|
|
||||||
|
resolvedConfigPath, err := resolve(cfg.Notarius.ConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("resolve pipeline.notarius.config_path: %w", err)
|
||||||
|
}
|
||||||
|
cfg.Notarius.ConfigPath = resolvedConfigPath
|
||||||
|
if strings.TrimSpace(cfg.Notarius.WorkingDirectory) == "" {
|
||||||
|
if resolvedConfigPath != "" {
|
||||||
|
cfg.Notarius.WorkingDirectory = filepath.Dir(resolvedConfigPath)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
workingDirectory, err := resolve(cfg.Notarius.WorkingDirectory)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("resolve pipeline.notarius.working_directory: %w", err)
|
||||||
|
}
|
||||||
|
cfg.Notarius.WorkingDirectory = workingDirectory
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func applyTrimDefaults(cfg *TrimConfig) {
|
func applyTrimDefaults(cfg *TrimConfig) {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package config
|
|||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
@@ -1049,6 +1050,11 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
|||||||
pipelineFile: "pipeline.full.annotated.yml",
|
pipelineFile: "pipeline.full.annotated.yml",
|
||||||
sessionFile: "session.local-audio.yml",
|
sessionFile: "session.local-audio.yml",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "extraction subset pipeline with local audio session",
|
||||||
|
pipelineFile: "pipeline.extraction-subset.yml",
|
||||||
|
sessionFile: "session.local-audio.yml",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -1068,6 +1074,47 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaintainedExtractionExamplesPreservePublishedContracts(t *testing.T) {
|
||||||
|
examplesDir := filepath.Join("..", "..", "examples")
|
||||||
|
full, err := LoadPipeline(filepath.Join(examplesDir, "pipeline.full.annotated.yml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load full example error = %v", err)
|
||||||
|
}
|
||||||
|
want := map[string]NotariusOutputConfig{
|
||||||
|
"item_registry": {LaneID: "item-registry", MediaType: "application/json", SchemaID: "notarius.dnd.item_registry", SchemaVersion: "v1", ModuleKey: "dnd/item-registry"},
|
||||||
|
"npc_registry": {LaneID: "npc-registry", MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry", SchemaVersion: "v1", ModuleKey: "dnd/npc-registry"},
|
||||||
|
"location_registry": {LaneID: "location-registry", MediaType: "application/json", SchemaID: "notarius.dnd.location_registry", SchemaVersion: "v1", ModuleKey: "dnd/location-registry"},
|
||||||
|
"scene_descriptions": {LaneID: "scene-descriptions", MediaType: "application/json", SchemaID: "notarius.dnd.scene_descriptions", SchemaVersion: "v1", ModuleKey: "dnd/scene-descriptions"},
|
||||||
|
"item_occurrences": {LaneID: "item-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.item_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/item-occurrences"},
|
||||||
|
"spells": {LaneID: "spells", MediaType: "application/json", SchemaID: "notarius.dnd.spells", SchemaVersion: "v1", ModuleKey: "dnd/spells"},
|
||||||
|
"combat_turns": {LaneID: "combat-turns", MediaType: "application/json", SchemaID: "notarius.dnd.combat_turns", SchemaVersion: "v1", ModuleKey: "dnd/combat-turns"},
|
||||||
|
"npc_occurrences": {LaneID: "npc-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.npc_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/npc-occurrences"},
|
||||||
|
"location_occurrences": {LaneID: "location-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.location_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/location-occurrences"},
|
||||||
|
"enemy_events": {LaneID: "enemy-events", MediaType: "application/json", SchemaID: "notarius.dnd.enemy_events", SchemaVersion: "v1", ModuleKey: "dnd/enemy-events"},
|
||||||
|
}
|
||||||
|
if full.Notarius == nil || !reflect.DeepEqual(full.Notarius.Outputs, want) {
|
||||||
|
t.Fatalf("full example outputs = %#v, want %#v", full.Notarius, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
subset, err := LoadPipeline(filepath.Join(examplesDir, "pipeline.extraction-subset.yml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load subset example error = %v", err)
|
||||||
|
}
|
||||||
|
brief := subset.Scriptorium.Artifacts["session_brief"]
|
||||||
|
wantSources := map[string]string{
|
||||||
|
"npcs": "narratio.extraction.npc_registry",
|
||||||
|
"locations": "narratio.extraction.location_registry",
|
||||||
|
"scenes": "narratio.extraction.scene_descriptions",
|
||||||
|
}
|
||||||
|
gotSources := make(map[string]string, len(brief.Inputs))
|
||||||
|
for name, input := range brief.Inputs {
|
||||||
|
gotSources[name] = input.Source
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(gotSources, wantSources) {
|
||||||
|
t.Fatalf("subset example sources = %#v, want %#v", gotSources, wantSources)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, string) {
|
func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if !strings.Contains(pipelineYAML, "\naudita:") && !strings.HasPrefix(pipelineYAML, "audita:") {
|
if !strings.Contains(pipelineYAML, "\naudita:") && !strings.HasPrefix(pipelineYAML, "audita:") {
|
||||||
|
|||||||
283
internal/config/notarius_test.go
Normal file
283
internal/config/notarius_test.go
Normal file
@@ -0,0 +1,283 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNotariusOmittedAndDisabledBehavior(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
omittedPath := filepath.Join(dir, "omitted.yml")
|
||||||
|
if err := os.WriteFile(omittedPath, []byte(testPipelineBaseYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write omitted pipeline: %v", err)
|
||||||
|
}
|
||||||
|
omitted, err := LoadPipeline(omittedPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadPipeline(omitted) error = %v", err)
|
||||||
|
}
|
||||||
|
if omitted.Notarius != nil {
|
||||||
|
t.Fatalf("Notarius = %#v, want nil when omitted", omitted.Notarius)
|
||||||
|
}
|
||||||
|
|
||||||
|
disabledPath := filepath.Join(dir, "disabled.yml")
|
||||||
|
disabledYAML := testPipelineBaseYAML + `
|
||||||
|
notarius:
|
||||||
|
enabled: false
|
||||||
|
config_path: relative/notarius.yml
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(disabledPath, []byte(disabledYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write disabled pipeline: %v", err)
|
||||||
|
}
|
||||||
|
disabled, err := LoadPipeline(disabledPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadPipeline(disabled) error = %v", err)
|
||||||
|
}
|
||||||
|
if disabled.Notarius == nil {
|
||||||
|
t.Fatal("Notarius = nil, want configured disabled section")
|
||||||
|
}
|
||||||
|
if disabled.Notarius.Enabled {
|
||||||
|
t.Fatal("Notarius.Enabled = true, want false")
|
||||||
|
}
|
||||||
|
if disabled.Notarius.Binary != DefaultNotariusBinary || disabled.Notarius.Timeout != DefaultNotariusTimeout {
|
||||||
|
t.Fatalf("disabled defaults = %#v", disabled.Notarius)
|
||||||
|
}
|
||||||
|
if disabled.Notarius.ConfigPath != "relative/notarius.yml" || disabled.Notarius.WorkingDirectory != "" {
|
||||||
|
t.Fatalf("disabled paths were resolved unexpectedly: %#v", disabled.Notarius)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotariusEnabledDefaultsAndPathResolution(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
pipelinePath := filepath.Join(dir, "deployment", "pipeline.yml")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(pipelinePath), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll() error = %v", err)
|
||||||
|
}
|
||||||
|
pipelineYAML := testPipelineBaseYAML + `
|
||||||
|
notarius:
|
||||||
|
enabled: true
|
||||||
|
config_path: notarius/config.yml
|
||||||
|
pipeline_id: dnd-session
|
||||||
|
outputs:
|
||||||
|
npc_registry:
|
||||||
|
lane_id: npc-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_registry
|
||||||
|
schema_version: v1
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err := LoadPipeline(pipelinePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadPipeline() error = %v", err)
|
||||||
|
}
|
||||||
|
wantConfigPath := filepath.Join(filepath.Dir(pipelinePath), "notarius", "config.yml")
|
||||||
|
if cfg.Notarius.Binary != "notarius" || cfg.Notarius.Timeout != "3h" {
|
||||||
|
t.Fatalf("defaults = %#v", cfg.Notarius)
|
||||||
|
}
|
||||||
|
if cfg.Notarius.ConfigPath != wantConfigPath {
|
||||||
|
t.Fatalf("config_path = %q, want %q", cfg.Notarius.ConfigPath, wantConfigPath)
|
||||||
|
}
|
||||||
|
if cfg.Notarius.WorkingDirectory != filepath.Dir(wantConfigPath) {
|
||||||
|
t.Fatalf("working_directory = %q, want %q", cfg.Notarius.WorkingDirectory, filepath.Dir(wantConfigPath))
|
||||||
|
}
|
||||||
|
|
||||||
|
explicitPath := filepath.Join(dir, "explicit.yml")
|
||||||
|
explicitYAML := strings.Replace(pipelineYAML, " pipeline_id: dnd-session\n", " pipeline_id: dnd-session\n working_directory: runtime\n", 1)
|
||||||
|
if err := os.WriteFile(explicitPath, []byte(explicitYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write explicit pipeline: %v", err)
|
||||||
|
}
|
||||||
|
explicit, err := LoadPipeline(explicitPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadPipeline(explicit working directory) error = %v", err)
|
||||||
|
}
|
||||||
|
if explicit.Notarius.WorkingDirectory != filepath.Join(dir, "runtime") {
|
||||||
|
t.Fatalf("explicit working_directory = %q, want %q", explicit.Notarius.WorkingDirectory, filepath.Join(dir, "runtime"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotariusStrictYAML(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
yaml string
|
||||||
|
}{
|
||||||
|
{name: "unknown section field", yaml: "notarius:\n unknown: true\n"},
|
||||||
|
{name: "unknown output field", yaml: "notarius:\n outputs:\n npc_registry:\n lane_id: npc-registry\n unknown: true\n"},
|
||||||
|
{name: "unsupported session id", yaml: "notarius:\n session_id: forbidden\n"},
|
||||||
|
{name: "unsupported model", yaml: "notarius:\n model: forbidden\n"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "pipeline.yml")
|
||||||
|
if err := os.WriteFile(path, []byte(testPipelineBaseYAML+"\n"+tt.yaml), 0o644); err != nil {
|
||||||
|
t.Fatalf("write pipeline: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := LoadPipeline(path); err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
||||||
|
t.Fatalf("LoadPipeline() error = %v, want strict decode failure", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotariusEnabledValidation(t *testing.T) {
|
||||||
|
valid := `notarius:
|
||||||
|
enabled: true
|
||||||
|
config_path: ./notarius.yml
|
||||||
|
pipeline_id: dnd-session
|
||||||
|
timeout: 45m
|
||||||
|
outputs:
|
||||||
|
npc_registry:
|
||||||
|
lane_id: npc-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/npc-registry
|
||||||
|
`
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
section string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{name: "valid", section: valid},
|
||||||
|
{name: "blank binary", section: strings.Replace(valid, " enabled: true\n", " enabled: true\n binary: \" \"\n", 1), wantErr: "pipeline.notarius.binary is required"},
|
||||||
|
{name: "missing config path", section: strings.Replace(valid, " config_path: ./notarius.yml\n", "", 1), wantErr: "pipeline.notarius.config_path is required"},
|
||||||
|
{name: "missing pipeline id", section: strings.Replace(valid, " pipeline_id: dnd-session\n", "", 1), wantErr: "pipeline.notarius.pipeline_id is required"},
|
||||||
|
{name: "missing outputs", section: strings.Split(valid, " outputs:\n")[0], wantErr: "pipeline.notarius.outputs must contain at least one output"},
|
||||||
|
{name: "zero timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: 0s", 1), wantErr: "pipeline.notarius.timeout must be positive"},
|
||||||
|
{name: "negative timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: -1s", 1), wantErr: "pipeline.notarius.timeout must be positive"},
|
||||||
|
{name: "invalid timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: later", 1), wantErr: "pipeline.notarius.timeout must be a valid duration"},
|
||||||
|
{name: "invalid output key", section: strings.Replace(valid, " npc_registry:", " npc-registry:", 1), wantErr: "outputs keys must match"},
|
||||||
|
{name: "missing lane", section: strings.Replace(valid, " lane_id: npc-registry\n", "", 1), wantErr: "lane_id is required"},
|
||||||
|
{name: "missing media type", section: strings.Replace(valid, " media_type: application/json\n", "", 1), wantErr: "media_type is required"},
|
||||||
|
{name: "missing schema id", section: strings.Replace(valid, " schema_id: notarius.dnd.npc_registry\n", "", 1), wantErr: "schema_id is required"},
|
||||||
|
{name: "missing schema version", section: strings.Replace(valid, " schema_version: v1\n", "", 1), wantErr: "schema_version is required"},
|
||||||
|
{
|
||||||
|
name: "normalized key collision",
|
||||||
|
section: strings.Replace(valid, " module_key: dnd/npc-registry\n", ` module_key: dnd/npc-registry
|
||||||
|
" npc_registry ":
|
||||||
|
lane_id: npc-registry-two
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: two
|
||||||
|
schema_version: v1
|
||||||
|
`, 1),
|
||||||
|
wantErr: "normalize to duplicate source",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate normalized lane",
|
||||||
|
section: strings.Replace(valid, " module_key: dnd/npc-registry\n", ` module_key: dnd/npc-registry
|
||||||
|
spells:
|
||||||
|
lane_id: " npc-registry "
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: two
|
||||||
|
schema_version: v1
|
||||||
|
`, 1),
|
||||||
|
wantErr: "duplicates pipeline.notarius.outputs.npc_registry.lane_id",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML+"\n"+tt.section, testSessionBaseYAML)
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
err = Validate(cfg)
|
||||||
|
if tt.wantErr == "" {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
output := cfg.Pipeline.Notarius.Outputs["npc_registry"]
|
||||||
|
if output.LaneID != "npc-registry" || output.ModuleKey != "dnd/npc-registry" {
|
||||||
|
t.Fatalf("normalized output = %#v", output)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("Validate() error = %v, want containing %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractionReferencesRequireDeclaredOutput(t *testing.T) {
|
||||||
|
declared := `notarius:
|
||||||
|
enabled: false
|
||||||
|
outputs:
|
||||||
|
npc_registry: {}
|
||||||
|
`
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
body string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "scriptorium declared extraction",
|
||||||
|
body: declared + `scriptorium:
|
||||||
|
artifacts:
|
||||||
|
recap:
|
||||||
|
inputs:
|
||||||
|
npcs:
|
||||||
|
source: narratio.extraction.npc_registry
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "scriptorium unknown extraction",
|
||||||
|
body: declared + `scriptorium:
|
||||||
|
artifacts:
|
||||||
|
recap:
|
||||||
|
inputs:
|
||||||
|
npcs:
|
||||||
|
source: narratio.extraction.unknown
|
||||||
|
`,
|
||||||
|
wantErr: `references unknown extraction output "unknown"`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "publish declared extraction",
|
||||||
|
body: declared + `publish:
|
||||||
|
outputs:
|
||||||
|
- source: narratio.extraction.npc_registry
|
||||||
|
dest: artifacts/npc-registry.json
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "publish unknown extraction",
|
||||||
|
body: declared + `publish:
|
||||||
|
outputs:
|
||||||
|
- source: narratio.extraction.unknown
|
||||||
|
dest: artifacts/unknown.json
|
||||||
|
`,
|
||||||
|
wantErr: `extraction output "unknown" is not defined`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "publish lock declared extraction",
|
||||||
|
body: declared + `publish:
|
||||||
|
locks:
|
||||||
|
- source: narratio.extraction.npc_registry
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML+"\n"+tt.body, testSessionBaseYAML)
|
||||||
|
cfg, err := Load(pipelinePath, sessionPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
err = Validate(cfg)
|
||||||
|
if tt.wantErr == "" {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("Validate() error = %v, want containing %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -537,7 +537,7 @@ func TestPublishLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
|
|||||||
store, err := LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
store, err := LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
reason: reviewed
|
reason: reviewed
|
||||||
`), nil)
|
`), nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -548,7 +548,7 @@ func TestPublishLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
|
|||||||
_, err = LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
_, err = LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
dest: transcripts/final.trimmed.json
|
dest: transcripts/final.trimmed.json
|
||||||
`), nil)
|
`), nil, nil)
|
||||||
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
||||||
t.Fatalf("unknown field error = %v, want strict decode failed", err)
|
t.Fatalf("unknown field error = %v, want strict decode failed", err)
|
||||||
}
|
}
|
||||||
@@ -556,7 +556,7 @@ func TestPublishLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
|
|||||||
_, err = LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
_, err = LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
- source: narratio.transcript.final_trimmed
|
- source: narratio.transcript.final_trimmed
|
||||||
`), nil)
|
`), nil, nil)
|
||||||
if err == nil || !strings.Contains(err.Error(), "duplicates another publish lock source") {
|
if err == nil || !strings.Contains(err.Error(), "duplicates another publish lock source") {
|
||||||
t.Fatalf("duplicate error = %v", err)
|
t.Fatalf("duplicate error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,9 +6,11 @@ import (
|
|||||||
"net/url"
|
"net/url"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||||
)
|
)
|
||||||
@@ -85,7 +87,10 @@ func validatePipeline(cfg *PipelineConfig) error {
|
|||||||
if err := validateCache(cfg.Cache); err != nil {
|
if err := validateCache(cfg.Cache); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validatePublish(cfg.Publish, cfg.Scriptorium); err != nil {
|
if err := validateNotarius(cfg.Notarius, cfg.Scriptorium); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := validatePublish(cfg.Publish, cfg.Scriptorium, cfg.Notarius); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateWhisperX(cfg.WhisperX); err != nil {
|
if err := validateWhisperX(cfg.WhisperX); err != nil {
|
||||||
@@ -106,7 +111,7 @@ func validatePipeline(cfg *PipelineConfig) error {
|
|||||||
if err := validateRender(cfg.Render); err != nil {
|
if err := validateRender(cfg.Render); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateScriptorium(cfg.Scriptorium); err != nil {
|
if err := validateScriptorium(cfg.Scriptorium, cfg.Notarius); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validateDuration("pipeline.notification.timeout", cfg.Notification.Timeout); err != nil {
|
if err := validateDuration("pipeline.notification.timeout", cfg.Notification.Timeout); err != nil {
|
||||||
@@ -149,11 +154,12 @@ func validateCache(cfg CacheConfig) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
|
func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notarius *NotariusConfig) error {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
||||||
|
extractionOutputs := notariusOutputKeySet(notarius)
|
||||||
seenDest := map[string]struct{}{}
|
seenDest := map[string]struct{}{}
|
||||||
for i, item := range cfg.Outputs {
|
for i, item := range cfg.Outputs {
|
||||||
prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i)
|
prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i)
|
||||||
@@ -161,12 +167,12 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
|
|||||||
if source == "" {
|
if source == "" {
|
||||||
return fmt.Errorf("%s.source is required", prefix)
|
return fmt.Errorf("%s.source is required", prefix)
|
||||||
}
|
}
|
||||||
if _, err := artifactpolicy.ValidatePublishSource(source, configuredOutputs); err != nil {
|
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
|
||||||
return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||||
}
|
}
|
||||||
dest := strings.TrimSpace(item.Dest)
|
dest := strings.TrimSpace(item.Dest)
|
||||||
if dest == "" {
|
if dest == "" {
|
||||||
derivedDest, err := artifactpolicy.ResolvePublishedDestination(source, "", configuredOutputs)
|
derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
|
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
|
||||||
}
|
}
|
||||||
@@ -189,7 +195,7 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
|
|||||||
}
|
}
|
||||||
seenDest[normalizedDest] = struct{}{}
|
seenDest[normalizedDest] = struct{}{}
|
||||||
}
|
}
|
||||||
locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, "pipeline.publish.locks")
|
locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, notarius, "pipeline.publish.locks")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -198,10 +204,11 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ValidatePublishLockRules validates and normalizes source-based publish locks.
|
// ValidatePublishLockRules validates and normalizes source-based publish locks.
|
||||||
func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, label string) ([]PublishLockRule, error) {
|
func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, notarius *NotariusConfig, label string) ([]PublishLockRule, error) {
|
||||||
seenLocks := map[string]struct{}{}
|
seenLocks := map[string]struct{}{}
|
||||||
out := make([]PublishLockRule, 0, len(locks))
|
out := make([]PublishLockRule, 0, len(locks))
|
||||||
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
||||||
|
extractionOutputs := notariusOutputKeySet(notarius)
|
||||||
if strings.TrimSpace(label) == "" {
|
if strings.TrimSpace(label) == "" {
|
||||||
label = "publish.locks"
|
label = "publish.locks"
|
||||||
}
|
}
|
||||||
@@ -211,7 +218,7 @@ func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumC
|
|||||||
if source == "" {
|
if source == "" {
|
||||||
return nil, fmt.Errorf("%s.source is required", prefix)
|
return nil, fmt.Errorf("%s.source is required", prefix)
|
||||||
}
|
}
|
||||||
if _, err := artifactpolicy.ValidatePublishSource(source, configuredOutputs); err != nil {
|
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
|
||||||
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||||
}
|
}
|
||||||
if _, ok := seenLocks[source]; ok {
|
if _, ok := seenLocks[source]; ok {
|
||||||
@@ -264,6 +271,20 @@ func scriptoriumOutputPathMap(scriptorium *ScriptoriumConfig) map[string]string
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func notariusOutputKeySet(notarius *NotariusConfig) map[string]struct{} {
|
||||||
|
out := map[string]struct{}{}
|
||||||
|
if notarius == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for key := range notarius.Outputs {
|
||||||
|
trimmed := strings.TrimSpace(key)
|
||||||
|
if artifactpolicy.IsConfiguredKey(trimmed) {
|
||||||
|
out[trimmed] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func validateSecrets(cfg *SecretsConfig) error {
|
func validateSecrets(cfg *SecretsConfig) error {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -474,7 +495,103 @@ func validateAudita(cfg AuditaConfig) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateScriptorium(cfg *ScriptoriumConfig) error {
|
func validateNotarius(cfg *NotariusConfig, scriptorium *ScriptoriumConfig) error {
|
||||||
|
if cfg == nil || !cfg.Enabled {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.Binary) == "" {
|
||||||
|
return fmt.Errorf("pipeline.notarius.binary is required when pipeline.notarius.enabled is true")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.ConfigPath) == "" {
|
||||||
|
return fmt.Errorf("pipeline.notarius.config_path is required when pipeline.notarius.enabled is true")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.PipelineID) == "" {
|
||||||
|
return fmt.Errorf("pipeline.notarius.pipeline_id is required when pipeline.notarius.enabled is true")
|
||||||
|
}
|
||||||
|
if len(cfg.Outputs) == 0 {
|
||||||
|
return fmt.Errorf("pipeline.notarius.outputs must contain at least one output when pipeline.notarius.enabled is true")
|
||||||
|
}
|
||||||
|
duration, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("pipeline.notarius.timeout must be a valid duration: %w", err)
|
||||||
|
}
|
||||||
|
if duration <= 0 {
|
||||||
|
return fmt.Errorf("pipeline.notarius.timeout must be positive")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(cfg.WorkingDirectory) == "" {
|
||||||
|
return fmt.Errorf("pipeline.notarius.working_directory is required when pipeline.notarius.enabled is true")
|
||||||
|
}
|
||||||
|
|
||||||
|
reservedSources := map[string]string{}
|
||||||
|
for _, spec := range artifactmodel.RuntimeTranscriptArtifacts() {
|
||||||
|
reservedSources[spec.SourceID] = "built-in source"
|
||||||
|
}
|
||||||
|
reservedSources[artifactpolicy.SourceBoundsSession] = "built-in source"
|
||||||
|
if scriptorium != nil {
|
||||||
|
for key := range scriptorium.Artifacts {
|
||||||
|
normalizedKey := strings.TrimSpace(key)
|
||||||
|
reservedSources[artifactpolicy.ConfiguredSourceID(normalizedKey)] = "configured Scriptorium source"
|
||||||
|
reservedSources[artifactpolicy.PreviousSessionSourceID(normalizedKey)] = "previous-session source"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rawKeys := make([]string, 0, len(cfg.Outputs))
|
||||||
|
for key := range cfg.Outputs {
|
||||||
|
rawKeys = append(rawKeys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(rawKeys)
|
||||||
|
normalizedOutputs := make(map[string]NotariusOutputConfig, len(cfg.Outputs))
|
||||||
|
sourceOwners := map[string]string{}
|
||||||
|
laneOwners := map[string]string{}
|
||||||
|
for _, rawKey := range rawKeys {
|
||||||
|
output := cfg.Outputs[rawKey]
|
||||||
|
key := strings.TrimSpace(rawKey)
|
||||||
|
if !artifactpolicy.IsConfiguredKey(key) {
|
||||||
|
return fmt.Errorf("pipeline.notarius.outputs keys must match ^[a-z][a-z0-9_]*$")
|
||||||
|
}
|
||||||
|
sourceID := artifactpolicy.ExtractionSourceID(key)
|
||||||
|
if previousKey, ok := sourceOwners[sourceID]; ok {
|
||||||
|
return fmt.Errorf("pipeline.notarius.outputs keys %q and %q normalize to duplicate source %q", previousKey, rawKey, sourceID)
|
||||||
|
}
|
||||||
|
if owner, ok := reservedSources[sourceID]; ok {
|
||||||
|
return fmt.Errorf("pipeline.notarius.outputs.%s source %q collides with %s", key, sourceID, owner)
|
||||||
|
}
|
||||||
|
sourceOwners[sourceID] = rawKey
|
||||||
|
|
||||||
|
output.LaneID = strings.TrimSpace(output.LaneID)
|
||||||
|
output.MediaType = strings.TrimSpace(output.MediaType)
|
||||||
|
output.SchemaID = strings.TrimSpace(output.SchemaID)
|
||||||
|
output.SchemaVersion = strings.TrimSpace(output.SchemaVersion)
|
||||||
|
output.ModuleKey = strings.TrimSpace(output.ModuleKey)
|
||||||
|
prefix := "pipeline.notarius.outputs." + key
|
||||||
|
if output.LaneID == "" {
|
||||||
|
return fmt.Errorf("%s.lane_id is required", prefix)
|
||||||
|
}
|
||||||
|
if previousKey, ok := laneOwners[output.LaneID]; ok {
|
||||||
|
return fmt.Errorf("%s.lane_id %q duplicates pipeline.notarius.outputs.%s.lane_id", prefix, output.LaneID, previousKey)
|
||||||
|
}
|
||||||
|
laneOwners[output.LaneID] = key
|
||||||
|
if output.MediaType == "" {
|
||||||
|
return fmt.Errorf("%s.media_type is required", prefix)
|
||||||
|
}
|
||||||
|
if output.SchemaID == "" {
|
||||||
|
return fmt.Errorf("%s.schema_id is required", prefix)
|
||||||
|
}
|
||||||
|
if output.SchemaVersion == "" {
|
||||||
|
return fmt.Errorf("%s.schema_version is required", prefix)
|
||||||
|
}
|
||||||
|
normalizedOutputs[key] = output
|
||||||
|
}
|
||||||
|
cfg.Binary = strings.TrimSpace(cfg.Binary)
|
||||||
|
cfg.ConfigPath = filepath.Clean(cfg.ConfigPath)
|
||||||
|
cfg.PipelineID = strings.TrimSpace(cfg.PipelineID)
|
||||||
|
cfg.Timeout = strings.TrimSpace(cfg.Timeout)
|
||||||
|
cfg.WorkingDirectory = filepath.Clean(cfg.WorkingDirectory)
|
||||||
|
cfg.Outputs = normalizedOutputs
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateScriptorium(cfg *ScriptoriumConfig, notarius *NotariusConfig) error {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -491,7 +608,7 @@ func validateScriptorium(cfg *ScriptoriumConfig) error {
|
|||||||
configuredArtifacts := make(map[string]struct{}, len(cfg.Artifacts))
|
configuredArtifacts := make(map[string]struct{}, len(cfg.Artifacts))
|
||||||
referencedArtifacts := make(map[string]struct{})
|
referencedArtifacts := make(map[string]struct{})
|
||||||
for artifactName := range cfg.Artifacts {
|
for artifactName := range cfg.Artifacts {
|
||||||
if !scriptoriumArtifactKeyRE.MatchString(strings.TrimSpace(artifactName)) {
|
if !artifactpolicy.IsConfiguredKey(artifactName) {
|
||||||
return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$")
|
return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$")
|
||||||
}
|
}
|
||||||
configuredArtifacts[artifactName] = struct{}{}
|
configuredArtifacts[artifactName] = struct{}{}
|
||||||
@@ -544,7 +661,7 @@ func validateScriptorium(cfg *ScriptoriumConfig) error {
|
|||||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName)
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName)
|
||||||
}
|
}
|
||||||
|
|
||||||
referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts)
|
referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts, notariusOutputKeySet(notarius))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -684,9 +801,12 @@ func publishUploadConfiguredForS3(pipeline *PipelineConfig) bool {
|
|||||||
|
|
||||||
var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
|
var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
|
||||||
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
var scriptoriumArtifactKeyRE = regexp.MustCompile(`^[a-z][a-z0-9_]*$`)
|
|
||||||
|
|
||||||
func validateScriptoriumInputSource(artifactName, inputName, source string, configuredArtifacts map[string]struct{}) (string, error) {
|
func validateScriptoriumInputSource(
|
||||||
|
artifactName, inputName, source string,
|
||||||
|
configuredArtifacts map[string]struct{},
|
||||||
|
extractionOutputs map[string]struct{},
|
||||||
|
) (string, error) {
|
||||||
trimmedSource := strings.TrimSpace(source)
|
trimmedSource := strings.TrimSpace(source)
|
||||||
descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(trimmedSource)
|
descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(trimmedSource)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -705,7 +825,7 @@ func validateScriptoriumInputSource(artifactName, inputName, source string, conf
|
|||||||
source,
|
source,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
if err := artifactpolicy.ValidateInputConfiguredReference(descriptor, configuredArtifacts); err != nil {
|
if err := artifactpolicy.ValidateInputReference(descriptor, configuredArtifacts, extractionOutputs); err != nil {
|
||||||
var unknownConfigured *artifactpolicy.UnknownConfiguredArtifactError
|
var unknownConfigured *artifactpolicy.UnknownConfiguredArtifactError
|
||||||
if errors.As(err, &unknownConfigured) {
|
if errors.As(err, &unknownConfigured) {
|
||||||
return "", fmt.Errorf(
|
return "", fmt.Errorf(
|
||||||
@@ -716,6 +836,16 @@ func validateScriptoriumInputSource(artifactName, inputName, source string, conf
|
|||||||
unknownConfigured.ConfiguredKey,
|
unknownConfigured.ConfiguredKey,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
var unknownExtraction *artifactpolicy.UnknownExtractionArtifactError
|
||||||
|
if errors.As(err, &unknownExtraction) {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown extraction output %q",
|
||||||
|
artifactName,
|
||||||
|
inputName,
|
||||||
|
source,
|
||||||
|
unknownExtraction.ConfiguredKey,
|
||||||
|
)
|
||||||
|
}
|
||||||
return "", fmt.Errorf(
|
return "", fmt.Errorf(
|
||||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
|
||||||
artifactName,
|
artifactName,
|
||||||
|
|||||||
308
internal/fileops/directory.go
Normal file
308
internal/fileops/directory.go
Normal file
@@ -0,0 +1,308 @@
|
|||||||
|
package fileops
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
promotedDirectoryMode = 0o755
|
||||||
|
promotedFileMode = 0o644
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrAtomicDirectoryPromotionUnsupported indicates that the current operating
|
||||||
|
// system lacks the atomic no-replace primitive required by PromoteDirectory.
|
||||||
|
var ErrAtomicDirectoryPromotionUnsupported = errors.New("atomic no-replace directory promotion is unsupported")
|
||||||
|
|
||||||
|
// PromoteDirectory copies an existing regular-file tree into a new directory
|
||||||
|
// and installs the complete copy atomically. It never removes the source or
|
||||||
|
// replaces an existing destination.
|
||||||
|
func PromoteDirectory(src, dst string) error {
|
||||||
|
if err := checkAtomicDirectoryPromotionSupport(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return promoteDirectory(src, dst, renameDirectoryNoReplace)
|
||||||
|
}
|
||||||
|
|
||||||
|
func promoteDirectory(src, dst string, install func(string, string) error) error {
|
||||||
|
return promoteDirectoryWithHooks(src, dst, install, sourceTraversalHooks{})
|
||||||
|
}
|
||||||
|
|
||||||
|
type sourceTraversalHooks struct {
|
||||||
|
afterRootInspect func()
|
||||||
|
afterEntryInspect func(string)
|
||||||
|
}
|
||||||
|
|
||||||
|
func promoteDirectoryWithHooks(
|
||||||
|
src, dst string,
|
||||||
|
install func(string, string) error,
|
||||||
|
hooks sourceTraversalHooks,
|
||||||
|
) error {
|
||||||
|
if strings.TrimSpace(src) == "" || strings.TrimSpace(dst) == "" {
|
||||||
|
return fmt.Errorf("source and destination directory paths are required")
|
||||||
|
}
|
||||||
|
|
||||||
|
sourceInfo, err := os.Lstat(src)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("inspect source directory: %w", err)
|
||||||
|
}
|
||||||
|
if !sourceInfo.IsDir() {
|
||||||
|
return fmt.Errorf("source path %q is not a directory", src)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Lstat(dst); err == nil {
|
||||||
|
return fmt.Errorf("destination path %q already exists", dst)
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return fmt.Errorf("inspect destination path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
insideSource, err := pathWithin(src, dst)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if insideSource {
|
||||||
|
return fmt.Errorf("destination path %q must not be inside source directory %q", dst, src)
|
||||||
|
}
|
||||||
|
|
||||||
|
destinationParent := filepath.Dir(dst)
|
||||||
|
parentInfo, err := os.Lstat(destinationParent)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("inspect destination parent: %w", err)
|
||||||
|
}
|
||||||
|
if !parentInfo.IsDir() {
|
||||||
|
return fmt.Errorf("destination parent %q is not a directory", destinationParent)
|
||||||
|
}
|
||||||
|
|
||||||
|
temporary, err := os.MkdirTemp(destinationParent, "."+filepath.Base(dst)+".tmp-*")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create temporary destination directory: %w", err)
|
||||||
|
}
|
||||||
|
removeTemporary := true
|
||||||
|
defer func() {
|
||||||
|
if removeTemporary {
|
||||||
|
_ = os.RemoveAll(temporary)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
sourceRoot, err := openVerifiedSourceRoot(src, sourceInfo, hooks)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = sourceRoot.Close() }()
|
||||||
|
|
||||||
|
if err := copyRegularTree(sourceRoot, src, temporary, hooks); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Chmod(temporary, promotedDirectoryMode); err != nil {
|
||||||
|
return fmt.Errorf("set temporary root permissions: %w", err)
|
||||||
|
}
|
||||||
|
if err := syncDirectory(temporary); err != nil {
|
||||||
|
return fmt.Errorf("sync temporary root: %w", err)
|
||||||
|
}
|
||||||
|
if err := install(temporary, dst); err != nil {
|
||||||
|
return fmt.Errorf("install promoted directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
removeTemporary = false
|
||||||
|
if err := syncDirectory(destinationParent); err != nil {
|
||||||
|
return fmt.Errorf("sync destination parent: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func openVerifiedSourceRoot(path string, inspected os.FileInfo, hooks sourceTraversalHooks) (*os.Root, error) {
|
||||||
|
if hooks.afterRootInspect != nil {
|
||||||
|
hooks.afterRootInspect()
|
||||||
|
}
|
||||||
|
|
||||||
|
root, err := os.OpenRoot(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open source directory %q: %w", path, err)
|
||||||
|
}
|
||||||
|
verified := false
|
||||||
|
defer func() {
|
||||||
|
if !verified {
|
||||||
|
_ = root.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
opened, err := root.Stat(".")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("inspect opened source directory %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if !opened.IsDir() || !os.SameFile(inspected, opened) {
|
||||||
|
return nil, fmt.Errorf("source directory %q changed while being opened", path)
|
||||||
|
}
|
||||||
|
current, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reinspect source directory %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if current.Mode()&os.ModeSymlink != 0 || !current.IsDir() || !os.SameFile(opened, current) {
|
||||||
|
return nil, fmt.Errorf("source directory %q changed while being opened", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
verified = true
|
||||||
|
return root, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyRegularTree(src *os.Root, sourcePath, dst string, hooks sourceTraversalHooks) error {
|
||||||
|
directory, err := src.Open(".")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("open source directory %q for traversal: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = directory.Close() }()
|
||||||
|
|
||||||
|
entries, err := directory.ReadDir(-1)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read source directory %q: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
sort.Slice(entries, func(i, j int) bool {
|
||||||
|
return entries[i].Name() < entries[j].Name()
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
entryPath := filepath.Join(sourcePath, entry.Name())
|
||||||
|
destinationPath := filepath.Join(dst, entry.Name())
|
||||||
|
info, err := src.Lstat(entry.Name())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("inspect source entry %q: %w", entryPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case info.Mode().IsRegular():
|
||||||
|
if hooks.afterEntryInspect != nil {
|
||||||
|
hooks.afterEntryInspect(entryPath)
|
||||||
|
}
|
||||||
|
if err := copyRegularFile(src, entry.Name(), entryPath, destinationPath, info); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case info.IsDir():
|
||||||
|
if hooks.afterEntryInspect != nil {
|
||||||
|
hooks.afterEntryInspect(entryPath)
|
||||||
|
}
|
||||||
|
if err := copyRegularDirectory(src, entry.Name(), entryPath, destinationPath, info, hooks); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("source entry %q has unsupported file type %s", entryPath, info.Mode().Type())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyRegularDirectory(
|
||||||
|
parent *os.Root,
|
||||||
|
name, sourcePath, dst string,
|
||||||
|
inspected os.FileInfo,
|
||||||
|
hooks sourceTraversalHooks,
|
||||||
|
) error {
|
||||||
|
child, err := parent.OpenRoot(name)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("open source directory %q: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = child.Close() }()
|
||||||
|
|
||||||
|
opened, err := child.Stat(".")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("inspect opened source directory %q: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
if !opened.IsDir() || !os.SameFile(inspected, opened) {
|
||||||
|
return fmt.Errorf("source directory %q changed while being copied", sourcePath)
|
||||||
|
}
|
||||||
|
current, err := parent.Lstat(name)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reinspect source directory %q: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
if current.Mode()&os.ModeSymlink != 0 || !current.IsDir() || !os.SameFile(opened, current) {
|
||||||
|
return fmt.Errorf("source directory %q changed while being copied", sourcePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Mkdir(dst, promotedDirectoryMode); err != nil {
|
||||||
|
return fmt.Errorf("create destination directory %q: %w", dst, err)
|
||||||
|
}
|
||||||
|
if err := copyRegularTree(child, sourcePath, dst, hooks); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Chmod(dst, promotedDirectoryMode); err != nil {
|
||||||
|
return fmt.Errorf("set destination directory permissions %q: %w", dst, err)
|
||||||
|
}
|
||||||
|
if err := syncDirectory(dst); err != nil {
|
||||||
|
return fmt.Errorf("sync destination directory %q: %w", dst, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyRegularFile(
|
||||||
|
root *os.Root,
|
||||||
|
name, sourcePath, dst string,
|
||||||
|
inspected os.FileInfo,
|
||||||
|
) error {
|
||||||
|
in, err := root.Open(name)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("open source file %q: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = in.Close() }()
|
||||||
|
|
||||||
|
opened, err := in.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("inspect opened source file %q: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
if !opened.Mode().IsRegular() || !os.SameFile(inspected, opened) {
|
||||||
|
return fmt.Errorf("source file %q changed while being copied", sourcePath)
|
||||||
|
}
|
||||||
|
current, err := root.Lstat(name)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reinspect source file %q: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
if current.Mode()&os.ModeSymlink != 0 || !current.Mode().IsRegular() || !os.SameFile(opened, current) {
|
||||||
|
return fmt.Errorf("source file %q changed while being copied", sourcePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, promotedFileMode)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create destination file %q: %w", dst, err)
|
||||||
|
}
|
||||||
|
closed := false
|
||||||
|
defer func() {
|
||||||
|
if !closed {
|
||||||
|
_ = out.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if _, err := io.Copy(out, in); err != nil {
|
||||||
|
return fmt.Errorf("copy source file %q: %w", sourcePath, err)
|
||||||
|
}
|
||||||
|
if err := out.Chmod(promotedFileMode); err != nil {
|
||||||
|
return fmt.Errorf("set destination file permissions %q: %w", dst, err)
|
||||||
|
}
|
||||||
|
if err := out.Sync(); err != nil {
|
||||||
|
return fmt.Errorf("sync destination file %q: %w", dst, err)
|
||||||
|
}
|
||||||
|
if err := out.Close(); err != nil {
|
||||||
|
return fmt.Errorf("close destination file %q: %w", dst, err)
|
||||||
|
}
|
||||||
|
closed = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathWithin(parent, candidate string) (bool, error) {
|
||||||
|
absoluteParent, err := filepath.Abs(parent)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("resolve source directory: %w", err)
|
||||||
|
}
|
||||||
|
absoluteCandidate, err := filepath.Abs(candidate)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("resolve destination directory: %w", err)
|
||||||
|
}
|
||||||
|
relative, err := filepath.Rel(absoluteParent, absoluteCandidate)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("compare source and destination directories: %w", err)
|
||||||
|
}
|
||||||
|
return relative != "." && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)), nil
|
||||||
|
}
|
||||||
35
internal/fileops/directory_special_unix_test.go
Normal file
35
internal/fileops/directory_special_unix_test.go
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
//go:build linux || darwin
|
||||||
|
|
||||||
|
package fileops
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsNamedPipe(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
if err := os.Mkdir(src, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(source) error = %v", err)
|
||||||
|
}
|
||||||
|
pipe := filepath.Join(src, "events.pipe")
|
||||||
|
if err := syscall.Mkfifo(pipe, 0o600); err != nil {
|
||||||
|
t.Skipf("Mkfifo() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := PromoteDirectory(src, dst); err == nil {
|
||||||
|
t.Fatal("PromoteDirectory() error = nil, want named pipe rejection")
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(pipe)
|
||||||
|
if err != nil || info.Mode()&os.ModeNamedPipe == 0 {
|
||||||
|
t.Fatalf("source named pipe was not preserved: info=%v err=%v", info, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(dst); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("Lstat(destination) error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
501
internal/fileops/directory_test.go
Normal file
501
internal/fileops/directory_test.go
Normal file
@@ -0,0 +1,501 @@
|
|||||||
|
//go:build linux || darwin || windows
|
||||||
|
|
||||||
|
package fileops
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPromoteDirectoryCopiesNestedRegularTree(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
dst := filepath.Join(root, "promoted")
|
||||||
|
mustWriteFile(t, filepath.Join(src, "z-last.txt"), []byte("last"), 0o777)
|
||||||
|
mustWriteFile(t, filepath.Join(src, "nested", "binary.dat"), []byte{0, 1, 2, 0xff}, 0o600)
|
||||||
|
mustWriteFile(t, filepath.Join(src, "a-first.txt"), []byte("first"), 0o400)
|
||||||
|
if err := os.Mkdir(filepath.Join(src, "empty"), 0o700); err != nil {
|
||||||
|
t.Fatalf("Mkdir(empty) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := PromoteDirectory(src, dst); err != nil {
|
||||||
|
t.Fatalf("PromoteDirectory() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantLayout := []string{".", "a-first.txt", "empty", "nested", "nested/binary.dat", "z-last.txt"}
|
||||||
|
if got := treeLayout(t, dst); !reflect.DeepEqual(got, wantLayout) {
|
||||||
|
t.Fatalf("promoted layout = %#v, want %#v", got, wantLayout)
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(dst, "a-first.txt"), []byte("first"))
|
||||||
|
assertFileBytes(t, filepath.Join(dst, "nested", "binary.dat"), []byte{0, 1, 2, 0xff})
|
||||||
|
assertFileBytes(t, filepath.Join(dst, "z-last.txt"), []byte("last"))
|
||||||
|
|
||||||
|
if runtime.GOOS != "windows" {
|
||||||
|
for _, path := range []string{dst, filepath.Join(dst, "nested"), filepath.Join(dst, "empty")} {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Stat(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
if got := info.Mode().Perm(); got != promotedDirectoryMode {
|
||||||
|
t.Fatalf("directory mode for %q = %o, want %o", path, got, promotedDirectoryMode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, path := range []string{filepath.Join(dst, "a-first.txt"), filepath.Join(dst, "nested", "binary.dat"), filepath.Join(dst, "z-last.txt")} {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Stat(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
if got := info.Mode().Perm(); got != promotedFileMode {
|
||||||
|
t.Fatalf("file mode for %q = %o, want %o", path, got, promotedFileMode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assertFileBytes(t, filepath.Join(src, "nested", "binary.dat"), []byte{0, 1, 2, 0xff})
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".promoted.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsInvalidPaths(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
if err := os.Mkdir(src, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(source) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
src string
|
||||||
|
dst string
|
||||||
|
}{
|
||||||
|
{name: "empty source", src: "", dst: filepath.Join(root, "out-a")},
|
||||||
|
{name: "empty destination", src: src, dst: " "},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if err := PromoteDirectory(test.src, test.dst); err == nil {
|
||||||
|
t.Fatal("PromoteDirectory() error = nil, want path validation failure")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsExistingDestination(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("source"), 0o644)
|
||||||
|
mustWriteFile(t, filepath.Join(dst, "value.txt"), []byte("existing"), 0o644)
|
||||||
|
|
||||||
|
err := PromoteDirectory(src, dst)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "already exists") {
|
||||||
|
t.Fatalf("PromoteDirectory() error = %v, want existing destination error", err)
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(dst, "value.txt"), []byte("existing"))
|
||||||
|
assertFileBytes(t, filepath.Join(src, "value.txt"), []byte("source"))
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsNonDirectorySource(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source.txt")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, src, []byte("source"), 0o644)
|
||||||
|
|
||||||
|
if err := PromoteDirectory(src, dst); err == nil {
|
||||||
|
t.Fatal("PromoteDirectory() error = nil, want non-directory source error")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, src, []byte("source"))
|
||||||
|
if _, err := os.Lstat(dst); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("Lstat(destination) error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsSymlinksWithoutFollowingThem(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
externalFile := filepath.Join(root, "external.txt")
|
||||||
|
externalDirectory := filepath.Join(root, "external-directory")
|
||||||
|
mustWriteFile(t, externalFile, []byte("outside"), 0o644)
|
||||||
|
mustWriteFile(t, filepath.Join(externalDirectory, "secret.txt"), []byte("secret"), 0o644)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
target string
|
||||||
|
link string
|
||||||
|
}{
|
||||||
|
{name: "file", target: externalFile, link: "file-link"},
|
||||||
|
{name: "directory", target: externalDirectory, link: "directory-link"},
|
||||||
|
{name: "escaping", target: filepath.Join("..", "external.txt"), link: "escaping-link"},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
src := filepath.Join(root, "source-"+test.name)
|
||||||
|
dst := filepath.Join(root, "destination-"+test.name)
|
||||||
|
if err := os.Mkdir(src, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(source) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(test.target, filepath.Join(src, test.link)); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := PromoteDirectory(src, dst); err == nil {
|
||||||
|
t.Fatal("PromoteDirectory() error = nil, want symlink rejection")
|
||||||
|
}
|
||||||
|
linkInfo, err := os.Lstat(filepath.Join(src, test.link))
|
||||||
|
if err != nil || linkInfo.Mode()&os.ModeSymlink == 0 {
|
||||||
|
t.Fatalf("source symlink was not preserved: info=%v err=%v", linkInfo, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(dst); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("Lstat(destination) error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination-"+test.name+".tmp-")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
assertFileBytes(t, externalFile, []byte("outside"))
|
||||||
|
assertFileBytes(t, filepath.Join(externalDirectory, "secret.txt"), []byte("secret"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsSymlinkSourceRoot(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
target := filepath.Join(root, "target")
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, filepath.Join(target, "value.txt"), []byte("outside"), 0o644)
|
||||||
|
if err := os.Symlink(target, src); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := PromoteDirectory(src, dst); err == nil {
|
||||||
|
t.Fatal("PromoteDirectory() error = nil, want source-root symlink rejection")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(target, "value.txt"), []byte("outside"))
|
||||||
|
assertPathMissing(t, dst)
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsSourceRootReplacementBeforeOpen(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
preserved := filepath.Join(root, "source-preserved")
|
||||||
|
replacement := filepath.Join(root, "replacement")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("original"), 0o644)
|
||||||
|
mustWriteFile(t, filepath.Join(replacement, "value.txt"), []byte("replacement"), 0o644)
|
||||||
|
|
||||||
|
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||||
|
afterRootInspect: func() {
|
||||||
|
if err := os.Rename(src, preserved); err != nil {
|
||||||
|
t.Fatalf("Rename(original source) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(replacement, src); err != nil {
|
||||||
|
t.Fatalf("Rename(replacement source) error = %v", err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("promoteDirectoryWithHooks() error = nil, want source identity failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||||
|
assertFileBytes(t, filepath.Join(src, "value.txt"), []byte("replacement"))
|
||||||
|
assertPathMissing(t, dst)
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsIdentityPreservingSourceRootSymlinkReplacement(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
preserved := filepath.Join(root, "source-preserved")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("original"), 0o644)
|
||||||
|
|
||||||
|
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||||
|
afterRootInspect: func() {
|
||||||
|
if err := os.Rename(src, preserved); err != nil {
|
||||||
|
t.Fatalf("Rename(inspected source) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(filepath.Base(preserved), src); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("promoteDirectoryWithHooks() error = nil, want source-root symlink replacement failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||||
|
assertPathIsSymlink(t, src)
|
||||||
|
assertPathMissing(t, dst)
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsInspectedDirectorySymlinkReplacement(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
child := filepath.Join(src, "child")
|
||||||
|
preserved := filepath.Join(src, "child-preserved")
|
||||||
|
outside := filepath.Join(root, "outside")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, filepath.Join(child, "value.txt"), []byte("original"), 0o644)
|
||||||
|
mustWriteFile(t, filepath.Join(outside, "sentinel.txt"), []byte("outside"), 0o644)
|
||||||
|
|
||||||
|
replaced := false
|
||||||
|
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||||
|
afterEntryInspect: func(path string) {
|
||||||
|
if replaced || path != child {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
replaced = true
|
||||||
|
if err := os.Rename(child, preserved); err != nil {
|
||||||
|
t.Fatalf("Rename(inspected child) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(filepath.Join("..", "outside"), child); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Mkdir(dst, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(concurrent destination) error = %v", err)
|
||||||
|
}
|
||||||
|
mustWriteFile(t, filepath.Join(dst, "value.txt"), []byte("concurrent"), 0o644)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("promoteDirectoryWithHooks() error = nil, want symlink replacement failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||||
|
assertFileBytes(t, filepath.Join(outside, "sentinel.txt"), []byte("outside"))
|
||||||
|
assertFileBytes(t, filepath.Join(dst, "value.txt"), []byte("concurrent"))
|
||||||
|
assertPathMissing(t, filepath.Join(dst, "sentinel.txt"))
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsInspectedFileIdentityMismatch(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
file := filepath.Join(src, "value.txt")
|
||||||
|
preserved := filepath.Join(src, "value-preserved.txt")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, file, []byte("original"), 0o644)
|
||||||
|
|
||||||
|
replaced := false
|
||||||
|
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||||
|
afterEntryInspect: func(path string) {
|
||||||
|
if replaced || path != file {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
replaced = true
|
||||||
|
if err := os.Rename(file, preserved); err != nil {
|
||||||
|
t.Fatalf("Rename(inspected file) error = %v", err)
|
||||||
|
}
|
||||||
|
mustWriteFile(t, file, []byte("replacement"), 0o644)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("promoteDirectoryWithHooks() error = nil, want file identity failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, preserved, []byte("original"))
|
||||||
|
assertFileBytes(t, file, []byte("replacement"))
|
||||||
|
assertPathMissing(t, dst)
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsIdentityPreservingFileSymlinkReplacement(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
file := filepath.Join(src, "value.txt")
|
||||||
|
preserved := filepath.Join(src, "value-preserved.txt")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, file, []byte("original"), 0o644)
|
||||||
|
|
||||||
|
replaced := false
|
||||||
|
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||||
|
afterEntryInspect: func(path string) {
|
||||||
|
if replaced || path != file {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
replaced = true
|
||||||
|
if err := os.Rename(file, preserved); err != nil {
|
||||||
|
t.Fatalf("Rename(inspected file) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(filepath.Base(preserved), file); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("promoteDirectoryWithHooks() error = nil, want file symlink replacement failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, preserved, []byte("original"))
|
||||||
|
assertPathIsSymlink(t, file)
|
||||||
|
assertPathMissing(t, dst)
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsInspectedDirectoryIdentityMismatch(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
child := filepath.Join(src, "child")
|
||||||
|
preserved := filepath.Join(src, "child-preserved")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, filepath.Join(child, "value.txt"), []byte("original"), 0o644)
|
||||||
|
|
||||||
|
replaced := false
|
||||||
|
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||||
|
afterEntryInspect: func(path string) {
|
||||||
|
if replaced || path != child {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
replaced = true
|
||||||
|
if err := os.Rename(child, preserved); err != nil {
|
||||||
|
t.Fatalf("Rename(inspected directory) error = %v", err)
|
||||||
|
}
|
||||||
|
mustWriteFile(t, filepath.Join(child, "value.txt"), []byte("replacement"), 0o644)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("promoteDirectoryWithHooks() error = nil, want directory identity failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||||
|
assertFileBytes(t, filepath.Join(child, "value.txt"), []byte("replacement"))
|
||||||
|
assertPathMissing(t, dst)
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsIdentityPreservingDirectorySymlinkReplacement(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
child := filepath.Join(src, "child")
|
||||||
|
preserved := filepath.Join(src, "child-preserved")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, filepath.Join(child, "value.txt"), []byte("original"), 0o644)
|
||||||
|
|
||||||
|
replaced := false
|
||||||
|
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||||
|
afterEntryInspect: func(path string) {
|
||||||
|
if replaced || path != child {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
replaced = true
|
||||||
|
if err := os.Rename(child, preserved); err != nil {
|
||||||
|
t.Fatalf("Rename(inspected directory) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(filepath.Base(preserved), child); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("promoteDirectoryWithHooks() error = nil, want directory symlink replacement failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||||
|
assertPathIsSymlink(t, child)
|
||||||
|
assertPathMissing(t, dst)
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryDoesNotReplaceDestinationCreatedBeforeInstall(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("source"), 0o644)
|
||||||
|
|
||||||
|
err := promoteDirectory(src, dst, func(temporary, destination string) error {
|
||||||
|
if err := os.Mkdir(destination, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(concurrent destination) error = %v", err)
|
||||||
|
}
|
||||||
|
mustWriteFile(t, filepath.Join(destination, "value.txt"), []byte("concurrent"), 0o644)
|
||||||
|
return renameDirectoryNoReplace(temporary, destination)
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("promoteDirectory() error = nil, want no-replace install failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(dst, "value.txt"), []byte("concurrent"))
|
||||||
|
assertFileBytes(t, filepath.Join(src, "value.txt"), []byte("source"))
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsDestinationInsideSource(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
dst := filepath.Join(src, "nested", "destination")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(destination parent) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := PromoteDirectory(src, dst); err == nil {
|
||||||
|
t.Fatal("PromoteDirectory() error = nil, want nested destination rejection")
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(dst); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("Lstat(destination) error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustWriteFile(t *testing.T, path string, data []byte, mode os.FileMode) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(%q) error = %v", filepath.Dir(path), err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, data, mode); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFileBytes(t *testing.T, path string, want []byte) {
|
||||||
|
t.Helper()
|
||||||
|
got, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, want) {
|
||||||
|
t.Fatalf("ReadFile(%q) = %v, want %v", path, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertPathMissing(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := os.Lstat(path); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("Lstat(%q) error = %v, want not exist", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertPathIsSymlink(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil || info.Mode()&os.ModeSymlink == 0 {
|
||||||
|
t.Fatalf("Lstat(%q) info = %v, error = %v, want symlink", path, info, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func treeLayout(t *testing.T, root string) []string {
|
||||||
|
t.Helper()
|
||||||
|
var layout []string
|
||||||
|
err := filepath.WalkDir(root, func(path string, _ os.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
relative, err := filepath.Rel(root, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
layout = append(layout, filepath.ToSlash(relative))
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WalkDir(%q) error = %v", root, err)
|
||||||
|
}
|
||||||
|
return layout
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertNoMatchingTempDirectories(t *testing.T, parent, prefix string) {
|
||||||
|
t.Helper()
|
||||||
|
entries, err := os.ReadDir(parent)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadDir(%q) error = %v", parent, err)
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if strings.HasPrefix(entry.Name(), prefix) {
|
||||||
|
t.Fatalf("unexpected temporary directory residue: %s", filepath.Join(parent, entry.Name()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
37
internal/fileops/directory_unsupported_test.go
Normal file
37
internal/fileops/directory_unsupported_test.go
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
//go:build !linux && !darwin && !windows
|
||||||
|
|
||||||
|
package fileops
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPromoteDirectoryFailsBeforeCreatingTemporaryTree(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
if err := os.Mkdir(src, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(source) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(src, "value.txt"), []byte("source"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(source) error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := PromoteDirectory(src, dst)
|
||||||
|
if !errors.Is(err, ErrAtomicDirectoryPromotionUnsupported) {
|
||||||
|
t.Fatalf("PromoteDirectory() error = %v, want unsupported capability", err)
|
||||||
|
}
|
||||||
|
entries, readErr := os.ReadDir(root)
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatalf("ReadDir(root) error = %v", readErr)
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.Name() == filepath.Base(dst) || strings.HasPrefix(entry.Name(), ".destination.tmp-") {
|
||||||
|
t.Fatalf("unsupported promotion created %q", entry.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
11
internal/fileops/rename_noreplace_darwin.go
Normal file
11
internal/fileops/rename_noreplace_darwin.go
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
//go:build darwin
|
||||||
|
|
||||||
|
package fileops
|
||||||
|
|
||||||
|
import "golang.org/x/sys/unix"
|
||||||
|
|
||||||
|
func checkAtomicDirectoryPromotionSupport() error { return nil }
|
||||||
|
|
||||||
|
func renameDirectoryNoReplace(src, dst string) error {
|
||||||
|
return unix.RenamexNp(src, dst, unix.RENAME_EXCL)
|
||||||
|
}
|
||||||
11
internal/fileops/rename_noreplace_linux.go
Normal file
11
internal/fileops/rename_noreplace_linux.go
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
//go:build linux
|
||||||
|
|
||||||
|
package fileops
|
||||||
|
|
||||||
|
import "golang.org/x/sys/unix"
|
||||||
|
|
||||||
|
func checkAtomicDirectoryPromotionSupport() error { return nil }
|
||||||
|
|
||||||
|
func renameDirectoryNoReplace(src, dst string) error {
|
||||||
|
return unix.Renameat2(unix.AT_FDCWD, src, unix.AT_FDCWD, dst, unix.RENAME_NOREPLACE)
|
||||||
|
}
|
||||||
16
internal/fileops/rename_noreplace_other.go
Normal file
16
internal/fileops/rename_noreplace_other.go
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
//go:build !linux && !darwin && !windows
|
||||||
|
|
||||||
|
package fileops
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"runtime"
|
||||||
|
)
|
||||||
|
|
||||||
|
func checkAtomicDirectoryPromotionSupport() error {
|
||||||
|
return fmt.Errorf("%w on %s", ErrAtomicDirectoryPromotionUnsupported, runtime.GOOS)
|
||||||
|
}
|
||||||
|
|
||||||
|
func renameDirectoryNoReplace(_, _ string) error {
|
||||||
|
return checkAtomicDirectoryPromotionSupport()
|
||||||
|
}
|
||||||
26
internal/fileops/rename_noreplace_supported_test.go
Normal file
26
internal/fileops/rename_noreplace_supported_test.go
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
//go:build linux || darwin || windows
|
||||||
|
|
||||||
|
package fileops
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRenameDirectoryNoReplacePreservesExistingDestination(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("source"), 0o644)
|
||||||
|
mustWriteFile(t, filepath.Join(dst, "value.txt"), []byte("existing"), 0o644)
|
||||||
|
|
||||||
|
if err := renameDirectoryNoReplace(src, dst); err == nil {
|
||||||
|
t.Fatal("renameDirectoryNoReplace() error = nil, want existing destination failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(src, "value.txt"), []byte("source"))
|
||||||
|
assertFileBytes(t, filepath.Join(dst, "value.txt"), []byte("existing"))
|
||||||
|
if info, err := os.Stat(src); err != nil || !info.IsDir() {
|
||||||
|
t.Fatalf("source directory was not preserved: info=%v err=%v", info, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
19
internal/fileops/rename_noreplace_windows.go
Normal file
19
internal/fileops/rename_noreplace_windows.go
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package fileops
|
||||||
|
|
||||||
|
import "golang.org/x/sys/windows"
|
||||||
|
|
||||||
|
func checkAtomicDirectoryPromotionSupport() error { return nil }
|
||||||
|
|
||||||
|
func renameDirectoryNoReplace(src, dst string) error {
|
||||||
|
from, err := windows.UTF16PtrFromString(src)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
to, err := windows.UTF16PtrFromString(dst)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return windows.MoveFileEx(from, to, 0)
|
||||||
|
}
|
||||||
25
internal/fileops/sync_directory_unix.go
Normal file
25
internal/fileops/sync_directory_unix.go
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
//go:build linux || darwin
|
||||||
|
|
||||||
|
package fileops
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
func syncDirectory(path string) error {
|
||||||
|
directory, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = directory.Close() }()
|
||||||
|
|
||||||
|
err = directory.Sync()
|
||||||
|
// Some Unix filesystems do not implement directory syncing. Only their
|
||||||
|
// explicit unsupported-operation errors are safe to treat as best effort.
|
||||||
|
if errors.Is(err, syscall.EINVAL) || errors.Is(err, syscall.ENOTSUP) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
7
internal/fileops/sync_directory_unsupported.go
Normal file
7
internal/fileops/sync_directory_unsupported.go
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
//go:build !linux && !darwin && !windows
|
||||||
|
|
||||||
|
package fileops
|
||||||
|
|
||||||
|
func syncDirectory(string) error {
|
||||||
|
return checkAtomicDirectoryPromotionSupport()
|
||||||
|
}
|
||||||
40
internal/fileops/sync_directory_windows.go
Normal file
40
internal/fileops/sync_directory_windows.go
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package fileops
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"golang.org/x/sys/windows"
|
||||||
|
)
|
||||||
|
|
||||||
|
func syncDirectory(path string) error {
|
||||||
|
pathPointer, err := windows.UTF16PtrFromString(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
directory, err := windows.CreateFile(
|
||||||
|
pathPointer,
|
||||||
|
windows.GENERIC_WRITE,
|
||||||
|
windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE,
|
||||||
|
nil,
|
||||||
|
windows.OPEN_EXISTING,
|
||||||
|
windows.FILE_FLAG_BACKUP_SEMANTICS,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = windows.CloseHandle(directory) }()
|
||||||
|
|
||||||
|
err = windows.FlushFileBuffers(directory)
|
||||||
|
// Windows filesystems may reject flushing a directory handle even when it
|
||||||
|
// was opened correctly. Preserve every error except the documented forms
|
||||||
|
// that mean this operation is unavailable for the handle or filesystem.
|
||||||
|
if errors.Is(err, windows.ERROR_INVALID_FUNCTION) ||
|
||||||
|
errors.Is(err, windows.ERROR_INVALID_HANDLE) ||
|
||||||
|
errors.Is(err, windows.ERROR_NOT_SUPPORTED) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -3,6 +3,8 @@ package manifest
|
|||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrorRecord captures structured error metadata at run or stage scope.
|
// ErrorRecord captures structured error metadata at run or stage scope.
|
||||||
@@ -28,9 +30,11 @@ type InputRecord struct {
|
|||||||
|
|
||||||
// ArtifactRecord captures one produced artifact and optional remote metadata.
|
// ArtifactRecord captures one produced artifact and optional remote metadata.
|
||||||
type ArtifactRecord struct {
|
type ArtifactRecord struct {
|
||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
SourceID string `json:"source_id,omitempty"`
|
SourceID string `json:"source_id,omitempty"`
|
||||||
LocalPath string `json:"local_path"`
|
LocalPath string `json:"local_path"`
|
||||||
|
Contract *artifactmodel.ContractMetadata `json:"contract,omitempty"`
|
||||||
|
ExternalProvenance *artifactmodel.ExternalProvenance `json:"external_provenance,omitempty"`
|
||||||
// ProducerRunID identifies the run that produced this durable artifact.
|
// ProducerRunID identifies the run that produced this durable artifact.
|
||||||
ProducerRunID string `json:"producer_run_id,omitempty"`
|
ProducerRunID string `json:"producer_run_id,omitempty"`
|
||||||
RemoteKey string `json:"remote_key,omitempty"`
|
RemoteKey string `json:"remote_key,omitempty"`
|
||||||
@@ -84,6 +88,7 @@ func New(sessionID string, now time.Time) *Manifest {
|
|||||||
// MarkStageRunning marks a stage as running and updates timestamps.
|
// MarkStageRunning marks a stage as running and updates timestamps.
|
||||||
func (m *Manifest) MarkStageRunning(name string, at time.Time) {
|
func (m *Manifest) MarkStageRunning(name string, at time.Time) {
|
||||||
s := m.ensureStage(name, at)
|
s := m.ensureStage(name, at)
|
||||||
|
s.clearResultDetails()
|
||||||
s.Status = StatusRunning
|
s.Status = StatusRunning
|
||||||
s.StartedAt = timePtr(at)
|
s.StartedAt = timePtr(at)
|
||||||
s.CompletedAt = nil
|
s.CompletedAt = nil
|
||||||
@@ -106,6 +111,7 @@ func (m *Manifest) MarkStageSucceeded(name string, at time.Time, outputs []Artif
|
|||||||
// MarkStageFailed marks a stage as failed and records error metadata.
|
// MarkStageFailed marks a stage as failed and records error metadata.
|
||||||
func (m *Manifest) MarkStageFailed(name string, at time.Time, message string) {
|
func (m *Manifest) MarkStageFailed(name string, at time.Time, message string) {
|
||||||
s := m.ensureStage(name, at)
|
s := m.ensureStage(name, at)
|
||||||
|
s.clearResultDetails()
|
||||||
s.Status = StatusFailed
|
s.Status = StatusFailed
|
||||||
s.CompletedAt = timePtr(at)
|
s.CompletedAt = timePtr(at)
|
||||||
s.Error = &ErrorRecord{Message: strings.TrimSpace(message), At: timePtr(at)}
|
s.Error = &ErrorRecord{Message: strings.TrimSpace(message), At: timePtr(at)}
|
||||||
@@ -117,6 +123,7 @@ func (m *Manifest) MarkStageFailed(name string, at time.Time, message string) {
|
|||||||
// MarkStageSkipped marks a stage as skipped and records the skip reason.
|
// MarkStageSkipped marks a stage as skipped and records the skip reason.
|
||||||
func (m *Manifest) MarkStageSkipped(name string, at time.Time, reason string) {
|
func (m *Manifest) MarkStageSkipped(name string, at time.Time, reason string) {
|
||||||
s := m.ensureStage(name, at)
|
s := m.ensureStage(name, at)
|
||||||
|
s.clearResultDetails()
|
||||||
s.Status = StatusSkipped
|
s.Status = StatusSkipped
|
||||||
s.CompletedAt = timePtr(at)
|
s.CompletedAt = timePtr(at)
|
||||||
s.Error = &ErrorRecord{Message: strings.TrimSpace(reason), Code: "skipped", At: timePtr(at)}
|
s.Error = &ErrorRecord{Message: strings.TrimSpace(reason), Code: "skipped", At: timePtr(at)}
|
||||||
@@ -133,6 +140,13 @@ func (m *Manifest) MarkStageStale(name string, at time.Time, reason string) {
|
|||||||
m.UpdatedAt = at
|
m.UpdatedAt = at
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *StageRecord) clearResultDetails() {
|
||||||
|
s.Outputs = nil
|
||||||
|
s.Logs = nil
|
||||||
|
s.GeneratedConfigs = nil
|
||||||
|
s.Metadata = nil
|
||||||
|
}
|
||||||
|
|
||||||
func (m *Manifest) ensureStage(name string, at time.Time) *StageRecord {
|
func (m *Manifest) ensureStage(name string, at time.Time) *StageRecord {
|
||||||
if m.Stages == nil {
|
if m.Stages == nil {
|
||||||
m.Stages = map[string]*StageRecord{}
|
m.Stages = map[string]*StageRecord{}
|
||||||
|
|||||||
@@ -69,11 +69,12 @@ func TestStageMarkHelpers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMarkStageRunningClearsPriorCompletionAndError(t *testing.T) {
|
func TestMarkStageRunningClearsPriorCompletionErrorAndResultDetails(t *testing.T) {
|
||||||
m := New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
m := New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
|
|
||||||
failedAt := time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC)
|
failedAt := time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC)
|
||||||
m.MarkStageFailed("merge", failedAt, "boom")
|
m.MarkStageFailed("merge", failedAt, "boom")
|
||||||
|
setStageResultDetails(m.Stages["merge"])
|
||||||
|
|
||||||
runAt := failedAt.Add(1 * time.Minute)
|
runAt := failedAt.Add(1 * time.Minute)
|
||||||
m.MarkStageRunning("merge", runAt)
|
m.MarkStageRunning("merge", runAt)
|
||||||
@@ -91,6 +92,7 @@ func TestMarkStageRunningClearsPriorCompletionAndError(t *testing.T) {
|
|||||||
if stage.Error != nil {
|
if stage.Error != nil {
|
||||||
t.Fatalf("error = %#v, want nil while running", stage.Error)
|
t.Fatalf("error = %#v, want nil while running", stage.Error)
|
||||||
}
|
}
|
||||||
|
requireStageResultDetailsCleared(t, stage)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMarkStageSucceededClearsError(t *testing.T) {
|
func TestMarkStageSucceededClearsError(t *testing.T) {
|
||||||
@@ -110,3 +112,68 @@ func TestMarkStageSucceededClearsError(t *testing.T) {
|
|||||||
t.Fatalf("error = %#v, want nil on success", stage.Error)
|
t.Fatalf("error = %#v, want nil on success", stage.Error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMarkStageFailedClearsEarlierResultDetails(t *testing.T) {
|
||||||
|
m := New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
|
m.MarkStageSucceeded("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
|
setStageResultDetails(m.Stages["extract"])
|
||||||
|
|
||||||
|
m.MarkStageFailed("extract", time.Date(2026, 5, 3, 10, 2, 0, 0, time.UTC), "replacement failed")
|
||||||
|
|
||||||
|
stage := m.Stages["extract"]
|
||||||
|
if stage == nil || stage.Status != StatusFailed {
|
||||||
|
t.Fatalf("stage = %#v, want failed", stage)
|
||||||
|
}
|
||||||
|
requireStageResultDetailsCleared(t, stage)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarkStageSkippedClearsEarlierResultDetails(t *testing.T) {
|
||||||
|
m := New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
|
m.MarkStageSucceeded("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
|
setStageResultDetails(m.Stages["extract"])
|
||||||
|
|
||||||
|
m.MarkStageSkipped("extract", time.Date(2026, 5, 3, 10, 2, 0, 0, time.UTC), "integration_disabled")
|
||||||
|
|
||||||
|
stage := m.Stages["extract"]
|
||||||
|
if stage == nil {
|
||||||
|
t.Fatal("missing stage record")
|
||||||
|
}
|
||||||
|
if stage.Status != StatusSkipped {
|
||||||
|
t.Fatalf("status = %q, want %q", stage.Status, StatusSkipped)
|
||||||
|
}
|
||||||
|
requireStageResultDetailsCleared(t, stage)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarkStageStalePreservesResultDetails(t *testing.T) {
|
||||||
|
m := New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||||
|
m.MarkStageSucceeded("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||||
|
stage := m.Stages["extract"]
|
||||||
|
setStageResultDetails(stage)
|
||||||
|
|
||||||
|
m.MarkStageStale("extract", time.Date(2026, 5, 3, 10, 2, 0, 0, time.UTC), "result is not resumable")
|
||||||
|
|
||||||
|
if stage.Status != StatusStale {
|
||||||
|
t.Fatalf("status = %q, want %q", stage.Status, StatusStale)
|
||||||
|
}
|
||||||
|
if len(stage.Outputs) != 1 || len(stage.Logs) != 1 || len(stage.GeneratedConfigs) != 1 || len(stage.Metadata) != 1 {
|
||||||
|
t.Fatalf("result details were not preserved: %#v", stage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setStageResultDetails(stage *StageRecord) {
|
||||||
|
stage.Outputs = []ArtifactRecord{{
|
||||||
|
Kind: "structured_data",
|
||||||
|
SourceID: "narratio.example.characters",
|
||||||
|
LocalPath: "artifacts/characters.json",
|
||||||
|
}}
|
||||||
|
stage.Logs = []string{"logs/extract.log"}
|
||||||
|
stage.GeneratedConfigs = []string{"generated/extract.yaml"}
|
||||||
|
stage.Metadata = map[string]any{"bundle_path": "extract/results/run-1"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func requireStageResultDetailsCleared(t *testing.T, stage *StageRecord) {
|
||||||
|
t.Helper()
|
||||||
|
if len(stage.Outputs) != 0 || len(stage.Logs) != 0 || len(stage.GeneratedConfigs) != 0 || len(stage.Metadata) != 0 {
|
||||||
|
t.Fatalf("result details were not cleared: %#v", stage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -119,6 +119,7 @@ func (m *RunManifest) MarkStageSkipped(name string, at time.Time, reason string)
|
|||||||
s.Status = StatusSkipped
|
s.Status = StatusSkipped
|
||||||
s.CompletedAt = timePtr(at)
|
s.CompletedAt = timePtr(at)
|
||||||
s.Error = &ErrorRecord{Message: strings.TrimSpace(reason), Code: "skipped", At: timePtr(at)}
|
s.Error = &ErrorRecord{Message: strings.TrimSpace(reason), Code: "skipped", At: timePtr(at)}
|
||||||
|
s.Outputs = nil
|
||||||
s.UpdatedAt = at
|
s.UpdatedAt = at
|
||||||
m.UpdatedAt = at
|
m.UpdatedAt = at
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
|
func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
|
||||||
@@ -64,6 +66,76 @@ func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLocalStoreArtifactMetadataCompatibility(t *testing.T) {
|
||||||
|
store := &LocalStore{}
|
||||||
|
ctx := context.Background()
|
||||||
|
dir := t.TempDir()
|
||||||
|
oldPath := filepath.Join(dir, "old-manifest.json")
|
||||||
|
oldJSON := `{
|
||||||
|
"session_id": "2026-05-03",
|
||||||
|
"created_at": "2026-05-03T10:00:00Z",
|
||||||
|
"updated_at": "2026-05-03T10:01:00Z",
|
||||||
|
"stages": {
|
||||||
|
"analyze": {
|
||||||
|
"name": "analyze",
|
||||||
|
"status": "succeeded",
|
||||||
|
"created_at": "2026-05-03T10:00:00Z",
|
||||||
|
"updated_at": "2026-05-03T10:01:00Z",
|
||||||
|
"outputs": [{"kind":"scriptorium_artifact","local_path":"artifacts/recap.md"}]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}`
|
||||||
|
if err := os.WriteFile(oldPath, []byte(oldJSON), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile() old manifest error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded, err := store.Load(ctx, oldPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() old manifest error = %v", err)
|
||||||
|
}
|
||||||
|
oldOutput := loaded.Stages["analyze"].Outputs[0]
|
||||||
|
if oldOutput.Contract != nil || oldOutput.ExternalProvenance != nil {
|
||||||
|
t.Fatalf("old output metadata = %#v, %#v; want nil", oldOutput.Contract, oldOutput.ExternalProvenance)
|
||||||
|
}
|
||||||
|
if err := store.Save(ctx, oldPath, loaded); err != nil {
|
||||||
|
t.Fatalf("Save() old manifest error = %v", err)
|
||||||
|
}
|
||||||
|
roundTripped, err := os.ReadFile(oldPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile() round-tripped old manifest error = %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(roundTripped), `"contract"`) || strings.Contains(string(roundTripped), `"external_provenance"`) {
|
||||||
|
t.Fatalf("old manifest gained fabricated metadata:\n%s", roundTripped)
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded.Stages["analyze"].Outputs[0].Contract = &artifactmodel.ContractMetadata{
|
||||||
|
MediaType: "application/json",
|
||||||
|
SchemaID: "example.recap",
|
||||||
|
SchemaVersion: "v1",
|
||||||
|
}
|
||||||
|
loaded.Stages["analyze"].Outputs[0].ExternalProvenance = &artifactmodel.ExternalProvenance{
|
||||||
|
System: "example",
|
||||||
|
RunID: "external-run",
|
||||||
|
PipelineID: "pipeline",
|
||||||
|
ArtifactID: "recap",
|
||||||
|
}
|
||||||
|
metadataPath := filepath.Join(dir, "metadata-manifest.json")
|
||||||
|
if err := store.Save(ctx, metadataPath, loaded); err != nil {
|
||||||
|
t.Fatalf("Save() metadata manifest error = %v", err)
|
||||||
|
}
|
||||||
|
withMetadata, err := store.Load(ctx, metadataPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() metadata manifest error = %v", err)
|
||||||
|
}
|
||||||
|
got := withMetadata.Stages["analyze"].Outputs[0]
|
||||||
|
if got.Contract == nil || got.Contract.SchemaID != "example.recap" {
|
||||||
|
t.Fatalf("contract = %#v, want persisted contract", got.Contract)
|
||||||
|
}
|
||||||
|
if got.ExternalProvenance == nil || got.ExternalProvenance.RunID != "external-run" {
|
||||||
|
t.Fatalf("external provenance = %#v, want persisted provenance", got.ExternalProvenance)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLocalStoreSaveUpdatesTimestamp(t *testing.T) {
|
func TestLocalStoreSaveUpdatesTimestamp(t *testing.T) {
|
||||||
store := &LocalStore{}
|
store := &LocalStore{}
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|||||||
@@ -84,7 +84,13 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
}}, nil
|
}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
runtimeCatalog, err := buildAnalyzeRuntimeArtifactCatalog(paths, env.Config.Pipeline.Scriptorium, env.SelectedArtifactKeys)
|
runtimeCatalog, err := buildAnalyzeRuntimeArtifactCatalog(
|
||||||
|
paths,
|
||||||
|
m,
|
||||||
|
env.Config.Pipeline.Scriptorium,
|
||||||
|
env.Config.Pipeline.Notarius,
|
||||||
|
env.SelectedArtifactKeys,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("analyze: build runtime artifact catalog: %w", err)
|
return nil, fmt.Errorf("analyze: build runtime artifact catalog: %w", err)
|
||||||
}
|
}
|
||||||
@@ -668,6 +674,16 @@ func resolveScriptoriumInput(
|
|||||||
return resolved.Path, true, ©, nil
|
return resolved.Path, true, ©, nil
|
||||||
}
|
}
|
||||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
||||||
|
if descriptor.Source.Kind == artifactpolicy.SourceKindExtraction {
|
||||||
|
if inputCfg.Required {
|
||||||
|
return "", false, nil, fmt.Errorf(
|
||||||
|
"required extraction source %q is unavailable; enable and configure pipeline.notarius output %q, then rerun extract with --force",
|
||||||
|
source,
|
||||||
|
descriptor.Source.ConfiguredKey,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return "", false, nil, nil
|
||||||
|
}
|
||||||
if descriptor.Source.Kind == artifactpolicy.SourceKindConfiguredArtifact {
|
if descriptor.Source.Kind == artifactpolicy.SourceKindConfiguredArtifact {
|
||||||
if inputCfg.Required {
|
if inputCfg.Required {
|
||||||
return "", false, nil, fmt.Errorf("configured artifact source %q is unavailable", source)
|
return "", false, nil, fmt.Errorf("configured artifact source %q is unavailable", source)
|
||||||
@@ -727,13 +743,22 @@ func preparedStableInputFilename(sourceID string) (string, bool) {
|
|||||||
|
|
||||||
func buildAnalyzeRuntimeArtifactCatalog(
|
func buildAnalyzeRuntimeArtifactCatalog(
|
||||||
paths artifacts.SessionPaths,
|
paths artifacts.SessionPaths,
|
||||||
|
m *manifest.Manifest,
|
||||||
scriptoriumCfg *config.ScriptoriumConfig,
|
scriptoriumCfg *config.ScriptoriumConfig,
|
||||||
|
notariusCfg *config.NotariusConfig,
|
||||||
selectedArtifacts []string,
|
selectedArtifacts []string,
|
||||||
) (*artifacts.ArtifactCatalog, error) {
|
) (*artifacts.ArtifactCatalog, error) {
|
||||||
catalog := artifacts.NewArtifactCatalog()
|
catalog := artifacts.NewArtifactCatalog()
|
||||||
if err := catalog.RegisterBuiltIns(); err != nil {
|
if err := catalog.RegisterBuiltIns(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
extractionDefinitions := artifacts.ExtractionDefinitionsFromConfig(notariusCfg)
|
||||||
|
if err := catalog.RegisterExtractionArtifacts(extractionDefinitions); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if notariusCfg != nil && notariusCfg.Enabled {
|
||||||
|
catalog.HydrateExtractionArtifacts(paths, m, extractionDefinitions)
|
||||||
|
}
|
||||||
if scriptoriumCfg == nil {
|
if scriptoriumCfg == nil {
|
||||||
return catalog, nil
|
return catalog, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,12 +5,14 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
@@ -1358,6 +1360,149 @@ func TestAnalyzeSkipsWhenArtifactMapEmpty(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAnalyzePassesOnlyExplicitExtractionInputsToSelectedArtifact(t *testing.T) {
|
||||||
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "final.trimmed.json"), `{"segments":[]}`)
|
||||||
|
extractionPaths := configureAnalyzeExtractionFixture(t, env, m)
|
||||||
|
|
||||||
|
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
|
artifact.Inputs["encounters"] = config.ScriptoriumInputConfig{
|
||||||
|
Source: artifacts.ExtractionArtifactSourceID("encounters"), Required: true,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact
|
||||||
|
env.SelectedArtifactKeys = []string{"session_recap"}
|
||||||
|
tracker := &analyzeObjectStoreTracker{}
|
||||||
|
env.ObjectStore = tracker
|
||||||
|
|
||||||
|
if _, err := (analyzeStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.RunRequests) != 1 {
|
||||||
|
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
|
||||||
|
}
|
||||||
|
inputs := fake.RunRequests[0].InputPaths
|
||||||
|
if inputs["encounters"] != extractionPaths["encounters"] {
|
||||||
|
t.Fatalf("encounters input = %q, want %q", inputs["encounters"], extractionPaths["encounters"])
|
||||||
|
}
|
||||||
|
if _, exists := inputs["characters"]; exists {
|
||||||
|
t.Fatalf("unconfigured extraction input was added: %#v", inputs)
|
||||||
|
}
|
||||||
|
if len(inputs) != 2 {
|
||||||
|
t.Fatalf("input paths = %#v, want transcript plus explicit encounters", inputs)
|
||||||
|
}
|
||||||
|
if tracker.called {
|
||||||
|
t.Fatal("analyze extraction resolution called the object store")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequiredUnavailableExtractionFailsWithGuidance(t *testing.T) {
|
||||||
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "final.trimmed.json"), `{"segments":[]}`)
|
||||||
|
configureAnalyzeExtraction(t, env)
|
||||||
|
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
|
artifact.Inputs["encounters"] = config.ScriptoriumInputConfig{
|
||||||
|
Source: artifacts.ExtractionArtifactSourceID("encounters"), Required: true,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact
|
||||||
|
|
||||||
|
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "pipeline.notarius output \"encounters\"") || !strings.Contains(err.Error(), "rerun extract with --force") {
|
||||||
|
t.Fatalf("Run() error = %v, want actionable extraction guidance", err)
|
||||||
|
}
|
||||||
|
if len(fake.RunRequests) != 0 {
|
||||||
|
t.Fatalf("run requests = %d, want 0", len(fake.RunRequests))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeOmitsOptionalUnavailableExtraction(t *testing.T) {
|
||||||
|
env, m, fake := setupAnalyzeEnv(t)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "final.trimmed.json"), `{"segments":[]}`)
|
||||||
|
configureAnalyzeExtraction(t, env)
|
||||||
|
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||||
|
artifact.Inputs["encounters"] = config.ScriptoriumInputConfig{
|
||||||
|
Source: artifacts.ExtractionArtifactSourceID("encounters"), Required: false,
|
||||||
|
}
|
||||||
|
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact
|
||||||
|
|
||||||
|
if _, err := (analyzeStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.RunRequests) != 1 {
|
||||||
|
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
|
||||||
|
}
|
||||||
|
if _, exists := fake.RunRequests[0].InputPaths["encounters"]; exists {
|
||||||
|
t.Fatalf("optional unavailable extraction was passed: %#v", fake.RunRequests[0].InputPaths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func configureAnalyzeExtraction(t *testing.T, env *Env) {
|
||||||
|
t.Helper()
|
||||||
|
env.Config.Pipeline.Notarius = &config.NotariusConfig{
|
||||||
|
Enabled: true, PipelineID: "campaign.extract",
|
||||||
|
Outputs: map[string]config.NotariusOutputConfig{
|
||||||
|
"characters": {LaneID: "characters", MediaType: "application/json", SchemaID: "characters", SchemaVersion: "1"},
|
||||||
|
"encounters": {LaneID: "encounters", MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func configureAnalyzeExtractionFixture(t *testing.T, env *Env, m *manifest.Manifest) map[string]string {
|
||||||
|
t.Helper()
|
||||||
|
configureAnalyzeExtraction(t, env)
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
producerRunID := "extract-run-1"
|
||||||
|
bundleRoot := filepath.Join(paths.ArtifactsDir, "notarius", producerRunID)
|
||||||
|
keys := make([]string, 0, len(env.Config.Pipeline.Notarius.Outputs))
|
||||||
|
for key := range env.Config.Pipeline.Notarius.Outputs {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
|
||||||
|
outputPaths := make(map[string]string, len(keys))
|
||||||
|
outputs := make([]manifest.ArtifactRecord, 0, len(keys)+1)
|
||||||
|
for _, key := range keys {
|
||||||
|
definition := env.Config.Pipeline.Notarius.Outputs[key]
|
||||||
|
outputPath := filepath.Join(bundleRoot, "lanes", key+".json")
|
||||||
|
writeAnalyzeFile(t, outputPath, `{"items":[]}`)
|
||||||
|
checksum, err := artifacts.SHA256File(outputPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
outputPaths[key] = outputPath
|
||||||
|
outputs = append(outputs, manifest.ArtifactRecord{
|
||||||
|
Kind: "notarius_lane", SourceID: artifacts.ExtractionArtifactSourceID(key), LocalPath: outputPath,
|
||||||
|
ProducerRunID: producerRunID, Checksum: checksum,
|
||||||
|
Contract: &artifactmodel.ContractMetadata{
|
||||||
|
MediaType: definition.MediaType, SchemaID: definition.SchemaID,
|
||||||
|
SchemaVersion: definition.SchemaVersion, ModuleKey: definition.ModuleKey,
|
||||||
|
},
|
||||||
|
ExternalProvenance: &artifactmodel.ExternalProvenance{
|
||||||
|
System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: definition.LaneID,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
indexPath := filepath.Join(bundleRoot, "index.json")
|
||||||
|
writeAnalyzeFile(t, indexPath, `{"lanes":[]}`)
|
||||||
|
indexChecksum, err := artifacts.SHA256File(indexPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
outputs = append(outputs, manifest.ArtifactRecord{
|
||||||
|
Kind: "notarius_index", LocalPath: indexPath, ProducerRunID: producerRunID, Checksum: indexChecksum,
|
||||||
|
})
|
||||||
|
m.Stages["extract"] = &manifest.StageRecord{
|
||||||
|
Name: "extract", Status: manifest.StatusSucceeded, Outputs: outputs,
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"narratio_run_id": producerRunID, "bundle_root": bundleRoot,
|
||||||
|
"receipt": map[string]any{"run_id": "notarius-run-1", "pipeline_id": "campaign.extract"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return outputPaths
|
||||||
|
}
|
||||||
|
|
||||||
func setupAnalyzeEnv(t *testing.T) (*Env, *manifest.Manifest, *scriptorium.FakeRunner) {
|
func setupAnalyzeEnv(t *testing.T) (*Env, *manifest.Manifest, *scriptorium.FakeRunner) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
workspace := t.TempDir()
|
workspace := t.TempDir()
|
||||||
|
|||||||
462
internal/stage/extract.go
Normal file
462
internal/stage/extract.go
Normal file
@@ -0,0 +1,462 @@
|
|||||||
|
package stage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
extractSkipReason = "notarius_disabled"
|
||||||
|
extractLaneOutputKind = "notarius_lane"
|
||||||
|
extractIndexOutputKind = "notarius_index"
|
||||||
|
maxDiagnosticSummaries = 100
|
||||||
|
)
|
||||||
|
|
||||||
|
type extractStage struct{}
|
||||||
|
|
||||||
|
func (extractStage) Name() string { return "extract" }
|
||||||
|
|
||||||
|
func (extractStage) Declares() IODecl {
|
||||||
|
return IODecl{
|
||||||
|
Inputs: []artifacts.Ref{{
|
||||||
|
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed,
|
||||||
|
SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
|
||||||
|
Category: "transcripts",
|
||||||
|
RelativePath: artifactmodel.TranscriptPathFinalTrimmed,
|
||||||
|
}},
|
||||||
|
Outputs: []artifacts.Ref{
|
||||||
|
{Kind: extractLaneOutputKind, Category: "artifacts", RelativePath: "artifacts/notarius/<run-id>/lanes/*.json"},
|
||||||
|
{Kind: extractIndexOutputKind, Category: "artifacts", RelativePath: "artifacts/notarius/<run-id>/index.json"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (extractStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
|
||||||
|
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolved stage environment config is required")
|
||||||
|
}
|
||||||
|
notariusConfig := env.Config.Pipeline.Notarius
|
||||||
|
if notariusConfig == nil || !notariusConfig.Enabled {
|
||||||
|
return &StageResult{
|
||||||
|
Disposition: StageDispositionSkipped,
|
||||||
|
SkipReason: extractSkipReason,
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"stage": "extract",
|
||||||
|
"notarius_enabled": false,
|
||||||
|
"reason": extractSkipReason,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
if env.ArtifactStore == nil {
|
||||||
|
return nil, fmt.Errorf("extract: artifact store is required")
|
||||||
|
}
|
||||||
|
if env.Notarius == nil {
|
||||||
|
return nil, fmt.Errorf("extract: notarius adapter is required")
|
||||||
|
}
|
||||||
|
if m == nil {
|
||||||
|
return nil, fmt.Errorf("extract: session manifest is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionID := strings.TrimSpace(m.SessionID)
|
||||||
|
if sessionID == "" {
|
||||||
|
sessionID = strings.TrimSpace(env.Config.Session.SessionID)
|
||||||
|
}
|
||||||
|
campaign := strings.TrimSpace(m.Campaign)
|
||||||
|
if campaign == "" {
|
||||||
|
campaign = strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
}
|
||||||
|
runID := strings.TrimSpace(m.RunID)
|
||||||
|
if sessionID == "" || campaign == "" || runID == "" {
|
||||||
|
return nil, fmt.Errorf("extract: session id, campaign, and run id are required")
|
||||||
|
}
|
||||||
|
if !safePathSegment(runID) {
|
||||||
|
return nil, fmt.Errorf("extract: run id %q is not a safe path segment", runID)
|
||||||
|
}
|
||||||
|
|
||||||
|
paths := sessionPathsForEnv(env, sessionID)
|
||||||
|
input, err := artifacts.ResolveSessionArtifact(paths, m, artifacts.ArtifactTranscriptFinalTrimmed)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve final-trimmed transcript: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeout, err := time.ParseDuration(strings.TrimSpace(notariusConfig.Timeout))
|
||||||
|
if err != nil || timeout <= 0 {
|
||||||
|
return nil, fmt.Errorf("extract: invalid notarius timeout %q", notariusConfig.Timeout)
|
||||||
|
}
|
||||||
|
resolvedBinary, err := resolveExecutable(notariusConfig.Binary)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve notarius binary: %w", err)
|
||||||
|
}
|
||||||
|
configPath, err := absolutePath(notariusConfig.ConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve notarius config path: %w", err)
|
||||||
|
}
|
||||||
|
inputPath, err := absolutePath(input.Path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve transcript input path: %w", err)
|
||||||
|
}
|
||||||
|
workingDirectory, err := absolutePath(notariusConfig.WorkingDirectory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve notarius working directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
workspaceRoot := strings.TrimSpace(paths.WorkspaceRoot)
|
||||||
|
if workspaceRoot == "" {
|
||||||
|
workspaceRoot = env.Config.Pipeline.Workspace.Root
|
||||||
|
}
|
||||||
|
receiptPath, err := absolutePath(artifacts.SessionRunNotariusReceiptPathForCampaign(workspaceRoot, campaign, sessionID, runID))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve receipt path: %w", err)
|
||||||
|
}
|
||||||
|
logPath, err := absolutePath(artifacts.SessionRunNotariusLogPathForCampaign(workspaceRoot, campaign, sessionID, runID))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve stderr path: %w", err)
|
||||||
|
}
|
||||||
|
outputRoot, err := absolutePath(artifacts.SessionRunNotariusOutputRootForCampaign(workspaceRoot, campaign, sessionID, runID))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve output root: %w", err)
|
||||||
|
}
|
||||||
|
durableBundle, err := absolutePath(artifacts.SessionNotariusBundleDirForCampaign(workspaceRoot, campaign, sessionID, runID))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve durable bundle path: %w", err)
|
||||||
|
}
|
||||||
|
for _, directory := range []string{filepath.Dir(receiptPath), outputRoot, filepath.Dir(durableBundle)} {
|
||||||
|
if err := os.MkdirAll(directory, 0o755); err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: create directory %q: %w", directory, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fingerprint, err := extractionFingerprint(resolvedBinary, configPath, notariusConfig, timeout, workingDirectory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: build configuration fingerprint: %w", err)
|
||||||
|
}
|
||||||
|
request := notarius.RunRequest{
|
||||||
|
Binary: resolvedBinary, ConfigPath: configPath, PipelineID: notariusConfig.PipelineID,
|
||||||
|
InputPath: inputPath, OutputRoot: outputRoot, WorkingDirectory: workingDirectory,
|
||||||
|
ReceiptPath: receiptPath, LogPath: logPath, Timeout: timeout,
|
||||||
|
}
|
||||||
|
adapterResult, err := env.Notarius.Run(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: run notarius: %w", err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(adapterResult.BundleRoot) == "" || strings.TrimSpace(adapterResult.Index.Path) == "" {
|
||||||
|
return nil, fmt.Errorf("extract: notarius result is missing bundle or index path")
|
||||||
|
}
|
||||||
|
if adapterResult.Receipt.RunID == "" || adapterResult.Receipt.PipelineID != notariusConfig.PipelineID {
|
||||||
|
return nil, fmt.Errorf("extract: notarius receipt identity is missing or incompatible")
|
||||||
|
}
|
||||||
|
|
||||||
|
selected, err := selectRequiredNotariusLanes(env.ArtifactStore, notariusConfig.Outputs, adapterResult)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
indexRelative, err := pathsafe.SlashRelativeFromRoot(adapterResult.BundleRoot, adapterResult.Index.Path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve staging index relative path: %w", err)
|
||||||
|
}
|
||||||
|
rejectionsRelative, err := pathsafe.SlashRelativeFromRoot(adapterResult.BundleRoot, adapterResult.Index.RejectedPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve staging rejections relative path: %w", err)
|
||||||
|
}
|
||||||
|
warningsRelative, err := pathsafe.SlashRelativeFromRoot(adapterResult.BundleRoot, adapterResult.Index.WarningsPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve staging warnings relative path: %w", err)
|
||||||
|
}
|
||||||
|
stagingIndexChecksum, err := checksumRegularFile(adapterResult.Index.Path, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: validate staging index: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := fileops.PromoteDirectory(adapterResult.BundleRoot, durableBundle); err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: promote notarius bundle: %w", err)
|
||||||
|
}
|
||||||
|
promotedIndexPath, err := pathsafe.JoinSlashRelativeUnderRoot(durableBundle, indexRelative)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve promoted index: %w", err)
|
||||||
|
}
|
||||||
|
promotedIndexChecksum, err := checksumRegularFile(promotedIndexPath, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: checksum promoted index: %w", err)
|
||||||
|
}
|
||||||
|
if promotedIndexChecksum != stagingIndexChecksum {
|
||||||
|
return nil, fmt.Errorf("extract: promoted index checksum differs from staging index")
|
||||||
|
}
|
||||||
|
promotedRejectionsPath, err := pathsafe.JoinSlashRelativeUnderRoot(durableBundle, rejectionsRelative)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve promoted rejections: %w", err)
|
||||||
|
}
|
||||||
|
promotedWarningsPath, err := pathsafe.JoinSlashRelativeUnderRoot(durableBundle, warningsRelative)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve promoted warnings: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
outputs := make([]artifacts.Ref, 0, len(selected)+1)
|
||||||
|
for _, lane := range selected {
|
||||||
|
promotedPath, err := pathsafe.JoinSlashRelativeUnderRoot(durableBundle, lane.RelativePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve promoted lane %q: %w", lane.Descriptor.LaneID, err)
|
||||||
|
}
|
||||||
|
checksum, err := checksumRegularFile(promotedPath, true)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: validate promoted lane %q: %w", lane.Descriptor.LaneID, err)
|
||||||
|
}
|
||||||
|
if checksum != lane.StagingChecksum {
|
||||||
|
return nil, fmt.Errorf("extract: promoted lane %q checksum differs from staging payload", lane.Descriptor.LaneID)
|
||||||
|
}
|
||||||
|
relativePath, err := pathsafe.SlashRelativeFromRoot(paths.Root, promotedPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: derive lane %q session-relative path: %w", lane.Descriptor.LaneID, err)
|
||||||
|
}
|
||||||
|
outputs = append(outputs, artifacts.Ref{
|
||||||
|
Kind: extractLaneOutputKind, SourceID: artifacts.ExtractionArtifactSourceID(lane.Key),
|
||||||
|
Category: "artifacts", SessionID: sessionID, RelativePath: relativePath,
|
||||||
|
AbsolutePath: promotedPath, Checksum: checksum,
|
||||||
|
Contract: &artifactmodel.ContractMetadata{
|
||||||
|
MediaType: lane.Descriptor.MediaType, SchemaID: lane.Descriptor.SchemaID,
|
||||||
|
SchemaVersion: lane.Descriptor.SchemaVersion, ModuleKey: lane.Descriptor.ModuleKey,
|
||||||
|
},
|
||||||
|
ExternalProvenance: &artifactmodel.ExternalProvenance{
|
||||||
|
System: "notarius", RunID: adapterResult.Receipt.RunID,
|
||||||
|
PipelineID: adapterResult.Receipt.PipelineID, ArtifactID: lane.Descriptor.LaneID,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
indexSessionRelative, err := pathsafe.SlashRelativeFromRoot(paths.Root, promotedIndexPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: derive index session-relative path: %w", err)
|
||||||
|
}
|
||||||
|
outputs = append(outputs, artifacts.Ref{
|
||||||
|
Kind: extractIndexOutputKind, Category: "artifacts", SessionID: sessionID,
|
||||||
|
RelativePath: indexSessionRelative, AbsolutePath: promotedIndexPath, Checksum: promotedIndexChecksum,
|
||||||
|
})
|
||||||
|
|
||||||
|
metadata := map[string]any{
|
||||||
|
"stage": "extract",
|
||||||
|
"notarius_enabled": true,
|
||||||
|
"bundle_root": durableBundle,
|
||||||
|
"receipt_path": receiptPath,
|
||||||
|
"diagnostic_path": logPath,
|
||||||
|
"rejections_path": promotedRejectionsPath,
|
||||||
|
"warnings_path": promotedWarningsPath,
|
||||||
|
"narratio_run_id": runID,
|
||||||
|
"configuration_fingerprint": fingerprint,
|
||||||
|
"receipt": map[string]any{
|
||||||
|
"run_id": adapterResult.Receipt.RunID, "pipeline_id": adapterResult.Receipt.PipelineID,
|
||||||
|
"normalized_output_count": adapterResult.Receipt.NormalizedOutputCount,
|
||||||
|
"rejected_output_count": adapterResult.Receipt.RejectedOutputCount,
|
||||||
|
"warning_count": adapterResult.Receipt.WarningCount,
|
||||||
|
"validation_status": adapterResult.Receipt.ValidationStatus,
|
||||||
|
},
|
||||||
|
"rejections": boundedRejectionMetadata(adapterResult.Rejections),
|
||||||
|
"warnings": boundedWarningMetadata(adapterResult.Warnings),
|
||||||
|
}
|
||||||
|
return &StageResult{
|
||||||
|
Outputs: outputs,
|
||||||
|
Logs: []string{receiptPath, logPath},
|
||||||
|
Metadata: metadata,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type selectedNotariusLane struct {
|
||||||
|
Key string
|
||||||
|
Descriptor notarius.LaneDescriptor
|
||||||
|
RelativePath string
|
||||||
|
StagingChecksum string
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectRequiredNotariusLanes(
|
||||||
|
store artifacts.Store,
|
||||||
|
required map[string]config.NotariusOutputConfig,
|
||||||
|
result notarius.RunResult,
|
||||||
|
) ([]selectedNotariusLane, error) {
|
||||||
|
keys := make([]string, 0, len(required))
|
||||||
|
for key := range required {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
selected := make([]selectedNotariusLane, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
expected := required[key]
|
||||||
|
for _, rejection := range result.Rejections {
|
||||||
|
if rejection.LaneID == expected.LaneID {
|
||||||
|
return nil, fmt.Errorf("extract: required lane %q was rejected (reason_code=%q)", expected.LaneID, rejection.ReasonCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
matches := make([]notarius.LaneDescriptor, 0, 1)
|
||||||
|
for _, descriptor := range result.Index.Lanes {
|
||||||
|
if descriptor.LaneID == expected.LaneID {
|
||||||
|
matches = append(matches, descriptor)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(matches) != 1 {
|
||||||
|
return nil, fmt.Errorf("extract: required lane %q has %d descriptors, want exactly one", expected.LaneID, len(matches))
|
||||||
|
}
|
||||||
|
descriptor := matches[0]
|
||||||
|
if descriptor.MediaType != expected.MediaType || descriptor.SchemaID != expected.SchemaID ||
|
||||||
|
descriptor.SchemaVersion != expected.SchemaVersion ||
|
||||||
|
(expected.ModuleKey != "" && descriptor.ModuleKey != expected.ModuleKey) {
|
||||||
|
return nil, fmt.Errorf("extract: required lane %q descriptor contract is incompatible", expected.LaneID)
|
||||||
|
}
|
||||||
|
checksum, err := checksumRegularFile(descriptor.Path, true)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: validate required lane %q: %w", expected.LaneID, err)
|
||||||
|
}
|
||||||
|
if store != nil {
|
||||||
|
storeChecksum, err := store.Checksum(descriptor.Path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: checksum required lane %q: %w", expected.LaneID, err)
|
||||||
|
}
|
||||||
|
if storeChecksum != checksum {
|
||||||
|
return nil, fmt.Errorf("extract: inconsistent staging checksum for required lane %q", expected.LaneID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
relative, err := pathsafe.SlashRelativeFromRoot(result.BundleRoot, descriptor.Path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("extract: resolve required lane %q relative path: %w", expected.LaneID, err)
|
||||||
|
}
|
||||||
|
selected = append(selected, selectedNotariusLane{
|
||||||
|
Key: key, Descriptor: descriptor, RelativePath: relative, StagingChecksum: checksum,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return selected, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checksumRegularFile(path string, requireJSON bool) (string, error) {
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||||
|
return "", fmt.Errorf("path %q must be a regular file without symlinks", path)
|
||||||
|
}
|
||||||
|
if info.Size() == 0 {
|
||||||
|
return "", fmt.Errorf("path %q must be non-empty", path)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if requireJSON && !json.Valid(data) {
|
||||||
|
return "", fmt.Errorf("path %q is not valid JSON", path)
|
||||||
|
}
|
||||||
|
digest := sha256.Sum256(data)
|
||||||
|
return hex.EncodeToString(digest[:]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fingerprintOutput struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
LaneID string `json:"lane_id"`
|
||||||
|
MediaType string `json:"media_type"`
|
||||||
|
SchemaID string `json:"schema_id"`
|
||||||
|
SchemaVersion string `json:"schema_version"`
|
||||||
|
ModuleKey string `json:"module_key,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type fingerprintDocument struct {
|
||||||
|
Binary string `json:"binary"`
|
||||||
|
ConfigPath string `json:"config_path"`
|
||||||
|
PipelineID string `json:"pipeline_id"`
|
||||||
|
Timeout string `json:"timeout"`
|
||||||
|
WorkingDirectory string `json:"working_directory"`
|
||||||
|
Outputs []fingerprintOutput `json:"outputs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractionFingerprint(
|
||||||
|
binary, configPath string,
|
||||||
|
cfg *config.NotariusConfig,
|
||||||
|
timeout time.Duration,
|
||||||
|
workingDirectory string,
|
||||||
|
) (string, error) {
|
||||||
|
keys := make([]string, 0, len(cfg.Outputs))
|
||||||
|
for key := range cfg.Outputs {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
outputs := make([]fingerprintOutput, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
output := cfg.Outputs[key]
|
||||||
|
outputs = append(outputs, fingerprintOutput{
|
||||||
|
Key: key, LaneID: output.LaneID, MediaType: output.MediaType,
|
||||||
|
SchemaID: output.SchemaID, SchemaVersion: output.SchemaVersion, ModuleKey: output.ModuleKey,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(fingerprintDocument{
|
||||||
|
Binary: binary, ConfigPath: configPath, PipelineID: cfg.PipelineID,
|
||||||
|
Timeout: timeout.String(), WorkingDirectory: workingDirectory, Outputs: outputs,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
digest := sha256.Sum256(payload)
|
||||||
|
return hex.EncodeToString(digest[:]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveExecutable(value string) (string, error) {
|
||||||
|
resolved, err := exec.LookPath(strings.TrimSpace(value))
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return absolutePath(resolved)
|
||||||
|
}
|
||||||
|
|
||||||
|
func absolutePath(value string) (string, error) {
|
||||||
|
trimmed := strings.TrimSpace(value)
|
||||||
|
if trimmed == "" {
|
||||||
|
return "", fmt.Errorf("path is required")
|
||||||
|
}
|
||||||
|
resolved, err := filepath.Abs(trimmed)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.Clean(resolved), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func safePathSegment(value string) bool {
|
||||||
|
return value != "" && value != "." && value != ".." && filepath.Base(value) == value &&
|
||||||
|
!strings.ContainsAny(value, `/\\`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func boundedRejectionMetadata(values []notarius.RejectionSummary) []map[string]any {
|
||||||
|
limit := len(values)
|
||||||
|
if limit > maxDiagnosticSummaries {
|
||||||
|
limit = maxDiagnosticSummaries
|
||||||
|
}
|
||||||
|
result := make([]map[string]any, 0, limit)
|
||||||
|
for _, value := range values[:limit] {
|
||||||
|
result = append(result, map[string]any{
|
||||||
|
"stage": value.Stage, "step_id": value.StepID, "lane_id": value.LaneID,
|
||||||
|
"module_key": value.ModuleKey, "chunk_id": value.ChunkID,
|
||||||
|
"validator_name": value.ValidatorName, "reason_code": value.ReasonCode,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func boundedWarningMetadata(values []notarius.WarningSummary) []map[string]any {
|
||||||
|
limit := len(values)
|
||||||
|
if limit > maxDiagnosticSummaries {
|
||||||
|
limit = maxDiagnosticSummaries
|
||||||
|
}
|
||||||
|
result := make([]map[string]any, 0, limit)
|
||||||
|
for _, value := range values[:limit] {
|
||||||
|
result = append(result, map[string]any{"scope": value.Scope, "reason_code": value.ReasonCode})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
250
internal/stage/extract_resume.go
Normal file
250
internal/stage/extract_resume.go
Normal file
@@ -0,0 +1,250 @@
|
|||||||
|
package stage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (extractStage) ValidateResume(_ context.Context, env *Env, m *manifest.Manifest) (ResumeValidation, error) {
|
||||||
|
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: resolved stage environment config is required")
|
||||||
|
}
|
||||||
|
cfg := env.Config.Pipeline.Notarius
|
||||||
|
if cfg == nil || !cfg.Enabled {
|
||||||
|
return NonResumable("Notarius extraction is disabled"), nil
|
||||||
|
}
|
||||||
|
if m == nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: session manifest is required")
|
||||||
|
}
|
||||||
|
record := m.Stages[(extractStage{}).Name()]
|
||||||
|
if record == nil || record.Status != manifest.StatusSucceeded {
|
||||||
|
return NonResumable("extract stage has no succeeded result"), nil
|
||||||
|
}
|
||||||
|
if record.Name != (extractStage{}).Name() {
|
||||||
|
return NonResumable("extract stage record identity is inconsistent"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
producerRunID := metadataString(record.Metadata, "narratio_run_id")
|
||||||
|
if producerRunID == "" {
|
||||||
|
return NonResumable("extract result is missing its producing run ID"), nil
|
||||||
|
}
|
||||||
|
if !safePathSegment(producerRunID) {
|
||||||
|
return NonResumable("extract result has an invalid producing run ID"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
timeout, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout))
|
||||||
|
if err != nil || timeout <= 0 {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: invalid Notarius timeout %q", cfg.Timeout)
|
||||||
|
}
|
||||||
|
resolvedBinary, err := resolveExecutable(cfg.Binary)
|
||||||
|
if err != nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: resolve Notarius binary: %w", err)
|
||||||
|
}
|
||||||
|
configPath, err := absolutePath(cfg.ConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: resolve Notarius config path: %w", err)
|
||||||
|
}
|
||||||
|
workingDirectory, err := absolutePath(cfg.WorkingDirectory)
|
||||||
|
if err != nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: resolve Notarius working directory: %w", err)
|
||||||
|
}
|
||||||
|
fingerprint, err := extractionFingerprint(resolvedBinary, configPath, cfg, timeout, workingDirectory)
|
||||||
|
if err != nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: build configuration fingerprint: %w", err)
|
||||||
|
}
|
||||||
|
if metadataString(record.Metadata, "configuration_fingerprint") != fingerprint {
|
||||||
|
return NonResumable("Notarius invocation contract changed"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionID := strings.TrimSpace(m.SessionID)
|
||||||
|
if sessionID == "" {
|
||||||
|
sessionID = strings.TrimSpace(env.Config.Session.SessionID)
|
||||||
|
}
|
||||||
|
campaign := strings.TrimSpace(m.Campaign)
|
||||||
|
if campaign == "" {
|
||||||
|
campaign = strings.TrimSpace(env.Config.Session.Campaign)
|
||||||
|
}
|
||||||
|
if sessionID == "" || campaign == "" {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: session ID and campaign are required")
|
||||||
|
}
|
||||||
|
paths := sessionPathsForEnv(env, sessionID)
|
||||||
|
workspaceRoot := strings.TrimSpace(paths.WorkspaceRoot)
|
||||||
|
if workspaceRoot == "" {
|
||||||
|
workspaceRoot = env.Config.Pipeline.Workspace.Root
|
||||||
|
}
|
||||||
|
bundleRoot, err := absolutePath(artifacts.SessionNotariusBundleDirForCampaign(workspaceRoot, campaign, sessionID, producerRunID))
|
||||||
|
if err != nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: resolve durable bundle path: %w", err)
|
||||||
|
}
|
||||||
|
storedBundleRoot := metadataString(record.Metadata, "bundle_root")
|
||||||
|
if storedBundleRoot == "" || !filepath.IsAbs(storedBundleRoot) || filepath.Clean(storedBundleRoot) != bundleRoot {
|
||||||
|
return NonResumable("extract result does not identify its canonical immutable bundle"), nil
|
||||||
|
}
|
||||||
|
bundleRelative, err := pathsafe.SlashRelativeFromRoot(paths.Root, bundleRoot)
|
||||||
|
if err != nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: immutable bundle path is unsafe: %w", err)
|
||||||
|
}
|
||||||
|
if err := rejectSymlinkComponents(paths.Root, bundleRelative); err != nil {
|
||||||
|
return ResumeValidation{}, err
|
||||||
|
}
|
||||||
|
bundleInfo, err := os.Lstat(bundleRoot)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return NonResumable("immutable Notarius bundle is missing"), nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: inspect immutable bundle: %w", err)
|
||||||
|
}
|
||||||
|
if bundleInfo.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: immutable bundle must not be a symlink")
|
||||||
|
}
|
||||||
|
if !bundleInfo.IsDir() {
|
||||||
|
return NonResumable("immutable Notarius bundle is not a directory"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
receiptRunID, receiptPipelineID := receiptIdentity(record.Metadata)
|
||||||
|
if receiptRunID == "" || receiptPipelineID != cfg.PipelineID {
|
||||||
|
return NonResumable("extract result has incompatible Notarius receipt identity"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
expectedSources := make(map[string]config.NotariusOutputConfig, len(cfg.Outputs))
|
||||||
|
for key, output := range cfg.Outputs {
|
||||||
|
expectedSources[artifacts.ExtractionArtifactSourceID(key)] = output
|
||||||
|
}
|
||||||
|
seenSources := make(map[string]struct{}, len(expectedSources))
|
||||||
|
indexSeen := false
|
||||||
|
for _, output := range record.Outputs {
|
||||||
|
if output.ProducerRunID != producerRunID {
|
||||||
|
return NonResumable("extract output producer identity is inconsistent"), nil
|
||||||
|
}
|
||||||
|
if output.SourceID == "" {
|
||||||
|
if indexSeen || output.Kind != extractIndexOutputKind {
|
||||||
|
return NonResumable("extract result has an unexpected non-selectable output"), nil
|
||||||
|
}
|
||||||
|
indexSeen = true
|
||||||
|
expectedIndex := filepath.Join(bundleRoot, "index.json")
|
||||||
|
if filepath.Clean(output.LocalPath) != expectedIndex {
|
||||||
|
return NonResumable("extract index path is not canonical"), nil
|
||||||
|
}
|
||||||
|
validation, err := validateResumePayload(bundleRoot, output.LocalPath, output.Checksum)
|
||||||
|
if err != nil || !validation.Resumable {
|
||||||
|
return validation, err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
expected, ok := expectedSources[output.SourceID]
|
||||||
|
if !ok || output.Kind != extractLaneOutputKind {
|
||||||
|
return NonResumable("extract result source set differs from current configuration"), nil
|
||||||
|
}
|
||||||
|
if _, duplicate := seenSources[output.SourceID]; duplicate {
|
||||||
|
return NonResumable("extract result contains a duplicate configured source"), nil
|
||||||
|
}
|
||||||
|
seenSources[output.SourceID] = struct{}{}
|
||||||
|
if !compatibleExtractionContract(output.Contract, expected) {
|
||||||
|
return NonResumable("extract output contract is incompatible with current configuration"), nil
|
||||||
|
}
|
||||||
|
if !compatibleExtractionProvenance(output.ExternalProvenance, receiptRunID, receiptPipelineID, expected.LaneID) {
|
||||||
|
return NonResumable("extract output has incompatible Notarius provenance"), nil
|
||||||
|
}
|
||||||
|
validation, err := validateResumePayload(bundleRoot, output.LocalPath, output.Checksum)
|
||||||
|
if err != nil || !validation.Resumable {
|
||||||
|
return validation, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !indexSeen {
|
||||||
|
return NonResumable("extract result is missing its canonical index"), nil
|
||||||
|
}
|
||||||
|
if len(seenSources) != len(expectedSources) || len(record.Outputs) != len(expectedSources)+1 {
|
||||||
|
return NonResumable("extract result is missing configured sources"), nil
|
||||||
|
}
|
||||||
|
return Resumable(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateResumePayload(bundleRoot, path, checksum string) (ResumeValidation, error) {
|
||||||
|
if !filepath.IsAbs(path) {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: persisted output path must be absolute")
|
||||||
|
}
|
||||||
|
relative, err := pathsafe.SlashRelativeFromRoot(bundleRoot, path)
|
||||||
|
if err != nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: persisted output path is unsafe: %w", err)
|
||||||
|
}
|
||||||
|
if err := rejectSymlinkComponents(bundleRoot, relative); err != nil {
|
||||||
|
return ResumeValidation{}, err
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return NonResumable("extract output is missing"), nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: inspect output %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: output %q must be a regular file without symlinks", path)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(checksum) == "" {
|
||||||
|
return NonResumable("extract output is missing its checksum"), nil
|
||||||
|
}
|
||||||
|
actual, err := artifacts.SHA256File(path)
|
||||||
|
if err != nil {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: checksum output %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if actual != checksum {
|
||||||
|
return NonResumable("extract output checksum does not match durable bytes"), nil
|
||||||
|
}
|
||||||
|
return Resumable(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func rejectSymlinkComponents(root, slashRelative string) error {
|
||||||
|
current := filepath.Clean(root)
|
||||||
|
parts := strings.Split(filepath.FromSlash(slashRelative), string(filepath.Separator))
|
||||||
|
for _, part := range parts[:len(parts)-1] {
|
||||||
|
current = filepath.Join(current, part)
|
||||||
|
info, err := os.Lstat(current)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("extract resume: inspect output directory %q: %w", current, err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return fmt.Errorf("extract resume: output directory %q must not be a symlink", current)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func compatibleExtractionContract(got *artifactmodel.ContractMetadata, want config.NotariusOutputConfig) bool {
|
||||||
|
return got != nil && got.MediaType == want.MediaType && got.SchemaID == want.SchemaID &&
|
||||||
|
got.SchemaVersion == want.SchemaVersion && (want.ModuleKey == "" || got.ModuleKey == want.ModuleKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func compatibleExtractionProvenance(got *artifactmodel.ExternalProvenance, runID, pipelineID, laneID string) bool {
|
||||||
|
return got != nil && got.System == "notarius" && got.RunID == runID &&
|
||||||
|
got.PipelineID == pipelineID && got.ArtifactID == laneID
|
||||||
|
}
|
||||||
|
|
||||||
|
func metadataString(metadata map[string]any, key string) string {
|
||||||
|
if metadata == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
value, _ := metadata[key].(string)
|
||||||
|
return strings.TrimSpace(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func receiptIdentity(metadata map[string]any) (string, string) {
|
||||||
|
if metadata == nil {
|
||||||
|
return "", ""
|
||||||
|
}
|
||||||
|
receipt, _ := metadata["receipt"].(map[string]any)
|
||||||
|
return metadataString(receipt, "run_id"), metadataString(receipt, "pipeline_id")
|
||||||
|
}
|
||||||
575
internal/stage/extract_test.go
Normal file
575
internal/stage/extract_test.go
Normal file
@@ -0,0 +1,575 @@
|
|||||||
|
package stage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExtractStageDisabledReturnsExplicitSkip(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
absent bool
|
||||||
|
}{
|
||||||
|
{name: "disabled"},
|
||||||
|
{name: "absent", absent: true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
env, m, fake := setupExtractEnv(t)
|
||||||
|
if test.absent {
|
||||||
|
env.Config.Pipeline.Notarius = nil
|
||||||
|
} else {
|
||||||
|
env.Config.Pipeline.Notarius.Enabled = false
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if result.Disposition != StageDispositionSkipped || result.SkipReason != extractSkipReason {
|
||||||
|
t.Fatalf("result disposition = %q reason = %q", result.Disposition, result.SkipReason)
|
||||||
|
}
|
||||||
|
if len(result.Outputs) != 0 || len(fake.Requests) != 0 {
|
||||||
|
t.Fatalf("outputs = %#v; requests = %#v", result.Outputs, fake.Requests)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractStageResolvesManifestInputAndBuildsExactRequest(t *testing.T) {
|
||||||
|
env, m, fake := setupExtractEnv(t)
|
||||||
|
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fake.Requests) != 1 {
|
||||||
|
t.Fatalf("requests = %d, want 1", len(fake.Requests))
|
||||||
|
}
|
||||||
|
req := fake.Requests[0]
|
||||||
|
fixture := extractFixtureFromEnv(t, env, m)
|
||||||
|
if req.InputPath != fixture.inputPath {
|
||||||
|
t.Fatalf("input path = %q, want manifest path %q", req.InputPath, fixture.inputPath)
|
||||||
|
}
|
||||||
|
wantReceipt := artifacts.SessionRunNotariusReceiptPathForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID)
|
||||||
|
wantLog := artifacts.SessionRunNotariusLogPathForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID)
|
||||||
|
wantOutputRoot := artifacts.SessionRunNotariusOutputRootForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID)
|
||||||
|
if req.ConfigPath != env.Config.Pipeline.Notarius.ConfigPath || req.PipelineID != "dnd-session" ||
|
||||||
|
req.OutputRoot != wantOutputRoot || req.ReceiptPath != wantReceipt || req.LogPath != wantLog ||
|
||||||
|
req.WorkingDirectory != env.Config.Pipeline.Notarius.WorkingDirectory || req.Timeout != 45*time.Minute {
|
||||||
|
t.Fatalf("request = %#v", req)
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(req.Binary) {
|
||||||
|
t.Fatalf("binary = %q, want absolute resolved path", req.Binary)
|
||||||
|
}
|
||||||
|
if len(result.Outputs) != 2 {
|
||||||
|
t.Fatalf("outputs = %#v, want lane and index", result.Outputs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractStageProducesImmutableManifestReadyOutputs(t *testing.T) {
|
||||||
|
env, m, fake := setupExtractEnv(t)
|
||||||
|
fixture := extractFixtureFromEnv(t, env, m)
|
||||||
|
stagingBundle := fake.Result.BundleRoot
|
||||||
|
|
||||||
|
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
durableBundle := artifacts.SessionNotariusBundleDirForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID)
|
||||||
|
if result.Metadata["bundle_root"] != durableBundle || result.Metadata["narratio_run_id"] != fixture.runID {
|
||||||
|
t.Fatalf("metadata = %#v", result.Metadata)
|
||||||
|
}
|
||||||
|
if fingerprint, _ := result.Metadata["configuration_fingerprint"].(string); len(fingerprint) != 64 {
|
||||||
|
t.Fatalf("configuration fingerprint = %#v", result.Metadata["configuration_fingerprint"])
|
||||||
|
}
|
||||||
|
if result.Metadata["receipt_path"] != artifacts.SessionRunNotariusReceiptPathForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID) ||
|
||||||
|
result.Metadata["diagnostic_path"] != artifacts.SessionRunNotariusLogPathForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID) ||
|
||||||
|
result.Metadata["rejections_path"] != filepath.Join(durableBundle, "rejected.json") ||
|
||||||
|
result.Metadata["warnings_path"] != filepath.Join(durableBundle, "warnings.json") {
|
||||||
|
t.Fatalf("diagnostic metadata = %#v", result.Metadata)
|
||||||
|
}
|
||||||
|
if len(result.Outputs) != 2 {
|
||||||
|
t.Fatalf("outputs = %#v", result.Outputs)
|
||||||
|
}
|
||||||
|
lane := result.Outputs[0]
|
||||||
|
if lane.Kind != extractLaneOutputKind || lane.SourceID != "narratio.extraction.npc_registry" {
|
||||||
|
t.Fatalf("lane identity = %#v", lane)
|
||||||
|
}
|
||||||
|
if lane.AbsolutePath != filepath.Join(durableBundle, "lanes", "npc.json") || lane.Checksum == "" {
|
||||||
|
t.Fatalf("lane path/checksum = %#v", lane)
|
||||||
|
}
|
||||||
|
wantContract := &artifactmodel.ContractMetadata{
|
||||||
|
MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry",
|
||||||
|
SchemaVersion: "v1", ModuleKey: "dnd/npc-registry",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(lane.Contract, wantContract) {
|
||||||
|
t.Fatalf("lane contract = %#v, want %#v", lane.Contract, wantContract)
|
||||||
|
}
|
||||||
|
wantProvenance := &artifactmodel.ExternalProvenance{
|
||||||
|
System: "notarius", RunID: "notarius-run-1", PipelineID: "dnd-session", ArtifactID: "npc-registry",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(lane.ExternalProvenance, wantProvenance) {
|
||||||
|
t.Fatalf("lane provenance = %#v, want %#v", lane.ExternalProvenance, wantProvenance)
|
||||||
|
}
|
||||||
|
index := result.Outputs[1]
|
||||||
|
if index.Kind != extractIndexOutputKind || index.SourceID != "" || index.AbsolutePath != filepath.Join(durableBundle, "index.json") || index.Checksum == "" {
|
||||||
|
t.Fatalf("index output = %#v", index)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(lane.AbsolutePath, durableBundle+string(filepath.Separator)) || strings.HasPrefix(lane.AbsolutePath, stagingBundle+string(filepath.Separator)) {
|
||||||
|
t.Fatalf("lane path was not re-resolved after promotion: %q", lane.AbsolutePath)
|
||||||
|
}
|
||||||
|
for _, relative := range []string{
|
||||||
|
"index.json", "manifest.json", "rejected.json", "warnings.json", "lanes/npc.json",
|
||||||
|
"lanes/unconfigured.json", "chunk-map.json", "evidence-context.json", "unknown/private-debug.json",
|
||||||
|
} {
|
||||||
|
if _, err := os.Stat(filepath.Join(durableBundle, filepath.FromSlash(relative))); err != nil {
|
||||||
|
t.Fatalf("promoted file %q missing: %v", relative, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(stagingBundle, "unknown", "private-debug.json")); err != nil {
|
||||||
|
t.Fatalf("source bundle was not preserved: %v", err)
|
||||||
|
}
|
||||||
|
for _, output := range result.Outputs {
|
||||||
|
if strings.Contains(output.AbsolutePath, "unconfigured") || strings.Contains(output.AbsolutePath, "private-debug") {
|
||||||
|
t.Fatalf("unknown file registered as output: %#v", output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rejections, _ := result.Metadata["rejections"].([]map[string]any)
|
||||||
|
warnings, _ := result.Metadata["warnings"].([]map[string]any)
|
||||||
|
if len(rejections) != 1 || rejections[0]["reason_code"] != "optional_rejected" || len(warnings) != 1 {
|
||||||
|
t.Fatalf("diagnostic summaries = rejections %#v warnings %#v", rejections, warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractStageRejectsMissingOrInvalidFinalTrimmedInputBeforeInvocation(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
content string
|
||||||
|
remove bool
|
||||||
|
}{
|
||||||
|
{name: "missing", remove: true},
|
||||||
|
{name: "invalid json", content: "not-json"},
|
||||||
|
{name: "missing segments", content: `{}`},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
env, m, fake := setupExtractEnv(t)
|
||||||
|
fixture := extractFixtureFromEnv(t, env, m)
|
||||||
|
if test.remove {
|
||||||
|
if err := os.Remove(fixture.inputPath); err != nil {
|
||||||
|
t.Fatalf("Remove(input) error = %v", err)
|
||||||
|
}
|
||||||
|
} else if err := os.WriteFile(fixture.inputPath, []byte(test.content), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(input) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := (extractStage{}).Run(context.Background(), env, m); err == nil {
|
||||||
|
t.Fatal("Run() error = nil")
|
||||||
|
}
|
||||||
|
if len(fake.Requests) != 0 {
|
||||||
|
t.Fatalf("adapter requests = %d, want 0", len(fake.Requests))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractStageEnforcesRequiredLanePolicy(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*notarius.RunResult)
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "missing", mutate: func(result *notarius.RunResult) { result.Index.Lanes = result.Index.Lanes[1:] }, want: "0 descriptors"},
|
||||||
|
{name: "rejected", mutate: func(result *notarius.RunResult) {
|
||||||
|
result.Rejections = append(result.Rejections, notarius.RejectionSummary{LaneID: "npc-registry", ReasonCode: "invalid_npc"})
|
||||||
|
}, want: "was rejected"},
|
||||||
|
{name: "duplicate", mutate: func(result *notarius.RunResult) {
|
||||||
|
result.Index.Lanes = append(result.Index.Lanes, result.Index.Lanes[0])
|
||||||
|
}, want: "2 descriptors"},
|
||||||
|
{name: "media type", mutate: func(result *notarius.RunResult) { result.Index.Lanes[0].MediaType = "text/plain" }, want: "incompatible"},
|
||||||
|
{name: "schema id", mutate: func(result *notarius.RunResult) { result.Index.Lanes[0].SchemaID = "other" }, want: "incompatible"},
|
||||||
|
{name: "schema version", mutate: func(result *notarius.RunResult) { result.Index.Lanes[0].SchemaVersion = "v2" }, want: "incompatible"},
|
||||||
|
{name: "module key", mutate: func(result *notarius.RunResult) { result.Index.Lanes[0].ModuleKey = "other" }, want: "incompatible"},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
env, m, fake := setupExtractEnv(t)
|
||||||
|
test.mutate(&fake.Result)
|
||||||
|
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||||
|
t.Fatalf("Run() result = %#v error = %v, want %q", result, err, test.want)
|
||||||
|
}
|
||||||
|
if result != nil && len(result.Outputs) != 0 {
|
||||||
|
t.Fatalf("failure returned outputs: %#v", result.Outputs)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractStageValidatesSelectedLaneJSON(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
content string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "empty", content: "", want: "non-empty"},
|
||||||
|
{name: "invalid", content: "not-json", want: "valid JSON"},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
env, m, fake := setupExtractEnv(t)
|
||||||
|
if err := os.WriteFile(fake.Result.Index.Lanes[0].Path, []byte(test.content), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(lane) error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := (extractStage{}).Run(context.Background(), env, m); err == nil || !strings.Contains(err.Error(), test.want) {
|
||||||
|
t.Fatalf("Run() error = %v, want %q", err, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractStageAdapterAndPromotionFailuresReturnNoOutputs(t *testing.T) {
|
||||||
|
t.Run("adapter", func(t *testing.T) {
|
||||||
|
env, m, fake := setupExtractEnv(t)
|
||||||
|
fake.Err = errors.New("adapter failed")
|
||||||
|
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "adapter failed") || result != nil {
|
||||||
|
t.Fatalf("Run() result = %#v error = %v", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("promotion", func(t *testing.T) {
|
||||||
|
env, m, fake := setupExtractEnv(t)
|
||||||
|
fixture := extractFixtureFromEnv(t, env, m)
|
||||||
|
destination := artifacts.SessionNotariusBundleDirForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID)
|
||||||
|
if err := os.MkdirAll(destination, 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(destination) error = %v", err)
|
||||||
|
}
|
||||||
|
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "promote") || result != nil {
|
||||||
|
t.Fatalf("Run() result = %#v error = %v", result, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(fake.Result.BundleRoot); err != nil {
|
||||||
|
t.Fatalf("failed promotion removed staging bundle: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractionFingerprintIsIndependentOfOutputMapOrder(t *testing.T) {
|
||||||
|
first := &config.NotariusConfig{PipelineID: "pipeline", Outputs: map[string]config.NotariusOutputConfig{
|
||||||
|
"zeta": {LaneID: "z", MediaType: "application/json", SchemaID: "z", SchemaVersion: "v1"},
|
||||||
|
"alpha": {LaneID: "a", MediaType: "application/json", SchemaID: "a", SchemaVersion: "v1"},
|
||||||
|
}}
|
||||||
|
second := &config.NotariusConfig{PipelineID: "pipeline", Outputs: map[string]config.NotariusOutputConfig{
|
||||||
|
"alpha": first.Outputs["alpha"], "zeta": first.Outputs["zeta"],
|
||||||
|
}}
|
||||||
|
one, err := extractionFingerprint("/bin/notarius", "/etc/notarius.yml", first, time.Minute, "/work")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("extractionFingerprint(first) error = %v", err)
|
||||||
|
}
|
||||||
|
two, err := extractionFingerprint("/bin/notarius", "/etc/notarius.yml", second, time.Minute, "/work")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("extractionFingerprint(second) error = %v", err)
|
||||||
|
}
|
||||||
|
if one != two {
|
||||||
|
t.Fatalf("fingerprints differ: %q != %q", one, two)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractStageResumeValidationAcceptsCurrentImmutableResult(t *testing.T) {
|
||||||
|
env, m, _ := setupExtractEnv(t)
|
||||||
|
seedSucceededExtractResult(t, env, m)
|
||||||
|
m.RunID = "20260810T020304Z-fedcba98"
|
||||||
|
|
||||||
|
validation, err := (extractStage{}).ValidateResume(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ValidateResume() error = %v", err)
|
||||||
|
}
|
||||||
|
if !validation.Resumable || validation.Reason != "" {
|
||||||
|
t.Fatalf("validation = %#v, want resumable", validation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractStageAdapterResultIsImmediatelyReusableAndCatalogVisible(t *testing.T) {
|
||||||
|
env, m, fake := setupExtractEnv(t)
|
||||||
|
producerRunID := m.RunID
|
||||||
|
receiptFixture := filepath.Join(t.TempDir(), "receipt.json")
|
||||||
|
receipt, err := json.Marshal(map[string]any{
|
||||||
|
"schema_version": notarius.ReceiptSchemaVersion,
|
||||||
|
"run_id": "notarius-run-1", "pipeline_id": "dnd-session",
|
||||||
|
"output_directory": fake.Result.BundleRoot, "index_file": "index.json",
|
||||||
|
"normalized_output_count": 2, "rejected_output_count": 0, "warning_count": 0,
|
||||||
|
"validation_status": "approved", "future_field": true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal(receipt) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(receiptFixture, receipt, 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(env.Config.Pipeline.Notarius.Binary, []byte("#!/bin/sh\ncat \"$NARRATIO_NOTARIUS_RECEIPT_FIXTURE\"\n"), 0o755); err != nil {
|
||||||
|
t.Fatalf("WriteFile(notarius helper) error = %v", err)
|
||||||
|
}
|
||||||
|
t.Setenv("NARRATIO_NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
||||||
|
env.Notarius = notarius.NewSubprocessRunner()
|
||||||
|
|
||||||
|
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
durableIndex := filepath.Join(result.Metadata["bundle_root"].(string), "index.json")
|
||||||
|
if len(result.Outputs) != 2 || result.Outputs[1].AbsolutePath != durableIndex {
|
||||||
|
t.Fatalf("outputs = %#v, want canonical durable index %q", result.Outputs, durableIndex)
|
||||||
|
}
|
||||||
|
recordSucceededExtractResult(m, producerRunID, result)
|
||||||
|
m.RunID = "20260810T020304Z-fedcba98"
|
||||||
|
|
||||||
|
validation, err := (extractStage{}).ValidateResume(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ValidateResume() error = %v", err)
|
||||||
|
}
|
||||||
|
if !validation.Resumable {
|
||||||
|
t.Fatalf("validation = %#v, want resumable", validation)
|
||||||
|
}
|
||||||
|
definition := env.Config.Pipeline.Notarius.Outputs["npc_registry"]
|
||||||
|
definitions := map[string]artifacts.ExtractionArtifactDefinition{
|
||||||
|
"npc_registry": {
|
||||||
|
LaneID: definition.LaneID, PipelineID: env.Config.Pipeline.Notarius.PipelineID,
|
||||||
|
MediaType: definition.MediaType, SchemaID: definition.SchemaID,
|
||||||
|
SchemaVersion: definition.SchemaVersion, ModuleKey: definition.ModuleKey,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
catalog := artifacts.NewArtifactCatalog()
|
||||||
|
if err := catalog.RegisterExtractionArtifacts(definitions); err != nil {
|
||||||
|
t.Fatalf("RegisterExtractionArtifacts() error = %v", err)
|
||||||
|
}
|
||||||
|
catalog.HydrateExtractionArtifacts(sessionPathsForEnv(env, m.SessionID), m, definitions)
|
||||||
|
entry, ok := catalog.Lookup(artifacts.ExtractionArtifactSourceID("npc_registry"))
|
||||||
|
if !ok || !entry.Available || entry.Path != result.Outputs[0].AbsolutePath || entry.ProducerRunID != producerRunID {
|
||||||
|
t.Fatalf("hydrated extraction entry = %#v, %v", entry, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractStageResumeValidationRejectsObsoleteResults(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*testing.T, *Env, *manifest.Manifest)
|
||||||
|
}{
|
||||||
|
{name: "disabled", mutate: func(_ *testing.T, env *Env, _ *manifest.Manifest) {
|
||||||
|
env.Config.Pipeline.Notarius.Enabled = false
|
||||||
|
}},
|
||||||
|
{name: "config changed", mutate: func(_ *testing.T, env *Env, _ *manifest.Manifest) {
|
||||||
|
env.Config.Pipeline.Notarius.PipelineID = "changed"
|
||||||
|
}},
|
||||||
|
{name: "missing lane", mutate: func(t *testing.T, _ *Env, m *manifest.Manifest) {
|
||||||
|
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
||||||
|
t.Fatalf("Remove(lane) error = %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "tampered lane", mutate: func(t *testing.T, _ *Env, m *manifest.Manifest) {
|
||||||
|
if err := os.WriteFile(m.Stages["extract"].Outputs[0].LocalPath, []byte(`{"tampered":true}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(lane) error = %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
{name: "incompatible contract", mutate: func(_ *testing.T, _ *Env, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs[0].Contract.SchemaVersion = "v2"
|
||||||
|
}},
|
||||||
|
{name: "missing source", mutate: func(_ *testing.T, _ *Env, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs = m.Stages["extract"].Outputs[1:]
|
||||||
|
}},
|
||||||
|
{name: "producer mismatch", mutate: func(_ *testing.T, _ *Env, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs[0].ProducerRunID = "different-run"
|
||||||
|
}},
|
||||||
|
{name: "provenance mismatch", mutate: func(_ *testing.T, _ *Env, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs[0].ExternalProvenance.RunID = "different-run"
|
||||||
|
}},
|
||||||
|
{name: "missing index", mutate: func(t *testing.T, _ *Env, m *manifest.Manifest) {
|
||||||
|
index := m.Stages["extract"].Outputs[1]
|
||||||
|
if err := os.Remove(index.LocalPath); err != nil {
|
||||||
|
t.Fatalf("Remove(index) error = %v", err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
env, m, _ := setupExtractEnv(t)
|
||||||
|
seedSucceededExtractResult(t, env, m)
|
||||||
|
test.mutate(t, env, m)
|
||||||
|
|
||||||
|
validation, err := (extractStage{}).ValidateResume(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ValidateResume() error = %v", err)
|
||||||
|
}
|
||||||
|
if validation.Resumable || validation.Reason == "" || len(validation.Reason) > maxResumeReasonLength {
|
||||||
|
t.Fatalf("validation = %#v, want bounded non-resumable result", validation)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractStageResumeValidationRejectsUnsafePathWithError(t *testing.T) {
|
||||||
|
env, m, _ := setupExtractEnv(t)
|
||||||
|
seedSucceededExtractResult(t, env, m)
|
||||||
|
prior := *m.Stages["extract"]
|
||||||
|
m.Stages["extract"].Outputs[0].LocalPath = filepath.Join(env.Config.Pipeline.Workspace.Root, "outside.json")
|
||||||
|
|
||||||
|
if _, err := (extractStage{}).ValidateResume(context.Background(), env, m); err == nil || !strings.Contains(err.Error(), "unsafe") {
|
||||||
|
t.Fatalf("ValidateResume() error = %v, want unsafe path error", err)
|
||||||
|
}
|
||||||
|
if m.Stages["extract"].Status != prior.Status || m.Stages["extract"].Error != prior.Error {
|
||||||
|
t.Fatalf("validation mutated stage record: %#v", m.Stages["extract"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func seedSucceededExtractResult(t *testing.T, env *Env, m *manifest.Manifest) {
|
||||||
|
t.Helper()
|
||||||
|
producerRunID := m.RunID
|
||||||
|
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
recordSucceededExtractResult(m, producerRunID, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
func recordSucceededExtractResult(m *manifest.Manifest, producerRunID string, result *StageResult) {
|
||||||
|
records := make([]manifest.ArtifactRecord, 0, len(result.Outputs))
|
||||||
|
for _, output := range result.Outputs {
|
||||||
|
records = append(records, manifest.ArtifactRecord{
|
||||||
|
Kind: output.Kind, SourceID: output.SourceID, LocalPath: output.AbsolutePath,
|
||||||
|
Contract: output.Contract, ExternalProvenance: output.ExternalProvenance,
|
||||||
|
ProducerRunID: producerRunID, Checksum: output.Checksum,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
m.MarkStageSucceeded("extract", time.Now().UTC(), records)
|
||||||
|
m.Stages["extract"].Metadata = result.Metadata
|
||||||
|
}
|
||||||
|
|
||||||
|
type extractFixture struct {
|
||||||
|
workspace string
|
||||||
|
campaign string
|
||||||
|
sessionID string
|
||||||
|
runID string
|
||||||
|
inputPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractFixtureFromEnv(t *testing.T, env *Env, m *manifest.Manifest) extractFixture {
|
||||||
|
t.Helper()
|
||||||
|
paths := sessionPathsForEnv(env, m.SessionID)
|
||||||
|
return extractFixture{
|
||||||
|
workspace: paths.WorkspaceRoot, campaign: m.Campaign, sessionID: m.SessionID,
|
||||||
|
runID: m.RunID, inputPath: filepath.Join(paths.ArtifactsDir, "trimmed.from-manifest.json"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupExtractEnv(t *testing.T) (*Env, *manifest.Manifest, *notarius.FakeRunner) {
|
||||||
|
t.Helper()
|
||||||
|
workspace := t.TempDir()
|
||||||
|
campaign := "campaign-a"
|
||||||
|
sessionID := "2026-08-09"
|
||||||
|
runID := "20260809T010203Z-abcdef12"
|
||||||
|
store := artifacts.NewLocalStore(workspace)
|
||||||
|
paths, err := store.EnsureLayoutFor(campaign, sessionID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("EnsureLayoutFor() error = %v", err)
|
||||||
|
}
|
||||||
|
inputPath := filepath.Join(paths.ArtifactsDir, "trimmed.from-manifest.json")
|
||||||
|
if err := os.WriteFile(inputPath, []byte(`{"segments":[{"id":1}]}`), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(input) error = %v", err)
|
||||||
|
}
|
||||||
|
configPath := filepath.Join(workspace, "notarius.yml")
|
||||||
|
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(config) error = %v", err)
|
||||||
|
}
|
||||||
|
workingDirectory := filepath.Join(workspace, "notarius-work")
|
||||||
|
if err := os.Mkdir(workingDirectory, 0o755); err != nil {
|
||||||
|
t.Fatalf("Mkdir(working directory) error = %v", err)
|
||||||
|
}
|
||||||
|
binary := filepath.Join(workspace, "notarius")
|
||||||
|
if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||||
|
t.Fatalf("WriteFile(binary) error = %v", err)
|
||||||
|
}
|
||||||
|
notariusConfig := &config.NotariusConfig{
|
||||||
|
Enabled: true, Binary: binary, ConfigPath: configPath, PipelineID: "dnd-session",
|
||||||
|
Timeout: "45m", WorkingDirectory: workingDirectory,
|
||||||
|
Outputs: map[string]config.NotariusOutputConfig{
|
||||||
|
"npc_registry": {
|
||||||
|
LaneID: "npc-registry", MediaType: "application/json",
|
||||||
|
SchemaID: "notarius.dnd.npc_registry", SchemaVersion: "v1", ModuleKey: "dnd/npc-registry",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
m := manifest.New(sessionID, time.Now().UTC())
|
||||||
|
m.Campaign = campaign
|
||||||
|
m.RunID = runID
|
||||||
|
m.MarkStageSucceeded("trim", time.Now().UTC(), []manifest.ArtifactRecord{{
|
||||||
|
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
|
||||||
|
LocalPath: inputPath,
|
||||||
|
}})
|
||||||
|
|
||||||
|
outputRoot := artifacts.SessionRunNotariusOutputRootForCampaign(workspace, campaign, sessionID, runID)
|
||||||
|
bundle := filepath.Join(outputRoot, "notarius-run-1")
|
||||||
|
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(bundle lanes) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Join(bundle, "unknown"), 0o755); err != nil {
|
||||||
|
t.Fatalf("MkdirAll(bundle unknown) error = %v", err)
|
||||||
|
}
|
||||||
|
files := map[string]string{
|
||||||
|
"index.json": `{"manifest_file":"manifest.json","output_files":[{"lane_id":"npc-registry","file":"lanes/npc.json","media_type":"application/json","module_key":"dnd/npc-registry","schema_id":"notarius.dnd.npc_registry","schema_version":"v1","future_field":true},{"lane_id":"unconfigured","file":"lanes/unconfigured.json"}],"rejected_file":"rejected.json","warnings_file":"warnings.json","future_field":true}`,
|
||||||
|
"manifest.json": `{}`,
|
||||||
|
"rejected.json": `{"rejected":[]}`,
|
||||||
|
"warnings.json": `{"warnings":[]}`,
|
||||||
|
"lanes/npc.json": `{"npcs":[]}`,
|
||||||
|
"lanes/unconfigured.json": `{"spells":[]}`,
|
||||||
|
"chunk-map.json": `{"chunks":[]}`,
|
||||||
|
"evidence-context.json": `{"source_units":[]}`,
|
||||||
|
"unknown/private-debug.json": `{"private":true}`,
|
||||||
|
}
|
||||||
|
for relative, content := range files {
|
||||||
|
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(relative)), []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q) error = %v", relative, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fake := ¬arius.FakeRunner{Result: notarius.RunResult{
|
||||||
|
Receipt: notarius.Receipt{
|
||||||
|
SchemaVersion: notarius.ReceiptSchemaVersion, RunID: "notarius-run-1", PipelineID: "dnd-session",
|
||||||
|
OutputDirectory: bundle, IndexFile: "index.json", NormalizedOutputCount: 2,
|
||||||
|
RejectedOutputCount: 1, WarningCount: 1, ValidationStatus: "rejected",
|
||||||
|
},
|
||||||
|
BundleRoot: bundle,
|
||||||
|
Index: notarius.Index{
|
||||||
|
Path: filepath.Join(bundle, "index.json"), RejectedPath: filepath.Join(bundle, "rejected.json"),
|
||||||
|
WarningsPath: filepath.Join(bundle, "warnings.json"),
|
||||||
|
Lanes: []notarius.LaneDescriptor{
|
||||||
|
{
|
||||||
|
LaneID: "npc-registry", File: "lanes/npc.json", Path: filepath.Join(bundle, "lanes", "npc.json"),
|
||||||
|
MediaType: "application/json", ModuleKey: "dnd/npc-registry",
|
||||||
|
SchemaID: "notarius.dnd.npc_registry", SchemaVersion: "v1",
|
||||||
|
},
|
||||||
|
{LaneID: "unconfigured", File: "lanes/unconfigured.json", Path: filepath.Join(bundle, "lanes", "unconfigured.json")},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Rejections: []notarius.RejectionSummary{{LaneID: "optional", ReasonCode: "optional_rejected"}},
|
||||||
|
Warnings: []notarius.WarningSummary{{Scope: "lane:npc-registry", ReasonCode: "normalized_name"}},
|
||||||
|
}}
|
||||||
|
env := &Env{
|
||||||
|
Config: &config.Config{
|
||||||
|
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}, Notarius: notariusConfig},
|
||||||
|
Session: &config.SessionConfig{SessionID: sessionID, Campaign: campaign},
|
||||||
|
},
|
||||||
|
ArtifactStore: store,
|
||||||
|
Notarius: fake,
|
||||||
|
}
|
||||||
|
return env, m, fake
|
||||||
|
}
|
||||||
@@ -74,6 +74,7 @@ func All() []Stage {
|
|||||||
polishStage{},
|
polishStage{},
|
||||||
normalizeStage{},
|
normalizeStage{},
|
||||||
trimStage{},
|
trimStage{},
|
||||||
|
extractStage{},
|
||||||
renderStage{},
|
renderStage{},
|
||||||
analyzeStage{},
|
analyzeStage{},
|
||||||
publishStage{},
|
publishStage{},
|
||||||
|
|||||||
@@ -113,7 +113,7 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
m := manifest.New("2026-05-03", time.Now().UTC())
|
m := manifest.New("2026-05-03", time.Now().UTC())
|
||||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||||
m.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
m.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||||
}
|
}
|
||||||
m.RunID = "20260516T000000Z-abcdef12"
|
m.RunID = "20260516T000000Z-abcdef12"
|
||||||
@@ -206,6 +206,12 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if s.Name() == "extract" {
|
||||||
|
if result.Disposition != StageDispositionSkipped || result.SkipReason != extractSkipReason {
|
||||||
|
t.Fatalf("extract result = %#v, want disabled self-skip", result)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
if s.Name() == "render" {
|
if s.Name() == "render" {
|
||||||
if result.Metadata["stage"] != "render" {
|
if result.Metadata["stage"] != "render" {
|
||||||
t.Fatalf("render metadata = %#v, want stage=render", result.Metadata)
|
t.Fatalf("render metadata = %#v, want stage=render", result.Metadata)
|
||||||
|
|||||||
@@ -124,7 +124,12 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("publish: collect previous files: %w", err)
|
return nil, fmt.Errorf("publish: collect previous files: %w", err)
|
||||||
}
|
}
|
||||||
runtimeCatalog, err := buildPublishRuntimeArtifactCatalog(sessionPaths, env.Config.Pipeline.Scriptorium)
|
runtimeCatalog, err := buildPublishRuntimeArtifactCatalog(
|
||||||
|
sessionPaths,
|
||||||
|
m,
|
||||||
|
env.Config.Pipeline.Scriptorium,
|
||||||
|
env.Config.Pipeline.Notarius,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("publish: build runtime artifact catalog: %w", err)
|
return nil, fmt.Errorf("publish: build runtime artifact catalog: %w", err)
|
||||||
}
|
}
|
||||||
@@ -361,10 +366,11 @@ func resolvePublishOutputs(
|
|||||||
lockSet := publishLockSet(locks)
|
lockSet := publishLockSet(locks)
|
||||||
selectedSet := publishSelectedArtifactSet(selectedArtifactKeys)
|
selectedSet := publishSelectedArtifactSet(selectedArtifactKeys)
|
||||||
configuredOutputs := configuredOutputPathMapFromCatalog(catalog)
|
configuredOutputs := configuredOutputPathMapFromCatalog(catalog)
|
||||||
|
extractionOutputs := extractionOutputSetFromCatalog(catalog)
|
||||||
for _, rule := range rules {
|
for _, rule := range rules {
|
||||||
source := strings.TrimSpace(rule.Source)
|
source := strings.TrimSpace(rule.Source)
|
||||||
required := rule.Required == nil || *rule.Required
|
required := rule.Required == nil || *rule.Required
|
||||||
dest, err := resolvePublishOutputDest(rule, configuredOutputs)
|
dest, err := resolvePublishOutputDest(rule, configuredOutputs, extractionOutputs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
|
return nil, nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
|
||||||
}
|
}
|
||||||
@@ -454,8 +460,8 @@ func publishLockSet(locks []config.PublishLockRule) map[string]config.PublishLoc
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolvePublishOutputDest(rule config.PublishOutputRule, configured map[string]string) (string, error) {
|
func resolvePublishOutputDest(rule config.PublishOutputRule, configured map[string]string, extractions map[string]struct{}) (string, error) {
|
||||||
return artifactpolicy.ResolvePublishedDestination(rule.Source, rule.Dest, configured)
|
return artifactpolicy.ResolvePublishedDestinationWithExtractions(rule.Source, rule.Dest, configured, extractions)
|
||||||
}
|
}
|
||||||
|
|
||||||
func configuredOutputPathMapFromCatalog(catalog *artifacts.ArtifactCatalog) map[string]string {
|
func configuredOutputPathMapFromCatalog(catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||||
@@ -472,14 +478,36 @@ func configuredOutputPathMapFromCatalog(catalog *artifacts.ArtifactCatalog) map[
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func extractionOutputSetFromCatalog(catalog *artifacts.ArtifactCatalog) map[string]struct{} {
|
||||||
|
out := map[string]struct{}{}
|
||||||
|
if catalog == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for _, entry := range catalog.ListExtraction() {
|
||||||
|
if strings.TrimSpace(entry.ExtractionKey) != "" {
|
||||||
|
out[entry.ExtractionKey] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func buildPublishRuntimeArtifactCatalog(
|
func buildPublishRuntimeArtifactCatalog(
|
||||||
paths artifacts.SessionPaths,
|
paths artifacts.SessionPaths,
|
||||||
|
m *manifest.Manifest,
|
||||||
scriptoriumCfg *config.ScriptoriumConfig,
|
scriptoriumCfg *config.ScriptoriumConfig,
|
||||||
|
notariusCfg *config.NotariusConfig,
|
||||||
) (*artifacts.ArtifactCatalog, error) {
|
) (*artifacts.ArtifactCatalog, error) {
|
||||||
catalog := artifacts.NewArtifactCatalog()
|
catalog := artifacts.NewArtifactCatalog()
|
||||||
if err := catalog.RegisterBuiltIns(); err != nil {
|
if err := catalog.RegisterBuiltIns(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
extractionDefinitions := artifacts.ExtractionDefinitionsFromConfig(notariusCfg)
|
||||||
|
if err := catalog.RegisterExtractionArtifacts(extractionDefinitions); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if notariusCfg != nil && notariusCfg.Enabled {
|
||||||
|
catalog.HydrateExtractionArtifacts(paths, m, extractionDefinitions)
|
||||||
|
}
|
||||||
if scriptoriumCfg == nil {
|
if scriptoriumCfg == nil {
|
||||||
return catalog, nil
|
return catalog, nil
|
||||||
}
|
}
|
||||||
@@ -545,19 +573,7 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile,
|
|||||||
if walkErr != nil {
|
if walkErr != nil {
|
||||||
return walkErr
|
return walkErr
|
||||||
}
|
}
|
||||||
if d.IsDir() {
|
if path == runRoot {
|
||||||
if path == runRoot {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
relDir, err := filepath.Rel(runRoot, path)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("relative dir from %q to %q: %w", runRoot, path, err)
|
|
||||||
}
|
|
||||||
relDir = filepath.ToSlash(relDir)
|
|
||||||
// Preserve existing behavior: audio is not uploaded in publish run record.
|
|
||||||
if relDir == "audio" || strings.HasPrefix(relDir, "audio/") {
|
|
||||||
return filepath.SkipDir
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
rel, err := filepath.Rel(runRoot, path)
|
rel, err := filepath.Rel(runRoot, path)
|
||||||
@@ -565,6 +581,15 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile,
|
|||||||
return fmt.Errorf("relative path from %q to %q: %w", runRoot, path, err)
|
return fmt.Errorf("relative path from %q to %q: %w", runRoot, path, err)
|
||||||
}
|
}
|
||||||
rel = filepath.ToSlash(rel)
|
rel = filepath.ToSlash(rel)
|
||||||
|
if publishRunPathExcluded(rel) {
|
||||||
|
if d.IsDir() {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if d.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
files = append(files, publishUploadFile{
|
files = append(files, publishUploadFile{
|
||||||
RelativePath: rel,
|
RelativePath: rel,
|
||||||
LocalPath: path,
|
LocalPath: path,
|
||||||
@@ -604,6 +629,11 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile,
|
|||||||
return files, nil
|
return files, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func publishRunPathExcluded(rel string) bool {
|
||||||
|
return rel == "audio" || strings.HasPrefix(rel, "audio/") ||
|
||||||
|
rel == "extract/notarius-output" || strings.HasPrefix(rel, "extract/notarius-output/")
|
||||||
|
}
|
||||||
|
|
||||||
func collectPublishPreviousFiles(previousDir string) ([]publishUploadFile, error) {
|
func collectPublishPreviousFiles(previousDir string) ([]publishUploadFile, error) {
|
||||||
previousDir = filepath.Clean(strings.TrimSpace(previousDir))
|
previousDir = filepath.Clean(strings.TrimSpace(previousDir))
|
||||||
if previousDir == "" {
|
if previousDir == "" {
|
||||||
|
|||||||
@@ -7,11 +7,13 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
@@ -63,8 +65,27 @@ func TestPublishFailsWhenPrerequisiteNotSucceeded(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T) {
|
func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T) {
|
||||||
env, m, _ := publishFixture(t)
|
env, m, runRoot := publishFixture(t)
|
||||||
fake := env.ObjectStore.(*storage.FakeBackend)
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||||
|
durableBundleRoot := filepath.Dir(filepath.Dir(lanePath))
|
||||||
|
runBundleRoot := filepath.Join(runRoot, "extract", "notarius-output", "notarius-run-1")
|
||||||
|
for rel, contents := range map[string]string{
|
||||||
|
"index.json": `{"lanes":[]}`,
|
||||||
|
"manifest.json": `{"run_id":"notarius-run-1"}`,
|
||||||
|
"rejected.json": `[]`,
|
||||||
|
"warnings.json": `[]`,
|
||||||
|
"lanes/encounters.json": `{"encounters":[]}`,
|
||||||
|
"pipeline/chunk-map.json": `{"chunks":[]}`,
|
||||||
|
"pipeline/evidence-context.json": `{"evidence":[]}`,
|
||||||
|
"unknown/notes.txt": "internal bundle note\n",
|
||||||
|
} {
|
||||||
|
writeStageTestFile(t, filepath.Join(runBundleRoot, filepath.FromSlash(rel)), contents)
|
||||||
|
}
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius.receipt.json"), `{"run_id":"notarius-run-1"}`)
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius.stderr.log"), "notarius diagnostic\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius-output-copy", "keep.json"), "{}\n")
|
||||||
|
writeStageTestFile(t, filepath.Join(runRoot, "archive", "notarius-output", "keep.json"), "{}\n")
|
||||||
|
|
||||||
result, err := publishStage{}.Run(context.Background(), env, m)
|
result, err := publishStage{}.Run(context.Background(), env, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -75,6 +96,10 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T)
|
|||||||
sessionPrefix := m.S3SessionPrefix
|
sessionPrefix := m.S3SessionPrefix
|
||||||
wantRunUploads := []string{
|
wantRunUploads := []string{
|
||||||
"analyze/outputs/artifacts/session_recap.md",
|
"analyze/outputs/artifacts/session_recap.md",
|
||||||
|
"archive/notarius-output/keep.json",
|
||||||
|
"extract/notarius-output-copy/keep.json",
|
||||||
|
"extract/notarius.receipt.json",
|
||||||
|
"extract/notarius.stderr.log",
|
||||||
"merge/config/seriatim.generated.yml",
|
"merge/config/seriatim.generated.yml",
|
||||||
"prepare/inputs/session.yml",
|
"prepare/inputs/session.yml",
|
||||||
"prepare/outputs/audio/speaker.flac",
|
"prepare/outputs/audio/speaker.flac",
|
||||||
@@ -84,12 +109,24 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T)
|
|||||||
"transcribe/outputs/transcripts/raw/speaker.json",
|
"transcribe/outputs/transcripts/raw/speaker.json",
|
||||||
"trim/outputs/transcripts/final.trimmed.json",
|
"trim/outputs/transcripts/final.trimmed.json",
|
||||||
}
|
}
|
||||||
|
sort.Strings(wantRunUploads)
|
||||||
for _, rel := range wantRunUploads {
|
for _, rel := range wantRunUploads {
|
||||||
key := runPrefix + rel
|
key := runPrefix + rel
|
||||||
if _, ok := fake.Objects[key]; !ok {
|
if _, ok := fake.Objects[key]; !ok {
|
||||||
t.Fatalf("missing run upload key %q", key)
|
t.Fatalf("missing run upload key %q", key)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for key := range fake.Objects {
|
||||||
|
if strings.HasPrefix(key, runPrefix+"extract/notarius-output/") {
|
||||||
|
t.Fatalf("run-local Notarius bundle member was uploaded at %q", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, upload := range fake.Uploads {
|
||||||
|
localPath := filepath.Clean(upload.LocalPath)
|
||||||
|
if localPath == durableBundleRoot || strings.HasPrefix(localPath, durableBundleRoot+string(filepath.Separator)) {
|
||||||
|
t.Fatalf("durable Notarius bundle member was implicitly uploaded from %q", upload.LocalPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
trimmedKey := sessionPrefix + "transcripts/final.trimmed.json"
|
trimmedKey := sessionPrefix + "transcripts/final.trimmed.json"
|
||||||
recapKey := sessionPrefix + "artifacts/session_recap.md"
|
recapKey := sessionPrefix + "artifacts/session_recap.md"
|
||||||
@@ -134,6 +171,12 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T)
|
|||||||
if result.Metadata["previous_files_uploaded"] != 0 {
|
if result.Metadata["previous_files_uploaded"] != 0 {
|
||||||
t.Fatalf("metadata previous_files_uploaded = %#v, want 0", result.Metadata["previous_files_uploaded"])
|
t.Fatalf("metadata previous_files_uploaded = %#v, want 0", result.Metadata["previous_files_uploaded"])
|
||||||
}
|
}
|
||||||
|
if result.Metadata["run_files_uploaded"] != len(wantRunUploads) {
|
||||||
|
t.Fatalf("metadata run_files_uploaded = %#v, want %d", result.Metadata["run_files_uploaded"], len(wantRunUploads))
|
||||||
|
}
|
||||||
|
if got := result.Metadata["run_uploaded_paths"]; !reflect.DeepEqual(got, wantRunUploads) {
|
||||||
|
t.Fatalf("metadata run_uploaded_paths = %#v, want %#v", got, wantRunUploads)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPublishUploadsPreviousCacheWhenPresent(t *testing.T) {
|
func TestPublishUploadsPreviousCacheWhenPresent(t *testing.T) {
|
||||||
@@ -198,6 +241,126 @@ func TestPublishUsesCustomOutputRules(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPublishUploadsExplicitExtractionAndPreservesManifestMetadata(t *testing.T) {
|
||||||
|
env, m, _ := publishFixture(t)
|
||||||
|
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||||
|
env.SelectedArtifactKeys = []string{"session_recap"}
|
||||||
|
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||||
|
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := publishStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
|
durableBundleRoot := filepath.Dir(filepath.Dir(lanePath))
|
||||||
|
publishedKey := m.S3SessionPrefix + "artifacts/encounters.json"
|
||||||
|
if got := string(fake.Objects[publishedKey].Data); got != `{"encounters":[]}` {
|
||||||
|
t.Fatalf("published extraction = %q", got)
|
||||||
|
}
|
||||||
|
if _, ok := fake.Objects[m.S3SessionPrefix+"artifacts/notarius/index.json"]; ok {
|
||||||
|
t.Fatal("Notarius index was implicitly published")
|
||||||
|
}
|
||||||
|
for key := range fake.Objects {
|
||||||
|
if strings.Contains(key, "/notarius/") {
|
||||||
|
t.Fatalf("Notarius bundle member was implicitly published at %q", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, upload := range fake.Uploads {
|
||||||
|
localPath := filepath.Clean(upload.LocalPath)
|
||||||
|
if (localPath == durableBundleRoot || strings.HasPrefix(localPath, durableBundleRoot+string(filepath.Separator))) && localPath != filepath.Clean(lanePath) {
|
||||||
|
t.Fatalf("durable Notarius bundle member other than selected lane was uploaded from %q", upload.LocalPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if result.Metadata["published_files_uploaded"] != 1 {
|
||||||
|
t.Fatalf("published_files_uploaded = %#v, want 1", result.Metadata["published_files_uploaded"])
|
||||||
|
}
|
||||||
|
if uploads := fake.Uploads; len(uploads) == 0 || uploads[len(uploads)-1].Key != m.S3SessionPrefix+"current/run_id.txt" {
|
||||||
|
t.Fatalf("last upload = %#v, want current run pointer", uploads)
|
||||||
|
}
|
||||||
|
|
||||||
|
var current manifest.Manifest
|
||||||
|
if err := json.Unmarshal(fake.Objects[m.S3SessionPrefix+"current/manifest.json"].Data, ¤t); err != nil {
|
||||||
|
t.Fatalf("unmarshal current manifest: %v", err)
|
||||||
|
}
|
||||||
|
extractRecord := current.Stages["extract"]
|
||||||
|
if extractRecord == nil || len(extractRecord.Outputs) != 2 {
|
||||||
|
t.Fatalf("extract record = %#v", extractRecord)
|
||||||
|
}
|
||||||
|
lane := extractRecord.Outputs[0]
|
||||||
|
if lane.LocalPath != lanePath || lane.Contract == nil || lane.Contract.SchemaID != "encounters" ||
|
||||||
|
lane.ExternalProvenance == nil || lane.ExternalProvenance.RunID != "notarius-run-1" {
|
||||||
|
t.Fatalf("serialized extraction metadata = %#v", lane)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishRequiredInvalidExtractionFails(t *testing.T) {
|
||||||
|
env, m, _ := publishFixture(t)
|
||||||
|
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||||
|
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||||
|
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||||
|
}
|
||||||
|
writeStageTestFile(t, lanePath, `{"tampered":true}`)
|
||||||
|
|
||||||
|
_, err := publishStage{}.Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), `required output source unavailable: "narratio.extraction.encounters"`) {
|
||||||
|
t.Fatalf("Run() error = %v, want unavailable extraction failure", err)
|
||||||
|
}
|
||||||
|
if len(env.ObjectStore.(*storage.FakeBackend).Uploads) != 0 {
|
||||||
|
t.Fatal("publish uploaded files after extraction validation failed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishDisabledExtractionConfigurationIsUnavailable(t *testing.T) {
|
||||||
|
env, m, _ := publishFixture(t)
|
||||||
|
configurePublishExtractionFixture(t, env, m)
|
||||||
|
env.Config.Pipeline.Notarius.Enabled = false
|
||||||
|
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||||
|
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := publishStage{}.Run(context.Background(), env, m)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), `required output source unavailable: "narratio.extraction.encounters"`) {
|
||||||
|
t.Fatalf("Run() error = %v, want unavailable disabled extraction", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishOptionalMissingExtractionIsSkipped(t *testing.T) {
|
||||||
|
env, m, _ := publishFixture(t)
|
||||||
|
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||||
|
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||||
|
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(false)},
|
||||||
|
}
|
||||||
|
if err := os.Remove(lanePath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := publishStage{}.Run(context.Background(), env, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if got := result.Metadata["skipped_optional_outputs"].([]string); !reflect.DeepEqual(got, []string{"artifacts/encounters.json"}) {
|
||||||
|
t.Fatalf("skipped_optional_outputs = %#v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishArtifactSelectionDoesNotFilterExtractionOutputs(t *testing.T) {
|
||||||
|
env, m, _ := publishFixture(t)
|
||||||
|
configurePublishExtractionFixture(t, env, m)
|
||||||
|
env.SelectedArtifactKeys = []string{"session_recap"}
|
||||||
|
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||||
|
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := (publishStage{}).Run(context.Background(), env, m); err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, ok := env.ObjectStore.(*storage.FakeBackend).Objects[m.S3SessionPrefix+"artifacts/encounters.json"]; !ok {
|
||||||
|
t.Fatal("explicit extraction output was filtered by Scriptorium artifact selection")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPublishSkipsOptionalMissingOutput(t *testing.T) {
|
func TestPublishSkipsOptionalMissingOutput(t *testing.T) {
|
||||||
env, m, _ := publishFixture(t)
|
env, m, _ := publishFixture(t)
|
||||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||||
@@ -591,6 +754,56 @@ func publishFixture(t *testing.T) (*Env, *manifest.Manifest, string) {
|
|||||||
return env, m, runRoot
|
return env, m, runRoot
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func configurePublishExtractionFixture(t *testing.T, env *Env, m *manifest.Manifest) string {
|
||||||
|
t.Helper()
|
||||||
|
env.Config.Pipeline.Notarius = &config.NotariusConfig{
|
||||||
|
Enabled: true, PipelineID: "campaign.extract",
|
||||||
|
Outputs: map[string]config.NotariusOutputConfig{
|
||||||
|
"encounters": {
|
||||||
|
LaneID: "encounters", MediaType: "application/json", SchemaID: "encounters",
|
||||||
|
SchemaVersion: "1", ModuleKey: "encounters",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
paths := publishSessionPaths(env, m)
|
||||||
|
producerRunID := "extract-run-1"
|
||||||
|
bundleRoot := filepath.Join(paths.ArtifactsDir, "notarius", producerRunID)
|
||||||
|
lanePath := filepath.Join(bundleRoot, "lanes", "encounters.json")
|
||||||
|
indexPath := filepath.Join(bundleRoot, "index.json")
|
||||||
|
writeStageTestFile(t, lanePath, `{"encounters":[]}`)
|
||||||
|
writeStageTestFile(t, indexPath, `{"lanes":[]}`)
|
||||||
|
writeStageTestFile(t, filepath.Join(bundleRoot, "manifest.json"), `{"run_id":"notarius-run-1"}`)
|
||||||
|
writeStageTestFile(t, filepath.Join(bundleRoot, "rejected.json"), `[]`)
|
||||||
|
writeStageTestFile(t, filepath.Join(bundleRoot, "warnings.json"), `[]`)
|
||||||
|
writeStageTestFile(t, filepath.Join(bundleRoot, "pipeline", "chunk-map.json"), `{"chunks":[]}`)
|
||||||
|
writeStageTestFile(t, filepath.Join(bundleRoot, "unknown", "notes.txt"), "internal bundle note\n")
|
||||||
|
laneChecksum, err := artifacts.SHA256File(lanePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
indexChecksum, err := artifacts.SHA256File(indexPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m.Stages["extract"] = &manifest.StageRecord{
|
||||||
|
Name: "extract", Status: manifest.StatusSucceeded,
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"narratio_run_id": producerRunID, "bundle_root": bundleRoot,
|
||||||
|
"receipt": map[string]any{"run_id": "notarius-run-1", "pipeline_id": "campaign.extract"},
|
||||||
|
},
|
||||||
|
Outputs: []manifest.ArtifactRecord{
|
||||||
|
{
|
||||||
|
Kind: "notarius_lane", SourceID: artifacts.ExtractionArtifactSourceID("encounters"), LocalPath: lanePath,
|
||||||
|
ProducerRunID: producerRunID, Checksum: laneChecksum,
|
||||||
|
Contract: &artifactmodel.ContractMetadata{MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters"},
|
||||||
|
ExternalProvenance: &artifactmodel.ExternalProvenance{System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: "encounters"},
|
||||||
|
},
|
||||||
|
{Kind: "notarius_index", LocalPath: indexPath, ProducerRunID: producerRunID, Checksum: indexChecksum},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return lanePath
|
||||||
|
}
|
||||||
|
|
||||||
type publishedOutputFailingStore struct {
|
type publishedOutputFailingStore struct {
|
||||||
delegate *storage.FakeBackend
|
delegate *storage.FakeBackend
|
||||||
failKey string
|
failKey string
|
||||||
|
|||||||
@@ -3,8 +3,11 @@ package stage
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/audita"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/audita"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notify"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notify"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
|
||||||
@@ -26,6 +29,7 @@ type Env struct {
|
|||||||
WhisperX whisperx.Client
|
WhisperX whisperx.Client
|
||||||
Seriatim seriatim.Runner
|
Seriatim seriatim.Runner
|
||||||
Audita audita.Runner
|
Audita audita.Runner
|
||||||
|
Notarius notarius.Runner
|
||||||
Scriptorium scriptorium.Runner
|
Scriptorium scriptorium.Runner
|
||||||
ObjectStore storage.ObjectStore
|
ObjectStore storage.ObjectStore
|
||||||
Notifier notify.Sender
|
Notifier notify.Sender
|
||||||
@@ -44,8 +48,61 @@ type Stage interface {
|
|||||||
Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error)
|
Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const maxResumeReasonLength = 512
|
||||||
|
|
||||||
|
// ResumeValidation reports whether a previously succeeded stage can be reused.
|
||||||
|
type ResumeValidation struct {
|
||||||
|
Resumable bool
|
||||||
|
Reason string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalized returns a result with a bounded reason and no reason on success.
|
||||||
|
func (r ResumeValidation) Normalized() ResumeValidation {
|
||||||
|
if r.Resumable {
|
||||||
|
return Resumable()
|
||||||
|
}
|
||||||
|
return NonResumable(r.Reason)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResumeValidator is implemented by stages that validate persisted success before reuse.
|
||||||
|
type ResumeValidator interface {
|
||||||
|
ValidateResume(ctx context.Context, env *Env, m *manifest.Manifest) (ResumeValidation, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resumable reports a successful resume validation.
|
||||||
|
func Resumable() ResumeValidation {
|
||||||
|
return ResumeValidation{Resumable: true}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NonResumable reports a bounded reason that persisted success must be rerun.
|
||||||
|
func NonResumable(reason string) ResumeValidation {
|
||||||
|
reason = strings.TrimSpace(reason)
|
||||||
|
if reason == "" {
|
||||||
|
reason = "persisted stage result is not reusable"
|
||||||
|
}
|
||||||
|
if len(reason) > maxResumeReasonLength {
|
||||||
|
cutoff := maxResumeReasonLength
|
||||||
|
for cutoff > 0 && !utf8.ValidString(reason[:cutoff]) {
|
||||||
|
cutoff--
|
||||||
|
}
|
||||||
|
reason = reason[:cutoff]
|
||||||
|
}
|
||||||
|
return ResumeValidation{Reason: reason}
|
||||||
|
}
|
||||||
|
|
||||||
|
// StageDisposition describes the outcome of a stage that returned without an error.
|
||||||
|
type StageDisposition string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// StageDispositionSucceeded is the zero value so existing stages remain successful.
|
||||||
|
StageDispositionSucceeded StageDisposition = ""
|
||||||
|
StageDispositionSkipped StageDisposition = "skipped"
|
||||||
|
)
|
||||||
|
|
||||||
// StageResult is the declared output of a stage execution.
|
// StageResult is the declared output of a stage execution.
|
||||||
type StageResult struct {
|
type StageResult struct {
|
||||||
|
Disposition StageDisposition
|
||||||
|
SkipReason string
|
||||||
Outputs []artifacts.Ref
|
Outputs []artifacts.Ref
|
||||||
Logs []string
|
Logs []string
|
||||||
GeneratedConfigs []string
|
GeneratedConfigs []string
|
||||||
|
|||||||
Reference in New Issue
Block a user