Compare commits
18 Commits
df58595d1e
...
v0.13.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 74e2d21de5 | |||
| 7cb18a1a40 | |||
| b556fc2f4f | |||
| b99bd38eb4 | |||
| 701b6726d7 | |||
| 665039f4dc | |||
| ef8dae776e | |||
| d01775b68a | |||
| 0d6f2dd0ce | |||
| df40cbec6e | |||
| 0341e0c7c0 | |||
| 39af7d4f3c | |||
| bba582b4ca | |||
| 1f16a85330 | |||
| f9482639d4 | |||
| dce721cdbd | |||
| 98734644d6 | |||
| 951383226c |
@@ -1,7 +1,8 @@
|
||||
# narratio
|
||||
|
||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
||||
polished transcripts and generated artifacts.
|
||||
polished transcripts, validated Notarius extraction lanes, and generated
|
||||
artifacts.
|
||||
|
||||
It runs a deterministic workflow with manifest-driven continuation, remote
|
||||
publish, and restore support.
|
||||
|
||||
15
docs/cli.md
15
docs/cli.md
@@ -75,7 +75,10 @@ narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common co
|
||||
Behavior:
|
||||
|
||||
- evaluates full stage order;
|
||||
- skips already-succeeded stages unless `--force` is set;
|
||||
- runs `extract` between `trim` and `render`; an omitted or disabled Notarius
|
||||
configuration records an explicit `notarius_disabled` self-skip;
|
||||
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
||||
resume check finds its durable result obsolete;
|
||||
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- writes session and run manifests.
|
||||
|
||||
@@ -93,6 +96,7 @@ Valid stage names:
|
||||
- `polish`
|
||||
- `normalize`
|
||||
- `trim`
|
||||
- `extract`
|
||||
- `render`
|
||||
- `analyze`
|
||||
- `publish`
|
||||
@@ -218,7 +222,10 @@ default restore scope, report location, and conflict-handling workflow.
|
||||
narratio session artifacts <session_id> [--remote] [...common config flags]
|
||||
```
|
||||
|
||||
Lists effective built-in and configured artifact sources, publish rules, lock state, and optional remote published-state availability.
|
||||
Lists effective built-in, configured Scriptorium, and configured extraction
|
||||
sources; reports planned, available, unavailable, and published state without
|
||||
reading payload bodies; and includes publish rules, lock state, and optional
|
||||
remote published-state availability.
|
||||
|
||||
### `session locks`
|
||||
|
||||
@@ -248,7 +255,9 @@ Effects:
|
||||
|
||||
- filters analyze execution to selected configured artifacts;
|
||||
- filters publish rules that source `narratio.artifact.<name>`;
|
||||
- does not filter built-in transcript/bounds publish sources.
|
||||
- does not filter built-in transcript/bounds or explicitly configured
|
||||
`narratio.extraction.<name>` publish sources; and
|
||||
- does not select or filter Notarius lanes.
|
||||
|
||||
## Common Workflows
|
||||
|
||||
|
||||
@@ -118,6 +118,9 @@ Rules:
|
||||
|
||||
- `outputs[].source` is required.
|
||||
- `outputs[].dest` may be omitted when derivable from source.
|
||||
- extraction sources require an explicit `outputs[].dest` and publish only when
|
||||
a rule names that source; the Notarius index and complete bundle are not
|
||||
publish sources.
|
||||
- `outputs[].required` defaults to `true`.
|
||||
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
||||
|
||||
@@ -196,6 +199,13 @@ Rules:
|
||||
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
||||
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
||||
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||
| `pipeline.notarius.enabled` | bool | No | `false` |
|
||||
| `pipeline.notarius.binary` | string | No | `notarius` |
|
||||
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
||||
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
||||
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
||||
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
||||
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
||||
| `pipeline.render.enabled` | bool | No | `true` |
|
||||
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
|
||||
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
|
||||
@@ -211,6 +221,26 @@ Rules:
|
||||
| `pipeline.notification.recipient` | string | No | empty |
|
||||
| `pipeline.notification.timeout` | duration | No | empty |
|
||||
|
||||
### Notarius Output Entries
|
||||
|
||||
For each `pipeline.notarius.outputs.<name>`:
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `lane_id` | string | Yes | unique Notarius lane ID |
|
||||
| `media_type` | string | Yes | exact accepted descriptor media type |
|
||||
| `schema_id` | string | Yes | exact accepted descriptor schema ID |
|
||||
| `schema_version` | string | Yes | exact accepted descriptor schema version |
|
||||
| `module_key` | string | No | exact accepted module key when set |
|
||||
|
||||
Output names must match `^[a-z][a-z0-9_]*$` and become selectable sources named
|
||||
`narratio.extraction.<name>`. Lane IDs must be unique. Every declared output is
|
||||
required from a successful Notarius result; a missing, rejected, duplicate, or
|
||||
contract-incompatible lane fails extraction. See the
|
||||
[complete maintained example](../examples/pipeline.full.annotated.yml) for the
|
||||
current ten-lane D&D mapping and the [Notarius contract](./integrations/notarius.md)
|
||||
for compatibility ownership.
|
||||
|
||||
### Scriptorium Artifact Entries
|
||||
|
||||
For each `pipeline.scriptorium.artifacts.<name>`:
|
||||
@@ -233,7 +263,7 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
|
||||
|
||||
| Field | Type | Required | Rule |
|
||||
| --- | --- | --- | --- |
|
||||
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
||||
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.extraction.<name>`, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
||||
| `artifact` | string | No | optional passthrough adapter field |
|
||||
| `path` | string | No | optional passthrough adapter field |
|
||||
| `required` | bool | No | optional input requirement |
|
||||
|
||||
@@ -19,6 +19,8 @@ focused stage documents.
|
||||
## Integration Contracts
|
||||
|
||||
- [Audita](./audita.md): transcript polishing (`audita process`).
|
||||
- [Notarius](./notarius.md): complete pipeline execution and safe JSON bundle
|
||||
discovery (`notarius run`).
|
||||
- [Seriatim](./seriatim.md): merge, normalize, trim, and render operations.
|
||||
- [Scriptorium](./scriptorium.md): artifact generation and debug rendering
|
||||
(`scriptorium run|render`).
|
||||
|
||||
83
docs/integrations/notarius.md
Normal file
83
docs/integrations/notarius.md
Normal file
@@ -0,0 +1,83 @@
|
||||
# Notarius Integration Contract
|
||||
|
||||
## Boundary
|
||||
|
||||
Narratio uses Notarius as a subprocess to extract configured structured JSON
|
||||
lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
|
||||
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
||||
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
||||
|
||||
Canonical Notarius references:
|
||||
|
||||
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/subprocess.md)
|
||||
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/dnd-pipeline.md)
|
||||
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/run-result.md)
|
||||
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/json-output.md)
|
||||
|
||||
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
||||
records the exact current constraints for all ten D&D lanes. Treat the linked
|
||||
Notarius documents as canonical when changing those values; Narratio does not
|
||||
duplicate the complete schemas.
|
||||
|
||||
## Invocation
|
||||
|
||||
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
||||
configuration path, input path, output directory, and working directory to
|
||||
absolute paths and invokes:
|
||||
|
||||
```text
|
||||
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> --json
|
||||
```
|
||||
|
||||
Standard output is reserved for the JSON receipt. Standard error is captured
|
||||
separately as diagnostic output. Narratio applies the configured timeout and
|
||||
does not interpret stdout as a receipt unless the subprocess exits successfully.
|
||||
It does not pass a Narratio session ID or run `notarius config validate`
|
||||
automatically; the configured working directory and inherited environment
|
||||
apply to the subprocess.
|
||||
|
||||
## Accepted Result
|
||||
|
||||
Narratio currently accepts receipt schema `notarius.run-result.v1`. The receipt
|
||||
must identify the configured pipeline, and its `index_file` must be exactly
|
||||
`index.json` beneath the reported bundle root. The production index must name
|
||||
the management files exactly as `manifest.json`, `rejected.json`, and
|
||||
`warnings.json`. All receipt, index, and lane paths must stay inside that
|
||||
bundle; symlinks and non-regular lane payloads are rejected.
|
||||
|
||||
Supported receipt and index shapes tolerate unknown fields for forward
|
||||
compatibility, while required identity, validation, count, manifest,
|
||||
rejection, warning, and lane-list fields remain mandatory. Narratio applies
|
||||
bounded reads to the receipt, index, rejection, and warning documents. Optional
|
||||
chunk-map and evidence-context descriptors must carry their complete generic
|
||||
contract metadata when present.
|
||||
|
||||
For every entry in `pipeline.notarius.outputs`, Narratio requires exactly one
|
||||
index descriptor with the configured lane ID, media type, schema ID, schema
|
||||
version, and, when configured, module key. Missing, duplicate, rejected, or
|
||||
incompatible required lanes fail extraction even if Notarius exited zero.
|
||||
Unconfigured lanes may remain in the preserved bundle but do not become
|
||||
selectable Narratio sources.
|
||||
|
||||
Each accepted configured lane is registered as
|
||||
`narratio.extraction.<output_key>`. The bundle index is retained for audit and
|
||||
resume validation but is not selectable. Scriptorium and publish rules consume
|
||||
only explicitly named lane sources; `--artifacts` never selects Notarius lanes.
|
||||
|
||||
## Failure And Compatibility Behavior
|
||||
|
||||
- Startup and nonzero-exit errors fail extraction and retain captured diagnostics.
|
||||
- Invalid receipt JSON or an unsupported receipt schema fails before bundle use.
|
||||
- Unsafe or incompatible index data and required-lane rejection fail before the
|
||||
staged bundle is promoted to durable storage.
|
||||
- Contract and external provenance metadata are preserved on lane artifact
|
||||
records and through explicit publication.
|
||||
|
||||
Rejection and warning summaries retain structured stage, scope, lane, and
|
||||
reason-code fields for diagnostics without exposing free-form external messages
|
||||
or reading lane payload bodies.
|
||||
|
||||
Configuration fields and defaults are in [Configuration](../config.md).
|
||||
Operator paths, rerun procedures, and bundle retention are in
|
||||
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
||||
for failure recovery.
|
||||
@@ -16,6 +16,7 @@ Primary adapters:
|
||||
- `seriatim.Runner`
|
||||
- `audita.Runner`
|
||||
- `scriptorium.Runner`
|
||||
- `notarius.Runner`
|
||||
- `storage.ObjectStore`
|
||||
- `notify.Sender`
|
||||
|
||||
@@ -40,9 +41,13 @@ Adapters do not own:
|
||||
- Seriatim subprocess runner.
|
||||
- Audita subprocess runner.
|
||||
- Scriptorium subprocess runner.
|
||||
- Notarius subprocess runner when extraction is enabled.
|
||||
- Noop notifier (`notify.NoopSender`).
|
||||
- Object store only when required by selected stages/config.
|
||||
|
||||
Notarius is composed only when extraction is enabled; the extract stage owns
|
||||
receipt, bundle, and configured-lane policy rather than the adapter.
|
||||
|
||||
Object-store construction goes through `newCommandObjectStore`, which loads
|
||||
configured filesystem secrets before adapter initialization.
|
||||
|
||||
@@ -56,16 +61,18 @@ configured filesystem secrets before adapter initialization.
|
||||
|
||||
- Composition: `internal/app/runner.go`, `internal/app/object_store.go`
|
||||
- Shared subprocess mechanics: `internal/adapters/subprocess`
|
||||
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,storage,notify}`
|
||||
- Focused adapters: `internal/adapters/{whisperx,seriatim,audita,scriptorium,notarius,storage,notify}`
|
||||
- `internal/adapters/whisperx/http_test.go`
|
||||
- `internal/adapters/seriatim/subprocess_test.go`
|
||||
- `internal/adapters/audita/subprocess_test.go`
|
||||
- `internal/adapters/scriptorium/subprocess_test.go`
|
||||
- `internal/adapters/notarius/subprocess_test.go`
|
||||
- `internal/adapters/storage/*_test.go`
|
||||
- `internal/app/runner_test.go`
|
||||
|
||||
See the [WhisperX](../integrations/whisperx.md),
|
||||
[Seriatim](../integrations/seriatim.md), [Audita](../integrations/audita.md),
|
||||
and [Scriptorium](../integrations/scriptorium.md) contracts before changing an
|
||||
[Scriptorium](../integrations/scriptorium.md), and
|
||||
[Notarius](../integrations/notarius.md) contracts before changing an
|
||||
externally visible boundary. Operator-selected values belong in
|
||||
[Configuration](../config.md).
|
||||
|
||||
@@ -24,12 +24,15 @@ Registry entries bind each ID to its producer, output kind, canonical fallback,
|
||||
and content validator. The focused stage documents own their input/output flow;
|
||||
[Configuration](../config.md) owns where operators may select these IDs.
|
||||
|
||||
## Configured and Previous-Session Sources
|
||||
## Configured, Extraction, And Previous-Session Sources
|
||||
|
||||
- configured source ID format: `narratio.artifact.<artifact_key>`
|
||||
- extraction source ID format: `narratio.extraction.<output_key>`
|
||||
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
||||
|
||||
Both formats are validated by strict source-policy rules.
|
||||
All formats are validated by strict source-policy rules. Extraction sources are
|
||||
registered only from `pipeline.notarius.outputs`; the Notarius index has no
|
||||
selectable source ID.
|
||||
|
||||
## Runtime Catalog
|
||||
|
||||
@@ -58,6 +61,15 @@ Configured sources (`narratio.artifact.*`):
|
||||
|
||||
- resolve only through runtime catalog availability.
|
||||
|
||||
Extraction sources (`narratio.extraction.*`):
|
||||
|
||||
- use the shared registration and manifest hydration path in
|
||||
`extraction_catalog.go`;
|
||||
- require a current successful extract record with the exact configured source,
|
||||
compatible contract and Notarius provenance, a confined regular durable
|
||||
payload, and matching checksum; and
|
||||
- are never inferred by scanning the Notarius bundle directory.
|
||||
|
||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||
|
||||
- resolve only from local `previous/` cache state;
|
||||
@@ -127,19 +139,23 @@ physical layout.
|
||||
|
||||
- source ID formats are stable contracts;
|
||||
- artifact resolution is deterministic and manifest-aware;
|
||||
- extraction sources are available only from a compatible successful manifest
|
||||
record;
|
||||
- previous-session source resolution in `analyze` is local-only;
|
||||
- remote current-state key construction remains centralized in artifacts helpers.
|
||||
|
||||
## Implementation And Tests
|
||||
|
||||
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
||||
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`
|
||||
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`,
|
||||
`internal/artifacts/extraction_catalog.go`
|
||||
- Current state: `internal/artifacts/current_state.go`
|
||||
- Paths and keys: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/s3_keys.go`
|
||||
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
||||
- Tests: `internal/artifacts/artifact_resolver_test.go`,
|
||||
`internal/artifacts/catalog_test.go`,
|
||||
`internal/artifacts/extraction_catalog_test.go`,
|
||||
`internal/artifacts/current_state_test.go`,
|
||||
`internal/artifacts/paths_model_test.go`,
|
||||
`internal/artifacts/previous_requirements_test.go`
|
||||
|
||||
@@ -50,13 +50,33 @@ The model admits these stage states:
|
||||
|
||||
The application runner marks an executing stage running and then succeeded or
|
||||
failed in both manifests, persisting each transition. On success it records
|
||||
outputs, logs, generated configuration references, and metadata. A successful
|
||||
forced rerun marks only succeeded downstream session-stage records stale.
|
||||
outputs, logs, generated configuration references, and metadata. Artifact
|
||||
records may include optional contract and external provenance objects; old
|
||||
manifests remain compatible when those fields are absent. A successful forced
|
||||
rerun marks only succeeded downstream session-stage records stale.
|
||||
|
||||
Starting an execution clears the current session-stage record's prior outputs,
|
||||
logs, generated configuration references, and metadata. Failed and skipped
|
||||
transitions enforce the same clearing rule directly, while success repopulates
|
||||
only fields returned by the new result. Marking a record stale does not clear
|
||||
those details because resume validation and diagnosis may still require them
|
||||
before execution begins. Invocation run manifests remain immutable audit
|
||||
records of their own outcomes.
|
||||
|
||||
A stage may explicitly return a skipped disposition and stable reason. The
|
||||
runner persists that outcome in both manifests, clears older outputs for the
|
||||
session-stage record along with older logs, generated configuration references,
|
||||
and metadata, then applies any bounded details from the current skip and
|
||||
continues. This self-skip is distinct from deciding not to execute an
|
||||
already-succeeded stage and is reconsidered on later runs. Skipped results
|
||||
cannot contain outputs.
|
||||
|
||||
When an already-succeeded stage is skipped, the invocation run manifest records
|
||||
the `skip` action and reason. The session manifest deliberately retains its
|
||||
existing succeeded record because it remains the cross-invocation progress
|
||||
authority.
|
||||
authority. Stages with a resume validator, currently extraction, may reject an
|
||||
otherwise eligible skip when the recorded durable result is obsolete; the
|
||||
runner marks it stale and executes it.
|
||||
|
||||
Session manifest is the authoritative stage-progress ledger across invocations.
|
||||
Run manifest is invocation-scoped audit state.
|
||||
@@ -64,6 +84,9 @@ Run manifest is invocation-scoped audit state.
|
||||
## Invariants
|
||||
|
||||
- stage resume/skip decisions are session-manifest driven.
|
||||
- running, failed, and self-skipped stages do not retain result payloads from
|
||||
an earlier success.
|
||||
- stale stages retain prior details until replacement execution starts.
|
||||
- force reruns stale downstream succeeded stages.
|
||||
- run manifest does not replace session manifest as progress authority.
|
||||
|
||||
|
||||
@@ -53,10 +53,11 @@ The implemented canonical order is:
|
||||
4. [`polish`](stage-polish.md)
|
||||
5. [`normalize`](stage-normalize.md)
|
||||
6. [`trim`](stage-trim.md)
|
||||
7. [`render`](stage-render.md)
|
||||
8. [`analyze`](stage-analyze.md)
|
||||
9. [`publish`](stage-publish.md)
|
||||
10. `notify` (placeholder)
|
||||
7. [`extract`](stage-extract.md)
|
||||
8. [`render`](stage-render.md)
|
||||
9. [`analyze`](stage-analyze.md)
|
||||
10. [`publish`](stage-publish.md)
|
||||
11. `notify` (placeholder)
|
||||
|
||||
`notify` currently has optional notifier call behavior and no persisted pipeline
|
||||
outputs; its default collaborator is a no-op sender. The focused stage
|
||||
@@ -83,6 +84,7 @@ and execution semantics.
|
||||
- [`polish`](stage-polish.md)
|
||||
- [`normalize`](stage-normalize.md)
|
||||
- [`trim`](stage-trim.md)
|
||||
- [`extract`](stage-extract.md)
|
||||
- [`render`](stage-render.md)
|
||||
- [`analyze`](stage-analyze.md)
|
||||
- [`publish`](stage-publish.md)
|
||||
|
||||
84
docs/internal/stage-extract.md
Normal file
84
docs/internal/stage-extract.md
Normal file
@@ -0,0 +1,84 @@
|
||||
# Internal: Extract Stage
|
||||
|
||||
## Responsibility
|
||||
|
||||
`extract` runs after `trim` and before `render`. It converts the canonical
|
||||
`narratio.transcript.final_trimmed` JSON into configured Notarius lane artifacts.
|
||||
An omitted or disabled Notarius section makes the stage explicitly self-skip
|
||||
with reason `notarius_disabled`, no outputs, and no Notarius runner.
|
||||
|
||||
The external protocol is documented in the
|
||||
[Notarius integration contract](../integrations/notarius.md). Configuration
|
||||
fields belong in [Configuration](../config.md), and physical paths and force
|
||||
procedures belong in [Operations](../operations.md).
|
||||
|
||||
## Lifecycle
|
||||
|
||||
`internal/stage/extract.go`:
|
||||
|
||||
1. resolves the final trimmed transcript from the shared artifact catalog;
|
||||
2. resolves and fingerprints the Notarius invocation contract;
|
||||
3. creates a run-local staging directory and invokes the injected
|
||||
`notarius.Runner`;
|
||||
4. validates the successful receipt, confined index, configured required lane
|
||||
descriptors, and regular payload files;
|
||||
5. atomically promotes the complete bundle to its immutable durable location;
|
||||
6. records one non-selectable `notarius_index` output and one selectable
|
||||
`notarius_lane` output per configured lane; and
|
||||
7. registers each lane as `narratio.extraction.<output_key>` for downstream
|
||||
Scriptorium and publish resolution.
|
||||
|
||||
Lane records retain checksum, contract, producer run ID, and Notarius system,
|
||||
run, pipeline, and lane provenance. Stage metadata retains the durable bundle
|
||||
root, receipt, diagnostic paths, rejection/warning summaries, producing
|
||||
Narratio run ID, and invocation fingerprint. Validation completes before
|
||||
promotion, so a rejected result cannot expose a partial durable bundle.
|
||||
|
||||
Any executed extraction outcome that replaces a different effective outcome
|
||||
marks succeeded downstream stages stale. Repeating the same disabled self-skip
|
||||
with no outputs is stable and does not repeatedly invalidate downstream stages.
|
||||
|
||||
## Resume Validation
|
||||
|
||||
`internal/stage/extract_resume.go` permits a skip only when the existing stage
|
||||
record succeeded and still matches the current invocation fingerprint. The
|
||||
fingerprint covers the resolved executable and config paths, pipeline ID,
|
||||
timeout, working directory, and sorted configured output contracts.
|
||||
|
||||
The validator then checks the producing run identity, canonical immutable
|
||||
bundle root, path confinement and absence of symlink components, receipt
|
||||
identity, exactly one canonical index, the exact configured source set,
|
||||
contracts and provenance, regular-file status, and stored checksums. Missing or
|
||||
obsolete results are non-resumable and run again; unsafe filesystem conditions
|
||||
return an error rather than silently accepting or replacing data.
|
||||
|
||||
The fingerprint cannot observe files imported by Notarius configuration,
|
||||
profile contents, prompt/module definitions, or other transitive inputs.
|
||||
Operators must force extraction after changing any such input.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
|
||||
index compatibility, required-lane rejection, payload inspection, checksum, or
|
||||
promotion errors fail the stage through ordinary manifest transition handling.
|
||||
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
|
||||
not given selectable extraction sources unless the complete configured result
|
||||
has passed validation and promotion.
|
||||
|
||||
When a replacement attempt begins, the current session-stage record no longer
|
||||
advertises payload from the previous success. A failed replacement therefore
|
||||
has no current outputs, logs, generated configuration references, or metadata,
|
||||
while the earlier invocation manifest and immutable promoted bundle remain
|
||||
available for audit and recovery.
|
||||
|
||||
## Implementation And Focused Tests
|
||||
|
||||
- Stage execution, selection, and resume validation: `internal/stage/extract.go`,
|
||||
`internal/stage/extract_resume.go`,
|
||||
`internal/stage/extract_test.go`
|
||||
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
||||
`internal/adapters/notarius/subprocess_test.go`
|
||||
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
||||
`internal/artifacts/extraction_catalog_test.go`
|
||||
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
||||
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
||||
@@ -26,8 +26,13 @@ Exact remote placement and the operator workflow belong in
|
||||
|
||||
- stage can self-skip when publish disabled or run upload disabled.
|
||||
- validates prerequisite stage success and object-store availability.
|
||||
- collects deterministic run file list plus run `manifest.json`.
|
||||
- collects a deterministic run file list plus run `manifest.json`, excluding
|
||||
`audio/**` and the run-local `extract/notarius-output/**` staging bundle.
|
||||
- keeps run-local Notarius receipt and stderr diagnostics eligible for the run
|
||||
archive.
|
||||
- resolves publish output sources through runtime artifact catalog and manifest-aware resolution.
|
||||
- publishes extraction lanes only through explicit configured output rules;
|
||||
neither run-local nor durable Notarius bundles are scanned or uploaded wholesale.
|
||||
- selected artifact filter applies to configured artifact sources only.
|
||||
- locked outputs are skipped intentionally (including required ones).
|
||||
- optional missing outputs are skipped; required missing unlocked outputs fail.
|
||||
@@ -48,7 +53,9 @@ Includes counts/lists for:
|
||||
## Invariants
|
||||
|
||||
- `current/run_id.txt` is the remote commit marker and is written last.
|
||||
- run upload excludes `audio/**`.
|
||||
- run upload excludes `audio/**` and `extract/notarius-output/**`.
|
||||
- `extract/notarius.receipt.json` and `extract/notarius.stderr.log` remain
|
||||
eligible run-record diagnostics.
|
||||
- publish locks are not overridden by `--force`.
|
||||
|
||||
The commit boundary and cleanup gate are normative architecture invariants; see
|
||||
|
||||
@@ -24,6 +24,16 @@ materialized into canonical session paths before stage success. Managed
|
||||
previous-session cache paths remain session-durable and are never redirected
|
||||
into run-local output space.
|
||||
|
||||
Extraction uses run-local receipt, stderr, and output-root helpers, then
|
||||
promotes the validated external bundle to the unique immutable Notarius bundle
|
||||
path supplied by `internal/artifacts`. `internal/fileops.PromoteDirectory`
|
||||
copies only regular files and directories to a same-filesystem temporary
|
||||
sibling. Source traversal uses confined directory handles and identity checks
|
||||
so replacing an inspected root, directory, or file is rejected rather than
|
||||
followed. The completed tree is atomically renamed without replacing an
|
||||
existing destination. Exact physical paths belong in
|
||||
[Operations](../operations.md#extraction-workflow).
|
||||
|
||||
## Locking
|
||||
|
||||
`artifacts.LocalStore` enforces the single-writer session lock via `.lock`
|
||||
@@ -54,9 +64,11 @@ deletion scope belong in [CLI](../cli.md#clean) and
|
||||
- Path model and local store: `internal/artifacts/paths.go`,
|
||||
`internal/artifacts/local.go`
|
||||
- Run-local materialization: `internal/stage/run_local.go`
|
||||
- Immutable bundle promotion: `internal/fileops/directory.go`
|
||||
- Cleanup confinement: `internal/app/cleanup_targets.go`,
|
||||
`internal/app/post_publish_cleanup.go`
|
||||
- Tests: `internal/artifacts/paths_model_test.go`,
|
||||
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
||||
`internal/fileops/directory_test.go`,
|
||||
`internal/app/cleanup_targets_test.go`,
|
||||
`internal/app/post_publish_cleanup_test.go`
|
||||
|
||||
@@ -75,16 +75,22 @@ Canonical stage order:
|
||||
4. `polish`
|
||||
5. `normalize`
|
||||
6. `trim`
|
||||
7. `render`
|
||||
8. `analyze`
|
||||
9. `publish`
|
||||
10. `notify`
|
||||
7. `extract`
|
||||
8. `render`
|
||||
9. `analyze`
|
||||
10. `publish`
|
||||
11. `notify`
|
||||
|
||||
Execution rules:
|
||||
|
||||
- succeeded stages are skipped unless `--force` is set;
|
||||
- `run` continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||
- force rerunning a succeeded upstream stage marks succeeded downstream stages as `stale`.
|
||||
- forcing an upstream stage marks succeeded downstream stages as `stale` before
|
||||
the replacement runs; and
|
||||
- an executed failure, changed self-skip, or success that replaces a different
|
||||
effective upstream outcome also marks succeeded downstream stages stale. A
|
||||
repeated self-skip with the same reason and no outputs is stable and does not
|
||||
perpetually rerun downstream work.
|
||||
|
||||
Single-stage execution:
|
||||
|
||||
@@ -101,7 +107,64 @@ Selection behavior:
|
||||
- validates names against `pipeline.scriptorium.artifacts`;
|
||||
- filters analyze execution to selected configured artifacts;
|
||||
- filters publish rules for `narratio.artifact.<name>` sources only;
|
||||
- does not suppress built-in transcript or bounds publish sources.
|
||||
- does not suppress built-in transcript, bounds, or explicitly configured
|
||||
`narratio.extraction.<name>` publish sources; and
|
||||
- never partially selects Notarius lanes.
|
||||
|
||||
## Extraction Workflow
|
||||
|
||||
When Notarius is omitted or disabled, `extract` records an explicit skipped
|
||||
outcome with reason `notarius_disabled` and no outputs. A later invocation
|
||||
reconsiders the skipped stage, so enabling Notarius does not require force.
|
||||
|
||||
When Notarius extraction is enabled, the stage consumes the final trimmed JSON
|
||||
and preserves the complete validated Notarius bundle at:
|
||||
|
||||
- `artifacts/notarius/{narratio_run_id}/`
|
||||
|
||||
The directory is immutable once promoted. Configured lanes become
|
||||
`narratio.extraction.<name>` sources for Scriptorium and explicit publish rules;
|
||||
the bundle and `index.json` are retained for audit and resume validation but
|
||||
are not selectable or published implicitly.
|
||||
|
||||
Starting a replacement clears the previous extraction payload from the current
|
||||
session-stage record. If that replacement fails or self-skips, the current
|
||||
record does not fall back to the earlier outputs. The earlier run manifest and
|
||||
immutable bundle remain available for inspection, but downstream resolution
|
||||
requires a new current successful extraction record.
|
||||
|
||||
Atomic Notarius bundle promotion is supported on Linux, macOS, and Windows.
|
||||
On other operating systems, extraction fails before copying the bundle into a
|
||||
temporary promotion tree because Narratio has no verified atomic no-replace
|
||||
directory primitive there. This is an extraction limitation, not a broader
|
||||
platform-support guarantee for every Narratio workflow.
|
||||
|
||||
Run-local diagnostics are:
|
||||
|
||||
- `runs/{run_id}/extract/notarius.receipt.json`
|
||||
- `runs/{run_id}/extract/notarius.stderr.log`
|
||||
- `runs/{run_id}/extract/notarius-output/` before durable promotion
|
||||
|
||||
The run-record upload excludes the complete
|
||||
`extract/notarius-output/**` subtree. The receipt and stderr files remain
|
||||
eligible run-record diagnostics. The durable bundle is never scanned for
|
||||
implicit publication; only lanes named by explicit `pipeline.publish.outputs`
|
||||
rules are uploaded.
|
||||
|
||||
To intentionally replace the current extraction result, run:
|
||||
|
||||
```bash
|
||||
narratio run-stage extract 2026-04-04 --force
|
||||
```
|
||||
|
||||
Narratio automatically reruns extraction when its recorded invocation contract
|
||||
or durable output validation changes. It cannot fingerprint configuration
|
||||
files, profiles, prompts, modules, or references loaded transitively by
|
||||
Notarius. Force extraction after changing any of those inputs, even when the
|
||||
top-level Narratio and Notarius config paths remain the same. A forced extract
|
||||
marks successful downstream stages stale. Ordinary extraction failures or
|
||||
outcome changes also stale affected downstream stages, while an identical
|
||||
repeated `notarius_disabled` self-skip does not repeatedly invalidate them.
|
||||
|
||||
## Publish Workflow
|
||||
|
||||
@@ -119,8 +182,10 @@ narratio run-stage publish 2026-04-04 --force
|
||||
|
||||
Publish commit model:
|
||||
|
||||
- uploads run files under `{session_prefix}/runs/{run_id}/`;
|
||||
- uploads configured published outputs;
|
||||
- uploads eligible run files under `{session_prefix}/runs/{run_id}/`, excluding
|
||||
audio and the run-local Notarius staging bundle;
|
||||
- uploads configured published outputs, including only explicitly configured
|
||||
extraction lanes;
|
||||
- uploads `previous/**` cache files when present;
|
||||
- writes `current/manifest.json`;
|
||||
- writes `current/run_id.txt` last.
|
||||
@@ -198,6 +263,10 @@ Durable session paths:
|
||||
- `config/**`
|
||||
- `runs/**`
|
||||
|
||||
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
||||
stderr, and pre-promotion output remain in the producing run's `extract`
|
||||
directory as described in [Extraction Workflow](#extraction-workflow).
|
||||
|
||||
Run-local layout:
|
||||
|
||||
- `runs/{run_id}/{stage}/outputs`
|
||||
|
||||
@@ -17,7 +17,8 @@ their domains:
|
||||
|
||||
- WhisperX performs transcription;
|
||||
- Seriatim performs deterministic transcript processing and rendering;
|
||||
- Audita performs transcript correction and polishing; and
|
||||
- Audita performs transcript correction and polishing;
|
||||
- Notarius extracts validated structured artifact bundles; and
|
||||
- Scriptorium executes prompts and produces configured artifacts.
|
||||
|
||||
Narratio owns orchestration, configuration resolution, session and run state,
|
||||
@@ -51,9 +52,9 @@ HTTP, subprocess, notification, and object-storage mechanics, including command
|
||||
construction, transport behavior, provider response handling, and external
|
||||
error adaptation. External dependency types must remain inside the adapter that
|
||||
owns them unless that dependency is the adapter's explicit public contract.
|
||||
WhisperX HTTP behavior, Seriatim, Audita, and Scriptorium command construction,
|
||||
notification transport, and object-storage SDK details remain behind these
|
||||
boundaries.
|
||||
WhisperX HTTP behavior, Seriatim, Audita, Notarius, and Scriptorium command
|
||||
construction, notification transport, and object-storage SDK details remain
|
||||
behind these boundaries.
|
||||
|
||||
State and path services must not infer stage policy. Storage implementations
|
||||
receive explicit bucket-relative keys and do not infer campaign, session, run,
|
||||
@@ -89,6 +90,13 @@ should preserve enough local state and diagnostics for inspection, recovery,
|
||||
and resume. Forcing an upstream stage invalidates succeeded downstream work
|
||||
according to the canonical stage order.
|
||||
|
||||
A stage may explicitly self-skip with a stable reason and no outputs. That
|
||||
outcome is persisted, clears older outputs owned by the stage, and is
|
||||
reconsidered on a later invocation. A stage may also validate whether an
|
||||
otherwise successful recorded result is still resumable; an obsolete result
|
||||
is staled and rerun, while an unsafe condition that prevents a sound decision
|
||||
stops execution.
|
||||
|
||||
Shared behavior should live behind a narrow service or helper with one clear
|
||||
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
|
||||
artifact, or path policy ad hoc.
|
||||
@@ -141,6 +149,9 @@ not reconstruct canonical paths through scattered string concatenation.
|
||||
Artifact resolution is deterministic and manifest-aware. Producers materialize
|
||||
canonical outputs before reporting success, and consumers resolve declared
|
||||
artifact identities rather than infer files from unrelated directory contents.
|
||||
External artifact bundles become current only through validated immutable
|
||||
promotion and manifest records; directory presence alone never establishes
|
||||
availability.
|
||||
|
||||
Writes, moves, replacements, and deletions must use narrow, explicit,
|
||||
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
||||
|
||||
@@ -1,379 +0,0 @@
|
||||
# Notarius Extraction Stage
|
||||
|
||||
## Status
|
||||
|
||||
Proposed.
|
||||
|
||||
## Purpose
|
||||
|
||||
Add a first-class Narratio `extract` stage that runs Notarius against the
|
||||
session's final trimmed transcript, validates and collects the resulting
|
||||
structured D&D artifacts, and registers those artifacts for later use by the
|
||||
`analyze` and `publish` stages.
|
||||
|
||||
This feature should integrate Notarius through Narratio's existing stage,
|
||||
adapter, manifest, workspace, and artifact-catalog boundaries. It must not turn
|
||||
Narratio into a generic workflow engine or a second configuration language for
|
||||
Notarius pipelines.
|
||||
|
||||
## User Outcome
|
||||
|
||||
An operator can enable one configured Notarius pipeline for a Narratio
|
||||
campaign. During a normal run, Narratio will:
|
||||
|
||||
1. finish producing the session transcript tiers;
|
||||
2. invoke Notarius once with the final trimmed Seriatim JSON transcript;
|
||||
3. collect and validate the configured structured artifact lanes;
|
||||
4. record their exact files and provenance in the Narratio manifest; and
|
||||
5. make those artifacts selectable as inputs to Scriptorium artifacts in the
|
||||
later `analyze` stage.
|
||||
|
||||
The maintained D&D example should demonstrate all ten lanes emitted by
|
||||
Notarius's complete `dnd-session` pipeline.
|
||||
|
||||
## Target Stage Architecture
|
||||
|
||||
### Canonical Order
|
||||
|
||||
The canonical stage order becomes:
|
||||
|
||||
```text
|
||||
prepare -> transcribe -> merge -> polish -> normalize -> trim -> render
|
||||
-> extract -> analyze -> publish -> notify
|
||||
```
|
||||
|
||||
`extract` is deliberately after all transcript-producing stages and before
|
||||
analysis. Its source document is the manifest-resolved
|
||||
`narratio.transcript.final_trimmed` artifact, normally
|
||||
`transcripts/final.trimmed.json`. It does not consume rendered Markdown.
|
||||
|
||||
Adding the stage must update full-plan construction, explicit stage selection,
|
||||
downstream invalidation, prerequisite checks, resume behavior, run manifests,
|
||||
CLI stage validation and help, and every canonical-stage inventory. Forcing an
|
||||
upstream transcript stage must stale a previously successful `extract` stage
|
||||
and its downstream stages. Forcing `extract` must stale `analyze`, `publish`,
|
||||
and `notify` according to existing rules.
|
||||
|
||||
### Stage Boundary
|
||||
|
||||
The stage owns Narratio policy and state transitions:
|
||||
|
||||
- resolve the final trimmed transcript through the runtime artifact catalog;
|
||||
- build a Narratio-level Notarius request from validated configuration and
|
||||
run-local paths;
|
||||
- call a narrow Notarius adapter;
|
||||
- apply the configured required-output policy;
|
||||
- materialize the validated bundle into its canonical session location;
|
||||
- return manifest-ready artifact references and bounded metadata; and
|
||||
- fail without marking the stage successful when any required contract or
|
||||
materialization step fails.
|
||||
|
||||
The stage must not construct subprocess arguments, infer Notarius output
|
||||
filenames, parse provider logs, or decode individual D&D payload bodies.
|
||||
|
||||
### Adapter Boundary
|
||||
|
||||
Add a dedicated Notarius adapter package with a small interface, production
|
||||
subprocess implementation, and test fake. Its request should contain only the
|
||||
resolved Notarius binary, configuration path, pipeline ID, transcript path,
|
||||
output root, working directory, timeout, and process-log destinations needed
|
||||
for one run.
|
||||
|
||||
The adapter owns:
|
||||
|
||||
- optional `notarius config validate` preflight for the configured pipeline;
|
||||
- exact `notarius run ... --json` argument construction;
|
||||
- stdout and stderr separation;
|
||||
- context cancellation and timeout propagation through Narratio's shared
|
||||
subprocess boundary;
|
||||
- exit-status handling;
|
||||
- decoding the `notarius.run-result.v1` success receipt;
|
||||
- receipt and index path-confinement checks;
|
||||
- decoding `index.json` and resolving descriptor paths safely beneath the
|
||||
reported output directory; and
|
||||
- returning a transport-neutral result containing the bundle location,
|
||||
receipt summary, lane descriptors, pipeline-wide descriptors, warnings and
|
||||
rejection locations, and diagnostic log paths.
|
||||
|
||||
Only exit status zero permits receipt decoding. Receipt, index, or descriptor
|
||||
paths that are absolute where a logical relative path is required, or that
|
||||
escape their owning root, are integration failures. Unknown fields in a
|
||||
supported receipt or index schema should be tolerated. Unsupported schema
|
||||
versions and incompatible descriptor metadata should fail clearly.
|
||||
|
||||
The adapter must not write Narratio manifests, choose required lanes, decide
|
||||
analysis inputs, or contain D&D domain logic.
|
||||
|
||||
## Configuration Contract
|
||||
|
||||
Add a strict optional `pipeline.notarius` configuration section. Omission or
|
||||
`enabled: false` keeps the current workflow usable and causes `extract` to
|
||||
self-skip without outputs.
|
||||
|
||||
The section should provide:
|
||||
|
||||
- `enabled`: explicit opt-in;
|
||||
- `binary`: Notarius executable, defaulting to `notarius`;
|
||||
- `config_path`: required when enabled;
|
||||
- `pipeline_id`: required when enabled;
|
||||
- `timeout`: a positive stage timeout with a documented default;
|
||||
- `working_directory`: optional explicit subprocess working directory,
|
||||
defaulting to the directory containing `config_path`; and
|
||||
- an `outputs` map defining the Notarius lane artifacts Narratio promises to
|
||||
collect.
|
||||
|
||||
Each output-map key is a stable Narratio extraction key. Each value must define:
|
||||
|
||||
- the exact Notarius `lane_id`;
|
||||
- the expected `media_type`;
|
||||
- the expected `schema_id`;
|
||||
- the expected `schema_version`; and
|
||||
- optionally an expected `module_key` when the operator needs to constrain the
|
||||
producing module as part of compatibility.
|
||||
|
||||
Narratio derives the downstream source ID
|
||||
`narratio.extraction.<output-key>` from the map key. Keys and lane IDs must be
|
||||
non-empty, unique after normalization, path-safe under the existing artifact
|
||||
policy, and collision-free with built-in and configured artifact identities.
|
||||
Every configured output is required: a successful Notarius process that omits
|
||||
one, rejects it, or reports incompatible descriptor metadata fails the
|
||||
`extract` stage.
|
||||
|
||||
This explicit map keeps Narratio's consumer contract stable when a Notarius
|
||||
lane ID or schema changes and avoids hard-coding the current D&D family into a
|
||||
generic adapter. It also replaces a separate `required_lanes` list, which would
|
||||
duplicate configuration.
|
||||
|
||||
Narratio should not reproduce Notarius lane selection, references, LLM
|
||||
profiles, model settings, retries, concurrency, or prompt configuration. Those
|
||||
remain in the referenced Notarius configuration. Narratio should not expose a
|
||||
runtime lane-selection flag for `extract`; one stage invocation runs the
|
||||
configured Notarius pipeline as a unit.
|
||||
|
||||
All configured paths should become absolute during Narratio configuration
|
||||
resolution. The deterministic default working directory allows a Notarius
|
||||
profile path relative to that directory, but operator documentation should
|
||||
still recommend absolute deployment paths where practical. Notarius reference
|
||||
paths continue to follow Notarius's own configuration-relative rules.
|
||||
|
||||
## Output And Artifact Model
|
||||
|
||||
### Canonical Bundle
|
||||
|
||||
Run Notarius against a run-local output root. After all configured descriptors
|
||||
are validated, materialize the contents of the exact run-specific Notarius
|
||||
bundle into a fixed canonical session directory:
|
||||
|
||||
```text
|
||||
artifacts/notarius/
|
||||
```
|
||||
|
||||
Preserve its relative layout, including `index.json`, `manifest.json`,
|
||||
`rejected.json`, `warnings.json`, `lanes/`, and any indexed `chunk-map.json` or
|
||||
`evidence-context.json`. Materialize the complete directory as one narrow,
|
||||
transactional replacement so a failed or interrupted rerun cannot mix files
|
||||
from different Notarius runs.
|
||||
|
||||
The raw subprocess receipt and stderr log belong in the run-local `extract`
|
||||
report and log directories. The raw receipt identifies the original run-local
|
||||
Notarius bundle and must not be rewritten to pretend that the canonical copy
|
||||
was its original `output_directory`. Narratio's manifest is the durable ledger
|
||||
for the canonical materialized paths.
|
||||
|
||||
### Registered Artifact Sources
|
||||
|
||||
For each configured output, locate the lane through the canonical copy of
|
||||
`index.json` and record a manifest artifact with:
|
||||
|
||||
- source ID `narratio.extraction.<output-key>`;
|
||||
- canonical lane-file path discovered from the index;
|
||||
- producer stage and Narratio run ID;
|
||||
- checksum;
|
||||
- Notarius lane ID; and
|
||||
- descriptor media type, schema identity/version, and module key when present.
|
||||
|
||||
If the current manifest model cannot carry descriptor compatibility metadata,
|
||||
extend its artifact metadata in a backward-tolerant way rather than encoding
|
||||
that information in filenames or source IDs.
|
||||
|
||||
Also record the canonical Notarius index as a stage output or stage metadata so
|
||||
operators can discover the complete bundle, including non-lane artifacts. The
|
||||
configured lane sources are the stable interface for analysis; the index and
|
||||
bundle remain the provenance and inspection interface.
|
||||
|
||||
## Analysis And Publish Integration
|
||||
|
||||
Extend the runtime artifact catalog and configured Scriptorium input validation
|
||||
so an enabled analysis artifact can declare, for example:
|
||||
|
||||
```yaml
|
||||
inputs:
|
||||
npc_registry:
|
||||
source: narratio.extraction.npc_registry
|
||||
```
|
||||
|
||||
Resolution must remain manifest-first and verify that the recorded artifact
|
||||
was produced by a successful current `extract` stage. A required extraction
|
||||
source that is unavailable must fail analysis with guidance to configure or
|
||||
rerun `extract`; an optional source may be omitted according to the existing
|
||||
Scriptorium input contract.
|
||||
|
||||
Publish source resolution should accept configured
|
||||
`narratio.extraction.<output-key>` sources through the same artifact catalog so
|
||||
operators may publish selected structured artifacts without manually copying
|
||||
paths. The existing `--artifacts` flag remains scoped to Scriptorium artifact
|
||||
selection and must not partially execute the Notarius pipeline.
|
||||
|
||||
No current-session analysis artifact should consume an incidental file from a
|
||||
failed, stale, skipped, or superseded extraction run.
|
||||
|
||||
## Failure, Skip, Resume, And Diagnostics
|
||||
|
||||
- Missing or invalid enabled Notarius configuration fails configuration
|
||||
validation before stage execution where statically discoverable.
|
||||
- A disabled or absent Notarius configuration makes `extract` skip with clear
|
||||
stage metadata and no new outputs.
|
||||
- A missing or invalid final trimmed transcript fails `extract` before starting
|
||||
Notarius.
|
||||
- Preflight failure, nonzero Notarius exit, cancellation, timeout, malformed or
|
||||
unsupported receipt/index data, unsafe paths, incompatible descriptors,
|
||||
rejected required outputs, or missing configured lanes fails the entire
|
||||
stage.
|
||||
- Process success does not override Narratio's required-output policy.
|
||||
- A failed run retains bounded run-local receipt bytes, stderr, and the
|
||||
unpublished Notarius bundle for diagnosis, subject to Narratio's existing
|
||||
sensitive-data and cleanup policies.
|
||||
- The canonical bundle and manifest artifacts are updated only after complete
|
||||
validation and materialization.
|
||||
- Resume skips a succeeded, non-stale `extract` stage only when its
|
||||
manifest-recorded canonical index and configured lane outputs still validate.
|
||||
- Force and staleness behavior follows the ordinary stage contract; it must not
|
||||
depend on merely finding `artifacts/notarius/` on disk.
|
||||
|
||||
Transcripts, Notarius outputs, evidence context, manifests, receipts, and logs
|
||||
are private campaign material. Subprocess arguments and manifest metadata must
|
||||
not contain secrets. Credentials remain in the environment or in mechanisms
|
||||
owned by Notarius and PromptKit.
|
||||
|
||||
## Maintained D&D Example
|
||||
|
||||
Add or update a Narratio example that enables Notarius's complete
|
||||
`dnd-session` pipeline and maps these ten required lanes to stable extraction
|
||||
keys:
|
||||
|
||||
| Output key | Notarius lane ID |
|
||||
| --- | --- |
|
||||
| `item_registry` | `item-registry` |
|
||||
| `npc_registry` | `npc-registry` |
|
||||
| `location_registry` | `location-registry` |
|
||||
| `scene_descriptions` | `scene-descriptions` |
|
||||
| `item_occurrences` | `item-occurrences` |
|
||||
| `spells` | `spells` |
|
||||
| `combat_turns` | `combat-turns` |
|
||||
| `npc_occurrences` | `npc-occurrences` |
|
||||
| `location_occurrences` | `location-occurrences` |
|
||||
| `enemy_events` | `enemy-events` |
|
||||
|
||||
The example must include each lane's current media type and schema identity
|
||||
from Notarius's published contracts. It should also demonstrate at least one
|
||||
Scriptorium analysis artifact consuming one or more
|
||||
`narratio.extraction.*` sources. The example must use placeholders and relative
|
||||
paths suitable for the example tree, contain no credentials, and pass the
|
||||
repository's configuration validation tests.
|
||||
|
||||
## Compatibility Policy
|
||||
|
||||
The initial integration baseline is the public subprocess contract available
|
||||
in Notarius v0.3.0:
|
||||
|
||||
- successful JSON receipt schema `notarius.run-result.v1`;
|
||||
- production JSON bundle discovery through `index.json`; and
|
||||
- the schema IDs and versions explicitly configured for required lanes.
|
||||
|
||||
Runtime compatibility should be decided from those published contracts, not
|
||||
from textual parsing of `notarius --version`. New optional receipt or index
|
||||
fields must not break Narratio. An unsupported receipt version or lane schema
|
||||
must fail before the artifact is registered for analysis.
|
||||
|
||||
## Documentation Deliverables When Implemented
|
||||
|
||||
Update current-behavior documentation in the same change that implements the
|
||||
feature:
|
||||
|
||||
- add `docs/integrations/notarius.md` for the external CLI, receipt, bundle,
|
||||
and adapter contract, linking to Notarius's canonical documentation;
|
||||
- add `docs/internal/stage-extract.md` for stage inputs, outputs, collaborators,
|
||||
state transitions, failures, and focused tests;
|
||||
- update `docs/internal/adapters.md`, `docs/internal/artifacts.md`,
|
||||
`docs/internal/manifest.md`, and the internal stage inventory;
|
||||
- update `docs/policy/architecture.md` to list Notarius among isolated external
|
||||
systems and preserve the adapter/stage boundary;
|
||||
- update `docs/config.md`, `docs/cli.md`, `docs/operations.md`,
|
||||
`docs/troubleshooting.md`, `README.md`, and maintained examples only to the
|
||||
extent their canonical scopes require; and
|
||||
- update `docs/development.md` only to the extent its canonical contributor
|
||||
routing scope requires.
|
||||
|
||||
Outside this roadmap, do not describe `extract`, Notarius configuration, or
|
||||
`narratio.extraction.*` sources as implemented until the code exists.
|
||||
|
||||
## Testing And Validation Expectations
|
||||
|
||||
Implementation should provide focused tests for:
|
||||
|
||||
- strict configuration decoding, defaults, required fields, path resolution,
|
||||
output-map validation, normalized-key collisions, and example loading;
|
||||
- exact stage order, selection, downstream staleness, resume, force, and
|
||||
prerequisite behavior;
|
||||
- adapter command construction, deterministic working directory, environment
|
||||
inheritance, stdout/stderr separation, cancellation, timeout, and nonzero
|
||||
exits;
|
||||
- supported and unsupported receipt versions, unknown optional fields,
|
||||
malformed receipts, index decoding, and path escapes at every boundary;
|
||||
- descriptor lookup by lane ID rather than filename, expected metadata checks,
|
||||
missing/rejected configured lanes, and tolerated unconfigured lanes;
|
||||
- run-local execution, transactional canonical-bundle replacement, checksums,
|
||||
failed-run preservation, and manifest recording;
|
||||
- artifact-catalog resolution from `narratio.extraction.*` into analysis and
|
||||
publish, including required, optional, missing, stale, and skipped cases; and
|
||||
- end-to-end stage execution with a fake Notarius adapter, without live LLM or
|
||||
external subprocess requirements in the ordinary test suite.
|
||||
|
||||
Run the repository-wide Go tests, vet, build, and maintained example validation
|
||||
after focused tests pass.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- `extract` is a first-class transactional stage between `render` and
|
||||
`analyze` everywhere Narratio models stage order or state.
|
||||
- Narratio invokes Notarius only through a narrow, tested adapter.
|
||||
- The stage consumes the manifest-resolved final trimmed Seriatim transcript.
|
||||
- The Notarius configuration remains owned by Notarius; Narratio configures
|
||||
only invocation and its downstream consumer contract.
|
||||
- Every configured output is discovered through the receipt and `index.json`,
|
||||
contract-checked, materialized transactionally, and recorded with a stable
|
||||
`narratio.extraction.*` source ID.
|
||||
- The complete D&D example maps all ten current lanes and passes strict config
|
||||
validation.
|
||||
- Analysis can consume extraction sources through the existing artifact input
|
||||
model, and publish can select them through the artifact catalog.
|
||||
- Failed, partial, rejected, unsafe, stale, or incompatible output never becomes
|
||||
a current analysis input.
|
||||
- Resume and force behavior remains manifest-driven.
|
||||
- Documentation accurately describes the implemented stage, adapter,
|
||||
configuration, operations, and compatibility boundary without duplicating
|
||||
Notarius's canonical schemas.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Reimplementing Notarius extraction, prompts, schemas, references, retries,
|
||||
profiles, or lane orchestration in Narratio.
|
||||
- Allowing one Narratio run to invoke arbitrary extractor programs or multiple
|
||||
Notarius pipelines.
|
||||
- Making `extract` a configurable DAG or folding it into the Scriptorium
|
||||
`analyze` stage.
|
||||
- Partially selecting Notarius lanes through Narratio's `--artifacts` flag.
|
||||
- Decoding D&D payload bodies in the generic Notarius adapter.
|
||||
- Supporting previous-session extraction artifacts in the initial feature.
|
||||
- Requiring live Notarius, PromptKit, an LLM provider, or external services in
|
||||
the ordinary unit test suite.
|
||||
@@ -117,6 +117,147 @@ Safe fix:
|
||||
|
||||
Relevant reference: [CLI artifact selection](./cli.md).
|
||||
|
||||
## Notarius executable missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails while resolving or starting the Notarius executable.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- `pipeline.notarius.binary` is not installed, executable, or on `PATH`;
|
||||
- a configured executable path is wrong.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- install a compatible Notarius release or correct the binary setting, then
|
||||
rerun extraction.
|
||||
|
||||
Relevant references: [Notarius configuration](./config.md#notarius-output-entries)
|
||||
and [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Notarius exits nonzero
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction reports a Notarius exit error instead of a receipt.
|
||||
|
||||
Diagnostics:
|
||||
|
||||
- inspect `runs/{run_id}/extract/notarius.stderr.log`; stdout is reserved for
|
||||
the receipt and is not merged with diagnostics.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- correct the reported Notarius pipeline, input, provider, or configuration
|
||||
failure and rerun extraction. Do not edit a staged output bundle into place.
|
||||
|
||||
After a failed replacement, an older immutable bundle may still exist even
|
||||
though the current session manifest has no successful extraction payload. This
|
||||
is expected audit state, not a signal to relink the old bundle manually.
|
||||
|
||||
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Atomic Notarius promotion unsupported
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails with `atomic no-replace directory promotion is unsupported`
|
||||
before a durable bundle or temporary promotion tree is created.
|
||||
|
||||
Likely cause:
|
||||
|
||||
- Narratio is running on an operating system other than Linux, macOS, or
|
||||
Windows, where the required atomic no-replace directory primitive has not
|
||||
been implemented and verified.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- run extraction on Linux, macOS, or Windows. Do not replace the atomic commit
|
||||
with a manual copy or move; the session manifest must never observe a partial
|
||||
or overwritten bundle.
|
||||
|
||||
This is an extraction-specific platform boundary, not a support statement for
|
||||
unrelated Narratio workflows. See
|
||||
[Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Notarius receipt or index incompatible
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction rejects the receipt schema, pipeline identity, bundle/index path,
|
||||
lane descriptor, or payload path even though Notarius exited successfully.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- Narratio and Notarius versions disagree on their consumer contract;
|
||||
- the configured pipeline or lane constraints are stale;
|
||||
- output paths escape the bundle or traverse symlinks.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- compare installed Notarius output with the canonical Notarius contracts,
|
||||
including receipt `index_file: index.json` and index management names
|
||||
`manifest.json`, `rejected.json`, and `warnings.json`; align
|
||||
`pipeline.notarius` constraints and rerun. Do not bypass confinement or schema
|
||||
checks.
|
||||
|
||||
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
||||
|
||||
## Required Notarius lane rejected or missing
|
||||
|
||||
Symptom:
|
||||
|
||||
- extraction fails because a configured lane is rejected, missing, duplicated,
|
||||
or incompatible, including after a zero exit.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- inspect the Notarius diagnostic log and bundle rejection/warning information;
|
||||
- correct the Notarius module or the exact declared lane contract;
|
||||
- remove an output declaration only if downstream consumers genuinely no longer
|
||||
require that source, then rerun extraction.
|
||||
|
||||
Every configured output is required. Narratio does not promote a partial result.
|
||||
|
||||
## Extraction resume invalidated
|
||||
|
||||
Symptom:
|
||||
|
||||
- a previously successful extraction runs again during ordinary continuation.
|
||||
|
||||
Likely causes:
|
||||
|
||||
- the executable/config path, pipeline ID, timeout, working directory, or
|
||||
configured output contracts changed;
|
||||
- the durable bundle, index, lane set, provenance, regular-file status, or
|
||||
checksum no longer validates.
|
||||
|
||||
Safe fix:
|
||||
|
||||
- allow the automatic rerun after verifying the current configuration. Treat
|
||||
an unsafe path or symlink error as filesystem corruption or tampering and
|
||||
investigate it rather than replacing files manually.
|
||||
|
||||
## Notarius transitive configuration changed
|
||||
|
||||
Symptom:
|
||||
|
||||
- Notarius profiles, prompts, modules, imported files, or references changed,
|
||||
but Narratio still considers the previous extraction resumable.
|
||||
|
||||
Safe fix:
|
||||
|
||||
```bash
|
||||
narratio run-stage extract 2026-04-04 --force
|
||||
```
|
||||
|
||||
Narratio fingerprints its invocation contract, not the contents of transitive
|
||||
Notarius inputs. Always force extraction after changing them; downstream
|
||||
successful stages are then marked stale normally.
|
||||
|
||||
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||
|
||||
## Previous-session artifact input missing
|
||||
|
||||
Symptom:
|
||||
|
||||
@@ -13,6 +13,8 @@ in the [configuration reference](../docs/config.md).
|
||||
external tools, and configured Scriptorium artifacts.
|
||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||
pipeline section with explanatory comments.
|
||||
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||
Scriptorium artifact consuming only three declared Notarius lanes.
|
||||
|
||||
The existing `internal/config` example test loads and validates each pipeline
|
||||
with the sample campaign and a compatible local- or S3-audio session.
|
||||
|
||||
55
examples/pipeline.extraction-subset.yml
Normal file
55
examples/pipeline.extraction-subset.yml
Normal file
@@ -0,0 +1,55 @@
|
||||
# Purpose-specific extraction example: a Scriptorium session brief consumes
|
||||
# only the three Notarius lanes it needs.
|
||||
|
||||
campaigns:
|
||||
root: /usr/local/share/narratio/campaigns
|
||||
default_campaign_id: sample-campaign
|
||||
|
||||
whisperx:
|
||||
transcribe_url: "https://transcription.example.com/transcribe"
|
||||
|
||||
notarius:
|
||||
enabled: true
|
||||
binary: notarius
|
||||
config_path: /usr/local/etc/notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
timeout: 3h
|
||||
outputs:
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-registry
|
||||
location_registry:
|
||||
lane_id: location-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/location-registry
|
||||
scene_descriptions:
|
||||
lane_id: scene-descriptions
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.scene_descriptions
|
||||
schema_version: v1
|
||||
module_key: dnd/scene-descriptions
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
artifacts:
|
||||
session_brief:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_brief
|
||||
output_path: artifacts/session_brief.md
|
||||
inputs:
|
||||
npcs:
|
||||
source: narratio.extraction.npc_registry
|
||||
required: true
|
||||
locations:
|
||||
source: narratio.extraction.location_registry
|
||||
required: true
|
||||
scenes:
|
||||
source: narratio.extraction.scene_descriptions
|
||||
required: true
|
||||
|
||||
@@ -60,6 +60,11 @@ publish:
|
||||
- source: narratio.artifact.player_handout
|
||||
dest: artifacts/player_handout.md
|
||||
required: false
|
||||
# Extraction lanes publish only when named explicitly; the bundle and index
|
||||
# are never implicit publish sources.
|
||||
- source: narratio.extraction.npc_registry
|
||||
dest: artifacts/extraction/npc-registry.json
|
||||
required: true
|
||||
|
||||
whisperx:
|
||||
# Required.
|
||||
@@ -123,6 +128,78 @@ trim:
|
||||
seriatim:
|
||||
report: false
|
||||
|
||||
notarius:
|
||||
# Optional structured extraction between trim and render.
|
||||
enabled: true
|
||||
binary: notarius
|
||||
config_path: /usr/local/etc/notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
timeout: 3h
|
||||
working_directory: /usr/local/etc/notarius
|
||||
# Each key creates source narratio.extraction.<key>. These constraints match
|
||||
# the current Notarius D&D lane contracts; update them with Notarius.
|
||||
outputs:
|
||||
item_registry:
|
||||
lane_id: item-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.item_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/item-registry
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-registry
|
||||
location_registry:
|
||||
lane_id: location-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/location-registry
|
||||
scene_descriptions:
|
||||
lane_id: scene-descriptions
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.scene_descriptions
|
||||
schema_version: v1
|
||||
module_key: dnd/scene-descriptions
|
||||
item_occurrences:
|
||||
lane_id: item-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.item_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/item-occurrences
|
||||
spells:
|
||||
lane_id: spells
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.spells
|
||||
schema_version: v1
|
||||
module_key: dnd/spells
|
||||
combat_turns:
|
||||
lane_id: combat-turns
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.combat_turns
|
||||
schema_version: v1
|
||||
module_key: dnd/combat-turns
|
||||
npc_occurrences:
|
||||
lane_id: npc-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-occurrences
|
||||
location_occurrences:
|
||||
lane_id: location-occurrences
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.location_occurrences
|
||||
schema_version: v1
|
||||
module_key: dnd/location-occurrences
|
||||
enemy_events:
|
||||
lane_id: enemy-events
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.enemy_events
|
||||
schema_version: v1
|
||||
module_key: dnd/enemy-events
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
config_path: /usr/local/etc/scriptorium/config.yml
|
||||
|
||||
1
go.mod
1
go.mod
@@ -7,6 +7,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.16
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
golang.org/x/sys v0.47.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
|
||||
2
go.sum
2
go.sum
@@ -34,6 +34,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOIt
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
22
internal/adapters/notarius/fake.go
Normal file
22
internal/adapters/notarius/fake.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package notarius
|
||||
|
||||
import "context"
|
||||
|
||||
// FakeRunner is a configurable in-memory runner for stage tests.
|
||||
type FakeRunner struct {
|
||||
Requests []RunRequest
|
||||
Result RunResult
|
||||
Err error
|
||||
}
|
||||
|
||||
// Run records the request and returns the configured result or error.
|
||||
func (f *FakeRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return RunResult{}, err
|
||||
}
|
||||
f.Requests = append(f.Requests, req)
|
||||
if f.Err != nil {
|
||||
return RunResult{}, f.Err
|
||||
}
|
||||
return f.Result, nil
|
||||
}
|
||||
108
internal/adapters/notarius/runner.go
Normal file
108
internal/adapters/notarius/runner.go
Normal file
@@ -0,0 +1,108 @@
|
||||
// Package notarius declares the adapter contract for Notarius CLI invocations.
|
||||
package notarius
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
const ReceiptSchemaVersion = "notarius.run-result.v1"
|
||||
|
||||
// Runner is the adapter boundary for a complete Notarius pipeline invocation.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, req RunRequest) (RunResult, error)
|
||||
}
|
||||
|
||||
// RunRequest contains the resolved inputs and diagnostic destinations for one invocation.
|
||||
type RunRequest struct {
|
||||
Binary string
|
||||
ConfigPath string
|
||||
PipelineID string
|
||||
InputPath string
|
||||
OutputRoot string
|
||||
WorkingDirectory string
|
||||
ReceiptPath string
|
||||
LogPath string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Receipt is the transport-neutral successful run receipt.
|
||||
type Receipt struct {
|
||||
SchemaVersion string
|
||||
RunID string
|
||||
PipelineID string
|
||||
OutputDirectory string
|
||||
IndexFile string
|
||||
NormalizedOutputCount int
|
||||
RejectedOutputCount int
|
||||
WarningCount int
|
||||
ValidationStatus string
|
||||
DebugDirectory string
|
||||
}
|
||||
|
||||
// LaneDescriptor identifies one normalized lane payload discovered through the index.
|
||||
type LaneDescriptor struct {
|
||||
LaneID string
|
||||
File string
|
||||
Path string
|
||||
MediaType string
|
||||
ModuleKey string
|
||||
SchemaID string
|
||||
SchemaName string
|
||||
SchemaVersion string
|
||||
}
|
||||
|
||||
// PipelineDescriptor identifies a pipeline-wide artifact discovered through the index.
|
||||
type PipelineDescriptor struct {
|
||||
ArtifactKind string
|
||||
File string
|
||||
Path string
|
||||
MediaType string
|
||||
SchemaID string
|
||||
SchemaName string
|
||||
SchemaVersion string
|
||||
}
|
||||
|
||||
// Index describes the validated bundle-management and artifact paths.
|
||||
type Index struct {
|
||||
Path string
|
||||
ManifestFile string
|
||||
ManifestPath string
|
||||
RejectedFile string
|
||||
RejectedPath string
|
||||
WarningsFile string
|
||||
WarningsPath string
|
||||
Lanes []LaneDescriptor
|
||||
ChunkMap *PipelineDescriptor
|
||||
EvidenceContext *PipelineDescriptor
|
||||
}
|
||||
|
||||
// RejectionSummary retains structured rejection identity without free-form messages.
|
||||
type RejectionSummary struct {
|
||||
Stage string
|
||||
StepID string
|
||||
LaneID string
|
||||
ModuleKey string
|
||||
ChunkID string
|
||||
ValidatorName string
|
||||
ReasonCode string
|
||||
}
|
||||
|
||||
// WarningSummary retains structured warning identity without free-form messages.
|
||||
type WarningSummary struct {
|
||||
Scope string
|
||||
ReasonCode string
|
||||
}
|
||||
|
||||
// RunResult describes a successfully decoded and validated Notarius bundle.
|
||||
type RunResult struct {
|
||||
Receipt Receipt
|
||||
Index Index
|
||||
BundleRoot string
|
||||
ReceiptPath string
|
||||
LogPath string
|
||||
ExitCode int
|
||||
Duration time.Duration
|
||||
Rejections []RejectionSummary
|
||||
Warnings []WarningSummary
|
||||
}
|
||||
524
internal/adapters/notarius/subprocess.go
Normal file
524
internal/adapters/notarius/subprocess.go
Normal file
@@ -0,0 +1,524 @@
|
||||
package notarius
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
maxReceiptBytes = 1 << 20
|
||||
maxIndexBytes = 4 << 20
|
||||
maxSummaryBytes = 4 << 20
|
||||
canonicalIndexFile = "index.json"
|
||||
canonicalManifestFile = "manifest.json"
|
||||
canonicalRejectedFile = "rejected.json"
|
||||
canonicalWarningsFile = "warnings.json"
|
||||
)
|
||||
|
||||
type subprocessRun func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error)
|
||||
|
||||
// SubprocessRunner invokes Notarius through its public CLI.
|
||||
type SubprocessRunner struct {
|
||||
run subprocessRun
|
||||
}
|
||||
|
||||
// NewSubprocessRunner constructs a production Notarius subprocess runner.
|
||||
func NewSubprocessRunner() *SubprocessRunner {
|
||||
return &SubprocessRunner{run: subprocess.Run}
|
||||
}
|
||||
|
||||
// Run executes a complete Notarius pipeline and discovers its published bundle.
|
||||
func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult, error) {
|
||||
if r == nil || r.run == nil {
|
||||
return RunResult{}, fmt.Errorf("notarius subprocess runner is nil")
|
||||
}
|
||||
if err := validateRunRequest(req); err != nil {
|
||||
return RunResult{}, err
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"run", req.PipelineID,
|
||||
"--config", req.ConfigPath,
|
||||
"--input", req.InputPath,
|
||||
"--output-dir", req.OutputRoot,
|
||||
"--json",
|
||||
}
|
||||
processResult, err := r.run(ctx, subprocess.RunRequest{
|
||||
Executable: req.Binary,
|
||||
Args: args,
|
||||
WorkingDir: req.WorkingDirectory,
|
||||
Timeout: req.Timeout,
|
||||
StdoutLogPath: req.ReceiptPath,
|
||||
StderrLogPath: req.LogPath,
|
||||
})
|
||||
baseResult := RunResult{
|
||||
ReceiptPath: req.ReceiptPath,
|
||||
LogPath: req.LogPath,
|
||||
ExitCode: processResult.ExitCode,
|
||||
Duration: processResult.Duration,
|
||||
}
|
||||
if err != nil {
|
||||
return baseResult, fmt.Errorf("run notarius pipeline %q: %w", req.PipelineID, err)
|
||||
}
|
||||
|
||||
receipt, err := loadReceipt(req.ReceiptPath, req.PipelineID)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
bundleRoot, err := validateBundleRoot(req.OutputRoot, receipt.OutputDirectory)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
indexPath, err := resolveRegularFile(bundleRoot, receipt.IndexFile)
|
||||
if err != nil {
|
||||
return baseResult, fmt.Errorf("resolve receipt index file: %w", err)
|
||||
}
|
||||
index, err := loadIndex(bundleRoot, indexPath)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
rejections, err := loadRejections(index.RejectedPath)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
warnings, err := loadWarnings(index.WarningsPath)
|
||||
if err != nil {
|
||||
return baseResult, err
|
||||
}
|
||||
|
||||
baseResult.Receipt = receipt
|
||||
baseResult.Index = index
|
||||
baseResult.BundleRoot = bundleRoot
|
||||
baseResult.Rejections = rejections
|
||||
baseResult.Warnings = warnings
|
||||
return baseResult, nil
|
||||
}
|
||||
|
||||
func validateRunRequest(req RunRequest) error {
|
||||
if strings.TrimSpace(req.Binary) == "" {
|
||||
return fmt.Errorf("notarius binary is required")
|
||||
}
|
||||
if strings.TrimSpace(req.PipelineID) == "" {
|
||||
return fmt.Errorf("notarius pipeline id is required")
|
||||
}
|
||||
if req.Timeout <= 0 {
|
||||
return fmt.Errorf("notarius timeout must be positive")
|
||||
}
|
||||
for label, path := range map[string]string{
|
||||
"config": req.ConfigPath,
|
||||
"input": req.InputPath,
|
||||
"output root": req.OutputRoot,
|
||||
"working directory": req.WorkingDirectory,
|
||||
"receipt": req.ReceiptPath,
|
||||
"log": req.LogPath,
|
||||
} {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fmt.Errorf("notarius %s path is required", label)
|
||||
}
|
||||
if !filepath.IsAbs(path) {
|
||||
return fmt.Errorf("notarius %s path must be absolute", label)
|
||||
}
|
||||
}
|
||||
if filepath.Clean(req.ReceiptPath) == filepath.Clean(req.LogPath) {
|
||||
return fmt.Errorf("notarius receipt and log paths must be different")
|
||||
}
|
||||
if err := requireRegularFile(req.ConfigPath); err != nil {
|
||||
return fmt.Errorf("validate notarius config path: %w", err)
|
||||
}
|
||||
if err := requireRegularFile(req.InputPath); err != nil {
|
||||
return fmt.Errorf("validate notarius input path: %w", err)
|
||||
}
|
||||
if err := requireDirectory(req.OutputRoot); err != nil {
|
||||
return fmt.Errorf("validate notarius output root: %w", err)
|
||||
}
|
||||
if err := requireDirectory(req.WorkingDirectory); err != nil {
|
||||
return fmt.Errorf("validate notarius working directory: %w", err)
|
||||
}
|
||||
if err := validateLogDestination(req.ReceiptPath); err != nil {
|
||||
return fmt.Errorf("validate notarius receipt path: %w", err)
|
||||
}
|
||||
if err := validateLogDestination(req.LogPath); err != nil {
|
||||
return fmt.Errorf("validate notarius log path: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type receiptDocument struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
RunID string `json:"run_id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
OutputDirectory string `json:"output_directory"`
|
||||
IndexFile string `json:"index_file"`
|
||||
NormalizedOutputCount *int `json:"normalized_output_count"`
|
||||
RejectedOutputCount *int `json:"rejected_output_count"`
|
||||
WarningCount *int `json:"warning_count"`
|
||||
ValidationStatus string `json:"validation_status"`
|
||||
DebugDirectory string `json:"debug_directory"`
|
||||
}
|
||||
|
||||
func loadReceipt(path, pipelineID string) (Receipt, error) {
|
||||
var document receiptDocument
|
||||
if err := decodeBoundedJSON(path, maxReceiptBytes, &document); err != nil {
|
||||
return Receipt{}, fmt.Errorf("decode notarius receipt: %w", err)
|
||||
}
|
||||
if document.SchemaVersion != ReceiptSchemaVersion {
|
||||
return Receipt{}, fmt.Errorf("unsupported notarius receipt schema version %q", document.SchemaVersion)
|
||||
}
|
||||
if strings.TrimSpace(document.RunID) == "" || strings.TrimSpace(document.PipelineID) == "" ||
|
||||
strings.TrimSpace(document.OutputDirectory) == "" || strings.TrimSpace(document.ValidationStatus) == "" ||
|
||||
document.NormalizedOutputCount == nil ||
|
||||
document.RejectedOutputCount == nil || document.WarningCount == nil {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt is missing required fields")
|
||||
}
|
||||
if document.IndexFile != canonicalIndexFile {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt index_file %q is incompatible; want %q", document.IndexFile, canonicalIndexFile)
|
||||
}
|
||||
if document.PipelineID != pipelineID {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt pipeline id %q does not match requested pipeline %q", document.PipelineID, pipelineID)
|
||||
}
|
||||
if *document.NormalizedOutputCount < 0 || *document.RejectedOutputCount < 0 || *document.WarningCount < 0 {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt counts must be non-negative")
|
||||
}
|
||||
if !filepath.IsAbs(document.OutputDirectory) {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt output directory must be absolute")
|
||||
}
|
||||
if document.DebugDirectory != "" && !filepath.IsAbs(document.DebugDirectory) {
|
||||
return Receipt{}, fmt.Errorf("notarius receipt debug directory must be absolute when present")
|
||||
}
|
||||
return Receipt{
|
||||
SchemaVersion: document.SchemaVersion,
|
||||
RunID: document.RunID,
|
||||
PipelineID: document.PipelineID,
|
||||
OutputDirectory: filepath.Clean(document.OutputDirectory),
|
||||
IndexFile: document.IndexFile,
|
||||
NormalizedOutputCount: *document.NormalizedOutputCount,
|
||||
RejectedOutputCount: *document.RejectedOutputCount,
|
||||
WarningCount: *document.WarningCount,
|
||||
ValidationStatus: document.ValidationStatus,
|
||||
DebugDirectory: document.DebugDirectory,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type indexDocument struct {
|
||||
ManifestFile string `json:"manifest_file"`
|
||||
OutputFiles *[]laneDocument `json:"output_files"`
|
||||
RejectedFile string `json:"rejected_file"`
|
||||
WarningsFile string `json:"warnings_file"`
|
||||
ChunkMap *pipelineDocument `json:"chunk_map"`
|
||||
EvidenceContext *pipelineDocument `json:"evidence_context"`
|
||||
}
|
||||
|
||||
type laneDocument struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
File string `json:"file"`
|
||||
MediaType string `json:"media_type"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
SchemaID string `json:"schema_id"`
|
||||
SchemaName string `json:"schema_name"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
}
|
||||
|
||||
type pipelineDocument struct {
|
||||
ArtifactKind string `json:"artifact_kind"`
|
||||
File string `json:"file"`
|
||||
MediaType string `json:"media_type"`
|
||||
SchemaID string `json:"schema_id"`
|
||||
SchemaName string `json:"schema_name"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
}
|
||||
|
||||
func loadIndex(bundleRoot, indexPath string) (Index, error) {
|
||||
var document indexDocument
|
||||
if err := decodeBoundedJSON(indexPath, maxIndexBytes, &document); err != nil {
|
||||
return Index{}, fmt.Errorf("decode notarius index: %w", err)
|
||||
}
|
||||
for _, field := range []struct {
|
||||
name string
|
||||
got string
|
||||
want string
|
||||
}{
|
||||
{name: "manifest_file", got: document.ManifestFile, want: canonicalManifestFile},
|
||||
{name: "rejected_file", got: document.RejectedFile, want: canonicalRejectedFile},
|
||||
{name: "warnings_file", got: document.WarningsFile, want: canonicalWarningsFile},
|
||||
} {
|
||||
if field.got != field.want {
|
||||
return Index{}, fmt.Errorf("notarius index %s %q is incompatible; want %q", field.name, field.got, field.want)
|
||||
}
|
||||
}
|
||||
if document.OutputFiles == nil {
|
||||
return Index{}, fmt.Errorf("notarius index is missing required output_files")
|
||||
}
|
||||
|
||||
index := Index{
|
||||
Path: indexPath,
|
||||
ManifestFile: document.ManifestFile,
|
||||
RejectedFile: document.RejectedFile,
|
||||
WarningsFile: document.WarningsFile,
|
||||
}
|
||||
var err error
|
||||
if index.ManifestPath, err = resolveRegularFile(bundleRoot, index.ManifestFile); err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius manifest file: %w", err)
|
||||
}
|
||||
if index.RejectedPath, err = resolveRegularFile(bundleRoot, index.RejectedFile); err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius rejection file: %w", err)
|
||||
}
|
||||
if index.WarningsPath, err = resolveRegularFile(bundleRoot, index.WarningsFile); err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius warning file: %w", err)
|
||||
}
|
||||
|
||||
seenLanes := make(map[string]struct{}, len(*document.OutputFiles))
|
||||
for _, lane := range *document.OutputFiles {
|
||||
if strings.TrimSpace(lane.LaneID) == "" || strings.TrimSpace(lane.File) == "" {
|
||||
return Index{}, fmt.Errorf("notarius lane descriptors require lane_id and file")
|
||||
}
|
||||
if _, exists := seenLanes[lane.LaneID]; exists {
|
||||
return Index{}, fmt.Errorf("notarius index contains duplicate lane id %q", lane.LaneID)
|
||||
}
|
||||
seenLanes[lane.LaneID] = struct{}{}
|
||||
path, err := resolveRegularFile(bundleRoot, lane.File)
|
||||
if err != nil {
|
||||
return Index{}, fmt.Errorf("resolve notarius lane %q file: %w", lane.LaneID, err)
|
||||
}
|
||||
index.Lanes = append(index.Lanes, LaneDescriptor{
|
||||
LaneID: lane.LaneID, File: lane.File, Path: path, MediaType: lane.MediaType,
|
||||
ModuleKey: lane.ModuleKey, SchemaID: lane.SchemaID, SchemaName: lane.SchemaName,
|
||||
SchemaVersion: lane.SchemaVersion,
|
||||
})
|
||||
}
|
||||
if document.ChunkMap != nil {
|
||||
index.ChunkMap, err = resolvePipelineDescriptor(bundleRoot, "chunk_map", *document.ChunkMap)
|
||||
if err != nil {
|
||||
return Index{}, err
|
||||
}
|
||||
}
|
||||
if document.EvidenceContext != nil {
|
||||
index.EvidenceContext, err = resolvePipelineDescriptor(bundleRoot, "evidence_context", *document.EvidenceContext)
|
||||
if err != nil {
|
||||
return Index{}, err
|
||||
}
|
||||
}
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func resolvePipelineDescriptor(bundleRoot, label string, document pipelineDocument) (*PipelineDescriptor, error) {
|
||||
if strings.TrimSpace(document.ArtifactKind) == "" || strings.TrimSpace(document.File) == "" ||
|
||||
strings.TrimSpace(document.MediaType) == "" || strings.TrimSpace(document.SchemaID) == "" ||
|
||||
strings.TrimSpace(document.SchemaName) == "" || strings.TrimSpace(document.SchemaVersion) == "" {
|
||||
return nil, fmt.Errorf("notarius %s descriptor is missing required fields", label)
|
||||
}
|
||||
path, err := resolveRegularFile(bundleRoot, document.File)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve notarius %s file: %w", label, err)
|
||||
}
|
||||
return &PipelineDescriptor{
|
||||
ArtifactKind: document.ArtifactKind, File: document.File, Path: path,
|
||||
MediaType: document.MediaType, SchemaID: document.SchemaID,
|
||||
SchemaName: document.SchemaName, SchemaVersion: document.SchemaVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type rejectionDocument struct {
|
||||
Rejected *[]struct {
|
||||
Stage string `json:"stage"`
|
||||
StepID string `json:"step_id"`
|
||||
LaneID string `json:"lane_id"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"rejected"`
|
||||
}
|
||||
|
||||
func loadRejections(path string) ([]RejectionSummary, error) {
|
||||
var document rejectionDocument
|
||||
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
||||
return nil, fmt.Errorf("decode notarius rejections: %w", err)
|
||||
}
|
||||
if document.Rejected == nil {
|
||||
return nil, fmt.Errorf("notarius rejection document is missing rejected array")
|
||||
}
|
||||
summaries := make([]RejectionSummary, 0, len(*document.Rejected))
|
||||
for _, item := range *document.Rejected {
|
||||
if strings.TrimSpace(item.Stage) == "" || strings.TrimSpace(item.Message) == "" {
|
||||
return nil, fmt.Errorf("notarius rejection entries require stage and message")
|
||||
}
|
||||
summaries = append(summaries, RejectionSummary{
|
||||
Stage: item.Stage, StepID: item.StepID, LaneID: item.LaneID,
|
||||
ModuleKey: item.ModuleKey, ChunkID: item.ChunkID,
|
||||
ValidatorName: item.ValidatorName, ReasonCode: item.ReasonCode,
|
||||
})
|
||||
}
|
||||
return summaries, nil
|
||||
}
|
||||
|
||||
type warningDocument struct {
|
||||
Warnings *[]struct {
|
||||
Scope string `json:"scope"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"warnings"`
|
||||
}
|
||||
|
||||
func loadWarnings(path string) ([]WarningSummary, error) {
|
||||
var document warningDocument
|
||||
if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil {
|
||||
return nil, fmt.Errorf("decode notarius warnings: %w", err)
|
||||
}
|
||||
if document.Warnings == nil {
|
||||
return nil, fmt.Errorf("notarius warning document is missing warnings array")
|
||||
}
|
||||
summaries := make([]WarningSummary, 0, len(*document.Warnings))
|
||||
for _, item := range *document.Warnings {
|
||||
if strings.TrimSpace(item.ReasonCode) == "" || strings.TrimSpace(item.Message) == "" {
|
||||
return nil, fmt.Errorf("notarius warning entries require reason_code and message")
|
||||
}
|
||||
summaries = append(summaries, WarningSummary{Scope: item.Scope, ReasonCode: item.ReasonCode})
|
||||
}
|
||||
return summaries, nil
|
||||
}
|
||||
|
||||
func decodeBoundedJSON(path string, limit int64, destination any) error {
|
||||
inspected, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if inspected.Mode()&os.ModeSymlink != 0 || !inspected.Mode().IsRegular() {
|
||||
return fmt.Errorf("path %q must be a regular file without symlinks", path)
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
opened, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !opened.Mode().IsRegular() || !os.SameFile(inspected, opened) {
|
||||
return fmt.Errorf("file %q changed before it could be read", path)
|
||||
}
|
||||
reader := io.LimitReader(file, limit+1)
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if int64(len(data)) > limit {
|
||||
return fmt.Errorf("file %q exceeds %d-byte limit", path, limit)
|
||||
}
|
||||
if err := json.Unmarshal(data, destination); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBundleRoot(outputRoot, bundleRoot string) (string, error) {
|
||||
root := filepath.Clean(outputRoot)
|
||||
bundle := filepath.Clean(bundleRoot)
|
||||
relative, err := filepath.Rel(root, bundle)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("compare notarius output paths: %w", err)
|
||||
}
|
||||
if relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("notarius output directory %q is not beneath output root %q", bundleRoot, outputRoot)
|
||||
}
|
||||
if err := requireDirectoryTree(root, relative); err != nil {
|
||||
return "", fmt.Errorf("validate notarius output directory: %w", err)
|
||||
}
|
||||
return bundle, nil
|
||||
}
|
||||
|
||||
func resolveRegularFile(root, logicalPath string) (string, error) {
|
||||
resolved, err := pathsafe.JoinSlashRelativeUnderRoot(root, logicalPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
relative, err := filepath.Rel(root, resolved)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := requireRegularFileTree(root, relative); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func requireDirectoryTree(root, relative string) error {
|
||||
if err := requireDirectory(root); err != nil {
|
||||
return err
|
||||
}
|
||||
current := root
|
||||
for _, component := range strings.Split(relative, string(filepath.Separator)) {
|
||||
current = filepath.Join(current, component)
|
||||
if err := requireDirectory(current); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireRegularFileTree(root, relative string) error {
|
||||
components := strings.Split(relative, string(filepath.Separator))
|
||||
if len(components) == 0 {
|
||||
return fmt.Errorf("regular file path is required")
|
||||
}
|
||||
if err := requireDirectory(root); err != nil {
|
||||
return err
|
||||
}
|
||||
current := root
|
||||
for _, component := range components[:len(components)-1] {
|
||||
current = filepath.Join(current, component)
|
||||
if err := requireDirectory(current); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return requireRegularFile(filepath.Join(current, components[len(components)-1]))
|
||||
}
|
||||
|
||||
func requireDirectory(path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return fmt.Errorf("path %q must be a directory without symlinks", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireRegularFile(path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("path %q must be a regular file without symlinks", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateLogDestination(path string) error {
|
||||
if err := requireDirectory(filepath.Dir(path)); err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("path %q must be absent or a regular file without symlinks", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
572
internal/adapters/notarius/subprocess_test.go
Normal file
572
internal/adapters/notarius/subprocess_test.go
Normal file
@@ -0,0 +1,572 @@
|
||||
package notarius
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sharedsubprocess "gitea.maximumdirect.net/eric/narratio/internal/adapters/subprocess"
|
||||
)
|
||||
|
||||
func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
var captured sharedsubprocess.RunRequest
|
||||
runner := &SubprocessRunner{run: func(_ context.Context, processReq sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||
captured = processReq
|
||||
writeValidBundleAndReceipt(t, req, true)
|
||||
return sharedsubprocess.RunResult{ExitCode: 0, Duration: 2 * time.Second}, nil
|
||||
}}
|
||||
|
||||
result, err := runner.Run(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
wantArgs := []string{
|
||||
"run", "dnd-session", "--config", req.ConfigPath, "--input", req.InputPath,
|
||||
"--output-dir", req.OutputRoot, "--json",
|
||||
}
|
||||
if !reflect.DeepEqual(captured.Args, wantArgs) {
|
||||
t.Fatalf("subprocess args = %#v, want %#v", captured.Args, wantArgs)
|
||||
}
|
||||
if captured.Executable != req.Binary || captured.WorkingDir != req.WorkingDirectory || captured.Timeout != req.Timeout {
|
||||
t.Fatalf("subprocess request = %#v", captured)
|
||||
}
|
||||
if captured.StdoutLogPath != req.ReceiptPath || captured.StderrLogPath != req.LogPath {
|
||||
t.Fatalf("stream paths = stdout %q stderr %q", captured.StdoutLogPath, captured.StderrLogPath)
|
||||
}
|
||||
if captured.EnvOverrides != nil {
|
||||
t.Fatalf("environment overrides = %#v, want inherited environment only", captured.EnvOverrides)
|
||||
}
|
||||
for _, arg := range captured.Args {
|
||||
if arg == "--session-id" {
|
||||
t.Fatal("subprocess args unexpectedly contain --session-id")
|
||||
}
|
||||
}
|
||||
|
||||
if result.Receipt.SchemaVersion != ReceiptSchemaVersion || result.Receipt.RunID != "notarius-run-1" {
|
||||
t.Fatalf("receipt = %#v", result.Receipt)
|
||||
}
|
||||
if len(result.Index.Lanes) != 1 || result.Index.Lanes[0].LaneID != "npc-registry" {
|
||||
t.Fatalf("lanes = %#v", result.Index.Lanes)
|
||||
}
|
||||
if result.Index.ChunkMap == nil || result.Index.ChunkMap.ArtifactKind != "chunk_map" {
|
||||
t.Fatalf("chunk map = %#v", result.Index.ChunkMap)
|
||||
}
|
||||
if result.Index.EvidenceContext == nil || result.Index.EvidenceContext.ArtifactKind != "evidence_context" {
|
||||
t.Fatalf("evidence context = %#v", result.Index.EvidenceContext)
|
||||
}
|
||||
if len(result.Rejections) != 1 || result.Rejections[0].LaneID != "spells" || result.Rejections[0].ReasonCode != "invalid_spell" {
|
||||
t.Fatalf("rejections = %#v", result.Rejections)
|
||||
}
|
||||
if len(result.Warnings) != 1 || result.Warnings[0].Scope != "lane:npc-registry" || result.Warnings[0].ReasonCode != "normalized_name" {
|
||||
t.Fatalf("warnings = %#v", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerInheritsEnvironmentAndSeparatesStreams(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
writeValidBundleAndReceipt(t, req, false)
|
||||
receiptFixture := req.ReceiptPath + ".fixture"
|
||||
data, err := os.ReadFile(req.ReceiptPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(receiptFixture, data, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
|
||||
}
|
||||
if err := os.Remove(req.ReceiptPath); err != nil {
|
||||
t.Fatalf("Remove(receipt) error = %v", err)
|
||||
}
|
||||
|
||||
captureDir := filepath.Join(filepath.Dir(req.ReceiptPath), "capture")
|
||||
if err := os.Mkdir(captureDir, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(capture) error = %v", err)
|
||||
}
|
||||
script := writeShellScript(t, `#!/bin/sh
|
||||
pwd > "$NOTARIUS_CAPTURE_DIR/working-directory"
|
||||
printf '%s' "$NOTARIUS_INHERITED_VALUE" > "$NOTARIUS_CAPTURE_DIR/environment"
|
||||
printf 'diagnostic stream\n' >&2
|
||||
cat "$NOTARIUS_RECEIPT_FIXTURE"
|
||||
`)
|
||||
req.Binary = script
|
||||
t.Setenv("NOTARIUS_CAPTURE_DIR", captureDir)
|
||||
t.Setenv("NOTARIUS_INHERITED_VALUE", "inherited-value")
|
||||
t.Setenv("NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
||||
|
||||
if _, err := NewSubprocessRunner().Run(context.Background(), req); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
assertTextFile(t, filepath.Join(captureDir, "working-directory"), req.WorkingDirectory+"\n")
|
||||
assertTextFile(t, filepath.Join(captureDir, "environment"), "inherited-value")
|
||||
assertTextFile(t, req.LogPath, "diagnostic stream\n")
|
||||
receiptBytes, err := os.ReadFile(req.ReceiptPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(receipt) error = %v", err)
|
||||
}
|
||||
if strings.Contains(string(receiptBytes), "diagnostic stream") {
|
||||
t.Fatal("receipt contains stderr output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerReturnsProcessFailuresWithoutParsingStdout(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
scriptBody string
|
||||
timeout time.Duration
|
||||
cancel bool
|
||||
want string
|
||||
}{
|
||||
{name: "nonzero", scriptBody: "printf '{malformed receipt'; printf 'failed\\n' >&2; exit 7\n", timeout: time.Second, want: "exit code 7"},
|
||||
{name: "timeout", scriptBody: "sleep 5\n", timeout: 20 * time.Millisecond, want: "timed out"},
|
||||
{name: "cancellation", scriptBody: "sleep 5\n", timeout: time.Second, cancel: true, want: "canceled"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
req.Binary = writeShellScript(t, "#!/bin/sh\n"+test.scriptBody)
|
||||
req.Timeout = test.timeout
|
||||
ctx := context.Background()
|
||||
if test.cancel {
|
||||
cancelCtx, cancel := context.WithCancel(ctx)
|
||||
ctx = cancelCtx
|
||||
time.AfterFunc(20*time.Millisecond, cancel)
|
||||
}
|
||||
_, err := NewSubprocessRunner().Run(ctx, req)
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Run() error = %v, want fragment %q", err, test.want)
|
||||
}
|
||||
if strings.Contains(err.Error(), "decode notarius receipt") {
|
||||
t.Fatalf("Run() parsed stdout after process failure: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRunnerReturnsSharedSubprocessErrorWithoutReadingReceipt(t *testing.T) {
|
||||
req := validRunRequest(t)
|
||||
if err := os.WriteFile(req.ReceiptPath, []byte("not json"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(receipt) error = %v", err)
|
||||
}
|
||||
wantErr := errors.New("process failed")
|
||||
runner := &SubprocessRunner{run: func(context.Context, sharedsubprocess.RunRequest) (sharedsubprocess.RunResult, error) {
|
||||
return sharedsubprocess.RunResult{ExitCode: 9}, wantErr
|
||||
}}
|
||||
_, err := runner.Run(context.Background(), req)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Run() error = %v, want wrapped process error", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "decode") {
|
||||
t.Fatalf("Run() parsed receipt after failure: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadReceiptValidation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
valid := map[string]any{
|
||||
"schema_version": ReceiptSchemaVersion, "run_id": "run-1", "pipeline_id": "pipeline-1",
|
||||
"output_directory": filepath.Join(root, "outputs", "run-1"), "index_file": "index.json",
|
||||
"normalized_output_count": 1, "rejected_output_count": 0, "warning_count": 0,
|
||||
"validation_status": "approved", "future_field": true,
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(map[string]any)
|
||||
raw []byte
|
||||
wantOK bool
|
||||
wantError string
|
||||
}{
|
||||
{name: "unknown fields tolerated", wantOK: true},
|
||||
{name: "malformed", raw: []byte("{")},
|
||||
{name: "unsupported version", mutate: func(v map[string]any) { v["schema_version"] = "notarius.run-result.v2" }},
|
||||
{name: "missing field", mutate: func(v map[string]any) { delete(v, "run_id") }},
|
||||
{name: "pipeline mismatch", mutate: func(v map[string]any) { v["pipeline_id"] = "other" }},
|
||||
{name: "relative output", mutate: func(v map[string]any) { v["output_directory"] = "run-1" }},
|
||||
{name: "negative count", mutate: func(v map[string]any) { v["warning_count"] = -1 }},
|
||||
{
|
||||
name: "nested index", mutate: func(v map[string]any) { v["index_file"] = "nested/index.json" },
|
||||
wantError: `index_file "nested/index.json"`,
|
||||
},
|
||||
{
|
||||
name: "cleanable index", mutate: func(v map[string]any) { v["index_file"] = "./index.json" },
|
||||
wantError: `index_file "./index.json"`,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
path := filepath.Join(root, strings.ReplaceAll(test.name, " ", "-")+".json")
|
||||
values := cloneMap(valid)
|
||||
if test.mutate != nil {
|
||||
test.mutate(values)
|
||||
}
|
||||
if test.raw != nil {
|
||||
if err := os.WriteFile(path, test.raw, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
} else {
|
||||
writeJSONFile(t, path, values)
|
||||
}
|
||||
_, err := loadReceipt(path, "pipeline-1")
|
||||
if test.wantOK && err != nil {
|
||||
t.Fatalf("loadReceipt() error = %v", err)
|
||||
}
|
||||
if !test.wantOK && err == nil {
|
||||
t.Fatal("loadReceipt() error = nil, want validation failure")
|
||||
}
|
||||
if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
|
||||
t.Fatalf("loadReceipt() error = %v, want fragment %q", err, test.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
oversized := filepath.Join(root, "oversized.json")
|
||||
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxReceiptBytes+1)), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(oversized) error = %v", err)
|
||||
}
|
||||
if _, err := loadReceipt(oversized, "pipeline-1"); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadReceipt(oversized) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateBundleRootRejectsEscapesAndSymlinks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outputRoot := filepath.Join(root, "output")
|
||||
if err := os.Mkdir(outputRoot, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(output root) error = %v", err)
|
||||
}
|
||||
validBundle := filepath.Join(outputRoot, "run-1")
|
||||
if err := os.Mkdir(validBundle, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(bundle) error = %v", err)
|
||||
}
|
||||
if _, err := validateBundleRoot(outputRoot, validBundle); err != nil {
|
||||
t.Fatalf("validateBundleRoot(valid) error = %v", err)
|
||||
}
|
||||
|
||||
outside := filepath.Join(root, "output-other")
|
||||
if err := os.Mkdir(outside, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(outside) error = %v", err)
|
||||
}
|
||||
for name, candidate := range map[string]string{"equal root": outputRoot, "escape": root, "prefix confusion": outside} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := validateBundleRoot(outputRoot, candidate); err == nil {
|
||||
t.Fatalf("validateBundleRoot(%q) error = nil", candidate)
|
||||
}
|
||||
})
|
||||
}
|
||||
symlink := filepath.Join(outputRoot, "linked")
|
||||
if err := os.Symlink(outside, symlink); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
if _, err := validateBundleRoot(outputRoot, symlink); err == nil {
|
||||
t.Fatal("validateBundleRoot(symlink) error = nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadIndexRejectsMalformedUnsafeAndUnsupportedDocuments(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
indexValue any
|
||||
prepare func(*testing.T, string)
|
||||
wantError string
|
||||
}{
|
||||
{name: "malformed", indexValue: json.RawMessage(`{"manifest_file":`)},
|
||||
{name: "unsupported output shape", indexValue: map[string]any{"manifest_file": "manifest.json", "output_files": map[string]any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
|
||||
{name: "missing management path", indexValue: map[string]any{"output_files": []any{}, "rejected_file": "rejected.json", "warnings_file": "warnings.json"}},
|
||||
{name: "renamed manifest", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["manifest_file"] = "metadata.json"
|
||||
return value
|
||||
}(), wantError: `manifest_file "metadata.json"`},
|
||||
{name: "cleanable manifest", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["manifest_file"] = "./manifest.json"
|
||||
return value
|
||||
}(), wantError: `manifest_file "./manifest.json"`},
|
||||
{name: "renamed rejections", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["rejected_file"] = "rejections.json"
|
||||
return value
|
||||
}(), wantError: `rejected_file "rejections.json"`},
|
||||
{name: "renamed warnings", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["warnings_file"] = "diagnostics/warnings.json"
|
||||
return value
|
||||
}(), wantError: `warnings_file "diagnostics/warnings.json"`},
|
||||
{name: "duplicate lane", indexValue: validIndexValue([]any{
|
||||
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
|
||||
map[string]any{"lane_id": "npc", "file": "lanes/npc.json"},
|
||||
})},
|
||||
{name: "absolute logical path", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "/tmp/npc.json"}})},
|
||||
{name: "lexical traversal", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../outside.json"}})},
|
||||
{name: "root prefix confusion", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "../bundle-other/npc.json"}})},
|
||||
{name: "file symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "lanes/npc.json"}}), prepare: func(t *testing.T, bundle string) {
|
||||
if err := os.Symlink(filepath.Join(bundle, "manifest.json"), filepath.Join(bundle, "lanes", "npc.json")); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "directory symlink", indexValue: validIndexValue([]any{map[string]any{"lane_id": "npc", "file": "linked/npc.json"}}), prepare: func(t *testing.T, bundle string) {
|
||||
if err := os.Symlink(filepath.Join(bundle, "lanes"), filepath.Join(bundle, "linked")); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "missing management file", indexValue: validIndexValue([]any{}), prepare: func(t *testing.T, bundle string) {
|
||||
if err := os.Remove(filepath.Join(bundle, "manifest.json")); err != nil {
|
||||
t.Fatalf("Remove(manifest) error = %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "incomplete pipeline descriptor", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["chunk_map"] = map[string]any{"artifact_kind": "chunk_map", "file": "chunk-map.json"}
|
||||
return value
|
||||
}()},
|
||||
{name: "pipeline descriptor escape", indexValue: func() any {
|
||||
value := validIndexValue([]any{})
|
||||
value["evidence_context"] = map[string]any{
|
||||
"artifact_kind": "evidence_context", "file": "../evidence.json", "media_type": "application/json",
|
||||
"schema_id": "evidence", "schema_name": "Evidence", "schema_version": "v1",
|
||||
}
|
||||
return value
|
||||
}()},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
bundle := createBundleSkeleton(t)
|
||||
indexPath := filepath.Join(bundle, "index.json")
|
||||
if raw, ok := test.indexValue.(json.RawMessage); ok {
|
||||
if err := os.WriteFile(indexPath, raw, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(index) error = %v", err)
|
||||
}
|
||||
} else {
|
||||
writeJSONFile(t, indexPath, test.indexValue)
|
||||
}
|
||||
if test.prepare != nil {
|
||||
test.prepare(t, bundle)
|
||||
}
|
||||
if _, err := loadIndex(bundle, indexPath); err == nil {
|
||||
t.Fatal("loadIndex() error = nil, want failure")
|
||||
} else if test.wantError != "" && !strings.Contains(err.Error(), test.wantError) {
|
||||
t.Fatalf("loadIndex() error = %v, want fragment %q", err, test.wantError)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
bundle := createBundleSkeleton(t)
|
||||
oversizedIndex := filepath.Join(bundle, "index.json")
|
||||
if err := os.WriteFile(oversizedIndex, []byte(strings.Repeat("x", maxIndexBytes+1)), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(oversized index) error = %v", err)
|
||||
}
|
||||
if _, err := loadIndex(bundle, oversizedIndex); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadIndex(oversized) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
rejectedPath := filepath.Join(root, "rejected.json")
|
||||
warningsPath := filepath.Join(root, "warnings.json")
|
||||
writeJSONFile(t, rejectedPath, map[string]any{"rejected": []any{map[string]any{
|
||||
"stage": "validate", "lane_id": "spells", "reason_code": "invalid", "message": "do not retain this", "future": true,
|
||||
}}, "future": true})
|
||||
writeJSONFile(t, warningsPath, map[string]any{"warnings": []any{map[string]any{
|
||||
"scope": "lane:spells", "reason_code": "bounded", "message": "do not retain this", "future": true,
|
||||
}}, "future": true})
|
||||
rejections, err := loadRejections(rejectedPath)
|
||||
if err != nil || len(rejections) != 1 || rejections[0].ReasonCode != "invalid" {
|
||||
t.Fatalf("loadRejections() = %#v, %v", rejections, err)
|
||||
}
|
||||
warnings, err := loadWarnings(warningsPath)
|
||||
if err != nil || len(warnings) != 1 || warnings[0].Scope != "lane:spells" {
|
||||
t.Fatalf("loadWarnings() = %#v, %v", warnings, err)
|
||||
}
|
||||
|
||||
for name, path := range map[string]string{"rejections": rejectedPath, "warnings": warningsPath} {
|
||||
t.Run("malformed "+name, func(t *testing.T) {
|
||||
if err := os.WriteFile(path, []byte("{"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
var err error
|
||||
if name == "rejections" {
|
||||
_, err = loadRejections(path)
|
||||
} else {
|
||||
_, err = loadWarnings(path)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("summary decoder error = nil")
|
||||
}
|
||||
})
|
||||
}
|
||||
oversized := filepath.Join(root, "oversized.json")
|
||||
if err := os.WriteFile(oversized, []byte(strings.Repeat("x", maxSummaryBytes+1)), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(oversized) error = %v", err)
|
||||
}
|
||||
if _, err := loadWarnings(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadWarnings(oversized) error = %v", err)
|
||||
}
|
||||
if _, err := loadRejections(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("loadRejections(oversized) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) {
|
||||
req := RunRequest{PipelineID: "pipeline"}
|
||||
want := RunResult{BundleRoot: "/bundle"}
|
||||
fake := &FakeRunner{Result: want}
|
||||
got, err := fake.Run(context.Background(), req)
|
||||
if err != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(fake.Requests, []RunRequest{req}) {
|
||||
t.Fatalf("Run() = %#v, %v; requests = %#v", got, err, fake.Requests)
|
||||
}
|
||||
|
||||
wantErr := errors.New("configured failure")
|
||||
fake.Err = wantErr
|
||||
if _, err := fake.Run(context.Background(), req); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Run(configured error) = %v", err)
|
||||
}
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
before := len(fake.Requests)
|
||||
if _, err := fake.Run(canceled, req); !errors.Is(err, context.Canceled) || len(fake.Requests) != before {
|
||||
t.Fatalf("Run(canceled) error = %v; requests = %d", err, len(fake.Requests))
|
||||
}
|
||||
}
|
||||
|
||||
func validRunRequest(t *testing.T) RunRequest {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
configPath := filepath.Join(root, "notarius.yml")
|
||||
inputPath := filepath.Join(root, "input.json")
|
||||
outputRoot := filepath.Join(root, "outputs")
|
||||
workingDirectory := filepath.Join(root, "work")
|
||||
diagnostics := filepath.Join(root, "diagnostics")
|
||||
for _, directory := range []string{outputRoot, workingDirectory, diagnostics} {
|
||||
if err := os.Mkdir(directory, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(%q) error = %v", directory, err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(config) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(inputPath, []byte("{}\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(input) error = %v", err)
|
||||
}
|
||||
return RunRequest{
|
||||
Binary: "notarius", ConfigPath: configPath, PipelineID: "dnd-session", InputPath: inputPath,
|
||||
OutputRoot: outputRoot, WorkingDirectory: workingDirectory,
|
||||
ReceiptPath: filepath.Join(diagnostics, "receipt.json"), LogPath: filepath.Join(diagnostics, "stderr.log"),
|
||||
Timeout: time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown bool) {
|
||||
t.Helper()
|
||||
bundle := filepath.Join(req.OutputRoot, "notarius-run-1")
|
||||
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(bundle) error = %v", err)
|
||||
}
|
||||
for path, data := range map[string]string{
|
||||
"manifest.json": `{}`,
|
||||
"lanes/npc.json": `{}`,
|
||||
"chunk-map.json": `{}`,
|
||||
"evidence-context.json": `{}`,
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(path)), []byte(data), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||
}
|
||||
}
|
||||
rejection := map[string]any{"stage": "validate", "lane_id": "spells", "reason_code": "invalid_spell", "message": strings.Repeat("external detail", 20)}
|
||||
warning := map[string]any{"scope": "lane:npc-registry", "reason_code": "normalized_name", "message": strings.Repeat("external warning", 20)}
|
||||
if includeUnknown {
|
||||
rejection["future"] = true
|
||||
warning["future"] = true
|
||||
}
|
||||
writeJSONFile(t, filepath.Join(bundle, "rejected.json"), map[string]any{"rejected": []any{rejection}, "future": true})
|
||||
writeJSONFile(t, filepath.Join(bundle, "warnings.json"), map[string]any{"warnings": []any{warning}, "future": true})
|
||||
index := validIndexValue([]any{map[string]any{
|
||||
"lane_id": "npc-registry", "file": "lanes/npc.json", "media_type": "application/json",
|
||||
"module_key": "dnd/npc-registry", "schema_id": "notarius.dnd.npc_registry",
|
||||
"schema_name": "NPCRegistry", "schema_version": "v1", "future": true,
|
||||
}})
|
||||
index["chunk_map"] = map[string]any{
|
||||
"artifact_kind": "chunk_map", "file": "chunk-map.json", "media_type": "application/json",
|
||||
"schema_id": "notarius.chunk_map", "schema_name": "ChunkMap", "schema_version": "v1", "future": true,
|
||||
}
|
||||
index["evidence_context"] = map[string]any{
|
||||
"artifact_kind": "evidence_context", "file": "evidence-context.json", "media_type": "application/json",
|
||||
"schema_id": "notarius.evidence_context", "schema_name": "EvidenceContext", "schema_version": "v1", "future": true,
|
||||
}
|
||||
index["future"] = true
|
||||
writeJSONFile(t, filepath.Join(bundle, "index.json"), index)
|
||||
receipt := map[string]any{
|
||||
"schema_version": ReceiptSchemaVersion, "run_id": "notarius-run-1", "pipeline_id": req.PipelineID,
|
||||
"output_directory": bundle, "index_file": "index.json", "normalized_output_count": 1,
|
||||
"rejected_output_count": 1, "warning_count": 1, "validation_status": "rejected",
|
||||
}
|
||||
if includeUnknown {
|
||||
receipt["future"] = true
|
||||
}
|
||||
writeJSONFile(t, req.ReceiptPath, receipt)
|
||||
}
|
||||
|
||||
func createBundleSkeleton(t *testing.T) string {
|
||||
t.Helper()
|
||||
bundle := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(bundle) error = %v", err)
|
||||
}
|
||||
for _, name := range []string{"manifest.json", "rejected.json", "warnings.json", "lanes/npc.json", "chunk-map.json"} {
|
||||
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(name)), []byte("{}"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", name, err)
|
||||
}
|
||||
}
|
||||
return bundle
|
||||
}
|
||||
|
||||
func validIndexValue(lanes []any) map[string]any {
|
||||
return map[string]any{
|
||||
"manifest_file": "manifest.json", "output_files": lanes,
|
||||
"rejected_file": "rejected.json", "warnings_file": "warnings.json",
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSONFile(t *testing.T, path string, value any) {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeShellScript(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "notarius-helper")
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(script) error = %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func assertTextFile(t *testing.T, path, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q) error = %v", path, err)
|
||||
}
|
||||
if string(data) != want {
|
||||
t.Fatalf("ReadFile(%q) = %q, want %q", path, string(data), want)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneMap(source map[string]any) map[string]any {
|
||||
result := make(map[string]any, len(source))
|
||||
for key, value := range source {
|
||||
result[key] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -21,7 +21,7 @@ func TestExecuteRunStageArtifactsUnsupportedStageFails(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute(
|
||||
[]string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
[]string{"run-stage", "extract", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&stdout,
|
||||
&stderr,
|
||||
)
|
||||
@@ -130,6 +130,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
seed.MarkStageSkipped("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), "notarius_disabled")
|
||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||
t.Fatalf("save manifest: %v", err)
|
||||
}
|
||||
@@ -143,7 +144,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "executed=0 skipped=10") {
|
||||
if !strings.Contains(out.String(), "executed=1 skipped=11") {
|
||||
t.Fatalf("output = %q, want all stages skipped", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,8 +31,8 @@ func TestExecuteValidCommands(t *testing.T) {
|
||||
args []string
|
||||
wantOut string
|
||||
}{
|
||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=10 skipped=0; manifest="},
|
||||
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nrender: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
||||
{name: "run", args: []string{"run", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run: session 2026-05-03; executed=11 skipped=1; manifest="},
|
||||
{name: "session plan", args: []string{"session", "plan", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "prepare: skip\ntranscribe: skip\nmerge: skip\npolish: skip\nnormalize: skip\ntrim: skip\nextract: run\nrender: skip\nanalyze: skip\npublish: skip\nnotify: skip"},
|
||||
{name: "session status", args: []string{"session", "status", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "Session: 2026-05-03"},
|
||||
{name: "run-stage", args: []string{"run-stage", "polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, wantOut: "narratio run-stage: stage=polish executed=0 skipped=1 force=false; manifest="},
|
||||
}
|
||||
@@ -335,7 +335,7 @@ func TestExecuteUsesDefaultPipelineConfigPathWhenConfigFlagOmitted(t *testing.T)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=10 skipped=0; manifest=") {
|
||||
if !strings.Contains(stdout.String(), "narratio run: session 2026-05-03; executed=11 skipped=1; manifest=") {
|
||||
t.Fatalf("stdout = %q, want successful run output", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
416
internal/app/extract_lifecycle_test.go
Normal file
416
internal/app/extract_lifecycle_test.go
Normal file
@@ -0,0 +1,416 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type materializingNotariusRunner struct {
|
||||
cfg *config.NotariusConfig
|
||||
requests []notarius.RunRequest
|
||||
failuresRemaining int
|
||||
}
|
||||
|
||||
func (r *materializingNotariusRunner) Run(_ context.Context, req notarius.RunRequest) (notarius.RunResult, error) {
|
||||
r.requests = append(r.requests, req)
|
||||
if r.failuresRemaining > 0 {
|
||||
r.failuresRemaining--
|
||||
return notarius.RunResult{}, errors.New("notarius execution failed")
|
||||
}
|
||||
externalRunID := fmt.Sprintf("notarius-run-%d", len(r.requests))
|
||||
bundle := filepath.Join(req.OutputRoot, externalRunID)
|
||||
lanesDir := filepath.Join(bundle, "lanes")
|
||||
if err := os.MkdirAll(lanesDir, 0o755); err != nil {
|
||||
return notarius.RunResult{}, err
|
||||
}
|
||||
for path, content := range map[string]string{
|
||||
filepath.Join(bundle, "index.json"): `{"manifest_file":"manifest.json"}`,
|
||||
filepath.Join(bundle, "manifest.json"): `{}`,
|
||||
filepath.Join(bundle, "rejected.json"): `{"rejected":[]}`,
|
||||
filepath.Join(bundle, "warnings.json"): `{"warnings":[]}`,
|
||||
filepath.Join(lanesDir, "npcs.json"): `{"npcs":[]}`,
|
||||
} {
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
return notarius.RunResult{}, err
|
||||
}
|
||||
}
|
||||
output := r.cfg.Outputs["npc_registry"]
|
||||
return notarius.RunResult{
|
||||
Receipt: notarius.Receipt{
|
||||
SchemaVersion: notarius.ReceiptSchemaVersion, RunID: externalRunID,
|
||||
PipelineID: req.PipelineID, OutputDirectory: bundle, IndexFile: "index.json",
|
||||
NormalizedOutputCount: 1, ValidationStatus: "valid",
|
||||
},
|
||||
BundleRoot: bundle,
|
||||
Index: notarius.Index{
|
||||
Path: filepath.Join(bundle, "index.json"), RejectedPath: filepath.Join(bundle, "rejected.json"),
|
||||
WarningsPath: filepath.Join(bundle, "warnings.json"),
|
||||
Lanes: []notarius.LaneDescriptor{{
|
||||
LaneID: output.LaneID, File: "lanes/npcs.json", Path: filepath.Join(lanesDir, "npcs.json"),
|
||||
MediaType: output.MediaType, SchemaID: output.SchemaID,
|
||||
SchemaVersion: output.SchemaVersion, ModuleKey: output.ModuleKey,
|
||||
}},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestExtractLifecycleDisabledThenEnabled(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||
plan, err := BuildSingleStagePlan("extract")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
|
||||
}
|
||||
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("disabled executeStages() error = %v", err)
|
||||
}
|
||||
if len(first.Executed) != 1 || len(first.Skipped) != 1 || first.Skipped[0] != "extract" || len(runner.requests) != 0 {
|
||||
t.Fatalf("disabled summary = %#v requests=%d", first, len(runner.requests))
|
||||
}
|
||||
|
||||
cfg.Pipeline.Notarius.Enabled = true
|
||||
second, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("enabled executeStages() error = %v", err)
|
||||
}
|
||||
if len(second.Executed) != 1 || len(second.Skipped) != 0 || len(runner.requests) != 1 {
|
||||
t.Fatalf("enabled summary = %#v requests=%d", second, len(runner.requests))
|
||||
}
|
||||
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), second.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if loaded.Stages["extract"] == nil || loaded.Stages["extract"].Status != manifest.StatusSucceeded || len(loaded.Stages["extract"].Outputs) != 2 {
|
||||
t.Fatalf("extract record = %#v, want succeeded manifest-ready outputs", loaded.Stages["extract"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleChangedOutcomeRerunsSucceededDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("disabled executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 1 || len(runner.requests) != 0 {
|
||||
t.Fatalf("disabled run analyze=%d Notarius=%d, want 1 and 0", analyzeRuns, len(runner.requests))
|
||||
}
|
||||
|
||||
cfg.Pipeline.Notarius.Enabled = true
|
||||
summary, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("enabled executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 2 || len(runner.requests) != 1 {
|
||||
t.Fatalf("enabled run analyze=%d Notarius=%d, want 2 and 1", analyzeRuns, len(runner.requests))
|
||||
}
|
||||
if len(summary.Executed) != 2 || len(summary.Skipped) != 0 {
|
||||
t.Fatalf("enabled summary = %#v, want extract and analyze executed", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleFailureInvalidatesAndOrdinaryRetryRerunsDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
runner.failuresRemaining = 1
|
||||
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil || !strings.Contains(err.Error(), "notarius execution failed") {
|
||||
t.Fatalf("failed executeStages() error = %v", err)
|
||||
}
|
||||
failed := loadLifecycleManifest(t, cfg)
|
||||
if failed.Stages["extract"].Status != manifest.StatusFailed || failed.Stages["analyze"].Status != manifest.StatusStale {
|
||||
t.Fatalf("failed lifecycle extract=%#v analyze=%#v", failed.Stages["extract"], failed.Stages["analyze"])
|
||||
}
|
||||
if failed.Stages["analyze"].Error == nil || failed.Stages["analyze"].Error.Message != staleReasonFailure {
|
||||
t.Fatalf("analyze stale reason = %#v, want %q", failed.Stages["analyze"].Error, staleReasonFailure)
|
||||
}
|
||||
|
||||
summary, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("retry executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 1 || len(runner.requests) != 2 || len(summary.Executed) != 2 {
|
||||
t.Fatalf("retry analyze=%d Notarius=%d summary=%#v", analyzeRuns, len(runner.requests), summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleForcedSelfSkipInvalidatesDownstream(t *testing.T) {
|
||||
cfg, env, _ := extractionLifecycleFixture(t, false)
|
||||
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||
plan, _ := BuildSingleStagePlan("extract")
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
if loaded.Stages["extract"].Status != manifest.StatusSkipped || loaded.Stages["analyze"].Status != manifest.StatusStale {
|
||||
t.Fatalf("forced self-skip extract=%#v analyze=%#v", loaded.Stages["extract"], loaded.Stages["analyze"])
|
||||
}
|
||||
if loaded.Stages["analyze"].Error == nil || loaded.Stages["analyze"].Error.Message != staleReasonForcedReplacement {
|
||||
t.Fatalf("analyze stale reason = %#v, want %q", loaded.Stages["analyze"].Error, staleReasonForcedReplacement)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleForcedFailureInvalidatesDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
plan, _ := BuildSingleStagePlan("extract")
|
||||
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("initial executeStages() error = %v", err)
|
||||
}
|
||||
succeeded := loadLifecycleManifest(t, cfg).Stages["extract"]
|
||||
if succeeded == nil || succeeded.Status != manifest.StatusSucceeded || len(succeeded.Outputs) == 0 || len(succeeded.Logs) == 0 || len(succeeded.Metadata) == 0 {
|
||||
t.Fatalf("initial extraction result = %#v, want succeeded result details", succeeded)
|
||||
}
|
||||
|
||||
markLifecycleStageSucceeded(t, cfg, "analyze")
|
||||
runner.failuresRemaining = 1
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env, Force: true}); err == nil {
|
||||
t.Fatal("executeStages() error = nil, want forced extraction failure")
|
||||
}
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
if loaded.Stages["extract"].Status != manifest.StatusFailed || loaded.Stages["analyze"].Status != manifest.StatusStale {
|
||||
t.Fatalf("forced failure extract=%#v analyze=%#v", loaded.Stages["extract"], loaded.Stages["analyze"])
|
||||
}
|
||||
if loaded.Stages["analyze"].Error == nil || loaded.Stages["analyze"].Error.Message != staleReasonForcedReplacement {
|
||||
t.Fatalf("analyze stale reason = %#v, want %q", loaded.Stages["analyze"].Error, staleReasonForcedReplacement)
|
||||
}
|
||||
failed := loaded.Stages["extract"]
|
||||
if len(failed.Outputs) != 0 || len(failed.Logs) != 0 || len(failed.GeneratedConfigs) != 0 || len(failed.Metadata) != 0 {
|
||||
t.Fatalf("failed replacement inherited extraction result details: %#v", failed)
|
||||
}
|
||||
historical, err := (&manifest.LocalStore{}).LoadRun(context.Background(), first.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadRun(initial) error = %v", err)
|
||||
}
|
||||
historicalExtract := historical.Stages["extract"]
|
||||
if historicalExtract == nil || historicalExtract.Status != manifest.StatusSucceeded || len(historicalExtract.Outputs) == 0 || len(historicalExtract.Logs) == 0 || len(historicalExtract.Metadata) == 0 {
|
||||
t.Fatalf("historical extraction result = %#v, want preserved succeeded details", historicalExtract)
|
||||
}
|
||||
if _, err := os.Stat(succeeded.Outputs[0].LocalPath); err != nil {
|
||||
t.Fatalf("durable extraction output was not preserved: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleRepeatedSelfSkipPreservesSucceededDownstream(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, false)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
second, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("second executeStages() error = %v", err)
|
||||
}
|
||||
if analyzeRuns != 1 || len(runner.requests) != 0 {
|
||||
t.Fatalf("repeated disabled run analyze=%d Notarius=%d, want 1 and 0", analyzeRuns, len(runner.requests))
|
||||
}
|
||||
if len(second.Executed) != 1 || len(second.Skipped) != 2 {
|
||||
t.Fatalf("second summary = %#v, want executed self-skip and skipped analyze", second)
|
||||
}
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
if loaded.Stages["analyze"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("analyze = %#v, want succeeded", loaded.Stages["analyze"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleSkipsCurrentResumableResult(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
resumed, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("resume executeStages() error = %v", err)
|
||||
}
|
||||
if len(resumed.Executed) != 0 || len(resumed.Skipped) != 2 || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||
t.Fatalf("resume summary = %#v requests=%d analyze=%d", resumed, len(runner.requests), analyzeRuns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleResumesAndRerunsObsoleteResults(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mutate func(*testing.T, *config.Config, *manifest.Manifest)
|
||||
}{
|
||||
{name: "configuration changed", mutate: func(_ *testing.T, cfg *config.Config, _ *manifest.Manifest) {
|
||||
output := cfg.Pipeline.Notarius.Outputs["npc_registry"]
|
||||
output.SchemaVersion = "v2"
|
||||
cfg.Pipeline.Notarius.Outputs["npc_registry"] = output
|
||||
}},
|
||||
{name: "payload missing", mutate: func(t *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
||||
t.Fatalf("Remove() error = %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "payload tampered", mutate: func(t *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||
if err := os.WriteFile(m.Stages["extract"].Outputs[0].LocalPath, []byte(`{"npcs":["tampered"]}`), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "record incompatible", mutate: func(_ *testing.T, _ *config.Config, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].Contract.SchemaID = "incompatible"
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
plan, _ := BuildSingleStagePlan("extract")
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
persisted, err := (&manifest.LocalStore{}).Load(context.Background(), first.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
test.mutate(t, cfg, persisted)
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), first.ManifestPath, persisted); err != nil {
|
||||
t.Fatalf("Save(mutated) error = %v", err)
|
||||
}
|
||||
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("rerun executeStages() error = %v", err)
|
||||
}
|
||||
if len(rerun.Executed) != 1 || len(rerun.Skipped) != 0 || len(runner.requests) != 2 {
|
||||
t.Fatalf("rerun summary = %#v requests=%d", rerun, len(runner.requests))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleUnsafeResumeErrorPreservesSuccess(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
first, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("first executeStages() error = %v", err)
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
persisted, err := store.Load(context.Background(), first.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
persisted.Stages["extract"].Outputs[0].LocalPath = filepath.Join(cfg.Pipeline.Workspace.Root, "outside.json")
|
||||
if err := store.Save(context.Background(), first.ManifestPath, persisted); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
before, _ := json.Marshal(map[string]*manifest.StageRecord{
|
||||
"extract": persisted.Stages["extract"],
|
||||
"analyze": persisted.Stages["analyze"],
|
||||
})
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil || !strings.Contains(err.Error(), "unsafe") {
|
||||
t.Fatalf("executeStages() error = %v, want unsafe resume failure", err)
|
||||
}
|
||||
afterManifest, err := store.Load(context.Background(), first.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load(after) error = %v", err)
|
||||
}
|
||||
after, _ := json.Marshal(map[string]*manifest.StageRecord{
|
||||
"extract": afterManifest.Stages["extract"],
|
||||
"analyze": afterManifest.Stages["analyze"],
|
||||
})
|
||||
if string(before) != string(after) || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||
t.Fatalf("successful records changed: before=%s after=%s requests=%d analyze=%d", before, after, len(runner.requests), analyzeRuns)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionLifecyclePlan(t *testing.T, analyzeRuns *int) []stage.Stage {
|
||||
t.Helper()
|
||||
plan, err := BuildSingleStagePlan("extract")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildSingleStagePlan(extract) error = %v", err)
|
||||
}
|
||||
return append(plan, countingStage{name: "analyze", runs: analyzeRuns})
|
||||
}
|
||||
|
||||
func loadLifecycleManifest(t *testing.T, cfg *config.Config) *manifest.Manifest {
|
||||
t.Helper()
|
||||
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
return loaded
|
||||
}
|
||||
|
||||
func markLifecycleStageSucceeded(t *testing.T, cfg *config.Config, name string) {
|
||||
t.Helper()
|
||||
loaded := loadLifecycleManifest(t, cfg)
|
||||
loaded.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), manifestPathFor(cfg), loaded); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionLifecycleFixture(t *testing.T, enabled bool) (*config.Config, *stage.Env, *materializingNotariusRunner) {
|
||||
t.Helper()
|
||||
cfg := testConfig(t)
|
||||
root := cfg.Pipeline.Workspace.Root
|
||||
binary := filepath.Join(root, "notarius")
|
||||
configPath := filepath.Join(root, "notarius.yml")
|
||||
workingDirectory := filepath.Join(root, "notarius-work")
|
||||
if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(binary) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(config) error = %v", err)
|
||||
}
|
||||
if err := os.Mkdir(workingDirectory, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(working directory) error = %v", err)
|
||||
}
|
||||
cfg.Pipeline.Notarius = &config.NotariusConfig{
|
||||
Enabled: enabled, Binary: binary, ConfigPath: configPath, PipelineID: "dnd-session",
|
||||
Timeout: "45m", WorkingDirectory: workingDirectory,
|
||||
Outputs: map[string]config.NotariusOutputConfig{
|
||||
"npc_registry": {
|
||||
LaneID: "npc-registry", MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry",
|
||||
SchemaVersion: "v1", ModuleKey: "dnd/npc-registry",
|
||||
},
|
||||
},
|
||||
}
|
||||
paths, err := artifacts.NewLocalStore(root).EnsureLayoutFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureLayoutFor() error = %v", err)
|
||||
}
|
||||
inputPath := filepath.Join(paths.ArtifactsDir, "trimmed.from-manifest.json")
|
||||
if err := os.WriteFile(inputPath, []byte(`{"segments":[]}`), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(input) error = %v", err)
|
||||
}
|
||||
m := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
m.Campaign = cfg.Session.Campaign
|
||||
m.MarkStageSucceeded("trim", time.Now().UTC(), []manifest.ArtifactRecord{{
|
||||
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
|
||||
LocalPath: inputPath,
|
||||
}})
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), paths.ManifestPath, m); err != nil {
|
||||
t.Fatalf("Save(seed) error = %v", err)
|
||||
}
|
||||
runner := &materializingNotariusRunner{cfg: cfg.Pipeline.Notarius}
|
||||
return cfg, &stage.Env{Notarius: runner}, runner
|
||||
}
|
||||
@@ -10,9 +10,10 @@ import (
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog, error) {
|
||||
func buildHelperArtifactCatalog(cfg *config.Config, m *manifest.Manifest) (*artifacts.ArtifactCatalog, error) {
|
||||
catalog := artifacts.NewArtifactCatalog()
|
||||
if err := catalog.RegisterBuiltIns(); err != nil {
|
||||
return nil, err
|
||||
@@ -26,6 +27,14 @@ func buildHelperArtifactCatalog(cfg *config.Config) (*artifacts.ArtifactCatalog,
|
||||
if err := catalog.RegisterConfiguredArtifacts(configured, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
extractionDefinitions := artifacts.ExtractionDefinitionsFromConfig(cfg.Pipeline.Notarius)
|
||||
if err := catalog.RegisterExtractionArtifacts(extractionDefinitions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
if cfg.Pipeline.Notarius != nil && cfg.Pipeline.Notarius.Enabled {
|
||||
catalog.HydrateExtractionArtifacts(paths, m, extractionDefinitions)
|
||||
}
|
||||
return catalog, nil
|
||||
}
|
||||
|
||||
@@ -40,6 +49,14 @@ func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.Art
|
||||
for _, entry := range catalog.ListConfigured() {
|
||||
writeArtifactLine(out, entry.SourceID, lockSet)
|
||||
}
|
||||
fmt.Fprintln(out, "Extraction:")
|
||||
for _, entry := range catalog.ListExtraction() {
|
||||
state := "unavailable"
|
||||
if entry.Available {
|
||||
state = "available"
|
||||
}
|
||||
writeExtractionArtifactLine(out, entry.SourceID, state, entry.Provenance, lockSet)
|
||||
}
|
||||
fmt.Fprintln(out, "Previous-session:")
|
||||
for _, req := range artifacts.CollectPreviousArtifactRequirements(configuredScriptoriumArtifacts(cfg)) {
|
||||
fmt.Fprintf(out, "- %s required=%t\n", artifactpolicy.PreviousSessionSourceID(req.Name), req.Required)
|
||||
@@ -50,6 +67,17 @@ func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.Art
|
||||
}
|
||||
}
|
||||
|
||||
func writeExtractionArtifactLine(out io.Writer, source, state, provenance string, lockSet map[string]config.PublishLockRule) {
|
||||
parts := []string{source, "planned", state}
|
||||
if strings.TrimSpace(provenance) != "" {
|
||||
parts = append(parts, "provenance="+strings.TrimSpace(provenance))
|
||||
}
|
||||
if _, ok := lockSet[source]; ok {
|
||||
parts = append(parts, "locked")
|
||||
}
|
||||
fmt.Fprintf(out, "- %s\n", strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
func writeArtifactLine(out io.Writer, source string, lockSet map[string]config.PublishLockRule) {
|
||||
parts := []string{source}
|
||||
if _, ok := lockSet[source]; ok {
|
||||
@@ -103,7 +131,12 @@ func remotePublishedOutputAvailability(ctx context.Context, cfg *config.Config,
|
||||
|
||||
func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts.ArtifactCatalog) (string, bool, error) {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
normalized, err := artifactpolicy.ResolvePublishedDestination(source, rule.Dest, helperConfiguredOutputPathMap(catalog))
|
||||
normalized, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(
|
||||
source,
|
||||
rule.Dest,
|
||||
helperConfiguredOutputPathMap(catalog),
|
||||
helperExtractionOutputSet(catalog),
|
||||
)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
@@ -112,6 +145,19 @@ func helperPublishedOutputDest(rule config.PublishOutputRule, catalog *artifacts
|
||||
return normalized, showDest, nil
|
||||
}
|
||||
|
||||
func helperExtractionOutputSet(catalog *artifacts.ArtifactCatalog) map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
if catalog == nil {
|
||||
return out
|
||||
}
|
||||
for _, entry := range catalog.ListExtraction() {
|
||||
if strings.TrimSpace(entry.ExtractionKey) != "" {
|
||||
out[entry.ExtractionKey] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func helperConfiguredOutputPathMap(catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||
out := map[string]string{}
|
||||
if catalog == nil {
|
||||
|
||||
@@ -22,11 +22,11 @@ func ArtifactsList(ctx context.Context, args []string, out io.Writer) error {
|
||||
if strings.TrimSpace(flags.sessionID) == "" {
|
||||
return fmt.Errorf("artifacts list: session_id is required")
|
||||
}
|
||||
cfg, store, locks, _, err := loadHelperContext(ctx, flags, remote)
|
||||
cfg, store, locks, m, err := loadHelperContext(ctx, flags, remote)
|
||||
if err != nil {
|
||||
return fmt.Errorf("artifacts list: %w", err)
|
||||
}
|
||||
catalog, err := buildHelperArtifactCatalog(cfg)
|
||||
catalog, err := buildHelperArtifactCatalog(cfg, m)
|
||||
if err != nil {
|
||||
return fmt.Errorf("artifacts list: %w", err)
|
||||
}
|
||||
|
||||
@@ -8,8 +8,10 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
@@ -508,7 +510,7 @@ func TestExecuteLocksAddListAndRemoveUseRemoteLockStore(t *testing.T) {
|
||||
if code != 0 {
|
||||
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
||||
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
||||
}
|
||||
@@ -772,6 +774,71 @@ func TestExecuteArtifactsListRemoteReportsPublishedAvailability(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteArtifactsListReportsExtractionLifecycleWithoutPayload(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
addExtractionOutputToPipeline(t, pipelinePath)
|
||||
addPublishOutputsToPipeline(t, pipelinePath, `
|
||||
outputs:
|
||||
- source: narratio.extraction.encounters
|
||||
dest: artifacts/encounters.json
|
||||
required: true
|
||||
`)
|
||||
lanePath := writeOperatorExtractionManifest(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
publishedKey := artifacts.S3PublishedOutputKey(
|
||||
artifacts.S3SessionPrefix("dnd", "sample-campaign", "2026-05-03"),
|
||||
"artifacts/encounters.json",
|
||||
)
|
||||
fake.SeedObject(storage.FakeObject{Key: publishedKey, Data: []byte(`{"secret":"DO_NOT_PRINT"}`)})
|
||||
var storeInitCalls int
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{
|
||||
"session", "artifacts", "2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
"--remote",
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
out := stdout.String()
|
||||
for _, want := range []string{
|
||||
"Extraction:",
|
||||
"narratio.extraction.encounters planned available provenance=manifest.current_extract_run",
|
||||
"narratio.extraction.encounters dest=artifacts/encounters.json remote=published",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("stdout = %q, want %q", out, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out, "DO_NOT_PRINT") {
|
||||
t.Fatalf("operator output exposed extraction payload: %q", out)
|
||||
}
|
||||
|
||||
if err := os.Remove(lanePath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = Execute([]string{
|
||||
"session", "artifacts", "2026-05-03",
|
||||
"--config", pipelinePath,
|
||||
"--campaign-file", campaignPath,
|
||||
"--session", sessionPath,
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("unavailable exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "narratio.extraction.encounters planned unavailable") {
|
||||
t.Fatalf("stdout = %q, want unavailable extraction state", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteArtifactsListRemoteUsesPublishOutputDestinations(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
@@ -1006,7 +1073,7 @@ func TestExecutePublishLoadsRemoteLocks(t *testing.T) {
|
||||
restoreAppConfigTestGlobals(t, fake, &storeInitCalls, []string{sessionPath})
|
||||
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||
// The publish stage only checks the manifest statuses and source files.
|
||||
_ = stageName
|
||||
}
|
||||
@@ -1041,6 +1108,72 @@ func addPublishOutputsToPipeline(t *testing.T, pipelinePath, publishYAML string)
|
||||
}
|
||||
}
|
||||
|
||||
func addExtractionOutputToPipeline(t *testing.T, pipelinePath string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data = append(data, []byte(`notarius:
|
||||
enabled: true
|
||||
config_path: notarius.yml
|
||||
pipeline_id: campaign.extract
|
||||
outputs:
|
||||
encounters:
|
||||
lane_id: encounters
|
||||
media_type: application/json
|
||||
schema_id: encounters
|
||||
schema_version: "1"
|
||||
module_key: encounters
|
||||
`)...)
|
||||
if err := os.WriteFile(pipelinePath, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(filepath.Dir(pipelinePath), "notarius.yml"), []byte("{}\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeOperatorExtractionManifest(t *testing.T, workspaceRoot string) string {
|
||||
t.Helper()
|
||||
paths := artifacts.NewLocalStore(workspaceRoot).SessionPathsFor("sample-campaign", "2026-05-03")
|
||||
bundleRoot := filepath.Join(paths.ArtifactsDir, "notarius", "extract-run-1")
|
||||
lanePath := filepath.Join(bundleRoot, "lanes", "encounters.json")
|
||||
indexPath := filepath.Join(bundleRoot, "index.json")
|
||||
mustWriteTestFile(t, lanePath, `{"secret":"DO_NOT_PRINT"}`)
|
||||
mustWriteTestFile(t, indexPath, `{"lanes":[]}`)
|
||||
laneChecksum, err := artifacts.SHA256File(lanePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
indexChecksum, err := artifacts.SHA256File(indexPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := manifest.New("2026-05-03", time.Now().UTC())
|
||||
m.Campaign = "sample-campaign"
|
||||
m.Stages["extract"] = &manifest.StageRecord{
|
||||
Name: "extract", Status: manifest.StatusSucceeded,
|
||||
Metadata: map[string]any{
|
||||
"narratio_run_id": "extract-run-1", "bundle_root": bundleRoot,
|
||||
"receipt": map[string]any{"run_id": "notarius-run-1", "pipeline_id": "campaign.extract"},
|
||||
},
|
||||
Outputs: []manifest.ArtifactRecord{
|
||||
{
|
||||
Kind: "notarius_lane", SourceID: artifacts.ExtractionArtifactSourceID("encounters"), LocalPath: lanePath,
|
||||
ProducerRunID: "extract-run-1", Checksum: laneChecksum,
|
||||
Contract: &artifactmodel.ContractMetadata{MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters"},
|
||||
ExternalProvenance: &artifactmodel.ExternalProvenance{System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: "encounters"},
|
||||
},
|
||||
{Kind: "notarius_index", LocalPath: indexPath, ProducerRunID: "extract-run-1", Checksum: indexChecksum},
|
||||
},
|
||||
}
|
||||
if err := (&manifest.LocalStore{}).Save(context.Background(), paths.ManifestPath, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return lanePath
|
||||
}
|
||||
|
||||
func replaceInFileOrFatal(t *testing.T, path, old, new string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
@@ -1076,7 +1209,7 @@ func writeValidPublishRunConfigFiles(t *testing.T, workspaceRoot string) (string
|
||||
m := manifest.New("2026-05-03", nowUTC())
|
||||
m.Campaign = "sample-campaign"
|
||||
m.RunID = "20260521T160000Z-test"
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||
m.MarkStageSucceeded(name, nowUTC(), nil)
|
||||
}
|
||||
path := artifacts.SessionManifestPathForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
|
||||
@@ -67,7 +67,7 @@ func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks add"); err != nil {
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks add"); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if _, ok := lockSourceSet(locks.Static)[source]; ok {
|
||||
@@ -79,7 +79,7 @@ func LocksAdd(ctx context.Context, args []string, out io.Writer) error {
|
||||
}
|
||||
remoteSet[source] = config.PublishLockRule{Source: source, Reason: strings.TrimSpace(reason)}
|
||||
remoteLocks := lockMapValues(remoteSet)
|
||||
if _, err := config.ValidatePublishLockRules(remoteLocks, cfg.Pipeline.Scriptorium, "locks"); err != nil {
|
||||
if _, err := config.ValidatePublishLockRules(remoteLocks, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks"); err != nil {
|
||||
return fmt.Errorf("locks add: %w", err)
|
||||
}
|
||||
if err := uploadRemoteLockStore(ctx, store, locks.Key, &config.PublishLockStore{Locks: remoteLocks}); err != nil {
|
||||
@@ -106,7 +106,7 @@ func LocksRemove(ctx context.Context, args []string, out io.Writer) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, "locks remove"); err != nil {
|
||||
if _, err := config.ValidatePublishLockRules([]config.PublishLockRule{{Source: source}}, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius, "locks remove"); err != nil {
|
||||
return fmt.Errorf("locks remove: %w", err)
|
||||
}
|
||||
remoteSet := lockSourceSet(locks.Remote)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
// Status reports effective local/remote session state.
|
||||
@@ -40,11 +41,13 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
writeStatusStableInputs(out, inspectStableInputs(cfg))
|
||||
writeStatusLocalAudio(out, inspectLocalAudioPresence(cfg))
|
||||
|
||||
var localManifest *manifest.Manifest
|
||||
if m, err := loadLocalManifest(ctx, paths.ManifestPath); err != nil {
|
||||
fmt.Fprintf(out, "Local manifest: error: %v\n", err)
|
||||
} else if m == nil {
|
||||
fmt.Fprintln(out, "Local manifest: missing")
|
||||
} else {
|
||||
localManifest = m
|
||||
fmt.Fprintf(out, "Local manifest: %s\n", paths.ManifestPath)
|
||||
writeStageStatuses(out, m)
|
||||
}
|
||||
@@ -72,7 +75,7 @@ func Status(ctx context.Context, args []string, out io.Writer) error {
|
||||
lockChecks := inspectEffectiveLocks(ctx, cfg, store)
|
||||
locks := lockChecks.Locks
|
||||
lockErr := lockChecks.Err
|
||||
if catalog, catalogErr := buildHelperArtifactCatalog(cfg); catalogErr != nil {
|
||||
if catalog, catalogErr := buildHelperArtifactCatalog(cfg, localManifest); catalogErr != nil {
|
||||
fmt.Fprintf(out, "Remote outputs: error: %v\n", catalogErr)
|
||||
} else if storeErr == nil {
|
||||
catalogLocks := locks
|
||||
|
||||
@@ -22,7 +22,7 @@ func Plan(ctx context.Context, args []string, out io.Writer) error {
|
||||
var flags commonConfigFlags
|
||||
var force bool
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||
fs.BoolVar(&force, "force", false, "show all stages as scheduled to rerun")
|
||||
|
||||
if err := parseSessionAwareFlags("plan", fs, args, &flags.sessionID); err != nil {
|
||||
return err
|
||||
|
||||
@@ -27,12 +27,12 @@ func TestPlanCreatesAndReusesWorkdir(t *testing.T) {
|
||||
if !strings.Contains(got, "narratio session plan: workdir prepared at") {
|
||||
t.Fatalf("first output = %q, want workdir prepared", got)
|
||||
}
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||
if !strings.Contains(got, name+": run") {
|
||||
t.Fatalf("first output = %q, missing stage %q", got, name)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(got, "totals: run=10 skip=0") {
|
||||
if !strings.Contains(got, "totals: run=11 skip=0") {
|
||||
t.Fatalf("first output = %q, want totals", got)
|
||||
}
|
||||
|
||||
@@ -84,8 +84,8 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||
if !strings.Contains(got, "trim: run") {
|
||||
t.Fatalf("output = %q, want trim run", got)
|
||||
}
|
||||
if !strings.Contains(got, "totals: run=8 skip=2") {
|
||||
t.Fatalf("output = %q, want totals run=8 skip=2", got)
|
||||
if !strings.Contains(got, "totals: run=9 skip=2") {
|
||||
t.Fatalf("output = %q, want totals run=9 skip=2", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import "testing"
|
||||
|
||||
func TestBuildFullPlanOrder(t *testing.T) {
|
||||
got := BuildFullPlan()
|
||||
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"}
|
||||
want := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("len(plan) = %d, want %d", len(got), len(want))
|
||||
}
|
||||
|
||||
@@ -355,7 +355,7 @@ func publishStageCleanupFixture(t *testing.T) (*config.Config, cleanupSeed, stri
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||
seedManifest.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
}
|
||||
seedManifest.S3SessionPrefix = artifacts.S3SessionPrefix("dnd", cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
|
||||
@@ -55,7 +55,7 @@ func loadRemoteLockStore(ctx context.Context, cfg *config.Config, store storage.
|
||||
if err != nil {
|
||||
return nil, key, fmt.Errorf("read remote locks %q: %w", key, err)
|
||||
}
|
||||
lockStore, err := config.LoadPublishLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium)
|
||||
lockStore, err := config.LoadPublishLockStoreBytes("s3://"+s3BucketName(cfg.Pipeline)+"/"+key, data, cfg.Pipeline.Scriptorium, cfg.Pipeline.Notarius)
|
||||
if err != nil {
|
||||
return nil, key, err
|
||||
}
|
||||
|
||||
@@ -9,8 +9,10 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
@@ -62,6 +64,55 @@ func TestExecuteRestoreNonDryRunRestoresDurableFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRestoreRoundTripsPublishedExtractionAndManifestMetadata(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
fake := &storage.FakeBackend{}
|
||||
cfg, sessionPrefix, manifestKey, runIDKey := seedRestoreCommittedState(t, fake, pipelinePath, campaignPath, sessionPath)
|
||||
seedRestoreObject(fake, sessionPrefix+"artifacts/encounters.json", []byte(`{"encounters":[]}`))
|
||||
seedRestoreObject(fake, runIDKey, []byte("20260519T010203Z-a1b2c3d4\n"))
|
||||
|
||||
remoteManifest := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
remoteManifest.Campaign = cfg.Session.Campaign
|
||||
remoteManifest.RunID = "20260519T010203Z-a1b2c3d4"
|
||||
remoteManifest.Stages["extract"] = &manifest.StageRecord{
|
||||
Name: "extract", Status: manifest.StatusSucceeded,
|
||||
Outputs: []manifest.ArtifactRecord{{
|
||||
Kind: "notarius_lane", SourceID: artifacts.ExtractionArtifactSourceID("encounters"),
|
||||
LocalPath: "/prior/workspace/artifacts/notarius/extract-run-1/lanes/encounters.json",
|
||||
Contract: &artifactmodel.ContractMetadata{
|
||||
MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1",
|
||||
},
|
||||
ExternalProvenance: &artifactmodel.ExternalProvenance{
|
||||
System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: "encounters",
|
||||
},
|
||||
}},
|
||||
}
|
||||
manifestBody, err := json.Marshal(remoteManifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seedRestoreObject(fake, manifestKey, manifestBody)
|
||||
restoreWithStoreAndRealPhases(t, fake)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := Execute([]string{"session", "restore", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &stdout, &stderr)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
sessionRoot := artifacts.SessionWorkDirForCampaign(workspaceRoot, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||
mustReadEquals(t, filepath.Join(sessionRoot, "artifacts", "encounters.json"), `{"encounters":[]}`)
|
||||
restored, err := (&manifest.LocalStore{}).Load(context.Background(), filepath.Join(sessionRoot, "manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("load restored manifest: %v", err)
|
||||
}
|
||||
lane := restored.Stages["extract"].Outputs[0]
|
||||
if lane.Contract == nil || lane.Contract.SchemaID != "encounters" || lane.ExternalProvenance == nil || lane.ExternalProvenance.RunID != "notarius-run-1" {
|
||||
t.Fatalf("restored extraction metadata = %#v", lane)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRestoreIncludeAudioRestoresAudio(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
@@ -18,7 +18,7 @@ func Run(ctx context.Context, args []string, out io.Writer) error {
|
||||
var force bool
|
||||
var selectedArtifacts artifactSelectionFlag
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||
fs.BoolVar(&force, "force", false, "rerun stages even when already succeeded")
|
||||
fs.Var(&selectedArtifacts, "artifacts", "configured artifact names to execute and publish (comma-separated or repeatable)")
|
||||
|
||||
if err := parseSessionAwareFlags("run", fs, args, &flags.sessionID); err != nil {
|
||||
|
||||
@@ -3,6 +3,7 @@ package app
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
@@ -23,22 +24,40 @@ type stageDecision struct {
|
||||
Action stageAction
|
||||
}
|
||||
|
||||
const (
|
||||
staleReasonForcedReplacement = "upstream stage was force-run"
|
||||
staleReasonChangedResult = "upstream stage result changed"
|
||||
staleReasonFailure = "upstream stage failed"
|
||||
staleReasonSelfSkip = "upstream stage self-skipped"
|
||||
staleReasonNotResumable = "upstream stage result was not resumable"
|
||||
)
|
||||
|
||||
type priorStageOutcome struct {
|
||||
exists bool
|
||||
status manifest.StageStatus
|
||||
skipReason string
|
||||
outputs int
|
||||
}
|
||||
|
||||
func decideStageActions(stages []stage.Stage, m *manifest.Manifest, force bool) []stageDecision {
|
||||
out := make([]stageDecision, 0, len(stages))
|
||||
for _, s := range stages {
|
||||
action := stageActionRun
|
||||
// TODO: incorporate stale detection once checksum/input change tracking is implemented.
|
||||
if !force && stageSucceeded(m, s.Name()) {
|
||||
action = stageActionSkip
|
||||
}
|
||||
out = append(out, stageDecision{
|
||||
Stage: s,
|
||||
Action: action,
|
||||
Action: decideStageAction(s, m, force),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func decideStageAction(s stage.Stage, m *manifest.Manifest, force bool) stageAction {
|
||||
// TODO: incorporate stale detection once checksum/input change tracking is implemented.
|
||||
if !force && stageSucceeded(m, s.Name()) {
|
||||
return stageActionSkip
|
||||
}
|
||||
return stageActionRun
|
||||
}
|
||||
|
||||
func stageSucceeded(m *manifest.Manifest, name string) bool {
|
||||
if m == nil || m.Stages == nil {
|
||||
return false
|
||||
@@ -47,6 +66,29 @@ func stageSucceeded(m *manifest.Manifest, name string) bool {
|
||||
return sr != nil && sr.Status == manifest.StatusSucceeded
|
||||
}
|
||||
|
||||
func capturePriorStageOutcome(m *manifest.Manifest, name string) priorStageOutcome {
|
||||
if m == nil || m.Stages == nil || m.Stages[name] == nil {
|
||||
return priorStageOutcome{}
|
||||
}
|
||||
record := m.Stages[name]
|
||||
outcome := priorStageOutcome{
|
||||
exists: true,
|
||||
status: record.Status,
|
||||
outputs: len(record.Outputs),
|
||||
}
|
||||
if record.Error != nil && record.Error.Code == "skipped" {
|
||||
outcome.skipReason = record.Error.Message
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
func (o priorStageOutcome) isSameSelfSkip(reason string) bool {
|
||||
return o.exists &&
|
||||
o.status == manifest.StatusSkipped &&
|
||||
o.outputs == 0 &&
|
||||
o.skipReason == strings.TrimSpace(reason)
|
||||
}
|
||||
|
||||
func loadManifestIfPresent(ctx context.Context, cfg *config.Config) (*manifest.Manifest, error) {
|
||||
path := artifacts.SessionManifestPathForCampaign(
|
||||
cfg.Pipeline.Workspace.Root,
|
||||
@@ -91,7 +133,7 @@ func downstreamStageNames(stageName string) []string {
|
||||
return nil
|
||||
}
|
||||
|
||||
func invalidateDownstreamSucceededStages(m *manifest.Manifest, upstreamStage string, at time.Time) []string {
|
||||
func invalidateDownstreamSucceededStagesWithReason(m *manifest.Manifest, upstreamStage string, at time.Time, reason string) []string {
|
||||
if m == nil || m.Stages == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -102,7 +144,7 @@ func invalidateDownstreamSucceededStages(m *manifest.Manifest, upstreamStage str
|
||||
if sr == nil || sr.Status != manifest.StatusSucceeded {
|
||||
continue
|
||||
}
|
||||
m.MarkStageStale(downstream, at, "upstream stage rerun with force")
|
||||
m.MarkStageStale(downstream, at, reason)
|
||||
invalidated = append(invalidated, downstream)
|
||||
}
|
||||
return invalidated
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestDecideStageActions(t *testing.T) {
|
||||
|
||||
func TestDownstreamStageNames(t *testing.T) {
|
||||
got := downstreamStageNames("polish")
|
||||
want := []string{"normalize", "trim", "render", "analyze", "publish", "notify"}
|
||||
want := []string{"normalize", "trim", "extract", "render", "analyze", "publish", "notify"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("downstreamStageNames(polish) = %#v, want %#v", got, want)
|
||||
}
|
||||
@@ -43,7 +43,7 @@ func TestDownstreamStageNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidateDownstreamSucceededStages(t *testing.T) {
|
||||
func TestInvalidateDownstreamSucceededStagesWithReason(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
m := manifest.New("2026-05-03", now)
|
||||
m.MarkStageSucceeded("prepare", now, nil)
|
||||
@@ -52,15 +52,16 @@ func TestInvalidateDownstreamSucceededStages(t *testing.T) {
|
||||
m.MarkStageSucceeded("polish", now, nil)
|
||||
m.MarkStageSucceeded("normalize", now, nil)
|
||||
m.MarkStageSucceeded("trim", now, nil)
|
||||
m.MarkStageSucceeded("extract", now, nil)
|
||||
m.MarkStageSucceeded("render", now, nil)
|
||||
m.MarkStageFailed("analyze", now, "analysis failed")
|
||||
m.MarkStageSucceeded("publish", now, nil)
|
||||
m.MarkStageSucceeded("notify", now, nil)
|
||||
|
||||
got := invalidateDownstreamSucceededStages(m, "polish", now.Add(1*time.Second))
|
||||
want := []string{"normalize", "trim", "render", "publish", "notify"}
|
||||
got := invalidateDownstreamSucceededStagesWithReason(m, "polish", now.Add(1*time.Second), staleReasonChangedResult)
|
||||
want := []string{"normalize", "trim", "extract", "render", "publish", "notify"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("invalidateDownstreamSucceededStages() = %#v, want %#v", got, want)
|
||||
t.Fatalf("invalidateDownstreamSucceededStagesWithReason() = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
for _, stageName := range want {
|
||||
@@ -75,3 +76,30 @@ func TestInvalidateDownstreamSucceededStages(t *testing.T) {
|
||||
t.Fatalf("prepare status = %q, want succeeded", m.Stages["prepare"].Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractionPositionControlsForceInvalidation(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
tests := []struct {
|
||||
upstream string
|
||||
want []string
|
||||
}{
|
||||
{upstream: "trim", want: []string{"extract", "render", "analyze", "publish", "notify"}},
|
||||
{upstream: "extract", want: []string{"render", "analyze", "publish", "notify"}},
|
||||
{upstream: "render", want: []string{"analyze", "publish", "notify"}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.upstream, func(t *testing.T) {
|
||||
m := manifest.New("2026-05-03", now)
|
||||
for _, name := range canonicalStageNames() {
|
||||
m.MarkStageSucceeded(name, now, nil)
|
||||
}
|
||||
got := invalidateDownstreamSucceededStagesWithReason(m, test.upstream, now.Add(time.Second), staleReasonForcedReplacement)
|
||||
if !reflect.DeepEqual(got, test.want) {
|
||||
t.Fatalf("invalidated = %#v, want %#v", got, test.want)
|
||||
}
|
||||
if test.upstream == "render" && m.Stages["extract"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("forcing render changed extract: %#v", m.Stages["extract"])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ func RunStage(ctx context.Context, args []string, out io.Writer) error {
|
||||
var force bool
|
||||
var selectedArtifacts artifactSelectionFlag
|
||||
addCommonConfigFlags(fs, &flags)
|
||||
fs.BoolVar(&force, "force", false, "force stage execution (reserved for future behavior)")
|
||||
fs.BoolVar(&force, "force", false, "rerun the stage even when already succeeded")
|
||||
fs.Var(&selectedArtifacts, "artifacts", "configured artifact names to execute or publish (comma-separated or repeatable)")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
|
||||
@@ -36,8 +36,8 @@ func TestRunContinuesAfterCompletedStages(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "executed=8 skipped=2") {
|
||||
t.Fatalf("output = %q, want executed=8 skipped=2", out.String())
|
||||
if !strings.Contains(out.String(), "executed=9 skipped=3") {
|
||||
t.Fatalf("output = %q, want executed=9 skipped=3", out.String())
|
||||
}
|
||||
|
||||
loaded, err := store.Load(context.Background(), manifestPath)
|
||||
@@ -59,6 +59,7 @@ func TestRunNoRemainingStagesRecordsSkippedStages(t *testing.T) {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
m.MarkStageSkipped("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), "notarius_disabled")
|
||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||
t.Fatalf("save manifest: %v", err)
|
||||
}
|
||||
@@ -68,8 +69,15 @@ func TestRunNoRemainingStagesRecordsSkippedStages(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "executed=0 skipped=10") {
|
||||
t.Fatalf("output = %q, want executed=0 skipped=10", out.String())
|
||||
if !strings.Contains(out.String(), "executed=1 skipped=11") {
|
||||
t.Fatalf("output = %q, want disabled extraction to self-skip", out.String())
|
||||
}
|
||||
loaded, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load migrated manifest: %v", err)
|
||||
}
|
||||
if loaded.Stages["extract"] == nil || loaded.Stages["extract"].Status != manifest.StatusSkipped {
|
||||
t.Fatalf("extract record = %#v, want stable disabled skip", loaded.Stages["extract"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,7 +93,7 @@ func TestRunForceRerunsSucceeded(t *testing.T) {
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||
@@ -97,7 +105,7 @@ func TestRunForceRerunsSucceeded(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "executed=10 skipped=0") {
|
||||
if !strings.Contains(out.String(), "executed=11 skipped=1") {
|
||||
t.Fatalf("output = %q, want forced full rerun", out.String())
|
||||
}
|
||||
}
|
||||
@@ -132,6 +140,27 @@ func TestRunStageExecutesOnlySelectedStage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStageExtractIsAcceptedAndSelfSkipsWhenDisabled(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
|
||||
var out bytes.Buffer
|
||||
err := RunStage(context.Background(), []string{"extract", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage(extract) error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "stage=extract executed=1 skipped=1 force=false") {
|
||||
t.Fatalf("output = %q, want disabled extraction self-skip", out.String())
|
||||
}
|
||||
loaded, err := (&manifest.LocalStore{}).Load(context.Background(), filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if loaded.Stages["extract"] == nil || loaded.Stages["extract"].Status != manifest.StatusSkipped {
|
||||
t.Fatalf("extract record = %#v, want skipped", loaded.Stages["extract"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStageSkipAndForce(t *testing.T) {
|
||||
workspaceRoot := t.TempDir()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspaceRoot)
|
||||
@@ -176,7 +205,7 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||
seed.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||
@@ -196,7 +225,7 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatalf("load manifest after force: %v", err)
|
||||
}
|
||||
for _, name := range []string{"normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
for _, name := range []string{"normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||
if afterForce.Stages[name] == nil || afterForce.Stages[name].Status != manifest.StatusStale {
|
||||
t.Fatalf("stage %q = %#v, want stale", name, afterForce.Stages[name])
|
||||
}
|
||||
@@ -207,7 +236,7 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "executed=6 skipped=4") {
|
||||
if !strings.Contains(out.String(), "executed=7 skipped=5") {
|
||||
t.Fatalf("output = %q, want run to execute stale downstream stages", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/audita"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notify"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
|
||||
@@ -80,6 +81,9 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
||||
}
|
||||
env.Audita = runner
|
||||
}
|
||||
if env.Notarius == nil && needsNotariusForRun(env.Config, stages) {
|
||||
env.Notarius = notarius.NewSubprocessRunner()
|
||||
}
|
||||
if env.Scriptorium == nil {
|
||||
env.Scriptorium = scriptorium.NewSubprocessRunner()
|
||||
}
|
||||
@@ -167,6 +171,29 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
||||
for _, d := range decisions {
|
||||
s := d.Stage
|
||||
runNames = append(runNames, s.Name())
|
||||
d.Action = decideStageAction(s, m, opts.Force)
|
||||
|
||||
if d.Action == stageActionSkip {
|
||||
if validator, ok := s.(stage.ResumeValidator); ok {
|
||||
validation, err := validator.ValidateResume(ctx, stageEnv, m)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("validate resume for stage %q: %w", s.Name(), err)
|
||||
}
|
||||
validation = validation.Normalized()
|
||||
if !validation.Resumable {
|
||||
staleAt := nowUTC()
|
||||
m.MarkStageStale(s.Name(), staleAt, validation.Reason)
|
||||
invalidateDownstreamSucceededStagesWithReason(
|
||||
m, s.Name(), staleAt, staleReasonNotResumable,
|
||||
)
|
||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||
return nil, fmt.Errorf("save manifest after resume validation for stage %q: %w", s.Name(), err)
|
||||
}
|
||||
env.Logger.Info("stage result is not resumable", "stage", s.Name(), "reason", validation.Reason)
|
||||
d.Action = stageActionRun
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if d.Action == stageActionSkip {
|
||||
skipped = append(skipped, s.Name())
|
||||
@@ -180,6 +207,7 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
||||
continue
|
||||
}
|
||||
executed = append(executed, s.Name())
|
||||
priorOutcome := capturePriorStageOutcome(m, s.Name())
|
||||
|
||||
now := nowUTC()
|
||||
runManifest.SetStageAction(s.Name(), manifest.RunStageActionRun, now)
|
||||
@@ -188,6 +216,9 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
||||
return nil, fmt.Errorf("save run manifest before stage %q: %w", s.Name(), err)
|
||||
}
|
||||
m.MarkStageRunning(s.Name(), now)
|
||||
if opts.Force {
|
||||
invalidateDownstreamSucceededStagesWithReason(m, s.Name(), now, staleReasonForcedReplacement)
|
||||
}
|
||||
env.Logger.Info("starting stage", "stage", s.Name())
|
||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||
return nil, fmt.Errorf("save manifest before stage %q: %w", s.Name(), err)
|
||||
@@ -195,9 +226,13 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
||||
env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "running", "path", manifestPath)
|
||||
|
||||
result, err := s.Run(ctx, stageEnv, m)
|
||||
if err == nil {
|
||||
err = validateStageResult(result)
|
||||
}
|
||||
if err != nil {
|
||||
failedAt := nowUTC()
|
||||
m.MarkStageFailed(s.Name(), failedAt, err.Error())
|
||||
invalidateDownstreamSucceededStagesWithReason(m, s.Name(), failedAt, staleReasonFailure)
|
||||
if saveErr := env.ManifestStore.Save(ctx, manifestPath, m); saveErr != nil {
|
||||
return nil, fmt.Errorf("stage %q failed (%v) and manifest save failed (%v)", s.Name(), err, saveErr)
|
||||
}
|
||||
@@ -209,13 +244,34 @@ func executeStages(ctx context.Context, cfg *config.Config, stages []stage.Stage
|
||||
env.Logger.Info("stage failed", "stage", s.Name(), "error", err)
|
||||
return nil, fmt.Errorf("stage %q failed: %w", s.Name(), err)
|
||||
}
|
||||
if result != nil && result.Disposition == stage.StageDispositionSkipped {
|
||||
skipped = append(skipped, s.Name())
|
||||
skippedAt := nowUTC()
|
||||
m.MarkStageSkipped(s.Name(), skippedAt, result.SkipReason)
|
||||
applyStageResultToManifest(m, s.Name(), result)
|
||||
if !priorOutcome.isSameSelfSkip(result.SkipReason) {
|
||||
invalidateDownstreamSucceededStagesWithReason(m, s.Name(), skippedAt, staleReasonSelfSkip)
|
||||
}
|
||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||
return nil, fmt.Errorf("save manifest after self-skip %q: %w", s.Name(), err)
|
||||
}
|
||||
runManifest.MarkStageSkipped(s.Name(), skippedAt, result.SkipReason)
|
||||
applyStageResultToRunManifest(runManifest, s.Name(), result)
|
||||
syncRunManifestIdentityFromSession(m, runManifest)
|
||||
if err := runManifestStore.SaveRun(ctx, runManifestPath, runManifest); err != nil {
|
||||
return nil, fmt.Errorf("save run manifest after self-skip %q: %w", s.Name(), err)
|
||||
}
|
||||
env.Logger.Debug("manifest saved", "stage", s.Name(), "transition", "skipped", "path", manifestPath)
|
||||
env.Logger.Info("stage skipped", "stage", s.Name(), "reason", result.SkipReason)
|
||||
continue
|
||||
}
|
||||
|
||||
outputs := mapResultOutputs(s.Name(), result, runID)
|
||||
succeededAt := nowUTC()
|
||||
m.MarkStageSucceeded(s.Name(), succeededAt, outputs)
|
||||
applyStageResultToManifest(m, s.Name(), result)
|
||||
if opts.Force {
|
||||
invalidateDownstreamSucceededStages(m, s.Name(), succeededAt)
|
||||
if !priorOutcome.exists || priorOutcome.status != manifest.StatusSucceeded {
|
||||
invalidateDownstreamSucceededStagesWithReason(m, s.Name(), succeededAt, staleReasonChangedResult)
|
||||
}
|
||||
|
||||
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
|
||||
@@ -407,26 +463,69 @@ func mapResultOutputs(stageName string, result *stage.StageResult, runID string)
|
||||
localPath = ref.RelativePath
|
||||
}
|
||||
kind := ref.Kind
|
||||
sourceID := ""
|
||||
if stageName == "analyze" {
|
||||
sourceID = artifacts.ConfiguredArtifactSourceID(ref.Kind)
|
||||
kind = "scriptorium_artifact"
|
||||
} else {
|
||||
sourceID = sourceIDForOutputKind(kind)
|
||||
sourceID := strings.TrimSpace(ref.SourceID)
|
||||
if sourceID == "" {
|
||||
if stageName == "analyze" {
|
||||
sourceID = artifacts.ConfiguredArtifactSourceID(ref.Kind)
|
||||
kind = "scriptorium_artifact"
|
||||
} else {
|
||||
sourceID = sourceIDForOutputKind(kind)
|
||||
}
|
||||
}
|
||||
out = append(out, manifest.ArtifactRecord{
|
||||
Kind: kind,
|
||||
SourceID: sourceID,
|
||||
LocalPath: localPath,
|
||||
ProducerRunID: runID,
|
||||
RemoteKey: ref.RemoteKey,
|
||||
Checksum: ref.Checksum,
|
||||
Kind: kind,
|
||||
SourceID: sourceID,
|
||||
LocalPath: localPath,
|
||||
Contract: cloneContractMetadata(ref.Contract),
|
||||
ExternalProvenance: cloneExternalProvenance(ref.ExternalProvenance),
|
||||
ProducerRunID: runID,
|
||||
RemoteKey: ref.RemoteKey,
|
||||
Checksum: ref.Checksum,
|
||||
})
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func validateStageResult(result *stage.StageResult) error {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
switch result.Disposition {
|
||||
case stage.StageDispositionSucceeded:
|
||||
if strings.TrimSpace(result.SkipReason) != "" {
|
||||
return fmt.Errorf("successful result contains a skip reason")
|
||||
}
|
||||
return nil
|
||||
case stage.StageDispositionSkipped:
|
||||
if strings.TrimSpace(result.SkipReason) == "" {
|
||||
return fmt.Errorf("skipped result requires a skip reason")
|
||||
}
|
||||
if len(result.Outputs) != 0 {
|
||||
return fmt.Errorf("skipped result contains %d output(s)", len(result.Outputs))
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unsupported stage result disposition %q", result.Disposition)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneContractMetadata(value *artifactmodel.ContractMetadata) *artifactmodel.ContractMetadata {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func cloneExternalProvenance(value *artifactmodel.ExternalProvenance) *artifactmodel.ExternalProvenance {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func sourceIDForOutputKind(kind string) string {
|
||||
trimmed := strings.TrimSpace(kind)
|
||||
if trimmed == "" {
|
||||
@@ -598,6 +697,18 @@ func needsObjectStoreForRun(cfg *config.Config, stages []stage.Stage) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func needsNotariusForRun(cfg *config.Config, stages []stage.Stage) bool {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Notarius == nil || !cfg.Pipeline.Notarius.Enabled {
|
||||
return false
|
||||
}
|
||||
for _, candidate := range stages {
|
||||
if candidate != nil && candidate.Name() == "extract" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func needsRemoteLocksForRun(cfg *config.Config, stages []stage.Stage) bool {
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return false
|
||||
|
||||
@@ -2,6 +2,7 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -16,6 +17,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
@@ -38,6 +40,25 @@ type countingStage struct {
|
||||
runs *int
|
||||
}
|
||||
|
||||
type resultStage struct {
|
||||
name string
|
||||
result *stage.StageResult
|
||||
runs *int
|
||||
order *[]string
|
||||
}
|
||||
|
||||
func (s resultStage) Name() string { return s.name }
|
||||
func (s resultStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||
func (s resultStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||
if s.runs != nil {
|
||||
*s.runs = *s.runs + 1
|
||||
}
|
||||
if s.order != nil {
|
||||
*s.order = append(*s.order, s.name)
|
||||
}
|
||||
return s.result, nil
|
||||
}
|
||||
|
||||
func (s countingStage) Name() string { return s.name }
|
||||
func (s countingStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||
func (s countingStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||
@@ -50,6 +71,34 @@ type captureSelectedArtifactsStage struct {
|
||||
captured *[]string
|
||||
}
|
||||
|
||||
type captureNotariusStage struct {
|
||||
captured *bool
|
||||
}
|
||||
|
||||
type resumeCheckingStage struct {
|
||||
name string
|
||||
validation stage.ResumeValidation
|
||||
validateErr error
|
||||
runs *int
|
||||
}
|
||||
|
||||
func (s resumeCheckingStage) Name() string { return s.name }
|
||||
func (s resumeCheckingStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||
func (s resumeCheckingStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||
*s.runs++
|
||||
return &stage.StageResult{}, nil
|
||||
}
|
||||
func (s resumeCheckingStage) ValidateResume(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (stage.ResumeValidation, error) {
|
||||
return s.validation, s.validateErr
|
||||
}
|
||||
|
||||
func (s captureNotariusStage) Name() string { return "extract" }
|
||||
func (s captureNotariusStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||
func (s captureNotariusStage) Run(_ context.Context, env *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||
*s.captured = env.Notarius != nil
|
||||
return &stage.StageResult{}, nil
|
||||
}
|
||||
|
||||
func (s captureSelectedArtifactsStage) Name() string { return s.name }
|
||||
func (s captureSelectedArtifactsStage) Declares() stage.IODecl { return stage.IODecl{} }
|
||||
func (s captureSelectedArtifactsStage) Run(_ context.Context, env *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
|
||||
@@ -134,6 +183,28 @@ func TestExecuteStagesPropagatesSelectedArtifactsToEnv(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesComposesNotariusOnlyForEnabledExtraction(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
cfg.Pipeline.Notarius = &config.NotariusConfig{Enabled: true}
|
||||
captured := false
|
||||
|
||||
_, err := executeStages(context.Background(), cfg, []stage.Stage{captureNotariusStage{captured: &captured}}, RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
if !captured {
|
||||
t.Fatal("extract stage did not receive the default Notarius runner")
|
||||
}
|
||||
|
||||
if needsNotariusForRun(cfg, []stage.Stage{countingStage{name: "analyze", runs: new(int)}}) {
|
||||
t.Fatal("Notarius runner requested without extract in the selected plan")
|
||||
}
|
||||
cfg.Pipeline.Notarius.Enabled = false
|
||||
if needsNotariusForRun(cfg, []stage.Stage{captureNotariusStage{captured: new(bool)}}) {
|
||||
t.Fatal("Notarius runner requested while extraction is disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesAnalyzeOutputsPersistAsScriptoriumArtifacts(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
storeForPaths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
|
||||
@@ -199,6 +270,61 @@ func TestExecuteStagesAnalyzeOutputsPersistAsScriptoriumArtifacts(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapResultOutputsPrefersExplicitSourceAndCopiesMetadata(t *testing.T) {
|
||||
contract := &artifactmodel.ContractMetadata{
|
||||
MediaType: "application/json",
|
||||
SchemaID: "notarius.dnd.npc_registry",
|
||||
SchemaVersion: "v1",
|
||||
}
|
||||
provenance := &artifactmodel.ExternalProvenance{
|
||||
System: "notarius",
|
||||
RunID: "external-run",
|
||||
PipelineID: "dnd-session",
|
||||
ArtifactID: "npc-registry",
|
||||
}
|
||||
result := &stage.StageResult{Outputs: []artifacts.Ref{{
|
||||
Kind: "structured_data",
|
||||
SourceID: "narratio.example.npcs",
|
||||
RelativePath: "artifacts/npcs.json",
|
||||
Contract: contract,
|
||||
ExternalProvenance: provenance,
|
||||
}}}
|
||||
|
||||
got := mapResultOutputs("analyze", result, "narratio-run")
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("outputs len = %d, want 1", len(got))
|
||||
}
|
||||
if got[0].SourceID != "narratio.example.npcs" {
|
||||
t.Fatalf("source_id = %q, want explicit source", got[0].SourceID)
|
||||
}
|
||||
if got[0].Kind != "structured_data" {
|
||||
t.Fatalf("kind = %q, want explicit output kind preserved", got[0].Kind)
|
||||
}
|
||||
if got[0].Contract == nil || *got[0].Contract != *contract {
|
||||
t.Fatalf("contract = %#v, want %#v", got[0].Contract, contract)
|
||||
}
|
||||
if got[0].ExternalProvenance == nil || *got[0].ExternalProvenance != *provenance {
|
||||
t.Fatalf("external provenance = %#v, want %#v", got[0].ExternalProvenance, provenance)
|
||||
}
|
||||
if got[0].Contract == contract || got[0].ExternalProvenance == provenance {
|
||||
t.Fatal("mapped metadata should not alias the stage result")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapResultOutputsRetainsFallbackInference(t *testing.T) {
|
||||
transcript := mapResultOutputs("trim", &stage.StageResult{Outputs: []artifacts.Ref{{
|
||||
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed,
|
||||
}}}, "run-id")
|
||||
if len(transcript) != 1 || transcript[0].SourceID != artifacts.ArtifactTranscriptFinalTrimmed {
|
||||
t.Fatalf("transcript fallback = %#v, want final-trimmed source", transcript)
|
||||
}
|
||||
|
||||
analyze := mapResultOutputs("analyze", &stage.StageResult{Outputs: []artifacts.Ref{{Kind: "session_recap"}}}, "run-id")
|
||||
if len(analyze) != 1 || analyze[0].SourceID != "narratio.artifact.session_recap" || analyze[0].Kind != "scriptorium_artifact" {
|
||||
t.Fatalf("analyze fallback = %#v, want configured artifact inference", analyze)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeedsObjectStoreForRunPrepareWithPreviousRequirements(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -273,7 +399,7 @@ func TestExecuteStagesPublishSkipsRequiredUnselectedConfiguredOutput(t *testing.
|
||||
manifestPath := manifestPathFor(cfg)
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
seed.Campaign = cfg.Session.Campaign
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render"} {
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render"} {
|
||||
seed.MarkStageSucceeded(stageName, time.Now().UTC(), nil)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||
@@ -332,8 +458,8 @@ func TestExecuteStagesPlaceholderSuccessUpdatesManifest(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
if len(summary.StageNames) != 10 || len(summary.Executed) != 10 || len(summary.Skipped) != 0 {
|
||||
t.Fatalf("summary = %#v, want all 10 executed", summary)
|
||||
if len(summary.StageNames) != 11 || len(summary.Executed) != 11 || len(summary.Skipped) != 1 || summary.Skipped[0] != "extract" {
|
||||
t.Fatalf("summary = %#v, want full plan with disabled extraction self-skip", summary)
|
||||
}
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
@@ -342,11 +468,17 @@ func TestExecuteStagesPlaceholderSuccessUpdatesManifest(t *testing.T) {
|
||||
t.Fatalf("Load manifest error = %v", err)
|
||||
}
|
||||
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||
sr := m.Stages[name]
|
||||
if sr == nil {
|
||||
t.Fatalf("missing stage record %q", name)
|
||||
}
|
||||
if name == "extract" {
|
||||
if sr.Status != manifest.StatusSkipped || sr.Error == nil || sr.Error.Message != "notarius_disabled" {
|
||||
t.Fatalf("extract stage = %#v, want disabled skip", sr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if sr.Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("stage %q status = %q, want %q", name, sr.Status, manifest.StatusSucceeded)
|
||||
}
|
||||
@@ -497,6 +629,91 @@ func TestExecuteStagesSkipSucceededWhenNotForced(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesUsesOptionalResumeValidation(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
validation stage.ResumeValidation
|
||||
wantRuns int
|
||||
wantSkipped int
|
||||
}{
|
||||
{name: "resumable", validation: stage.Resumable(), wantSkipped: 1},
|
||||
{name: "rerun", validation: stage.NonResumable("durable output changed"), wantRuns: 1},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
seed.MarkStageSucceeded("checked", time.Now().UTC(), nil)
|
||||
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
runs := 0
|
||||
candidate := resumeCheckingStage{name: "checked", validation: test.validation, runs: &runs}
|
||||
summary, err := executeStages(context.Background(), cfg, []stage.Stage{candidate}, RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
if runs != test.wantRuns || len(summary.Skipped) != test.wantSkipped {
|
||||
t.Fatalf("runs = %d summary = %#v", runs, summary)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesNonResumableResultRerunsSucceededDownstream(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
seed.MarkStageSucceeded("extract", time.Now().UTC(), nil)
|
||||
seed.MarkStageSucceeded("render", time.Now().UTC(), nil)
|
||||
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
extractRuns, renderRuns := 0, 0
|
||||
stages := []stage.Stage{
|
||||
resumeCheckingStage{name: "extract", validation: stage.NonResumable("checksum changed"), runs: &extractRuns},
|
||||
countingStage{name: "render", runs: &renderRuns},
|
||||
}
|
||||
summary, err := executeStages(context.Background(), cfg, stages, RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
if extractRuns != 1 || renderRuns != 1 || len(summary.Executed) != 2 || len(summary.Skipped) != 0 {
|
||||
t.Fatalf("extract runs=%d render runs=%d summary=%#v", extractRuns, renderRuns, summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesResumeValidationErrorPreservesSucceededRecord(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
|
||||
seed.MarkStageSucceeded("checked", time.Now().UTC(), []manifest.ArtifactRecord{{Kind: "kept", LocalPath: "kept.json"}})
|
||||
seed.Stages["checked"].Metadata = map[string]any{"kept": true}
|
||||
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||
t.Fatalf("Save() error = %v", err)
|
||||
}
|
||||
before, err := json.Marshal(seed.Stages["checked"])
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal(before) error = %v", err)
|
||||
}
|
||||
runs := 0
|
||||
candidate := resumeCheckingStage{name: "checked", validateErr: errors.New("inspection unavailable"), runs: &runs}
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{candidate}, RunOptions{}); err == nil || !strings.Contains(err.Error(), "inspection unavailable") {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
loaded, err := store.Load(context.Background(), manifestPathFor(cfg))
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
after, err := json.Marshal(loaded.Stages["checked"])
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal(after) error = %v", err)
|
||||
}
|
||||
if string(after) != string(before) || runs != 0 {
|
||||
t.Fatalf("succeeded record changed: before=%s after=%s runs=%d", before, after, runs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesForceRerunsSucceeded(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
manifestPath := manifestPathFor(cfg)
|
||||
@@ -525,13 +742,48 @@ func TestExecuteStagesForceRerunsSucceeded(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesSuccessfulReplacementDoesNotInheritResultDetails(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
manifestPath := manifestPathFor(cfg)
|
||||
store := &manifest.LocalStore{}
|
||||
|
||||
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
||||
existing.MarkStageSucceeded("transcribe", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), []manifest.ArtifactRecord{{
|
||||
Kind: "transcript_raw", LocalPath: "transcripts/old.json",
|
||||
}})
|
||||
existingStage := existing.Stages["transcribe"]
|
||||
existingStage.Logs = []string{"logs/old.log"}
|
||||
existingStage.GeneratedConfigs = []string{"generated/old.yaml"}
|
||||
existingStage.Metadata = map[string]any{"old_result": true}
|
||||
if err := store.Save(context.Background(), manifestPath, existing); err != nil {
|
||||
t.Fatalf("Save manifest error = %v", err)
|
||||
}
|
||||
|
||||
runs := 0
|
||||
replacement := resultStage{name: "transcribe", result: &stage.StageResult{}, runs: &runs}
|
||||
if _, err := executeStages(context.Background(), cfg, []stage.Stage{replacement}, RunOptions{Force: true}); err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
loaded, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load manifest error = %v", err)
|
||||
}
|
||||
record := loaded.Stages["transcribe"]
|
||||
if runs != 1 || record == nil || record.Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("runs = %d, record = %#v, want one successful replacement", runs, record)
|
||||
}
|
||||
if len(record.Outputs) != 0 || len(record.Logs) != 0 || len(record.GeneratedConfigs) != 0 || len(record.Metadata) != 0 {
|
||||
t.Fatalf("replacement inherited result details: %#v", record)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesForceSuccessInvalidatesDownstreamSucceededStages(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
manifestPath := manifestPathFor(cfg)
|
||||
store := &manifest.LocalStore{}
|
||||
|
||||
existing := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "publish", "notify"} {
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "publish", "notify"} {
|
||||
existing.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
existing.MarkStageFailed("analyze", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), "previous analyze failure")
|
||||
@@ -562,7 +814,7 @@ func TestExecuteStagesForceSuccessInvalidatesDownstreamSucceededStages(t *testin
|
||||
if loaded.Stages["polish"] == nil || loaded.Stages["polish"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("polish status = %#v, want succeeded", loaded.Stages["polish"])
|
||||
}
|
||||
for _, stageName := range []string{"normalize", "trim", "render", "publish", "notify"} {
|
||||
for _, stageName := range []string{"normalize", "trim", "extract", "render", "publish", "notify"} {
|
||||
if loaded.Stages[stageName] == nil || loaded.Stages[stageName].Status != manifest.StatusStale {
|
||||
t.Fatalf("%s status = %#v, want stale", stageName, loaded.Stages[stageName])
|
||||
}
|
||||
@@ -738,6 +990,126 @@ func TestExecuteStagesRunManifestRecordsSkippedStage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesPersistsSelfSkipAndContinues(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
store := &manifest.LocalStore{}
|
||||
manifestPath := manifestPathFor(cfg)
|
||||
seed := manifest.New(cfg.Session.SessionID, time.Date(2026, 5, 3, 1, 0, 0, 0, time.UTC))
|
||||
seed.MarkStageSucceeded("optional", time.Date(2026, 5, 3, 1, 1, 0, 0, time.UTC), []manifest.ArtifactRecord{{
|
||||
Kind: "old_output",
|
||||
SourceID: "narratio.example.old",
|
||||
LocalPath: "artifacts/old.json",
|
||||
}})
|
||||
seed.Stages["optional"].Logs = []string{"old.log"}
|
||||
seed.Stages["optional"].GeneratedConfigs = []string{"old.yml"}
|
||||
seed.Stages["optional"].Metadata = map[string]any{"old": true}
|
||||
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
if err := store.Save(context.Background(), manifestPath, seed); err != nil {
|
||||
t.Fatalf("Save() seed manifest error = %v", err)
|
||||
}
|
||||
|
||||
order := []string{}
|
||||
optionalRuns := 0
|
||||
stages := []stage.Stage{
|
||||
resultStage{
|
||||
name: "optional",
|
||||
runs: &optionalRuns,
|
||||
order: &order,
|
||||
result: &stage.StageResult{
|
||||
Disposition: stage.StageDispositionSkipped,
|
||||
SkipReason: "integration_disabled",
|
||||
Logs: []string{"runs/current/optional.log"},
|
||||
GeneratedConfigs: []string{"runs/current/optional.yml"},
|
||||
Metadata: map[string]any{"enabled": false},
|
||||
},
|
||||
},
|
||||
resultStage{name: "later", order: &order, result: &stage.StageResult{}},
|
||||
}
|
||||
summary, err := executeStages(context.Background(), cfg, stages, RunOptions{Force: true})
|
||||
if err != nil {
|
||||
t.Fatalf("executeStages() error = %v", err)
|
||||
}
|
||||
if strings.Join(order, ",") != "optional,later" {
|
||||
t.Fatalf("execution order = %v, want optional then later", order)
|
||||
}
|
||||
if len(summary.Executed) != 2 || len(summary.Skipped) != 1 || summary.Skipped[0] != "optional" {
|
||||
t.Fatalf("summary = %#v, want optional executed and self-skipped before later", summary)
|
||||
}
|
||||
|
||||
sessionManifest, err := store.Load(context.Background(), summary.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() session manifest error = %v", err)
|
||||
}
|
||||
selfSkipped := sessionManifest.Stages["optional"]
|
||||
if selfSkipped == nil || selfSkipped.Status != manifest.StatusSkipped {
|
||||
t.Fatalf("optional stage = %#v, want skipped", selfSkipped)
|
||||
}
|
||||
if len(selfSkipped.Outputs) != 0 {
|
||||
t.Fatalf("optional outputs = %#v, want old outputs cleared", selfSkipped.Outputs)
|
||||
}
|
||||
if selfSkipped.Error == nil || selfSkipped.Error.Message != "integration_disabled" {
|
||||
t.Fatalf("optional skip reason = %#v, want integration_disabled", selfSkipped.Error)
|
||||
}
|
||||
if len(selfSkipped.Logs) != 1 || selfSkipped.Logs[0] != "runs/current/optional.log" ||
|
||||
len(selfSkipped.GeneratedConfigs) != 1 || selfSkipped.GeneratedConfigs[0] != "runs/current/optional.yml" ||
|
||||
selfSkipped.Metadata["enabled"] != false || selfSkipped.Metadata["old"] != nil {
|
||||
t.Fatalf("optional result details = %#v, want current bounded diagnostics and metadata", selfSkipped)
|
||||
}
|
||||
if later := sessionManifest.Stages["later"]; later == nil || later.Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("later stage = %#v, want succeeded", later)
|
||||
}
|
||||
|
||||
runManifest, err := store.LoadRun(context.Background(), summary.RunManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadRun() error = %v", err)
|
||||
}
|
||||
runStage := runManifest.Stages["optional"]
|
||||
if runStage == nil || runStage.Action != manifest.RunStageActionRun || runStage.Status != manifest.StatusSkipped {
|
||||
t.Fatalf("run optional stage = %#v, want run action with skipped status", runStage)
|
||||
}
|
||||
if len(runStage.Logs) != 1 || runStage.Metadata["enabled"] != false {
|
||||
t.Fatalf("run optional stage details = %#v, want result diagnostics and metadata", runStage)
|
||||
}
|
||||
|
||||
_, err = executeStages(context.Background(), cfg, []stage.Stage{stages[0]}, RunOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("second executeStages() error = %v", err)
|
||||
}
|
||||
if optionalRuns != 2 {
|
||||
t.Fatalf("optional runs = %d, want self-skipped stage reconsidered", optionalRuns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesRejectsSkippedResultWithOutputs(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
invalid := resultStage{name: "optional", result: &stage.StageResult{
|
||||
Disposition: stage.StageDispositionSkipped,
|
||||
SkipReason: "integration_disabled",
|
||||
Outputs: []artifacts.Ref{{Kind: "unexpected"}},
|
||||
}}
|
||||
|
||||
summary, err := executeStages(context.Background(), cfg, []stage.Stage{invalid}, RunOptions{})
|
||||
if err == nil {
|
||||
t.Fatal("executeStages() error = nil, want invalid skipped result failure")
|
||||
}
|
||||
if summary != nil {
|
||||
t.Fatalf("summary = %#v, want nil", summary)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "skipped result contains 1 output") {
|
||||
t.Fatalf("error = %q, want skipped-output validation", err)
|
||||
}
|
||||
|
||||
loaded, loadErr := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
|
||||
if loadErr != nil {
|
||||
t.Fatalf("Load() session manifest error = %v", loadErr)
|
||||
}
|
||||
if got := loaded.Stages["optional"]; got == nil || got.Status != manifest.StatusFailed {
|
||||
t.Fatalf("optional stage = %#v, want failed", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteStagesRunLocalArtifactsAndCanonicalSync(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
stages := []stage.Stage{
|
||||
@@ -986,7 +1358,7 @@ func TestAdapterBackedStageFailureMarksManifestFailed(t *testing.T) {
|
||||
seed.S3Bucket = "my-dnd-archive"
|
||||
seed.S3SessionPrefix = "dnd/campaigns/" + cfg.Session.Campaign + "/sessions/" + cfg.Session.SessionID + "/"
|
||||
seed.S3RunPrefix = seed.S3SessionPrefix + "runs/" + runID + "/"
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
}
|
||||
if err := tc.env.ManifestStore.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
|
||||
|
||||
@@ -346,7 +346,7 @@ func TestExecuteSessionLocksMutationAcceptsPositionalSessionID(t *testing.T) {
|
||||
if code != 0 {
|
||||
t.Fatalf("locks remove exit code = %d, want 0; stderr=%q", code, stderr.String())
|
||||
}
|
||||
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil)
|
||||
store, err := config.LoadPublishLockStoreBytes("locks.yml", fake.Objects[key].Data, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
||||
}
|
||||
|
||||
17
internal/artifactmodel/metadata.go
Normal file
17
internal/artifactmodel/metadata.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package artifactmodel
|
||||
|
||||
// ContractMetadata identifies the data contract implemented by an artifact.
|
||||
type ContractMetadata struct {
|
||||
MediaType string `json:"media_type"`
|
||||
SchemaID string `json:"schema_id"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
}
|
||||
|
||||
// ExternalProvenance identifies an artifact produced by an external system.
|
||||
type ExternalProvenance struct {
|
||||
System string `json:"system"`
|
||||
RunID string `json:"run_id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
ArtifactID string `json:"artifact_id"`
|
||||
}
|
||||
56
internal/artifactmodel/metadata_test.go
Normal file
56
internal/artifactmodel/metadata_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package artifactmodel
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestArtifactMetadataJSON(t *testing.T) {
|
||||
type envelope struct {
|
||||
Contract *ContractMetadata `json:"contract,omitempty"`
|
||||
ExternalProvenance *ExternalProvenance `json:"external_provenance,omitempty"`
|
||||
}
|
||||
|
||||
complete, err := json.Marshal(envelope{
|
||||
Contract: &ContractMetadata{
|
||||
MediaType: "application/json",
|
||||
SchemaID: "notarius.dnd.npc_registry",
|
||||
SchemaVersion: "v1",
|
||||
ModuleKey: "dnd/npc-registry",
|
||||
},
|
||||
ExternalProvenance: &ExternalProvenance{
|
||||
System: "notarius",
|
||||
RunID: "run-123",
|
||||
PipelineID: "dnd-session",
|
||||
ArtifactID: "npc-registry",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() complete metadata error = %v", err)
|
||||
}
|
||||
wantComplete := `{"contract":{"media_type":"application/json","schema_id":"notarius.dnd.npc_registry","schema_version":"v1","module_key":"dnd/npc-registry"},"external_provenance":{"system":"notarius","run_id":"run-123","pipeline_id":"dnd-session","artifact_id":"npc-registry"}}`
|
||||
if string(complete) != wantComplete {
|
||||
t.Fatalf("complete metadata JSON = %s, want %s", complete, wantComplete)
|
||||
}
|
||||
|
||||
omitted, err := json.Marshal(envelope{})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() omitted metadata error = %v", err)
|
||||
}
|
||||
if string(omitted) != `{}` {
|
||||
t.Fatalf("omitted metadata JSON = %s, want {}", omitted)
|
||||
}
|
||||
|
||||
withoutModule, err := json.Marshal(envelope{Contract: &ContractMetadata{
|
||||
MediaType: "application/json",
|
||||
SchemaID: "notarius.dnd.npc_registry",
|
||||
SchemaVersion: "v1",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() contract without module key error = %v", err)
|
||||
}
|
||||
if strings.Contains(string(withoutModule), "module_key") {
|
||||
t.Fatalf("contract JSON unexpectedly contains omitted module_key: %s", withoutModule)
|
||||
}
|
||||
}
|
||||
@@ -18,11 +18,14 @@ const (
|
||||
SourceInputGlossary = "narratio.input.glossary"
|
||||
|
||||
configuredSourcePrefix = "narratio.artifact."
|
||||
extractionSourcePrefix = "narratio.extraction."
|
||||
previousConfiguredSrcPrefix = "narratio.previous_session.artifact."
|
||||
)
|
||||
|
||||
var configuredSourceRE = regexp.MustCompile(`^narratio\.artifact\.([a-z][a-z0-9_]*)$`)
|
||||
var extractionSourceRE = regexp.MustCompile(`^narratio\.extraction\.([a-z][a-z0-9_]*)$`)
|
||||
var previousSourceRE = regexp.MustCompile(`^narratio\.previous_session\.artifact\.([a-z][a-z0-9_]*)$`)
|
||||
var configuredKeyRE = regexp.MustCompile(`^[a-z][a-z0-9_]*$`)
|
||||
|
||||
var (
|
||||
ErrUnsupportedScriptoriumInputSource = errors.New("unsupported scriptorium input source")
|
||||
@@ -34,6 +37,7 @@ type SourceKind string
|
||||
const (
|
||||
SourceKindBuiltIn SourceKind = "built_in"
|
||||
SourceKindConfiguredArtifact SourceKind = "configured_artifact"
|
||||
SourceKindExtraction SourceKind = "extraction"
|
||||
SourceKindPreviousArtifact SourceKind = "previous_session_configured_artifact"
|
||||
SourceKindStableInput SourceKind = "stable_input"
|
||||
)
|
||||
@@ -67,11 +71,30 @@ func (e *UnknownConfiguredArtifactError) Error() string {
|
||||
return fmt.Sprintf("references unknown artifact %q", e.ConfiguredKey)
|
||||
}
|
||||
|
||||
// UnknownExtractionArtifactError reports a source that references an undefined extraction key.
|
||||
type UnknownExtractionArtifactError struct {
|
||||
ConfiguredKey string
|
||||
}
|
||||
|
||||
func (e *UnknownExtractionArtifactError) Error() string {
|
||||
return fmt.Sprintf("references unknown extraction output %q", e.ConfiguredKey)
|
||||
}
|
||||
|
||||
// IsConfiguredKey reports whether a key follows the configured-artifact key grammar.
|
||||
func IsConfiguredKey(key string) bool {
|
||||
return configuredKeyRE.MatchString(strings.TrimSpace(key))
|
||||
}
|
||||
|
||||
// ConfiguredSourceID converts a configured artifact key into source id form.
|
||||
func ConfiguredSourceID(key string) string {
|
||||
return configuredSourcePrefix + strings.TrimSpace(key)
|
||||
}
|
||||
|
||||
// ExtractionSourceID converts an extraction output key into source id form.
|
||||
func ExtractionSourceID(key string) string {
|
||||
return extractionSourcePrefix + strings.TrimSpace(key)
|
||||
}
|
||||
|
||||
// PreviousSessionSourceID converts a configured artifact key into previous-session source id form.
|
||||
func PreviousSessionSourceID(key string) string {
|
||||
return previousConfiguredSrcPrefix + strings.TrimSpace(key)
|
||||
@@ -86,6 +109,15 @@ func ParseConfiguredSource(source string) (string, bool) {
|
||||
return matches[1], true
|
||||
}
|
||||
|
||||
// ParseExtractionSource extracts configured key from narratio.extraction.<key>.
|
||||
func ParseExtractionSource(source string) (string, bool) {
|
||||
matches := extractionSourceRE.FindStringSubmatch(strings.TrimSpace(source))
|
||||
if len(matches) != 2 {
|
||||
return "", false
|
||||
}
|
||||
return matches[1], true
|
||||
}
|
||||
|
||||
// ParsePreviousSessionSource extracts configured key from narratio.previous_session.artifact.<key>.
|
||||
func ParsePreviousSessionSource(source string) (string, bool) {
|
||||
matches := previousSourceRE.FindStringSubmatch(strings.TrimSpace(source))
|
||||
@@ -110,6 +142,9 @@ func ClassifySource(source string) (Source, error) {
|
||||
if key, ok := ParseConfiguredSource(trimmed); ok {
|
||||
return Source{ID: trimmed, Kind: SourceKindConfiguredArtifact, ConfiguredKey: key}, nil
|
||||
}
|
||||
if key, ok := ParseExtractionSource(trimmed); ok {
|
||||
return Source{ID: trimmed, Kind: SourceKindExtraction, ConfiguredKey: key}, nil
|
||||
}
|
||||
if key, ok := ParsePreviousSessionSource(trimmed); ok {
|
||||
return Source{ID: trimmed, Kind: SourceKindPreviousArtifact, ConfiguredKey: key}, nil
|
||||
}
|
||||
@@ -186,24 +221,48 @@ func PreviousSessionSourceDescriptorForConfiguredKey(configuredKey string) (Prev
|
||||
func ValidateInputConfiguredReference(
|
||||
descriptor ScriptoriumInputSourceDescriptor,
|
||||
configured map[string]struct{},
|
||||
) error {
|
||||
return ValidateInputReference(descriptor, configured, nil)
|
||||
}
|
||||
|
||||
// ValidateInputReference checks that configured and extraction sources are declared
|
||||
// by the effective pipeline configuration.
|
||||
func ValidateInputReference(
|
||||
descriptor ScriptoriumInputSourceDescriptor,
|
||||
configured map[string]struct{},
|
||||
extractions map[string]struct{},
|
||||
) error {
|
||||
switch descriptor.Source.Kind {
|
||||
case SourceKindConfiguredArtifact, SourceKindPreviousArtifact:
|
||||
if _, ok := configured[descriptor.Source.ConfiguredKey]; !ok {
|
||||
return &UnknownConfiguredArtifactError{ConfiguredKey: descriptor.Source.ConfiguredKey}
|
||||
}
|
||||
case SourceKindExtraction:
|
||||
if _, ok := extractions[descriptor.Source.ConfiguredKey]; !ok {
|
||||
return &UnknownExtractionArtifactError{ConfiguredKey: descriptor.Source.ConfiguredKey}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidatePublishSource validates that a source is publish-compatible and references a known configured artifact.
|
||||
func ValidatePublishSource(source string, configured map[string]string) (Source, error) {
|
||||
return ValidatePublishSourceWithExtractions(source, configured, nil)
|
||||
}
|
||||
|
||||
// ValidatePublishSourceWithExtractions validates publish sources against the
|
||||
// configured Scriptorium artifacts and extraction outputs.
|
||||
func ValidatePublishSourceWithExtractions(
|
||||
source string,
|
||||
configured map[string]string,
|
||||
extractions map[string]struct{},
|
||||
) (Source, error) {
|
||||
classified, err := ClassifySource(source)
|
||||
if err != nil {
|
||||
return Source{}, fmt.Errorf("must be a built-in source id or narratio.artifact.<name>")
|
||||
return Source{}, fmt.Errorf("must be a built-in source id, narratio.artifact.<name>, or configured narratio.extraction.<name>")
|
||||
}
|
||||
if classified.Kind == SourceKindPreviousArtifact {
|
||||
return Source{}, fmt.Errorf("must be a built-in source id or narratio.artifact.<name>")
|
||||
return Source{}, fmt.Errorf("must be a built-in source id, narratio.artifact.<name>, or configured narratio.extraction.<name>")
|
||||
}
|
||||
if classified.Kind == SourceKindConfiguredArtifact {
|
||||
if configured == nil {
|
||||
@@ -213,6 +272,11 @@ func ValidatePublishSource(source string, configured map[string]string) (Source,
|
||||
return Source{}, fmt.Errorf("configured artifact %q is not defined in pipeline.scriptorium.artifacts", classified.ConfiguredKey)
|
||||
}
|
||||
}
|
||||
if classified.Kind == SourceKindExtraction {
|
||||
if _, ok := extractions[classified.ConfiguredKey]; !ok {
|
||||
return Source{}, fmt.Errorf("extraction output %q is not defined in pipeline.notarius.outputs", classified.ConfiguredKey)
|
||||
}
|
||||
}
|
||||
return classified, nil
|
||||
}
|
||||
|
||||
@@ -247,7 +311,17 @@ func DeriveDefaultPublishedDestination(source Source, configured map[string]stri
|
||||
// ResolvePublishedDestination validates and normalizes an explicit destination,
|
||||
// or derives one when omitted.
|
||||
func ResolvePublishedDestination(sourceID, explicitDest string, configured map[string]string) (string, error) {
|
||||
source, err := ValidatePublishSource(sourceID, configured)
|
||||
return ResolvePublishedDestinationWithExtractions(sourceID, explicitDest, configured, nil)
|
||||
}
|
||||
|
||||
// ResolvePublishedDestinationWithExtractions validates and normalizes a destination
|
||||
// while accepting extraction sources declared by the effective Notarius configuration.
|
||||
func ResolvePublishedDestinationWithExtractions(
|
||||
sourceID, explicitDest string,
|
||||
configured map[string]string,
|
||||
extractions map[string]struct{},
|
||||
) (string, error) {
|
||||
source, err := ValidatePublishSourceWithExtractions(sourceID, configured, extractions)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ func TestClassifySource(t *testing.T) {
|
||||
{name: "built in transcript", source: "narratio.transcript.final_trimmed", wantKind: SourceKindBuiltIn},
|
||||
{name: "built in bounds", source: "narratio.bounds.session", wantKind: SourceKindBuiltIn},
|
||||
{name: "configured artifact", source: "narratio.artifact.session_recap", wantKind: SourceKindConfiguredArtifact, wantKey: "session_recap"},
|
||||
{name: "extraction", source: "narratio.extraction.npc_registry", wantKind: SourceKindExtraction, wantKey: "npc_registry"},
|
||||
{name: "previous session configured", source: "narratio.previous_session.artifact.session_recap", wantKind: SourceKindPreviousArtifact, wantKey: "session_recap"},
|
||||
{name: "unsupported", source: "narratio.unknown", wantErrLike: "unsupported artifact source"},
|
||||
}
|
||||
@@ -56,6 +57,60 @@ func TestValidatePublishSource(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractionSourcePolicy(t *testing.T) {
|
||||
if got := ExtractionSourceID(" npc_registry "); got != "narratio.extraction.npc_registry" {
|
||||
t.Fatalf("ExtractionSourceID() = %q, want narratio.extraction.npc_registry", got)
|
||||
}
|
||||
if key, ok := ParseExtractionSource(" narratio.extraction.npc_registry "); !ok || key != "npc_registry" {
|
||||
t.Fatalf("ParseExtractionSource() = %q, %t; want npc_registry, true", key, ok)
|
||||
}
|
||||
for _, source := range []string{
|
||||
"narratio.extraction.",
|
||||
"narratio.extraction.NPC",
|
||||
"narratio.extraction.npc-registry",
|
||||
"narratio.extraction.npc_registry.extra",
|
||||
} {
|
||||
if _, ok := ParseExtractionSource(source); ok {
|
||||
t.Fatalf("ParseExtractionSource(%q) unexpectedly matched", source)
|
||||
}
|
||||
}
|
||||
|
||||
descriptor, err := DescribeScriptoriumInputSource("narratio.extraction.npc_registry")
|
||||
if err != nil {
|
||||
t.Fatalf("DescribeScriptoriumInputSource(extraction) error = %v", err)
|
||||
}
|
||||
if descriptor.Source.Kind != SourceKindExtraction || descriptor.Source.ConfiguredKey != "npc_registry" {
|
||||
t.Fatalf("extraction descriptor = %#v", descriptor)
|
||||
}
|
||||
declared := map[string]struct{}{"npc_registry": {}}
|
||||
if err := ValidateInputReference(descriptor, nil, declared); err != nil {
|
||||
t.Fatalf("ValidateInputReference(declared extraction) error = %v", err)
|
||||
}
|
||||
if err := ValidateInputReference(descriptor, nil, nil); err == nil {
|
||||
t.Fatal("ValidateInputReference(unknown extraction) error = nil, want error")
|
||||
}
|
||||
|
||||
if _, err := ValidatePublishSourceWithExtractions("narratio.extraction.npc_registry", nil, declared); err != nil {
|
||||
t.Fatalf("ValidatePublishSourceWithExtractions(declared) error = %v", err)
|
||||
}
|
||||
if _, err := ValidatePublishSourceWithExtractions("narratio.extraction.unknown", nil, declared); err == nil {
|
||||
t.Fatal("ValidatePublishSourceWithExtractions(unknown) error = nil, want error")
|
||||
}
|
||||
|
||||
identities := map[string]struct{}{}
|
||||
for _, sourceID := range []string{
|
||||
ExtractionSourceID("npc_registry"),
|
||||
ConfiguredSourceID("npc_registry"),
|
||||
PreviousSessionSourceID("npc_registry"),
|
||||
SourceBoundsSession,
|
||||
} {
|
||||
if _, exists := identities[sourceID]; exists {
|
||||
t.Fatalf("source identity collision at %q", sourceID)
|
||||
}
|
||||
identities[sourceID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePublishedDestination(t *testing.T) {
|
||||
configured := map[string]string{"session_recap": "artifacts/session_recap.md"}
|
||||
|
||||
|
||||
@@ -144,6 +144,22 @@ func ConfiguredArtifactName(source string) (string, bool) {
|
||||
return artifactpolicy.ParseConfiguredSource(source)
|
||||
}
|
||||
|
||||
// ExtractionArtifactSourceID returns narratio.extraction.<name> for a configured key.
|
||||
func ExtractionArtifactSourceID(key string) string {
|
||||
return artifactpolicy.ExtractionSourceID(key)
|
||||
}
|
||||
|
||||
// IsExtractionArtifactSource reports whether source is narratio.extraction.<name>.
|
||||
func IsExtractionArtifactSource(source string) bool {
|
||||
_, ok := artifactpolicy.ParseExtractionSource(source)
|
||||
return ok
|
||||
}
|
||||
|
||||
// ExtractionArtifactName extracts <name> from narratio.extraction.<name>.
|
||||
func ExtractionArtifactName(source string) (string, bool) {
|
||||
return artifactpolicy.ParseExtractionSource(source)
|
||||
}
|
||||
|
||||
// IsPreviousSessionArtifactSource returns true when source is narratio.previous_session.artifact.<name>.
|
||||
func IsPreviousSessionArtifactSource(source string) bool {
|
||||
_, ok := artifactpolicy.ParsePreviousSessionSource(source)
|
||||
@@ -204,17 +220,19 @@ func ResolveSessionArtifact(paths SessionPaths, m *manifest.Manifest, source str
|
||||
}
|
||||
|
||||
// ResolveSessionArtifactWithCatalog resolves built-in sources using existing rules and resolves
|
||||
// configured narratio.artifact.<name> sources through runtime catalog availability.
|
||||
// configured artifact and extraction sources through runtime catalog availability.
|
||||
func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest, source string, catalog *ArtifactCatalog) (ResolvedSessionArtifact, error) {
|
||||
normalized := strings.TrimSpace(source)
|
||||
if IsPreviousSessionArtifactSource(normalized) {
|
||||
return ResolvePreviousSessionArtifactWithCatalog(paths, m, normalized, catalog)
|
||||
}
|
||||
if !IsConfiguredArtifactSource(normalized) {
|
||||
configuredSource := IsConfiguredArtifactSource(normalized)
|
||||
extractionSource := IsExtractionArtifactSource(normalized)
|
||||
if !configuredSource && !extractionSource {
|
||||
return ResolveSessionArtifact(paths, m, normalized)
|
||||
}
|
||||
if catalog == nil {
|
||||
return ResolvedSessionArtifact{}, fmt.Errorf("configured artifact source %q requires runtime artifact catalog", source)
|
||||
return ResolvedSessionArtifact{}, fmt.Errorf("catalog-backed artifact source %q requires runtime artifact catalog", source)
|
||||
}
|
||||
entry, ok := catalog.Lookup(normalized)
|
||||
if !ok {
|
||||
@@ -223,7 +241,11 @@ func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest,
|
||||
if !entry.Available {
|
||||
return ResolvedSessionArtifact{}, &SessionArtifactNotFoundError{ArtifactID: normalized}
|
||||
}
|
||||
if err := validateResolvedContent(entry.Path, contentText); err != nil {
|
||||
contentKind := contentText
|
||||
if extractionSource {
|
||||
contentKind = contentJSON
|
||||
}
|
||||
if err := validateResolvedContent(entry.Path, contentKind); err != nil {
|
||||
return ResolvedSessionArtifact{}, fmt.Errorf("validate %q: %w", normalized, err)
|
||||
}
|
||||
return ResolvedSessionArtifact{
|
||||
@@ -231,6 +253,7 @@ func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest,
|
||||
Path: filepath.Clean(entry.Path),
|
||||
ProducerStage: entry.ProducerStage,
|
||||
OutputKind: entry.OutputKind,
|
||||
ProducerRunID: entry.ProducerRunID,
|
||||
Provenance: entry.Provenance,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -150,6 +150,23 @@ func TestPreviousSessionArtifactSourceHelpers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractionArtifactSourceHelpers(t *testing.T) {
|
||||
if got := ExtractionArtifactSourceID("npc_registry"); got != "narratio.extraction.npc_registry" {
|
||||
t.Fatalf("ExtractionArtifactSourceID() = %q", got)
|
||||
}
|
||||
if !IsExtractionArtifactSource(" narratio.extraction.npc_registry ") {
|
||||
t.Fatal("IsExtractionArtifactSource(valid) = false")
|
||||
}
|
||||
if key, ok := ExtractionArtifactName("narratio.extraction.npc_registry"); !ok || key != "npc_registry" {
|
||||
t.Fatalf("ExtractionArtifactName() = %q, %t", key, ok)
|
||||
}
|
||||
for _, source := range []string{"narratio.extraction.", "narratio.extraction.npc-registry", "narratio.artifact.npc_registry"} {
|
||||
if IsExtractionArtifactSource(source) {
|
||||
t.Fatalf("IsExtractionArtifactSource(%q) = true, want false", source)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveSessionArtifactPrefersManifestOutput(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
paths := buildSessionPaths(workspace, "campaign", "session")
|
||||
|
||||
@@ -6,11 +6,13 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
)
|
||||
|
||||
const (
|
||||
ArtifactProvenanceGeneratedCurrentAnalyzeRun = "generated.current_analyze_run"
|
||||
ArtifactProvenanceDisabledFromDisk = "filesystem.disabled_artifact_output"
|
||||
ArtifactProvenanceCurrentExtractManifest = "manifest.current_extract_run"
|
||||
)
|
||||
|
||||
// ConfiguredArtifactDefinition describes one configured analyze artifact.
|
||||
@@ -19,10 +21,40 @@ type ConfiguredArtifactDefinition struct {
|
||||
OutputPath string
|
||||
}
|
||||
|
||||
// ExtractionArtifactDefinition describes one configured Notarius output lane.
|
||||
type ExtractionArtifactDefinition struct {
|
||||
LaneID string
|
||||
PipelineID string
|
||||
MediaType string
|
||||
SchemaID string
|
||||
SchemaVersion string
|
||||
ModuleKey string
|
||||
}
|
||||
|
||||
// ExtractionDefinitionsFromConfig converts the effective Notarius output map into catalog definitions.
|
||||
func ExtractionDefinitionsFromConfig(cfg *config.NotariusConfig) map[string]ExtractionArtifactDefinition {
|
||||
if cfg == nil || len(cfg.Outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
definitions := make(map[string]ExtractionArtifactDefinition, len(cfg.Outputs))
|
||||
for key, output := range cfg.Outputs {
|
||||
definitions[key] = ExtractionArtifactDefinition{
|
||||
LaneID: output.LaneID,
|
||||
PipelineID: cfg.PipelineID,
|
||||
MediaType: output.MediaType,
|
||||
SchemaID: output.SchemaID,
|
||||
SchemaVersion: output.SchemaVersion,
|
||||
ModuleKey: output.ModuleKey,
|
||||
}
|
||||
}
|
||||
return definitions
|
||||
}
|
||||
|
||||
// CatalogEntry is one runtime catalog entry resolved by source ID.
|
||||
type CatalogEntry struct {
|
||||
SourceID string
|
||||
ConfiguredKey string
|
||||
ExtractionKey string
|
||||
CanonicalRelPath string
|
||||
ProducerStage string
|
||||
OutputKind string
|
||||
@@ -31,12 +63,14 @@ type CatalogEntry struct {
|
||||
Available bool
|
||||
Path string
|
||||
Provenance string
|
||||
ProducerRunID string
|
||||
}
|
||||
|
||||
// ArtifactCatalog tracks built-in and configured artifact definitions and runtime state.
|
||||
// ArtifactCatalog tracks built-in, configured, and extraction artifact definitions and runtime state.
|
||||
type ArtifactCatalog struct {
|
||||
entries map[string]CatalogEntry
|
||||
configuredIndex map[string]string
|
||||
extractionIndex map[string]string
|
||||
}
|
||||
|
||||
// NewArtifactCatalog returns an empty runtime artifact catalog.
|
||||
@@ -44,9 +78,41 @@ func NewArtifactCatalog() *ArtifactCatalog {
|
||||
return &ArtifactCatalog{
|
||||
entries: map[string]CatalogEntry{},
|
||||
configuredIndex: map[string]string{},
|
||||
extractionIndex: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterExtractionArtifacts registers the configured Notarius output lanes.
|
||||
func (c *ArtifactCatalog) RegisterExtractionArtifacts(configured map[string]ExtractionArtifactDefinition) error {
|
||||
keys := make([]string, 0, len(configured))
|
||||
for key := range configured {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
for _, key := range keys {
|
||||
trimmed := strings.TrimSpace(key)
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("extraction artifact keys must be non-empty")
|
||||
}
|
||||
if _, exists := c.extractionIndex[trimmed]; exists {
|
||||
return fmt.Errorf("duplicate extraction artifact key %q", trimmed)
|
||||
}
|
||||
sourceID := ExtractionArtifactSourceID(trimmed)
|
||||
if err := c.addEntry(CatalogEntry{
|
||||
SourceID: sourceID,
|
||||
ExtractionKey: trimmed,
|
||||
ProducerStage: "extract",
|
||||
OutputKind: "notarius_lane",
|
||||
Planned: true,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("register extraction artifact %q: %w", trimmed, err)
|
||||
}
|
||||
c.extractionIndex[trimmed] = sourceID
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ConfiguredArtifactSourceID converts a configured artifact key into canonical source ID.
|
||||
func ConfiguredArtifactSourceID(key string) string {
|
||||
return artifactpolicy.ConfiguredSourceID(key)
|
||||
@@ -151,6 +217,15 @@ func (c *ArtifactCatalog) SourceIDForConfiguredKey(key string) (string, bool) {
|
||||
return sourceID, ok
|
||||
}
|
||||
|
||||
// SourceIDForExtractionKey returns the canonical source ID for one Notarius output key.
|
||||
func (c *ArtifactCatalog) SourceIDForExtractionKey(key string) (string, bool) {
|
||||
if c == nil {
|
||||
return "", false
|
||||
}
|
||||
sourceID, ok := c.extractionIndex[strings.TrimSpace(key)]
|
||||
return sourceID, ok
|
||||
}
|
||||
|
||||
// ListConfigured returns configured entries sorted by configured key.
|
||||
func (c *ArtifactCatalog) ListConfigured() []CatalogEntry {
|
||||
if c == nil || len(c.configuredIndex) == 0 {
|
||||
@@ -169,6 +244,23 @@ func (c *ArtifactCatalog) ListConfigured() []CatalogEntry {
|
||||
return out
|
||||
}
|
||||
|
||||
// ListExtraction returns extraction entries sorted by configured output key.
|
||||
func (c *ArtifactCatalog) ListExtraction() []CatalogEntry {
|
||||
if c == nil || len(c.extractionIndex) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(c.extractionIndex))
|
||||
for key := range c.extractionIndex {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := make([]CatalogEntry, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
out = append(out, c.entries[c.extractionIndex[key]])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// MarkAvailableGenerated marks one source as available in current analyze execution.
|
||||
func (c *ArtifactCatalog) MarkAvailableGenerated(sourceID, path string) error {
|
||||
return c.markAvailable(sourceID, path, ArtifactProvenanceGeneratedCurrentAnalyzeRun)
|
||||
@@ -179,6 +271,16 @@ func (c *ArtifactCatalog) MarkAvailableFromDisk(sourceID, path string) error {
|
||||
return c.markAvailable(sourceID, path, ArtifactProvenanceDisabledFromDisk)
|
||||
}
|
||||
|
||||
func (c *ArtifactCatalog) markAvailableFromExtractManifest(sourceID, path, producerRunID string) error {
|
||||
if err := c.markAvailable(sourceID, path, ArtifactProvenanceCurrentExtractManifest); err != nil {
|
||||
return err
|
||||
}
|
||||
entry := c.entries[strings.TrimSpace(sourceID)]
|
||||
entry.ProducerRunID = strings.TrimSpace(producerRunID)
|
||||
c.entries[entry.SourceID] = entry
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ArtifactCatalog) markAvailable(sourceID, path, provenance string) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("artifact catalog is nil")
|
||||
|
||||
180
internal/artifacts/extraction_catalog.go
Normal file
180
internal/artifacts/extraction_catalog.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
const (
|
||||
extractStageName = "extract"
|
||||
extractionLaneKind = "notarius_lane"
|
||||
extractionIndexKind = "notarius_index"
|
||||
extractionMetadataRun = "narratio_run_id"
|
||||
extractionMetadataRoot = "bundle_root"
|
||||
)
|
||||
|
||||
type hydratedExtraction struct {
|
||||
sourceID string
|
||||
path string
|
||||
}
|
||||
|
||||
// HydrateExtractionArtifacts marks extraction sources available only when the current
|
||||
// manifest contains one complete, internally consistent, succeeded extraction bundle.
|
||||
// Invalid, stale, incomplete, or unsafe records leave every extraction source unavailable.
|
||||
func (c *ArtifactCatalog) HydrateExtractionArtifacts(
|
||||
paths SessionPaths,
|
||||
m *manifest.Manifest,
|
||||
configured map[string]ExtractionArtifactDefinition,
|
||||
) {
|
||||
if c == nil || m == nil || len(configured) == 0 {
|
||||
return
|
||||
}
|
||||
record := m.Stages[extractStageName]
|
||||
if record == nil || record.Name != extractStageName || record.Status != manifest.StatusSucceeded {
|
||||
return
|
||||
}
|
||||
producerRunID := extractionMetadataString(record.Metadata, extractionMetadataRun)
|
||||
if !safeExtractionPathSegment(producerRunID) {
|
||||
return
|
||||
}
|
||||
bundleRoot := filepath.Clean(filepath.Join(paths.ArtifactsDir, "notarius", producerRunID))
|
||||
if !filepath.IsAbs(bundleRoot) || extractionMetadataString(record.Metadata, extractionMetadataRoot) != bundleRoot {
|
||||
return
|
||||
}
|
||||
if !safeExistingExtractionDirectory(paths.Root, bundleRoot) {
|
||||
return
|
||||
}
|
||||
receiptRunID, receiptPipelineID := extractionReceiptIdentity(record.Metadata)
|
||||
if receiptRunID == "" || receiptPipelineID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
expected := make(map[string]ExtractionArtifactDefinition, len(configured))
|
||||
for key, definition := range configured {
|
||||
sourceID, ok := c.SourceIDForExtractionKey(key)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
expected[sourceID] = definition
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{}, len(expected))
|
||||
hydrated := make([]hydratedExtraction, 0, len(expected))
|
||||
indexSeen := false
|
||||
for _, output := range record.Outputs {
|
||||
if strings.TrimSpace(output.ProducerRunID) != producerRunID {
|
||||
return
|
||||
}
|
||||
if output.SourceID == "" {
|
||||
if indexSeen || output.Kind != extractionIndexKind || filepath.Clean(output.LocalPath) != filepath.Join(bundleRoot, "index.json") ||
|
||||
!validExtractionPayload(bundleRoot, output.LocalPath, output.Checksum) {
|
||||
return
|
||||
}
|
||||
indexSeen = true
|
||||
continue
|
||||
}
|
||||
|
||||
definition, ok := expected[output.SourceID]
|
||||
if !ok || output.Kind != extractionLaneKind {
|
||||
return
|
||||
}
|
||||
if _, duplicate := seen[output.SourceID]; duplicate {
|
||||
return
|
||||
}
|
||||
if !compatibleCatalogExtractionContract(output.Contract, definition) ||
|
||||
!compatibleCatalogExtractionProvenance(output.ExternalProvenance, receiptRunID, receiptPipelineID, definition) ||
|
||||
!validExtractionPayload(bundleRoot, output.LocalPath, output.Checksum) {
|
||||
return
|
||||
}
|
||||
seen[output.SourceID] = struct{}{}
|
||||
hydrated = append(hydrated, hydratedExtraction{sourceID: output.SourceID, path: output.LocalPath})
|
||||
}
|
||||
if !indexSeen || len(seen) != len(expected) || len(record.Outputs) != len(expected)+1 {
|
||||
return
|
||||
}
|
||||
for _, item := range hydrated {
|
||||
_ = c.markAvailableFromExtractManifest(item.sourceID, item.path, producerRunID)
|
||||
}
|
||||
}
|
||||
|
||||
func compatibleCatalogExtractionContract(got *artifactmodel.ContractMetadata, want ExtractionArtifactDefinition) bool {
|
||||
return got != nil && got.MediaType == want.MediaType && got.SchemaID == want.SchemaID &&
|
||||
got.SchemaVersion == want.SchemaVersion && (want.ModuleKey == "" || got.ModuleKey == want.ModuleKey)
|
||||
}
|
||||
|
||||
func compatibleCatalogExtractionProvenance(
|
||||
got *artifactmodel.ExternalProvenance,
|
||||
runID, pipelineID string,
|
||||
want ExtractionArtifactDefinition,
|
||||
) bool {
|
||||
return got != nil && got.System == "notarius" && got.RunID == runID && got.PipelineID == pipelineID &&
|
||||
pipelineID == strings.TrimSpace(want.PipelineID) && got.ArtifactID == want.LaneID
|
||||
}
|
||||
|
||||
func validExtractionPayload(bundleRoot, path, checksum string) bool {
|
||||
if !filepath.IsAbs(path) || !pathWithinExtractionRoot(bundleRoot, path) || strings.TrimSpace(checksum) == "" {
|
||||
return false
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return false
|
||||
}
|
||||
if !safeExtractionComponents(bundleRoot, path) {
|
||||
return false
|
||||
}
|
||||
actual, err := SHA256File(path)
|
||||
if err != nil || actual != checksum {
|
||||
return false
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
return err == nil && json.Valid(body)
|
||||
}
|
||||
|
||||
func safeExistingExtractionDirectory(sessionRoot, bundleRoot string) bool {
|
||||
if !pathWithinExtractionRoot(sessionRoot, bundleRoot) || !safeExtractionComponents(sessionRoot, bundleRoot) {
|
||||
return false
|
||||
}
|
||||
info, err := os.Lstat(bundleRoot)
|
||||
return err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0
|
||||
}
|
||||
|
||||
func safeExtractionComponents(root, target string) bool {
|
||||
relative, err := filepath.Rel(filepath.Clean(root), filepath.Clean(target))
|
||||
if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
return false
|
||||
}
|
||||
current := filepath.Clean(root)
|
||||
for _, part := range strings.Split(relative, string(filepath.Separator)) {
|
||||
current = filepath.Join(current, part)
|
||||
info, err := os.Lstat(current)
|
||||
if err != nil || info.Mode()&os.ModeSymlink != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func pathWithinExtractionRoot(root, target string) bool {
|
||||
relative, err := filepath.Rel(filepath.Clean(root), filepath.Clean(target))
|
||||
return err == nil && relative != "." && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
|
||||
}
|
||||
|
||||
func safeExtractionPathSegment(value string) bool {
|
||||
return value != "" && value != "." && value != ".." && filepath.Base(value) == value &&
|
||||
!strings.ContainsAny(value, `/\\`)
|
||||
}
|
||||
|
||||
func extractionMetadataString(metadata map[string]any, key string) string {
|
||||
value, _ := metadata[key].(string)
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func extractionReceiptIdentity(metadata map[string]any) (string, string) {
|
||||
receipt, _ := metadata["receipt"].(map[string]any)
|
||||
return extractionMetadataString(receipt, "run_id"), extractionMetadataString(receipt, "pipeline_id")
|
||||
}
|
||||
222
internal/artifacts/extraction_catalog_test.go
Normal file
222
internal/artifacts/extraction_catalog_test.go
Normal file
@@ -0,0 +1,222 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func TestArtifactCatalogRegistersExtractionArtifactsDeterministically(t *testing.T) {
|
||||
catalog := NewArtifactCatalog()
|
||||
if err := catalog.RegisterConfiguredArtifacts(map[string]ConfiguredArtifactDefinition{
|
||||
"summary": {Enabled: true},
|
||||
}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := catalog.RegisterExtractionArtifacts(map[string]ExtractionArtifactDefinition{
|
||||
"zeta": {LaneID: "zeta"},
|
||||
"summary": {LaneID: "summary"},
|
||||
"alpha": {LaneID: "alpha"},
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterExtractionArtifacts() error = %v", err)
|
||||
}
|
||||
|
||||
entries := catalog.ListExtraction()
|
||||
if len(entries) != 3 || entries[0].ExtractionKey != "alpha" || entries[1].ExtractionKey != "summary" || entries[2].ExtractionKey != "zeta" {
|
||||
t.Fatalf("ListExtraction() = %#v, want alpha, summary, zeta", entries)
|
||||
}
|
||||
extractionID, ok := catalog.SourceIDForExtractionKey("summary")
|
||||
if !ok || extractionID != "narratio.extraction.summary" {
|
||||
t.Fatalf("SourceIDForExtractionKey(summary) = %q, %v", extractionID, ok)
|
||||
}
|
||||
configuredID, _ := catalog.SourceIDForConfiguredKey("summary")
|
||||
if configuredID == extractionID {
|
||||
t.Fatalf("configured and extraction source families collided at %q", extractionID)
|
||||
}
|
||||
entry, ok := catalog.Lookup(extractionID)
|
||||
if !ok || entry.ProducerStage != "extract" || entry.OutputKind != "notarius_lane" || !entry.Planned {
|
||||
t.Fatalf("Lookup(%q) = %#v, %v", extractionID, entry, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactCatalogRejectsDuplicateExtractionRegistration(t *testing.T) {
|
||||
catalog := NewArtifactCatalog()
|
||||
definitions := map[string]ExtractionArtifactDefinition{"summary": {LaneID: "summary"}}
|
||||
if err := catalog.RegisterExtractionArtifacts(definitions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := catalog.RegisterExtractionArtifacts(definitions); err == nil {
|
||||
t.Fatal("second RegisterExtractionArtifacts() error = nil, want collision error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHydrateExtractionArtifactsAcceptsOnlyCompleteCurrentBundle(t *testing.T) {
|
||||
paths, currentManifest, definitions := validExtractionCatalogFixture(t)
|
||||
catalog := registeredExtractionCatalog(t, definitions)
|
||||
catalog.HydrateExtractionArtifacts(paths, currentManifest, definitions)
|
||||
|
||||
entry, ok := catalog.Lookup(ExtractionArtifactSourceID("encounters"))
|
||||
if !ok || !entry.Available {
|
||||
t.Fatalf("hydrated entry = %#v, %v; want available", entry, ok)
|
||||
}
|
||||
if entry.Provenance != ArtifactProvenanceCurrentExtractManifest || entry.ProducerRunID != "extract-run-1" {
|
||||
t.Fatalf("hydrated provenance = %#v", entry)
|
||||
}
|
||||
resolved, err := ResolveSessionArtifactWithCatalog(paths, currentManifest, entry.SourceID, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveSessionArtifactWithCatalog() error = %v", err)
|
||||
}
|
||||
if resolved.Path != entry.Path || resolved.ProducerRunID != "extract-run-1" {
|
||||
t.Fatalf("resolved = %#v", resolved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHydrateExtractionArtifactsRejectsUntrustedManifestState(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(t *testing.T, paths SessionPaths, m *manifest.Manifest)
|
||||
}{
|
||||
{name: "missing stage", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) { delete(m.Stages, "extract") }},
|
||||
{name: "skipped", mutate: setExtractionStatus(manifest.StatusSkipped)},
|
||||
{name: "failed", mutate: setExtractionStatus(manifest.StatusFailed)},
|
||||
{name: "stale", mutate: setExtractionStatus(manifest.StatusStale)},
|
||||
{name: "interrupted", mutate: setExtractionStatus(manifest.StatusInterrupted)},
|
||||
{name: "missing source", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs = m.Stages["extract"].Outputs[1:]
|
||||
}},
|
||||
{name: "inconsistent producer identity", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].ProducerRunID = "another-run"
|
||||
}},
|
||||
{name: "incompatible contract", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].Contract.SchemaVersion = "99"
|
||||
}},
|
||||
{name: "incompatible provenance", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].ExternalProvenance.RunID = "another-notarius-run"
|
||||
}},
|
||||
{name: "missing file", mutate: func(t *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "tampered checksum", mutate: func(t *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
if err := os.WriteFile(m.Stages["extract"].Outputs[0].LocalPath, []byte(`{"tampered":true}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "unsafe path", mutate: func(t *testing.T, paths SessionPaths, m *manifest.Manifest) {
|
||||
outside := filepath.Join(paths.Root, "incidental.json")
|
||||
writeExtractionFixtureFile(t, outside, `{"incidental":true}`)
|
||||
m.Stages["extract"].Outputs[0].LocalPath = outside
|
||||
m.Stages["extract"].Outputs[0].Checksum = extractionFixtureChecksum(t, outside)
|
||||
}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
paths, currentManifest, definitions := validExtractionCatalogFixture(t)
|
||||
test.mutate(t, paths, currentManifest)
|
||||
catalog := registeredExtractionCatalog(t, definitions)
|
||||
catalog.HydrateExtractionArtifacts(paths, currentManifest, definitions)
|
||||
entry, _ := catalog.Lookup(ExtractionArtifactSourceID("encounters"))
|
||||
if entry.Available {
|
||||
t.Fatalf("entry became available from %s manifest", test.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveExtractionArtifactNeverDiscoversIncidentalBundleFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
paths := SessionPaths{Root: root, ArtifactsDir: filepath.Join(root, "artifacts")}
|
||||
incidental := filepath.Join(paths.ArtifactsDir, "notarius", "incidental", "lanes", "encounters.json")
|
||||
writeExtractionFixtureFile(t, incidental, `{"encounters":[]}`)
|
||||
definitions := extractionFixtureDefinitions()
|
||||
catalog := registeredExtractionCatalog(t, definitions)
|
||||
|
||||
_, err := ResolveSessionArtifactWithCatalog(paths, manifest.New("session", fixtureTime), ExtractionArtifactSourceID("encounters"), catalog)
|
||||
if err == nil || !errors.Is(err, ErrSessionArtifactNotFound) {
|
||||
t.Fatalf("ResolveSessionArtifactWithCatalog() error = %v, want not found", err)
|
||||
}
|
||||
}
|
||||
|
||||
var fixtureTime = mustFixtureTime()
|
||||
|
||||
func mustFixtureTime() (value time.Time) {
|
||||
return time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
func setExtractionStatus(status manifest.StageStatus) func(*testing.T, SessionPaths, *manifest.Manifest) {
|
||||
return func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) { m.Stages["extract"].Status = status }
|
||||
}
|
||||
|
||||
func extractionFixtureDefinitions() map[string]ExtractionArtifactDefinition {
|
||||
return map[string]ExtractionArtifactDefinition{
|
||||
"encounters": {
|
||||
LaneID: "encounters", PipelineID: "campaign.extract", MediaType: "application/json",
|
||||
SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func registeredExtractionCatalog(t *testing.T, definitions map[string]ExtractionArtifactDefinition) *ArtifactCatalog {
|
||||
t.Helper()
|
||||
catalog := NewArtifactCatalog()
|
||||
if err := catalog.RegisterExtractionArtifacts(definitions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
||||
func validExtractionCatalogFixture(t *testing.T) (SessionPaths, *manifest.Manifest, map[string]ExtractionArtifactDefinition) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
paths := SessionPaths{Root: root, ArtifactsDir: filepath.Join(root, "artifacts")}
|
||||
bundleRoot := filepath.Join(paths.ArtifactsDir, "notarius", "extract-run-1")
|
||||
lanePath := filepath.Join(bundleRoot, "lanes", "encounters.json")
|
||||
indexPath := filepath.Join(bundleRoot, "index.json")
|
||||
writeExtractionFixtureFile(t, lanePath, `{"encounters":[]}`)
|
||||
writeExtractionFixtureFile(t, indexPath, `{"lanes":[]}`)
|
||||
definitions := extractionFixtureDefinitions()
|
||||
m := manifest.New("session", fixtureTime)
|
||||
m.Stages["extract"] = &manifest.StageRecord{
|
||||
Name: "extract", Status: manifest.StatusSucceeded,
|
||||
Metadata: map[string]any{
|
||||
"narratio_run_id": "extract-run-1", "bundle_root": bundleRoot,
|
||||
"receipt": map[string]any{"run_id": "notarius-run-1", "pipeline_id": "campaign.extract"},
|
||||
},
|
||||
Outputs: []manifest.ArtifactRecord{
|
||||
{
|
||||
Kind: "notarius_lane", SourceID: ExtractionArtifactSourceID("encounters"), LocalPath: lanePath,
|
||||
ProducerRunID: "extract-run-1", Checksum: extractionFixtureChecksum(t, lanePath),
|
||||
Contract: &artifactmodel.ContractMetadata{MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters"},
|
||||
ExternalProvenance: &artifactmodel.ExternalProvenance{System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: "encounters"},
|
||||
},
|
||||
{Kind: "notarius_index", LocalPath: indexPath, ProducerRunID: "extract-run-1", Checksum: extractionFixtureChecksum(t, indexPath)},
|
||||
},
|
||||
}
|
||||
return paths, m, definitions
|
||||
}
|
||||
|
||||
func writeExtractionFixtureFile(t *testing.T, path, body string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionFixtureChecksum(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
checksum, err := SHA256File(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return checksum
|
||||
}
|
||||
@@ -86,6 +86,36 @@ func SessionRunStageDirForCampaign(rootDir, campaign, sessionID, runID, stageNam
|
||||
return filepath.Join(SessionRunRootForCampaign(rootDir, campaign, sessionID, runID), stageName)
|
||||
}
|
||||
|
||||
// SessionRunExtractDirForCampaign returns the invocation-local extraction directory.
|
||||
func SessionRunExtractDirForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||
return SessionRunStageDirForCampaign(rootDir, campaign, sessionID, runID, "extract")
|
||||
}
|
||||
|
||||
// SessionRunNotariusReceiptPathForCampaign returns the invocation-local receipt path.
|
||||
func SessionRunNotariusReceiptPathForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||
return filepath.Join(SessionRunExtractDirForCampaign(rootDir, campaign, sessionID, runID), "notarius.receipt.json")
|
||||
}
|
||||
|
||||
// SessionRunNotariusLogPathForCampaign returns the invocation-local stderr log path.
|
||||
func SessionRunNotariusLogPathForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||
return filepath.Join(SessionRunExtractDirForCampaign(rootDir, campaign, sessionID, runID), "notarius.stderr.log")
|
||||
}
|
||||
|
||||
// SessionRunNotariusOutputRootForCampaign returns the invocation-local Notarius output root.
|
||||
func SessionRunNotariusOutputRootForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||
return filepath.Join(SessionRunExtractDirForCampaign(rootDir, campaign, sessionID, runID), "notarius-output")
|
||||
}
|
||||
|
||||
// SessionNotariusBundleDirForCampaign returns one immutable durable bundle destination.
|
||||
func SessionNotariusBundleDirForCampaign(rootDir, campaign, sessionID, runID string) string {
|
||||
return filepath.Join(
|
||||
SessionWorkDirForCampaign(rootDir, campaign, sessionID),
|
||||
config.PathArtifactsDirSegment,
|
||||
"notarius",
|
||||
runID,
|
||||
)
|
||||
}
|
||||
|
||||
// SessionSpoolAudioDir returns the campaign/session/run scoped local spool audio path.
|
||||
func SessionSpoolAudioDir(spoolRoot, campaign, sessionID, runID string) string {
|
||||
return filepath.Join(spoolRoot, campaign, sessionID, runID, config.PathAudioDirSegment)
|
||||
|
||||
@@ -49,6 +49,33 @@ func TestSessionRunManifestPathForCampaign(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionNotariusPathsForCampaign(t *testing.T) {
|
||||
root := "/tmp/workspace"
|
||||
campaign := "forsaken"
|
||||
sessionID := "2026-04-19"
|
||||
runID := "20260515T031522Z-a1b2c3d4"
|
||||
extractDir := filepath.Join(root, "work", campaign, sessionID, "runs", runID, "extract")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
got string
|
||||
want string
|
||||
}{
|
||||
{name: "extract directory", got: SessionRunExtractDirForCampaign(root, campaign, sessionID, runID), want: extractDir},
|
||||
{name: "receipt", got: SessionRunNotariusReceiptPathForCampaign(root, campaign, sessionID, runID), want: filepath.Join(extractDir, "notarius.receipt.json")},
|
||||
{name: "stderr", got: SessionRunNotariusLogPathForCampaign(root, campaign, sessionID, runID), want: filepath.Join(extractDir, "notarius.stderr.log")},
|
||||
{name: "output root", got: SessionRunNotariusOutputRootForCampaign(root, campaign, sessionID, runID), want: filepath.Join(extractDir, "notarius-output")},
|
||||
{name: "durable bundle", got: SessionNotariusBundleDirForCampaign(root, campaign, sessionID, runID), want: filepath.Join(root, "work", campaign, sessionID, "artifacts", "notarius", runID)},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if test.got != test.want {
|
||||
t.Fatalf("path = %q, want %q", test.got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionPreviousPathsForCampaign(t *testing.T) {
|
||||
root := "/tmp/workspace"
|
||||
previousDir := SessionPreviousDirForCampaign(root, "forsaken", "2026-04-19")
|
||||
|
||||
@@ -1,16 +1,23 @@
|
||||
package artifacts
|
||||
|
||||
import "os"
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
)
|
||||
|
||||
// Ref identifies a pipeline artifact and its local/remote coordinates.
|
||||
type Ref struct {
|
||||
Kind string
|
||||
Category string
|
||||
SessionID string
|
||||
RelativePath string
|
||||
AbsolutePath string
|
||||
RemoteKey string
|
||||
Checksum string
|
||||
Kind string
|
||||
SourceID string
|
||||
Category string
|
||||
SessionID string
|
||||
RelativePath string
|
||||
AbsolutePath string
|
||||
RemoteKey string
|
||||
Checksum string
|
||||
Contract *artifactmodel.ContractMetadata
|
||||
ExternalProvenance *artifactmodel.ExternalProvenance
|
||||
}
|
||||
|
||||
// Store is the local artifact/workdir abstraction used by orchestration code.
|
||||
|
||||
@@ -30,6 +30,7 @@ type PipelineConfig struct {
|
||||
Trim *TrimConfig `yaml:"trim"`
|
||||
Render *RenderConfig `yaml:"render"`
|
||||
Scriptorium *ScriptoriumConfig `yaml:"scriptorium"`
|
||||
Notarius *NotariusConfig `yaml:"notarius"`
|
||||
Notification NotificationConfig `yaml:"notification"`
|
||||
}
|
||||
|
||||
@@ -252,6 +253,26 @@ type ScriptoriumInputConfig struct {
|
||||
Required bool `yaml:"required"`
|
||||
}
|
||||
|
||||
// NotariusConfig configures structured artifact extraction by Notarius.
|
||||
type NotariusConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Binary string `yaml:"binary"`
|
||||
ConfigPath string `yaml:"config_path"`
|
||||
PipelineID string `yaml:"pipeline_id"`
|
||||
Timeout string `yaml:"timeout"`
|
||||
WorkingDirectory string `yaml:"working_directory"`
|
||||
Outputs map[string]NotariusOutputConfig `yaml:"outputs"`
|
||||
}
|
||||
|
||||
// NotariusOutputConfig declares one required extraction lane contract.
|
||||
type NotariusOutputConfig struct {
|
||||
LaneID string `yaml:"lane_id"`
|
||||
MediaType string `yaml:"media_type"`
|
||||
SchemaID string `yaml:"schema_id"`
|
||||
SchemaVersion string `yaml:"schema_version"`
|
||||
ModuleKey string `yaml:"module_key"`
|
||||
}
|
||||
|
||||
// NotificationConfig configures notification backend settings.
|
||||
type NotificationConfig struct {
|
||||
Backend string `yaml:"backend"`
|
||||
|
||||
@@ -30,9 +30,11 @@ const (
|
||||
DefaultSeriatimCoalesceGap = 3.0
|
||||
DefaultSeriatimReport = true
|
||||
|
||||
DefaultAuditaBinary = "audita"
|
||||
DefaultAuditaTimeout = "3h"
|
||||
DefaultAuditaReport = true
|
||||
DefaultAuditaBinary = "audita"
|
||||
DefaultAuditaTimeout = "3h"
|
||||
DefaultAuditaReport = true
|
||||
DefaultNotariusBinary = "notarius"
|
||||
DefaultNotariusTimeout = "3h"
|
||||
|
||||
DefaultScriptoriumBinary = "scriptorium"
|
||||
DefaultScriptoriumTimeout = "10m"
|
||||
|
||||
@@ -19,6 +19,9 @@ func LoadPipeline(path string) (*PipelineConfig, error) {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
applyPipelineDefaults(&cfg)
|
||||
if err := resolveNotariusPaths(&cfg, path); err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
@@ -86,12 +89,17 @@ func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOpti
|
||||
|
||||
// LoadPublishLockStoreBytes loads a mutable session lock store with strict
|
||||
// field checking and source validation.
|
||||
func LoadPublishLockStoreBytes(label string, data []byte, scriptorium *ScriptoriumConfig) (*PublishLockStore, error) {
|
||||
func LoadPublishLockStoreBytes(
|
||||
label string,
|
||||
data []byte,
|
||||
scriptorium *ScriptoriumConfig,
|
||||
notarius *NotariusConfig,
|
||||
) (*PublishLockStore, error) {
|
||||
var store PublishLockStore
|
||||
if err := decodeStrictYAMLFromReader("publish lock store", label, strings.NewReader(string(data)), &store); err != nil {
|
||||
return nil, fmt.Errorf("load publish lock store: %w", err)
|
||||
}
|
||||
locks, err := ValidatePublishLockRules(store.Locks, scriptorium, "locks")
|
||||
locks, err := ValidatePublishLockRules(store.Locks, scriptorium, notarius, "locks")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load publish lock store: %w", err)
|
||||
}
|
||||
@@ -359,6 +367,7 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
|
||||
}
|
||||
applyRenderDefaults(&cfg.Render)
|
||||
applyScriptoriumDefaults(cfg.Scriptorium)
|
||||
applyNotariusDefaults(cfg.Notarius)
|
||||
}
|
||||
|
||||
func applyCampaignsDefaults(cfg *CampaignsConfig) {
|
||||
@@ -516,6 +525,57 @@ func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
|
||||
}
|
||||
}
|
||||
|
||||
func applyNotariusDefaults(cfg *NotariusConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = DefaultNotariusBinary
|
||||
}
|
||||
if cfg.Timeout == "" {
|
||||
cfg.Timeout = DefaultNotariusTimeout
|
||||
}
|
||||
}
|
||||
|
||||
func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
|
||||
if cfg == nil || cfg.Notarius == nil || !cfg.Notarius.Enabled {
|
||||
return nil
|
||||
}
|
||||
pipelineAbs, err := filepath.Abs(pipelinePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve pipeline config path %q: %w", pipelinePath, err)
|
||||
}
|
||||
baseDir := filepath.Dir(pipelineAbs)
|
||||
resolve := func(value string) (string, error) {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return "", nil
|
||||
}
|
||||
if !filepath.IsAbs(trimmed) {
|
||||
trimmed = filepath.Join(baseDir, trimmed)
|
||||
}
|
||||
return filepath.Abs(trimmed)
|
||||
}
|
||||
|
||||
resolvedConfigPath, err := resolve(cfg.Notarius.ConfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve pipeline.notarius.config_path: %w", err)
|
||||
}
|
||||
cfg.Notarius.ConfigPath = resolvedConfigPath
|
||||
if strings.TrimSpace(cfg.Notarius.WorkingDirectory) == "" {
|
||||
if resolvedConfigPath != "" {
|
||||
cfg.Notarius.WorkingDirectory = filepath.Dir(resolvedConfigPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
workingDirectory, err := resolve(cfg.Notarius.WorkingDirectory)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve pipeline.notarius.working_directory: %w", err)
|
||||
}
|
||||
cfg.Notarius.WorkingDirectory = workingDirectory
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyTrimDefaults(cfg *TrimConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
|
||||
@@ -3,6 +3,7 @@ package config
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -1049,6 +1050,11 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
||||
pipelineFile: "pipeline.full.annotated.yml",
|
||||
sessionFile: "session.local-audio.yml",
|
||||
},
|
||||
{
|
||||
name: "extraction subset pipeline with local audio session",
|
||||
pipelineFile: "pipeline.extraction-subset.yml",
|
||||
sessionFile: "session.local-audio.yml",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -1068,6 +1074,47 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintainedExtractionExamplesPreservePublishedContracts(t *testing.T) {
|
||||
examplesDir := filepath.Join("..", "..", "examples")
|
||||
full, err := LoadPipeline(filepath.Join(examplesDir, "pipeline.full.annotated.yml"))
|
||||
if err != nil {
|
||||
t.Fatalf("load full example error = %v", err)
|
||||
}
|
||||
want := map[string]NotariusOutputConfig{
|
||||
"item_registry": {LaneID: "item-registry", MediaType: "application/json", SchemaID: "notarius.dnd.item_registry", SchemaVersion: "v1", ModuleKey: "dnd/item-registry"},
|
||||
"npc_registry": {LaneID: "npc-registry", MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry", SchemaVersion: "v1", ModuleKey: "dnd/npc-registry"},
|
||||
"location_registry": {LaneID: "location-registry", MediaType: "application/json", SchemaID: "notarius.dnd.location_registry", SchemaVersion: "v1", ModuleKey: "dnd/location-registry"},
|
||||
"scene_descriptions": {LaneID: "scene-descriptions", MediaType: "application/json", SchemaID: "notarius.dnd.scene_descriptions", SchemaVersion: "v1", ModuleKey: "dnd/scene-descriptions"},
|
||||
"item_occurrences": {LaneID: "item-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.item_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/item-occurrences"},
|
||||
"spells": {LaneID: "spells", MediaType: "application/json", SchemaID: "notarius.dnd.spells", SchemaVersion: "v1", ModuleKey: "dnd/spells"},
|
||||
"combat_turns": {LaneID: "combat-turns", MediaType: "application/json", SchemaID: "notarius.dnd.combat_turns", SchemaVersion: "v1", ModuleKey: "dnd/combat-turns"},
|
||||
"npc_occurrences": {LaneID: "npc-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.npc_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/npc-occurrences"},
|
||||
"location_occurrences": {LaneID: "location-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.location_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/location-occurrences"},
|
||||
"enemy_events": {LaneID: "enemy-events", MediaType: "application/json", SchemaID: "notarius.dnd.enemy_events", SchemaVersion: "v1", ModuleKey: "dnd/enemy-events"},
|
||||
}
|
||||
if full.Notarius == nil || !reflect.DeepEqual(full.Notarius.Outputs, want) {
|
||||
t.Fatalf("full example outputs = %#v, want %#v", full.Notarius, want)
|
||||
}
|
||||
|
||||
subset, err := LoadPipeline(filepath.Join(examplesDir, "pipeline.extraction-subset.yml"))
|
||||
if err != nil {
|
||||
t.Fatalf("load subset example error = %v", err)
|
||||
}
|
||||
brief := subset.Scriptorium.Artifacts["session_brief"]
|
||||
wantSources := map[string]string{
|
||||
"npcs": "narratio.extraction.npc_registry",
|
||||
"locations": "narratio.extraction.location_registry",
|
||||
"scenes": "narratio.extraction.scene_descriptions",
|
||||
}
|
||||
gotSources := make(map[string]string, len(brief.Inputs))
|
||||
for name, input := range brief.Inputs {
|
||||
gotSources[name] = input.Source
|
||||
}
|
||||
if !reflect.DeepEqual(gotSources, wantSources) {
|
||||
t.Fatalf("subset example sources = %#v, want %#v", gotSources, wantSources)
|
||||
}
|
||||
}
|
||||
|
||||
func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, string) {
|
||||
t.Helper()
|
||||
if !strings.Contains(pipelineYAML, "\naudita:") && !strings.HasPrefix(pipelineYAML, "audita:") {
|
||||
|
||||
283
internal/config/notarius_test.go
Normal file
283
internal/config/notarius_test.go
Normal file
@@ -0,0 +1,283 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNotariusOmittedAndDisabledBehavior(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
omittedPath := filepath.Join(dir, "omitted.yml")
|
||||
if err := os.WriteFile(omittedPath, []byte(testPipelineBaseYAML), 0o644); err != nil {
|
||||
t.Fatalf("write omitted pipeline: %v", err)
|
||||
}
|
||||
omitted, err := LoadPipeline(omittedPath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPipeline(omitted) error = %v", err)
|
||||
}
|
||||
if omitted.Notarius != nil {
|
||||
t.Fatalf("Notarius = %#v, want nil when omitted", omitted.Notarius)
|
||||
}
|
||||
|
||||
disabledPath := filepath.Join(dir, "disabled.yml")
|
||||
disabledYAML := testPipelineBaseYAML + `
|
||||
notarius:
|
||||
enabled: false
|
||||
config_path: relative/notarius.yml
|
||||
`
|
||||
if err := os.WriteFile(disabledPath, []byte(disabledYAML), 0o644); err != nil {
|
||||
t.Fatalf("write disabled pipeline: %v", err)
|
||||
}
|
||||
disabled, err := LoadPipeline(disabledPath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPipeline(disabled) error = %v", err)
|
||||
}
|
||||
if disabled.Notarius == nil {
|
||||
t.Fatal("Notarius = nil, want configured disabled section")
|
||||
}
|
||||
if disabled.Notarius.Enabled {
|
||||
t.Fatal("Notarius.Enabled = true, want false")
|
||||
}
|
||||
if disabled.Notarius.Binary != DefaultNotariusBinary || disabled.Notarius.Timeout != DefaultNotariusTimeout {
|
||||
t.Fatalf("disabled defaults = %#v", disabled.Notarius)
|
||||
}
|
||||
if disabled.Notarius.ConfigPath != "relative/notarius.yml" || disabled.Notarius.WorkingDirectory != "" {
|
||||
t.Fatalf("disabled paths were resolved unexpectedly: %#v", disabled.Notarius)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotariusEnabledDefaultsAndPathResolution(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
pipelinePath := filepath.Join(dir, "deployment", "pipeline.yml")
|
||||
if err := os.MkdirAll(filepath.Dir(pipelinePath), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
pipelineYAML := testPipelineBaseYAML + `
|
||||
notarius:
|
||||
enabled: true
|
||||
config_path: notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
outputs:
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
`
|
||||
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := LoadPipeline(pipelinePath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPipeline() error = %v", err)
|
||||
}
|
||||
wantConfigPath := filepath.Join(filepath.Dir(pipelinePath), "notarius", "config.yml")
|
||||
if cfg.Notarius.Binary != "notarius" || cfg.Notarius.Timeout != "3h" {
|
||||
t.Fatalf("defaults = %#v", cfg.Notarius)
|
||||
}
|
||||
if cfg.Notarius.ConfigPath != wantConfigPath {
|
||||
t.Fatalf("config_path = %q, want %q", cfg.Notarius.ConfigPath, wantConfigPath)
|
||||
}
|
||||
if cfg.Notarius.WorkingDirectory != filepath.Dir(wantConfigPath) {
|
||||
t.Fatalf("working_directory = %q, want %q", cfg.Notarius.WorkingDirectory, filepath.Dir(wantConfigPath))
|
||||
}
|
||||
|
||||
explicitPath := filepath.Join(dir, "explicit.yml")
|
||||
explicitYAML := strings.Replace(pipelineYAML, " pipeline_id: dnd-session\n", " pipeline_id: dnd-session\n working_directory: runtime\n", 1)
|
||||
if err := os.WriteFile(explicitPath, []byte(explicitYAML), 0o644); err != nil {
|
||||
t.Fatalf("write explicit pipeline: %v", err)
|
||||
}
|
||||
explicit, err := LoadPipeline(explicitPath)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPipeline(explicit working directory) error = %v", err)
|
||||
}
|
||||
if explicit.Notarius.WorkingDirectory != filepath.Join(dir, "runtime") {
|
||||
t.Fatalf("explicit working_directory = %q, want %q", explicit.Notarius.WorkingDirectory, filepath.Join(dir, "runtime"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotariusStrictYAML(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
}{
|
||||
{name: "unknown section field", yaml: "notarius:\n unknown: true\n"},
|
||||
{name: "unknown output field", yaml: "notarius:\n outputs:\n npc_registry:\n lane_id: npc-registry\n unknown: true\n"},
|
||||
{name: "unsupported session id", yaml: "notarius:\n session_id: forbidden\n"},
|
||||
{name: "unsupported model", yaml: "notarius:\n model: forbidden\n"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "pipeline.yml")
|
||||
if err := os.WriteFile(path, []byte(testPipelineBaseYAML+"\n"+tt.yaml), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline: %v", err)
|
||||
}
|
||||
if _, err := LoadPipeline(path); err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
||||
t.Fatalf("LoadPipeline() error = %v, want strict decode failure", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotariusEnabledValidation(t *testing.T) {
|
||||
valid := `notarius:
|
||||
enabled: true
|
||||
config_path: ./notarius.yml
|
||||
pipeline_id: dnd-session
|
||||
timeout: 45m
|
||||
outputs:
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-registry
|
||||
`
|
||||
tests := []struct {
|
||||
name string
|
||||
section string
|
||||
wantErr string
|
||||
}{
|
||||
{name: "valid", section: valid},
|
||||
{name: "blank binary", section: strings.Replace(valid, " enabled: true\n", " enabled: true\n binary: \" \"\n", 1), wantErr: "pipeline.notarius.binary is required"},
|
||||
{name: "missing config path", section: strings.Replace(valid, " config_path: ./notarius.yml\n", "", 1), wantErr: "pipeline.notarius.config_path is required"},
|
||||
{name: "missing pipeline id", section: strings.Replace(valid, " pipeline_id: dnd-session\n", "", 1), wantErr: "pipeline.notarius.pipeline_id is required"},
|
||||
{name: "missing outputs", section: strings.Split(valid, " outputs:\n")[0], wantErr: "pipeline.notarius.outputs must contain at least one output"},
|
||||
{name: "zero timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: 0s", 1), wantErr: "pipeline.notarius.timeout must be positive"},
|
||||
{name: "negative timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: -1s", 1), wantErr: "pipeline.notarius.timeout must be positive"},
|
||||
{name: "invalid timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: later", 1), wantErr: "pipeline.notarius.timeout must be a valid duration"},
|
||||
{name: "invalid output key", section: strings.Replace(valid, " npc_registry:", " npc-registry:", 1), wantErr: "outputs keys must match"},
|
||||
{name: "missing lane", section: strings.Replace(valid, " lane_id: npc-registry\n", "", 1), wantErr: "lane_id is required"},
|
||||
{name: "missing media type", section: strings.Replace(valid, " media_type: application/json\n", "", 1), wantErr: "media_type is required"},
|
||||
{name: "missing schema id", section: strings.Replace(valid, " schema_id: notarius.dnd.npc_registry\n", "", 1), wantErr: "schema_id is required"},
|
||||
{name: "missing schema version", section: strings.Replace(valid, " schema_version: v1\n", "", 1), wantErr: "schema_version is required"},
|
||||
{
|
||||
name: "normalized key collision",
|
||||
section: strings.Replace(valid, " module_key: dnd/npc-registry\n", ` module_key: dnd/npc-registry
|
||||
" npc_registry ":
|
||||
lane_id: npc-registry-two
|
||||
media_type: application/json
|
||||
schema_id: two
|
||||
schema_version: v1
|
||||
`, 1),
|
||||
wantErr: "normalize to duplicate source",
|
||||
},
|
||||
{
|
||||
name: "duplicate normalized lane",
|
||||
section: strings.Replace(valid, " module_key: dnd/npc-registry\n", ` module_key: dnd/npc-registry
|
||||
spells:
|
||||
lane_id: " npc-registry "
|
||||
media_type: application/json
|
||||
schema_id: two
|
||||
schema_version: v1
|
||||
`, 1),
|
||||
wantErr: "duplicates pipeline.notarius.outputs.npc_registry.lane_id",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML+"\n"+tt.section, testSessionBaseYAML)
|
||||
cfg, err := Load(pipelinePath, sessionPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
err = Validate(cfg)
|
||||
if tt.wantErr == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
output := cfg.Pipeline.Notarius.Outputs["npc_registry"]
|
||||
if output.LaneID != "npc-registry" || output.ModuleKey != "dnd/npc-registry" {
|
||||
t.Fatalf("normalized output = %#v", output)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("Validate() error = %v, want containing %q", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractionReferencesRequireDeclaredOutput(t *testing.T) {
|
||||
declared := `notarius:
|
||||
enabled: false
|
||||
outputs:
|
||||
npc_registry: {}
|
||||
`
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "scriptorium declared extraction",
|
||||
body: declared + `scriptorium:
|
||||
artifacts:
|
||||
recap:
|
||||
inputs:
|
||||
npcs:
|
||||
source: narratio.extraction.npc_registry
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "scriptorium unknown extraction",
|
||||
body: declared + `scriptorium:
|
||||
artifacts:
|
||||
recap:
|
||||
inputs:
|
||||
npcs:
|
||||
source: narratio.extraction.unknown
|
||||
`,
|
||||
wantErr: `references unknown extraction output "unknown"`,
|
||||
},
|
||||
{
|
||||
name: "publish declared extraction",
|
||||
body: declared + `publish:
|
||||
outputs:
|
||||
- source: narratio.extraction.npc_registry
|
||||
dest: artifacts/npc-registry.json
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "publish unknown extraction",
|
||||
body: declared + `publish:
|
||||
outputs:
|
||||
- source: narratio.extraction.unknown
|
||||
dest: artifacts/unknown.json
|
||||
`,
|
||||
wantErr: `extraction output "unknown" is not defined`,
|
||||
},
|
||||
{
|
||||
name: "publish lock declared extraction",
|
||||
body: declared + `publish:
|
||||
locks:
|
||||
- source: narratio.extraction.npc_registry
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML+"\n"+tt.body, testSessionBaseYAML)
|
||||
cfg, err := Load(pipelinePath, sessionPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
err = Validate(cfg)
|
||||
if tt.wantErr == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("Validate() error = %v, want containing %q", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -537,7 +537,7 @@ func TestPublishLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
|
||||
store, err := LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
reason: reviewed
|
||||
`), nil)
|
||||
`), nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
|
||||
}
|
||||
@@ -548,7 +548,7 @@ func TestPublishLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
|
||||
_, err = LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
dest: transcripts/final.trimmed.json
|
||||
`), nil)
|
||||
`), nil, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
|
||||
t.Fatalf("unknown field error = %v, want strict decode failed", err)
|
||||
}
|
||||
@@ -556,7 +556,7 @@ func TestPublishLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
|
||||
_, err = LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
- source: narratio.transcript.final_trimmed
|
||||
`), nil)
|
||||
`), nil, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicates another publish lock source") {
|
||||
t.Fatalf("duplicate error = %v", err)
|
||||
}
|
||||
|
||||
@@ -6,9 +6,11 @@ import (
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||
)
|
||||
@@ -85,7 +87,10 @@ func validatePipeline(cfg *PipelineConfig) error {
|
||||
if err := validateCache(cfg.Cache); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validatePublish(cfg.Publish, cfg.Scriptorium); err != nil {
|
||||
if err := validateNotarius(cfg.Notarius, cfg.Scriptorium); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validatePublish(cfg.Publish, cfg.Scriptorium, cfg.Notarius); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateWhisperX(cfg.WhisperX); err != nil {
|
||||
@@ -106,7 +111,7 @@ func validatePipeline(cfg *PipelineConfig) error {
|
||||
if err := validateRender(cfg.Render); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateScriptorium(cfg.Scriptorium); err != nil {
|
||||
if err := validateScriptorium(cfg.Scriptorium, cfg.Notarius); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateDuration("pipeline.notification.timeout", cfg.Notification.Timeout); err != nil {
|
||||
@@ -149,11 +154,12 @@ func validateCache(cfg CacheConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
|
||||
func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notarius *NotariusConfig) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
||||
extractionOutputs := notariusOutputKeySet(notarius)
|
||||
seenDest := map[string]struct{}{}
|
||||
for i, item := range cfg.Outputs {
|
||||
prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i)
|
||||
@@ -161,12 +167,12 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
|
||||
if source == "" {
|
||||
return fmt.Errorf("%s.source is required", prefix)
|
||||
}
|
||||
if _, err := artifactpolicy.ValidatePublishSource(source, configuredOutputs); err != nil {
|
||||
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
|
||||
return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||
}
|
||||
dest := strings.TrimSpace(item.Dest)
|
||||
if dest == "" {
|
||||
derivedDest, err := artifactpolicy.ResolvePublishedDestination(source, "", configuredOutputs)
|
||||
derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
|
||||
}
|
||||
@@ -189,7 +195,7 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
|
||||
}
|
||||
seenDest[normalizedDest] = struct{}{}
|
||||
}
|
||||
locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, "pipeline.publish.locks")
|
||||
locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, notarius, "pipeline.publish.locks")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -198,10 +204,11 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
|
||||
}
|
||||
|
||||
// ValidatePublishLockRules validates and normalizes source-based publish locks.
|
||||
func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, label string) ([]PublishLockRule, error) {
|
||||
func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, notarius *NotariusConfig, label string) ([]PublishLockRule, error) {
|
||||
seenLocks := map[string]struct{}{}
|
||||
out := make([]PublishLockRule, 0, len(locks))
|
||||
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
||||
extractionOutputs := notariusOutputKeySet(notarius)
|
||||
if strings.TrimSpace(label) == "" {
|
||||
label = "publish.locks"
|
||||
}
|
||||
@@ -211,7 +218,7 @@ func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumC
|
||||
if source == "" {
|
||||
return nil, fmt.Errorf("%s.source is required", prefix)
|
||||
}
|
||||
if _, err := artifactpolicy.ValidatePublishSource(source, configuredOutputs); err != nil {
|
||||
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
|
||||
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||
}
|
||||
if _, ok := seenLocks[source]; ok {
|
||||
@@ -264,6 +271,20 @@ func scriptoriumOutputPathMap(scriptorium *ScriptoriumConfig) map[string]string
|
||||
return out
|
||||
}
|
||||
|
||||
func notariusOutputKeySet(notarius *NotariusConfig) map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
if notarius == nil {
|
||||
return out
|
||||
}
|
||||
for key := range notarius.Outputs {
|
||||
trimmed := strings.TrimSpace(key)
|
||||
if artifactpolicy.IsConfiguredKey(trimmed) {
|
||||
out[trimmed] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func validateSecrets(cfg *SecretsConfig) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
@@ -474,7 +495,103 @@ func validateAudita(cfg AuditaConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateScriptorium(cfg *ScriptoriumConfig) error {
|
||||
func validateNotarius(cfg *NotariusConfig, scriptorium *ScriptoriumConfig) error {
|
||||
if cfg == nil || !cfg.Enabled {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(cfg.Binary) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.binary is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
if strings.TrimSpace(cfg.ConfigPath) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.config_path is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
if strings.TrimSpace(cfg.PipelineID) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.pipeline_id is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
if len(cfg.Outputs) == 0 {
|
||||
return fmt.Errorf("pipeline.notarius.outputs must contain at least one output when pipeline.notarius.enabled is true")
|
||||
}
|
||||
duration, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout))
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline.notarius.timeout must be a valid duration: %w", err)
|
||||
}
|
||||
if duration <= 0 {
|
||||
return fmt.Errorf("pipeline.notarius.timeout must be positive")
|
||||
}
|
||||
if strings.TrimSpace(cfg.WorkingDirectory) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.working_directory is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
|
||||
reservedSources := map[string]string{}
|
||||
for _, spec := range artifactmodel.RuntimeTranscriptArtifacts() {
|
||||
reservedSources[spec.SourceID] = "built-in source"
|
||||
}
|
||||
reservedSources[artifactpolicy.SourceBoundsSession] = "built-in source"
|
||||
if scriptorium != nil {
|
||||
for key := range scriptorium.Artifacts {
|
||||
normalizedKey := strings.TrimSpace(key)
|
||||
reservedSources[artifactpolicy.ConfiguredSourceID(normalizedKey)] = "configured Scriptorium source"
|
||||
reservedSources[artifactpolicy.PreviousSessionSourceID(normalizedKey)] = "previous-session source"
|
||||
}
|
||||
}
|
||||
|
||||
rawKeys := make([]string, 0, len(cfg.Outputs))
|
||||
for key := range cfg.Outputs {
|
||||
rawKeys = append(rawKeys, key)
|
||||
}
|
||||
sort.Strings(rawKeys)
|
||||
normalizedOutputs := make(map[string]NotariusOutputConfig, len(cfg.Outputs))
|
||||
sourceOwners := map[string]string{}
|
||||
laneOwners := map[string]string{}
|
||||
for _, rawKey := range rawKeys {
|
||||
output := cfg.Outputs[rawKey]
|
||||
key := strings.TrimSpace(rawKey)
|
||||
if !artifactpolicy.IsConfiguredKey(key) {
|
||||
return fmt.Errorf("pipeline.notarius.outputs keys must match ^[a-z][a-z0-9_]*$")
|
||||
}
|
||||
sourceID := artifactpolicy.ExtractionSourceID(key)
|
||||
if previousKey, ok := sourceOwners[sourceID]; ok {
|
||||
return fmt.Errorf("pipeline.notarius.outputs keys %q and %q normalize to duplicate source %q", previousKey, rawKey, sourceID)
|
||||
}
|
||||
if owner, ok := reservedSources[sourceID]; ok {
|
||||
return fmt.Errorf("pipeline.notarius.outputs.%s source %q collides with %s", key, sourceID, owner)
|
||||
}
|
||||
sourceOwners[sourceID] = rawKey
|
||||
|
||||
output.LaneID = strings.TrimSpace(output.LaneID)
|
||||
output.MediaType = strings.TrimSpace(output.MediaType)
|
||||
output.SchemaID = strings.TrimSpace(output.SchemaID)
|
||||
output.SchemaVersion = strings.TrimSpace(output.SchemaVersion)
|
||||
output.ModuleKey = strings.TrimSpace(output.ModuleKey)
|
||||
prefix := "pipeline.notarius.outputs." + key
|
||||
if output.LaneID == "" {
|
||||
return fmt.Errorf("%s.lane_id is required", prefix)
|
||||
}
|
||||
if previousKey, ok := laneOwners[output.LaneID]; ok {
|
||||
return fmt.Errorf("%s.lane_id %q duplicates pipeline.notarius.outputs.%s.lane_id", prefix, output.LaneID, previousKey)
|
||||
}
|
||||
laneOwners[output.LaneID] = key
|
||||
if output.MediaType == "" {
|
||||
return fmt.Errorf("%s.media_type is required", prefix)
|
||||
}
|
||||
if output.SchemaID == "" {
|
||||
return fmt.Errorf("%s.schema_id is required", prefix)
|
||||
}
|
||||
if output.SchemaVersion == "" {
|
||||
return fmt.Errorf("%s.schema_version is required", prefix)
|
||||
}
|
||||
normalizedOutputs[key] = output
|
||||
}
|
||||
cfg.Binary = strings.TrimSpace(cfg.Binary)
|
||||
cfg.ConfigPath = filepath.Clean(cfg.ConfigPath)
|
||||
cfg.PipelineID = strings.TrimSpace(cfg.PipelineID)
|
||||
cfg.Timeout = strings.TrimSpace(cfg.Timeout)
|
||||
cfg.WorkingDirectory = filepath.Clean(cfg.WorkingDirectory)
|
||||
cfg.Outputs = normalizedOutputs
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateScriptorium(cfg *ScriptoriumConfig, notarius *NotariusConfig) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -491,7 +608,7 @@ func validateScriptorium(cfg *ScriptoriumConfig) error {
|
||||
configuredArtifacts := make(map[string]struct{}, len(cfg.Artifacts))
|
||||
referencedArtifacts := make(map[string]struct{})
|
||||
for artifactName := range cfg.Artifacts {
|
||||
if !scriptoriumArtifactKeyRE.MatchString(strings.TrimSpace(artifactName)) {
|
||||
if !artifactpolicy.IsConfiguredKey(artifactName) {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$")
|
||||
}
|
||||
configuredArtifacts[artifactName] = struct{}{}
|
||||
@@ -544,7 +661,7 @@ func validateScriptorium(cfg *ScriptoriumConfig) error {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName)
|
||||
}
|
||||
|
||||
referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts)
|
||||
referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts, notariusOutputKeySet(notarius))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -684,9 +801,12 @@ func publishUploadConfiguredForS3(pipeline *PipelineConfig) bool {
|
||||
|
||||
var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
|
||||
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
var scriptoriumArtifactKeyRE = regexp.MustCompile(`^[a-z][a-z0-9_]*$`)
|
||||
|
||||
func validateScriptoriumInputSource(artifactName, inputName, source string, configuredArtifacts map[string]struct{}) (string, error) {
|
||||
func validateScriptoriumInputSource(
|
||||
artifactName, inputName, source string,
|
||||
configuredArtifacts map[string]struct{},
|
||||
extractionOutputs map[string]struct{},
|
||||
) (string, error) {
|
||||
trimmedSource := strings.TrimSpace(source)
|
||||
descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(trimmedSource)
|
||||
if err != nil {
|
||||
@@ -705,7 +825,7 @@ func validateScriptoriumInputSource(artifactName, inputName, source string, conf
|
||||
source,
|
||||
)
|
||||
}
|
||||
if err := artifactpolicy.ValidateInputConfiguredReference(descriptor, configuredArtifacts); err != nil {
|
||||
if err := artifactpolicy.ValidateInputReference(descriptor, configuredArtifacts, extractionOutputs); err != nil {
|
||||
var unknownConfigured *artifactpolicy.UnknownConfiguredArtifactError
|
||||
if errors.As(err, &unknownConfigured) {
|
||||
return "", fmt.Errorf(
|
||||
@@ -716,6 +836,16 @@ func validateScriptoriumInputSource(artifactName, inputName, source string, conf
|
||||
unknownConfigured.ConfiguredKey,
|
||||
)
|
||||
}
|
||||
var unknownExtraction *artifactpolicy.UnknownExtractionArtifactError
|
||||
if errors.As(err, &unknownExtraction) {
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown extraction output %q",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
unknownExtraction.ConfiguredKey,
|
||||
)
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
|
||||
artifactName,
|
||||
|
||||
308
internal/fileops/directory.go
Normal file
308
internal/fileops/directory.go
Normal file
@@ -0,0 +1,308 @@
|
||||
package fileops
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
promotedDirectoryMode = 0o755
|
||||
promotedFileMode = 0o644
|
||||
)
|
||||
|
||||
// ErrAtomicDirectoryPromotionUnsupported indicates that the current operating
|
||||
// system lacks the atomic no-replace primitive required by PromoteDirectory.
|
||||
var ErrAtomicDirectoryPromotionUnsupported = errors.New("atomic no-replace directory promotion is unsupported")
|
||||
|
||||
// PromoteDirectory copies an existing regular-file tree into a new directory
|
||||
// and installs the complete copy atomically. It never removes the source or
|
||||
// replaces an existing destination.
|
||||
func PromoteDirectory(src, dst string) error {
|
||||
if err := checkAtomicDirectoryPromotionSupport(); err != nil {
|
||||
return err
|
||||
}
|
||||
return promoteDirectory(src, dst, renameDirectoryNoReplace)
|
||||
}
|
||||
|
||||
func promoteDirectory(src, dst string, install func(string, string) error) error {
|
||||
return promoteDirectoryWithHooks(src, dst, install, sourceTraversalHooks{})
|
||||
}
|
||||
|
||||
type sourceTraversalHooks struct {
|
||||
afterRootInspect func()
|
||||
afterEntryInspect func(string)
|
||||
}
|
||||
|
||||
func promoteDirectoryWithHooks(
|
||||
src, dst string,
|
||||
install func(string, string) error,
|
||||
hooks sourceTraversalHooks,
|
||||
) error {
|
||||
if strings.TrimSpace(src) == "" || strings.TrimSpace(dst) == "" {
|
||||
return fmt.Errorf("source and destination directory paths are required")
|
||||
}
|
||||
|
||||
sourceInfo, err := os.Lstat(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect source directory: %w", err)
|
||||
}
|
||||
if !sourceInfo.IsDir() {
|
||||
return fmt.Errorf("source path %q is not a directory", src)
|
||||
}
|
||||
|
||||
if _, err := os.Lstat(dst); err == nil {
|
||||
return fmt.Errorf("destination path %q already exists", dst)
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("inspect destination path: %w", err)
|
||||
}
|
||||
|
||||
insideSource, err := pathWithin(src, dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if insideSource {
|
||||
return fmt.Errorf("destination path %q must not be inside source directory %q", dst, src)
|
||||
}
|
||||
|
||||
destinationParent := filepath.Dir(dst)
|
||||
parentInfo, err := os.Lstat(destinationParent)
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect destination parent: %w", err)
|
||||
}
|
||||
if !parentInfo.IsDir() {
|
||||
return fmt.Errorf("destination parent %q is not a directory", destinationParent)
|
||||
}
|
||||
|
||||
temporary, err := os.MkdirTemp(destinationParent, "."+filepath.Base(dst)+".tmp-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create temporary destination directory: %w", err)
|
||||
}
|
||||
removeTemporary := true
|
||||
defer func() {
|
||||
if removeTemporary {
|
||||
_ = os.RemoveAll(temporary)
|
||||
}
|
||||
}()
|
||||
|
||||
sourceRoot, err := openVerifiedSourceRoot(src, sourceInfo, hooks)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = sourceRoot.Close() }()
|
||||
|
||||
if err := copyRegularTree(sourceRoot, src, temporary, hooks); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(temporary, promotedDirectoryMode); err != nil {
|
||||
return fmt.Errorf("set temporary root permissions: %w", err)
|
||||
}
|
||||
if err := syncDirectory(temporary); err != nil {
|
||||
return fmt.Errorf("sync temporary root: %w", err)
|
||||
}
|
||||
if err := install(temporary, dst); err != nil {
|
||||
return fmt.Errorf("install promoted directory: %w", err)
|
||||
}
|
||||
|
||||
removeTemporary = false
|
||||
if err := syncDirectory(destinationParent); err != nil {
|
||||
return fmt.Errorf("sync destination parent: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func openVerifiedSourceRoot(path string, inspected os.FileInfo, hooks sourceTraversalHooks) (*os.Root, error) {
|
||||
if hooks.afterRootInspect != nil {
|
||||
hooks.afterRootInspect()
|
||||
}
|
||||
|
||||
root, err := os.OpenRoot(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open source directory %q: %w", path, err)
|
||||
}
|
||||
verified := false
|
||||
defer func() {
|
||||
if !verified {
|
||||
_ = root.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
opened, err := root.Stat(".")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect opened source directory %q: %w", path, err)
|
||||
}
|
||||
if !opened.IsDir() || !os.SameFile(inspected, opened) {
|
||||
return nil, fmt.Errorf("source directory %q changed while being opened", path)
|
||||
}
|
||||
current, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reinspect source directory %q: %w", path, err)
|
||||
}
|
||||
if current.Mode()&os.ModeSymlink != 0 || !current.IsDir() || !os.SameFile(opened, current) {
|
||||
return nil, fmt.Errorf("source directory %q changed while being opened", path)
|
||||
}
|
||||
|
||||
verified = true
|
||||
return root, nil
|
||||
}
|
||||
|
||||
func copyRegularTree(src *os.Root, sourcePath, dst string, hooks sourceTraversalHooks) error {
|
||||
directory, err := src.Open(".")
|
||||
if err != nil {
|
||||
return fmt.Errorf("open source directory %q for traversal: %w", sourcePath, err)
|
||||
}
|
||||
defer func() { _ = directory.Close() }()
|
||||
|
||||
entries, err := directory.ReadDir(-1)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read source directory %q: %w", sourcePath, err)
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
return entries[i].Name() < entries[j].Name()
|
||||
})
|
||||
|
||||
for _, entry := range entries {
|
||||
entryPath := filepath.Join(sourcePath, entry.Name())
|
||||
destinationPath := filepath.Join(dst, entry.Name())
|
||||
info, err := src.Lstat(entry.Name())
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect source entry %q: %w", entryPath, err)
|
||||
}
|
||||
|
||||
switch {
|
||||
case info.Mode().IsRegular():
|
||||
if hooks.afterEntryInspect != nil {
|
||||
hooks.afterEntryInspect(entryPath)
|
||||
}
|
||||
if err := copyRegularFile(src, entry.Name(), entryPath, destinationPath, info); err != nil {
|
||||
return err
|
||||
}
|
||||
case info.IsDir():
|
||||
if hooks.afterEntryInspect != nil {
|
||||
hooks.afterEntryInspect(entryPath)
|
||||
}
|
||||
if err := copyRegularDirectory(src, entry.Name(), entryPath, destinationPath, info, hooks); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("source entry %q has unsupported file type %s", entryPath, info.Mode().Type())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyRegularDirectory(
|
||||
parent *os.Root,
|
||||
name, sourcePath, dst string,
|
||||
inspected os.FileInfo,
|
||||
hooks sourceTraversalHooks,
|
||||
) error {
|
||||
child, err := parent.OpenRoot(name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open source directory %q: %w", sourcePath, err)
|
||||
}
|
||||
defer func() { _ = child.Close() }()
|
||||
|
||||
opened, err := child.Stat(".")
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect opened source directory %q: %w", sourcePath, err)
|
||||
}
|
||||
if !opened.IsDir() || !os.SameFile(inspected, opened) {
|
||||
return fmt.Errorf("source directory %q changed while being copied", sourcePath)
|
||||
}
|
||||
current, err := parent.Lstat(name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reinspect source directory %q: %w", sourcePath, err)
|
||||
}
|
||||
if current.Mode()&os.ModeSymlink != 0 || !current.IsDir() || !os.SameFile(opened, current) {
|
||||
return fmt.Errorf("source directory %q changed while being copied", sourcePath)
|
||||
}
|
||||
|
||||
if err := os.Mkdir(dst, promotedDirectoryMode); err != nil {
|
||||
return fmt.Errorf("create destination directory %q: %w", dst, err)
|
||||
}
|
||||
if err := copyRegularTree(child, sourcePath, dst, hooks); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(dst, promotedDirectoryMode); err != nil {
|
||||
return fmt.Errorf("set destination directory permissions %q: %w", dst, err)
|
||||
}
|
||||
if err := syncDirectory(dst); err != nil {
|
||||
return fmt.Errorf("sync destination directory %q: %w", dst, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyRegularFile(
|
||||
root *os.Root,
|
||||
name, sourcePath, dst string,
|
||||
inspected os.FileInfo,
|
||||
) error {
|
||||
in, err := root.Open(name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open source file %q: %w", sourcePath, err)
|
||||
}
|
||||
defer func() { _ = in.Close() }()
|
||||
|
||||
opened, err := in.Stat()
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect opened source file %q: %w", sourcePath, err)
|
||||
}
|
||||
if !opened.Mode().IsRegular() || !os.SameFile(inspected, opened) {
|
||||
return fmt.Errorf("source file %q changed while being copied", sourcePath)
|
||||
}
|
||||
current, err := root.Lstat(name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reinspect source file %q: %w", sourcePath, err)
|
||||
}
|
||||
if current.Mode()&os.ModeSymlink != 0 || !current.Mode().IsRegular() || !os.SameFile(opened, current) {
|
||||
return fmt.Errorf("source file %q changed while being copied", sourcePath)
|
||||
}
|
||||
|
||||
out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, promotedFileMode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create destination file %q: %w", dst, err)
|
||||
}
|
||||
closed := false
|
||||
defer func() {
|
||||
if !closed {
|
||||
_ = out.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
return fmt.Errorf("copy source file %q: %w", sourcePath, err)
|
||||
}
|
||||
if err := out.Chmod(promotedFileMode); err != nil {
|
||||
return fmt.Errorf("set destination file permissions %q: %w", dst, err)
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
return fmt.Errorf("sync destination file %q: %w", dst, err)
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return fmt.Errorf("close destination file %q: %w", dst, err)
|
||||
}
|
||||
closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func pathWithin(parent, candidate string) (bool, error) {
|
||||
absoluteParent, err := filepath.Abs(parent)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("resolve source directory: %w", err)
|
||||
}
|
||||
absoluteCandidate, err := filepath.Abs(candidate)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("resolve destination directory: %w", err)
|
||||
}
|
||||
relative, err := filepath.Rel(absoluteParent, absoluteCandidate)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("compare source and destination directories: %w", err)
|
||||
}
|
||||
return relative != "." && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)), nil
|
||||
}
|
||||
35
internal/fileops/directory_special_unix_test.go
Normal file
35
internal/fileops/directory_special_unix_test.go
Normal file
@@ -0,0 +1,35 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fileops
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPromoteDirectoryRejectsNamedPipe(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
dst := filepath.Join(root, "destination")
|
||||
if err := os.Mkdir(src, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(source) error = %v", err)
|
||||
}
|
||||
pipe := filepath.Join(src, "events.pipe")
|
||||
if err := syscall.Mkfifo(pipe, 0o600); err != nil {
|
||||
t.Skipf("Mkfifo() unavailable: %v", err)
|
||||
}
|
||||
|
||||
if err := PromoteDirectory(src, dst); err == nil {
|
||||
t.Fatal("PromoteDirectory() error = nil, want named pipe rejection")
|
||||
}
|
||||
info, err := os.Lstat(pipe)
|
||||
if err != nil || info.Mode()&os.ModeNamedPipe == 0 {
|
||||
t.Fatalf("source named pipe was not preserved: info=%v err=%v", info, err)
|
||||
}
|
||||
if _, err := os.Lstat(dst); !os.IsNotExist(err) {
|
||||
t.Fatalf("Lstat(destination) error = %v, want not exist", err)
|
||||
}
|
||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||
}
|
||||
501
internal/fileops/directory_test.go
Normal file
501
internal/fileops/directory_test.go
Normal file
@@ -0,0 +1,501 @@
|
||||
//go:build linux || darwin || windows
|
||||
|
||||
package fileops
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPromoteDirectoryCopiesNestedRegularTree(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
dst := filepath.Join(root, "promoted")
|
||||
mustWriteFile(t, filepath.Join(src, "z-last.txt"), []byte("last"), 0o777)
|
||||
mustWriteFile(t, filepath.Join(src, "nested", "binary.dat"), []byte{0, 1, 2, 0xff}, 0o600)
|
||||
mustWriteFile(t, filepath.Join(src, "a-first.txt"), []byte("first"), 0o400)
|
||||
if err := os.Mkdir(filepath.Join(src, "empty"), 0o700); err != nil {
|
||||
t.Fatalf("Mkdir(empty) error = %v", err)
|
||||
}
|
||||
|
||||
if err := PromoteDirectory(src, dst); err != nil {
|
||||
t.Fatalf("PromoteDirectory() error = %v", err)
|
||||
}
|
||||
|
||||
wantLayout := []string{".", "a-first.txt", "empty", "nested", "nested/binary.dat", "z-last.txt"}
|
||||
if got := treeLayout(t, dst); !reflect.DeepEqual(got, wantLayout) {
|
||||
t.Fatalf("promoted layout = %#v, want %#v", got, wantLayout)
|
||||
}
|
||||
assertFileBytes(t, filepath.Join(dst, "a-first.txt"), []byte("first"))
|
||||
assertFileBytes(t, filepath.Join(dst, "nested", "binary.dat"), []byte{0, 1, 2, 0xff})
|
||||
assertFileBytes(t, filepath.Join(dst, "z-last.txt"), []byte("last"))
|
||||
|
||||
if runtime.GOOS != "windows" {
|
||||
for _, path := range []string{dst, filepath.Join(dst, "nested"), filepath.Join(dst, "empty")} {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Stat(%q) error = %v", path, err)
|
||||
}
|
||||
if got := info.Mode().Perm(); got != promotedDirectoryMode {
|
||||
t.Fatalf("directory mode for %q = %o, want %o", path, got, promotedDirectoryMode)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{filepath.Join(dst, "a-first.txt"), filepath.Join(dst, "nested", "binary.dat"), filepath.Join(dst, "z-last.txt")} {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Stat(%q) error = %v", path, err)
|
||||
}
|
||||
if got := info.Mode().Perm(); got != promotedFileMode {
|
||||
t.Fatalf("file mode for %q = %o, want %o", path, got, promotedFileMode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assertFileBytes(t, filepath.Join(src, "nested", "binary.dat"), []byte{0, 1, 2, 0xff})
|
||||
assertNoMatchingTempDirectories(t, root, ".promoted.tmp-")
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsInvalidPaths(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
if err := os.Mkdir(src, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(source) error = %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
src string
|
||||
dst string
|
||||
}{
|
||||
{name: "empty source", src: "", dst: filepath.Join(root, "out-a")},
|
||||
{name: "empty destination", src: src, dst: " "},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if err := PromoteDirectory(test.src, test.dst); err == nil {
|
||||
t.Fatal("PromoteDirectory() error = nil, want path validation failure")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsExistingDestination(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("source"), 0o644)
|
||||
mustWriteFile(t, filepath.Join(dst, "value.txt"), []byte("existing"), 0o644)
|
||||
|
||||
err := PromoteDirectory(src, dst)
|
||||
if err == nil || !strings.Contains(err.Error(), "already exists") {
|
||||
t.Fatalf("PromoteDirectory() error = %v, want existing destination error", err)
|
||||
}
|
||||
assertFileBytes(t, filepath.Join(dst, "value.txt"), []byte("existing"))
|
||||
assertFileBytes(t, filepath.Join(src, "value.txt"), []byte("source"))
|
||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsNonDirectorySource(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source.txt")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, src, []byte("source"), 0o644)
|
||||
|
||||
if err := PromoteDirectory(src, dst); err == nil {
|
||||
t.Fatal("PromoteDirectory() error = nil, want non-directory source error")
|
||||
}
|
||||
assertFileBytes(t, src, []byte("source"))
|
||||
if _, err := os.Lstat(dst); !os.IsNotExist(err) {
|
||||
t.Fatalf("Lstat(destination) error = %v, want not exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsSymlinksWithoutFollowingThem(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
externalFile := filepath.Join(root, "external.txt")
|
||||
externalDirectory := filepath.Join(root, "external-directory")
|
||||
mustWriteFile(t, externalFile, []byte("outside"), 0o644)
|
||||
mustWriteFile(t, filepath.Join(externalDirectory, "secret.txt"), []byte("secret"), 0o644)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
target string
|
||||
link string
|
||||
}{
|
||||
{name: "file", target: externalFile, link: "file-link"},
|
||||
{name: "directory", target: externalDirectory, link: "directory-link"},
|
||||
{name: "escaping", target: filepath.Join("..", "external.txt"), link: "escaping-link"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src := filepath.Join(root, "source-"+test.name)
|
||||
dst := filepath.Join(root, "destination-"+test.name)
|
||||
if err := os.Mkdir(src, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(source) error = %v", err)
|
||||
}
|
||||
if err := os.Symlink(test.target, filepath.Join(src, test.link)); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
|
||||
if err := PromoteDirectory(src, dst); err == nil {
|
||||
t.Fatal("PromoteDirectory() error = nil, want symlink rejection")
|
||||
}
|
||||
linkInfo, err := os.Lstat(filepath.Join(src, test.link))
|
||||
if err != nil || linkInfo.Mode()&os.ModeSymlink == 0 {
|
||||
t.Fatalf("source symlink was not preserved: info=%v err=%v", linkInfo, err)
|
||||
}
|
||||
if _, err := os.Lstat(dst); !os.IsNotExist(err) {
|
||||
t.Fatalf("Lstat(destination) error = %v, want not exist", err)
|
||||
}
|
||||
assertNoMatchingTempDirectories(t, root, ".destination-"+test.name+".tmp-")
|
||||
})
|
||||
}
|
||||
assertFileBytes(t, externalFile, []byte("outside"))
|
||||
assertFileBytes(t, filepath.Join(externalDirectory, "secret.txt"), []byte("secret"))
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsSymlinkSourceRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
target := filepath.Join(root, "target")
|
||||
src := filepath.Join(root, "source")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, filepath.Join(target, "value.txt"), []byte("outside"), 0o644)
|
||||
if err := os.Symlink(target, src); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
|
||||
if err := PromoteDirectory(src, dst); err == nil {
|
||||
t.Fatal("PromoteDirectory() error = nil, want source-root symlink rejection")
|
||||
}
|
||||
assertFileBytes(t, filepath.Join(target, "value.txt"), []byte("outside"))
|
||||
assertPathMissing(t, dst)
|
||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsSourceRootReplacementBeforeOpen(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
preserved := filepath.Join(root, "source-preserved")
|
||||
replacement := filepath.Join(root, "replacement")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("original"), 0o644)
|
||||
mustWriteFile(t, filepath.Join(replacement, "value.txt"), []byte("replacement"), 0o644)
|
||||
|
||||
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||
afterRootInspect: func() {
|
||||
if err := os.Rename(src, preserved); err != nil {
|
||||
t.Fatalf("Rename(original source) error = %v", err)
|
||||
}
|
||||
if err := os.Rename(replacement, src); err != nil {
|
||||
t.Fatalf("Rename(replacement source) error = %v", err)
|
||||
}
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("promoteDirectoryWithHooks() error = nil, want source identity failure")
|
||||
}
|
||||
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||
assertFileBytes(t, filepath.Join(src, "value.txt"), []byte("replacement"))
|
||||
assertPathMissing(t, dst)
|
||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsIdentityPreservingSourceRootSymlinkReplacement(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
preserved := filepath.Join(root, "source-preserved")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("original"), 0o644)
|
||||
|
||||
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||
afterRootInspect: func() {
|
||||
if err := os.Rename(src, preserved); err != nil {
|
||||
t.Fatalf("Rename(inspected source) error = %v", err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Base(preserved), src); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("promoteDirectoryWithHooks() error = nil, want source-root symlink replacement failure")
|
||||
}
|
||||
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||
assertPathIsSymlink(t, src)
|
||||
assertPathMissing(t, dst)
|
||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsInspectedDirectorySymlinkReplacement(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
child := filepath.Join(src, "child")
|
||||
preserved := filepath.Join(src, "child-preserved")
|
||||
outside := filepath.Join(root, "outside")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, filepath.Join(child, "value.txt"), []byte("original"), 0o644)
|
||||
mustWriteFile(t, filepath.Join(outside, "sentinel.txt"), []byte("outside"), 0o644)
|
||||
|
||||
replaced := false
|
||||
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||
afterEntryInspect: func(path string) {
|
||||
if replaced || path != child {
|
||||
return
|
||||
}
|
||||
replaced = true
|
||||
if err := os.Rename(child, preserved); err != nil {
|
||||
t.Fatalf("Rename(inspected child) error = %v", err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Join("..", "outside"), child); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
if err := os.Mkdir(dst, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(concurrent destination) error = %v", err)
|
||||
}
|
||||
mustWriteFile(t, filepath.Join(dst, "value.txt"), []byte("concurrent"), 0o644)
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("promoteDirectoryWithHooks() error = nil, want symlink replacement failure")
|
||||
}
|
||||
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||
assertFileBytes(t, filepath.Join(outside, "sentinel.txt"), []byte("outside"))
|
||||
assertFileBytes(t, filepath.Join(dst, "value.txt"), []byte("concurrent"))
|
||||
assertPathMissing(t, filepath.Join(dst, "sentinel.txt"))
|
||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsInspectedFileIdentityMismatch(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
file := filepath.Join(src, "value.txt")
|
||||
preserved := filepath.Join(src, "value-preserved.txt")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, file, []byte("original"), 0o644)
|
||||
|
||||
replaced := false
|
||||
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||
afterEntryInspect: func(path string) {
|
||||
if replaced || path != file {
|
||||
return
|
||||
}
|
||||
replaced = true
|
||||
if err := os.Rename(file, preserved); err != nil {
|
||||
t.Fatalf("Rename(inspected file) error = %v", err)
|
||||
}
|
||||
mustWriteFile(t, file, []byte("replacement"), 0o644)
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("promoteDirectoryWithHooks() error = nil, want file identity failure")
|
||||
}
|
||||
assertFileBytes(t, preserved, []byte("original"))
|
||||
assertFileBytes(t, file, []byte("replacement"))
|
||||
assertPathMissing(t, dst)
|
||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsIdentityPreservingFileSymlinkReplacement(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
file := filepath.Join(src, "value.txt")
|
||||
preserved := filepath.Join(src, "value-preserved.txt")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, file, []byte("original"), 0o644)
|
||||
|
||||
replaced := false
|
||||
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||
afterEntryInspect: func(path string) {
|
||||
if replaced || path != file {
|
||||
return
|
||||
}
|
||||
replaced = true
|
||||
if err := os.Rename(file, preserved); err != nil {
|
||||
t.Fatalf("Rename(inspected file) error = %v", err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Base(preserved), file); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("promoteDirectoryWithHooks() error = nil, want file symlink replacement failure")
|
||||
}
|
||||
assertFileBytes(t, preserved, []byte("original"))
|
||||
assertPathIsSymlink(t, file)
|
||||
assertPathMissing(t, dst)
|
||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsInspectedDirectoryIdentityMismatch(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
child := filepath.Join(src, "child")
|
||||
preserved := filepath.Join(src, "child-preserved")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, filepath.Join(child, "value.txt"), []byte("original"), 0o644)
|
||||
|
||||
replaced := false
|
||||
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||
afterEntryInspect: func(path string) {
|
||||
if replaced || path != child {
|
||||
return
|
||||
}
|
||||
replaced = true
|
||||
if err := os.Rename(child, preserved); err != nil {
|
||||
t.Fatalf("Rename(inspected directory) error = %v", err)
|
||||
}
|
||||
mustWriteFile(t, filepath.Join(child, "value.txt"), []byte("replacement"), 0o644)
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("promoteDirectoryWithHooks() error = nil, want directory identity failure")
|
||||
}
|
||||
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||
assertFileBytes(t, filepath.Join(child, "value.txt"), []byte("replacement"))
|
||||
assertPathMissing(t, dst)
|
||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsIdentityPreservingDirectorySymlinkReplacement(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
child := filepath.Join(src, "child")
|
||||
preserved := filepath.Join(src, "child-preserved")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, filepath.Join(child, "value.txt"), []byte("original"), 0o644)
|
||||
|
||||
replaced := false
|
||||
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||
afterEntryInspect: func(path string) {
|
||||
if replaced || path != child {
|
||||
return
|
||||
}
|
||||
replaced = true
|
||||
if err := os.Rename(child, preserved); err != nil {
|
||||
t.Fatalf("Rename(inspected directory) error = %v", err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Base(preserved), child); err != nil {
|
||||
t.Skipf("Symlink() unavailable: %v", err)
|
||||
}
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("promoteDirectoryWithHooks() error = nil, want directory symlink replacement failure")
|
||||
}
|
||||
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||
assertPathIsSymlink(t, child)
|
||||
assertPathMissing(t, dst)
|
||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryDoesNotReplaceDestinationCreatedBeforeInstall(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("source"), 0o644)
|
||||
|
||||
err := promoteDirectory(src, dst, func(temporary, destination string) error {
|
||||
if err := os.Mkdir(destination, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(concurrent destination) error = %v", err)
|
||||
}
|
||||
mustWriteFile(t, filepath.Join(destination, "value.txt"), []byte("concurrent"), 0o644)
|
||||
return renameDirectoryNoReplace(temporary, destination)
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("promoteDirectory() error = nil, want no-replace install failure")
|
||||
}
|
||||
assertFileBytes(t, filepath.Join(dst, "value.txt"), []byte("concurrent"))
|
||||
assertFileBytes(t, filepath.Join(src, "value.txt"), []byte("source"))
|
||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||
}
|
||||
|
||||
func TestPromoteDirectoryRejectsDestinationInsideSource(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
dst := filepath.Join(src, "nested", "destination")
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(destination parent) error = %v", err)
|
||||
}
|
||||
|
||||
if err := PromoteDirectory(src, dst); err == nil {
|
||||
t.Fatal("PromoteDirectory() error = nil, want nested destination rejection")
|
||||
}
|
||||
if _, err := os.Lstat(dst); !os.IsNotExist(err) {
|
||||
t.Fatalf("Lstat(destination) error = %v, want not exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustWriteFile(t *testing.T, path string, data []byte, mode os.FileMode) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(%q) error = %v", filepath.Dir(path), err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, mode); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFileBytes(t *testing.T, path string, want []byte) {
|
||||
t.Helper()
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q) error = %v", path, err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("ReadFile(%q) = %v, want %v", path, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertPathMissing(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Lstat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("Lstat(%q) error = %v, want not exist", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertPathIsSymlink(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil || info.Mode()&os.ModeSymlink == 0 {
|
||||
t.Fatalf("Lstat(%q) info = %v, error = %v, want symlink", path, info, err)
|
||||
}
|
||||
}
|
||||
|
||||
func treeLayout(t *testing.T, root string) []string {
|
||||
t.Helper()
|
||||
var layout []string
|
||||
err := filepath.WalkDir(root, func(path string, _ os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
relative, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
layout = append(layout, filepath.ToSlash(relative))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WalkDir(%q) error = %v", root, err)
|
||||
}
|
||||
return layout
|
||||
}
|
||||
|
||||
func assertNoMatchingTempDirectories(t *testing.T, parent, prefix string) {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(parent)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir(%q) error = %v", parent, err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if strings.HasPrefix(entry.Name(), prefix) {
|
||||
t.Fatalf("unexpected temporary directory residue: %s", filepath.Join(parent, entry.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
37
internal/fileops/directory_unsupported_test.go
Normal file
37
internal/fileops/directory_unsupported_test.go
Normal file
@@ -0,0 +1,37 @@
|
||||
//go:build !linux && !darwin && !windows
|
||||
|
||||
package fileops
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPromoteDirectoryFailsBeforeCreatingTemporaryTree(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
dst := filepath.Join(root, "destination")
|
||||
if err := os.Mkdir(src, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(source) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(src, "value.txt"), []byte("source"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(source) error = %v", err)
|
||||
}
|
||||
|
||||
err := PromoteDirectory(src, dst)
|
||||
if !errors.Is(err, ErrAtomicDirectoryPromotionUnsupported) {
|
||||
t.Fatalf("PromoteDirectory() error = %v, want unsupported capability", err)
|
||||
}
|
||||
entries, readErr := os.ReadDir(root)
|
||||
if readErr != nil {
|
||||
t.Fatalf("ReadDir(root) error = %v", readErr)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.Name() == filepath.Base(dst) || strings.HasPrefix(entry.Name(), ".destination.tmp-") {
|
||||
t.Fatalf("unsupported promotion created %q", entry.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
11
internal/fileops/rename_noreplace_darwin.go
Normal file
11
internal/fileops/rename_noreplace_darwin.go
Normal file
@@ -0,0 +1,11 @@
|
||||
//go:build darwin
|
||||
|
||||
package fileops
|
||||
|
||||
import "golang.org/x/sys/unix"
|
||||
|
||||
func checkAtomicDirectoryPromotionSupport() error { return nil }
|
||||
|
||||
func renameDirectoryNoReplace(src, dst string) error {
|
||||
return unix.RenamexNp(src, dst, unix.RENAME_EXCL)
|
||||
}
|
||||
11
internal/fileops/rename_noreplace_linux.go
Normal file
11
internal/fileops/rename_noreplace_linux.go
Normal file
@@ -0,0 +1,11 @@
|
||||
//go:build linux
|
||||
|
||||
package fileops
|
||||
|
||||
import "golang.org/x/sys/unix"
|
||||
|
||||
func checkAtomicDirectoryPromotionSupport() error { return nil }
|
||||
|
||||
func renameDirectoryNoReplace(src, dst string) error {
|
||||
return unix.Renameat2(unix.AT_FDCWD, src, unix.AT_FDCWD, dst, unix.RENAME_NOREPLACE)
|
||||
}
|
||||
16
internal/fileops/rename_noreplace_other.go
Normal file
16
internal/fileops/rename_noreplace_other.go
Normal file
@@ -0,0 +1,16 @@
|
||||
//go:build !linux && !darwin && !windows
|
||||
|
||||
package fileops
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
func checkAtomicDirectoryPromotionSupport() error {
|
||||
return fmt.Errorf("%w on %s", ErrAtomicDirectoryPromotionUnsupported, runtime.GOOS)
|
||||
}
|
||||
|
||||
func renameDirectoryNoReplace(_, _ string) error {
|
||||
return checkAtomicDirectoryPromotionSupport()
|
||||
}
|
||||
26
internal/fileops/rename_noreplace_supported_test.go
Normal file
26
internal/fileops/rename_noreplace_supported_test.go
Normal file
@@ -0,0 +1,26 @@
|
||||
//go:build linux || darwin || windows
|
||||
|
||||
package fileops
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRenameDirectoryNoReplacePreservesExistingDestination(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "source")
|
||||
dst := filepath.Join(root, "destination")
|
||||
mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("source"), 0o644)
|
||||
mustWriteFile(t, filepath.Join(dst, "value.txt"), []byte("existing"), 0o644)
|
||||
|
||||
if err := renameDirectoryNoReplace(src, dst); err == nil {
|
||||
t.Fatal("renameDirectoryNoReplace() error = nil, want existing destination failure")
|
||||
}
|
||||
assertFileBytes(t, filepath.Join(src, "value.txt"), []byte("source"))
|
||||
assertFileBytes(t, filepath.Join(dst, "value.txt"), []byte("existing"))
|
||||
if info, err := os.Stat(src); err != nil || !info.IsDir() {
|
||||
t.Fatalf("source directory was not preserved: info=%v err=%v", info, err)
|
||||
}
|
||||
}
|
||||
19
internal/fileops/rename_noreplace_windows.go
Normal file
19
internal/fileops/rename_noreplace_windows.go
Normal file
@@ -0,0 +1,19 @@
|
||||
//go:build windows
|
||||
|
||||
package fileops
|
||||
|
||||
import "golang.org/x/sys/windows"
|
||||
|
||||
func checkAtomicDirectoryPromotionSupport() error { return nil }
|
||||
|
||||
func renameDirectoryNoReplace(src, dst string) error {
|
||||
from, err := windows.UTF16PtrFromString(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
to, err := windows.UTF16PtrFromString(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return windows.MoveFileEx(from, to, 0)
|
||||
}
|
||||
25
internal/fileops/sync_directory_unix.go
Normal file
25
internal/fileops/sync_directory_unix.go
Normal file
@@ -0,0 +1,25 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fileops
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func syncDirectory(path string) error {
|
||||
directory, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = directory.Close() }()
|
||||
|
||||
err = directory.Sync()
|
||||
// Some Unix filesystems do not implement directory syncing. Only their
|
||||
// explicit unsupported-operation errors are safe to treat as best effort.
|
||||
if errors.Is(err, syscall.EINVAL) || errors.Is(err, syscall.ENOTSUP) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
7
internal/fileops/sync_directory_unsupported.go
Normal file
7
internal/fileops/sync_directory_unsupported.go
Normal file
@@ -0,0 +1,7 @@
|
||||
//go:build !linux && !darwin && !windows
|
||||
|
||||
package fileops
|
||||
|
||||
func syncDirectory(string) error {
|
||||
return checkAtomicDirectoryPromotionSupport()
|
||||
}
|
||||
40
internal/fileops/sync_directory_windows.go
Normal file
40
internal/fileops/sync_directory_windows.go
Normal file
@@ -0,0 +1,40 @@
|
||||
//go:build windows
|
||||
|
||||
package fileops
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func syncDirectory(path string) error {
|
||||
pathPointer, err := windows.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
directory, err := windows.CreateFile(
|
||||
pathPointer,
|
||||
windows.GENERIC_WRITE,
|
||||
windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE,
|
||||
nil,
|
||||
windows.OPEN_EXISTING,
|
||||
windows.FILE_FLAG_BACKUP_SEMANTICS,
|
||||
0,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = windows.CloseHandle(directory) }()
|
||||
|
||||
err = windows.FlushFileBuffers(directory)
|
||||
// Windows filesystems may reject flushing a directory handle even when it
|
||||
// was opened correctly. Preserve every error except the documented forms
|
||||
// that mean this operation is unavailable for the handle or filesystem.
|
||||
if errors.Is(err, windows.ERROR_INVALID_FUNCTION) ||
|
||||
errors.Is(err, windows.ERROR_INVALID_HANDLE) ||
|
||||
errors.Is(err, windows.ERROR_NOT_SUPPORTED) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package manifest
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
)
|
||||
|
||||
// ErrorRecord captures structured error metadata at run or stage scope.
|
||||
@@ -28,9 +30,11 @@ type InputRecord struct {
|
||||
|
||||
// ArtifactRecord captures one produced artifact and optional remote metadata.
|
||||
type ArtifactRecord struct {
|
||||
Kind string `json:"kind"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
LocalPath string `json:"local_path"`
|
||||
Kind string `json:"kind"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
LocalPath string `json:"local_path"`
|
||||
Contract *artifactmodel.ContractMetadata `json:"contract,omitempty"`
|
||||
ExternalProvenance *artifactmodel.ExternalProvenance `json:"external_provenance,omitempty"`
|
||||
// ProducerRunID identifies the run that produced this durable artifact.
|
||||
ProducerRunID string `json:"producer_run_id,omitempty"`
|
||||
RemoteKey string `json:"remote_key,omitempty"`
|
||||
@@ -84,6 +88,7 @@ func New(sessionID string, now time.Time) *Manifest {
|
||||
// MarkStageRunning marks a stage as running and updates timestamps.
|
||||
func (m *Manifest) MarkStageRunning(name string, at time.Time) {
|
||||
s := m.ensureStage(name, at)
|
||||
s.clearResultDetails()
|
||||
s.Status = StatusRunning
|
||||
s.StartedAt = timePtr(at)
|
||||
s.CompletedAt = nil
|
||||
@@ -106,6 +111,7 @@ func (m *Manifest) MarkStageSucceeded(name string, at time.Time, outputs []Artif
|
||||
// MarkStageFailed marks a stage as failed and records error metadata.
|
||||
func (m *Manifest) MarkStageFailed(name string, at time.Time, message string) {
|
||||
s := m.ensureStage(name, at)
|
||||
s.clearResultDetails()
|
||||
s.Status = StatusFailed
|
||||
s.CompletedAt = timePtr(at)
|
||||
s.Error = &ErrorRecord{Message: strings.TrimSpace(message), At: timePtr(at)}
|
||||
@@ -117,6 +123,7 @@ func (m *Manifest) MarkStageFailed(name string, at time.Time, message string) {
|
||||
// MarkStageSkipped marks a stage as skipped and records the skip reason.
|
||||
func (m *Manifest) MarkStageSkipped(name string, at time.Time, reason string) {
|
||||
s := m.ensureStage(name, at)
|
||||
s.clearResultDetails()
|
||||
s.Status = StatusSkipped
|
||||
s.CompletedAt = timePtr(at)
|
||||
s.Error = &ErrorRecord{Message: strings.TrimSpace(reason), Code: "skipped", At: timePtr(at)}
|
||||
@@ -133,6 +140,13 @@ func (m *Manifest) MarkStageStale(name string, at time.Time, reason string) {
|
||||
m.UpdatedAt = at
|
||||
}
|
||||
|
||||
func (s *StageRecord) clearResultDetails() {
|
||||
s.Outputs = nil
|
||||
s.Logs = nil
|
||||
s.GeneratedConfigs = nil
|
||||
s.Metadata = nil
|
||||
}
|
||||
|
||||
func (m *Manifest) ensureStage(name string, at time.Time) *StageRecord {
|
||||
if m.Stages == nil {
|
||||
m.Stages = map[string]*StageRecord{}
|
||||
|
||||
@@ -69,11 +69,12 @@ func TestStageMarkHelpers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkStageRunningClearsPriorCompletionAndError(t *testing.T) {
|
||||
func TestMarkStageRunningClearsPriorCompletionErrorAndResultDetails(t *testing.T) {
|
||||
m := New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
|
||||
failedAt := time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC)
|
||||
m.MarkStageFailed("merge", failedAt, "boom")
|
||||
setStageResultDetails(m.Stages["merge"])
|
||||
|
||||
runAt := failedAt.Add(1 * time.Minute)
|
||||
m.MarkStageRunning("merge", runAt)
|
||||
@@ -91,6 +92,7 @@ func TestMarkStageRunningClearsPriorCompletionAndError(t *testing.T) {
|
||||
if stage.Error != nil {
|
||||
t.Fatalf("error = %#v, want nil while running", stage.Error)
|
||||
}
|
||||
requireStageResultDetailsCleared(t, stage)
|
||||
}
|
||||
|
||||
func TestMarkStageSucceededClearsError(t *testing.T) {
|
||||
@@ -110,3 +112,68 @@ func TestMarkStageSucceededClearsError(t *testing.T) {
|
||||
t.Fatalf("error = %#v, want nil on success", stage.Error)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkStageFailedClearsEarlierResultDetails(t *testing.T) {
|
||||
m := New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
m.MarkStageSucceeded("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
setStageResultDetails(m.Stages["extract"])
|
||||
|
||||
m.MarkStageFailed("extract", time.Date(2026, 5, 3, 10, 2, 0, 0, time.UTC), "replacement failed")
|
||||
|
||||
stage := m.Stages["extract"]
|
||||
if stage == nil || stage.Status != StatusFailed {
|
||||
t.Fatalf("stage = %#v, want failed", stage)
|
||||
}
|
||||
requireStageResultDetailsCleared(t, stage)
|
||||
}
|
||||
|
||||
func TestMarkStageSkippedClearsEarlierResultDetails(t *testing.T) {
|
||||
m := New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
m.MarkStageSucceeded("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
setStageResultDetails(m.Stages["extract"])
|
||||
|
||||
m.MarkStageSkipped("extract", time.Date(2026, 5, 3, 10, 2, 0, 0, time.UTC), "integration_disabled")
|
||||
|
||||
stage := m.Stages["extract"]
|
||||
if stage == nil {
|
||||
t.Fatal("missing stage record")
|
||||
}
|
||||
if stage.Status != StatusSkipped {
|
||||
t.Fatalf("status = %q, want %q", stage.Status, StatusSkipped)
|
||||
}
|
||||
requireStageResultDetailsCleared(t, stage)
|
||||
}
|
||||
|
||||
func TestMarkStageStalePreservesResultDetails(t *testing.T) {
|
||||
m := New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
m.MarkStageSucceeded("extract", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
stage := m.Stages["extract"]
|
||||
setStageResultDetails(stage)
|
||||
|
||||
m.MarkStageStale("extract", time.Date(2026, 5, 3, 10, 2, 0, 0, time.UTC), "result is not resumable")
|
||||
|
||||
if stage.Status != StatusStale {
|
||||
t.Fatalf("status = %q, want %q", stage.Status, StatusStale)
|
||||
}
|
||||
if len(stage.Outputs) != 1 || len(stage.Logs) != 1 || len(stage.GeneratedConfigs) != 1 || len(stage.Metadata) != 1 {
|
||||
t.Fatalf("result details were not preserved: %#v", stage)
|
||||
}
|
||||
}
|
||||
|
||||
func setStageResultDetails(stage *StageRecord) {
|
||||
stage.Outputs = []ArtifactRecord{{
|
||||
Kind: "structured_data",
|
||||
SourceID: "narratio.example.characters",
|
||||
LocalPath: "artifacts/characters.json",
|
||||
}}
|
||||
stage.Logs = []string{"logs/extract.log"}
|
||||
stage.GeneratedConfigs = []string{"generated/extract.yaml"}
|
||||
stage.Metadata = map[string]any{"bundle_path": "extract/results/run-1"}
|
||||
}
|
||||
|
||||
func requireStageResultDetailsCleared(t *testing.T, stage *StageRecord) {
|
||||
t.Helper()
|
||||
if len(stage.Outputs) != 0 || len(stage.Logs) != 0 || len(stage.GeneratedConfigs) != 0 || len(stage.Metadata) != 0 {
|
||||
t.Fatalf("result details were not cleared: %#v", stage)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,6 +119,7 @@ func (m *RunManifest) MarkStageSkipped(name string, at time.Time, reason string)
|
||||
s.Status = StatusSkipped
|
||||
s.CompletedAt = timePtr(at)
|
||||
s.Error = &ErrorRecord{Message: strings.TrimSpace(reason), Code: "skipped", At: timePtr(at)}
|
||||
s.Outputs = nil
|
||||
s.UpdatedAt = at
|
||||
m.UpdatedAt = at
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
)
|
||||
|
||||
func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
|
||||
@@ -64,6 +66,76 @@ func TestLocalStoreCreateSaveLoadRoundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalStoreArtifactMetadataCompatibility(t *testing.T) {
|
||||
store := &LocalStore{}
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
oldPath := filepath.Join(dir, "old-manifest.json")
|
||||
oldJSON := `{
|
||||
"session_id": "2026-05-03",
|
||||
"created_at": "2026-05-03T10:00:00Z",
|
||||
"updated_at": "2026-05-03T10:01:00Z",
|
||||
"stages": {
|
||||
"analyze": {
|
||||
"name": "analyze",
|
||||
"status": "succeeded",
|
||||
"created_at": "2026-05-03T10:00:00Z",
|
||||
"updated_at": "2026-05-03T10:01:00Z",
|
||||
"outputs": [{"kind":"scriptorium_artifact","local_path":"artifacts/recap.md"}]
|
||||
}
|
||||
}
|
||||
}`
|
||||
if err := os.WriteFile(oldPath, []byte(oldJSON), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile() old manifest error = %v", err)
|
||||
}
|
||||
|
||||
loaded, err := store.Load(ctx, oldPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() old manifest error = %v", err)
|
||||
}
|
||||
oldOutput := loaded.Stages["analyze"].Outputs[0]
|
||||
if oldOutput.Contract != nil || oldOutput.ExternalProvenance != nil {
|
||||
t.Fatalf("old output metadata = %#v, %#v; want nil", oldOutput.Contract, oldOutput.ExternalProvenance)
|
||||
}
|
||||
if err := store.Save(ctx, oldPath, loaded); err != nil {
|
||||
t.Fatalf("Save() old manifest error = %v", err)
|
||||
}
|
||||
roundTripped, err := os.ReadFile(oldPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() round-tripped old manifest error = %v", err)
|
||||
}
|
||||
if strings.Contains(string(roundTripped), `"contract"`) || strings.Contains(string(roundTripped), `"external_provenance"`) {
|
||||
t.Fatalf("old manifest gained fabricated metadata:\n%s", roundTripped)
|
||||
}
|
||||
|
||||
loaded.Stages["analyze"].Outputs[0].Contract = &artifactmodel.ContractMetadata{
|
||||
MediaType: "application/json",
|
||||
SchemaID: "example.recap",
|
||||
SchemaVersion: "v1",
|
||||
}
|
||||
loaded.Stages["analyze"].Outputs[0].ExternalProvenance = &artifactmodel.ExternalProvenance{
|
||||
System: "example",
|
||||
RunID: "external-run",
|
||||
PipelineID: "pipeline",
|
||||
ArtifactID: "recap",
|
||||
}
|
||||
metadataPath := filepath.Join(dir, "metadata-manifest.json")
|
||||
if err := store.Save(ctx, metadataPath, loaded); err != nil {
|
||||
t.Fatalf("Save() metadata manifest error = %v", err)
|
||||
}
|
||||
withMetadata, err := store.Load(ctx, metadataPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() metadata manifest error = %v", err)
|
||||
}
|
||||
got := withMetadata.Stages["analyze"].Outputs[0]
|
||||
if got.Contract == nil || got.Contract.SchemaID != "example.recap" {
|
||||
t.Fatalf("contract = %#v, want persisted contract", got.Contract)
|
||||
}
|
||||
if got.ExternalProvenance == nil || got.ExternalProvenance.RunID != "external-run" {
|
||||
t.Fatalf("external provenance = %#v, want persisted provenance", got.ExternalProvenance)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalStoreSaveUpdatesTimestamp(t *testing.T) {
|
||||
store := &LocalStore{}
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -84,7 +84,13 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
}}, nil
|
||||
}
|
||||
|
||||
runtimeCatalog, err := buildAnalyzeRuntimeArtifactCatalog(paths, env.Config.Pipeline.Scriptorium, env.SelectedArtifactKeys)
|
||||
runtimeCatalog, err := buildAnalyzeRuntimeArtifactCatalog(
|
||||
paths,
|
||||
m,
|
||||
env.Config.Pipeline.Scriptorium,
|
||||
env.Config.Pipeline.Notarius,
|
||||
env.SelectedArtifactKeys,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("analyze: build runtime artifact catalog: %w", err)
|
||||
}
|
||||
@@ -668,6 +674,16 @@ func resolveScriptoriumInput(
|
||||
return resolved.Path, true, ©, nil
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
||||
if descriptor.Source.Kind == artifactpolicy.SourceKindExtraction {
|
||||
if inputCfg.Required {
|
||||
return "", false, nil, fmt.Errorf(
|
||||
"required extraction source %q is unavailable; enable and configure pipeline.notarius output %q, then rerun extract with --force",
|
||||
source,
|
||||
descriptor.Source.ConfiguredKey,
|
||||
)
|
||||
}
|
||||
return "", false, nil, nil
|
||||
}
|
||||
if descriptor.Source.Kind == artifactpolicy.SourceKindConfiguredArtifact {
|
||||
if inputCfg.Required {
|
||||
return "", false, nil, fmt.Errorf("configured artifact source %q is unavailable", source)
|
||||
@@ -727,13 +743,22 @@ func preparedStableInputFilename(sourceID string) (string, bool) {
|
||||
|
||||
func buildAnalyzeRuntimeArtifactCatalog(
|
||||
paths artifacts.SessionPaths,
|
||||
m *manifest.Manifest,
|
||||
scriptoriumCfg *config.ScriptoriumConfig,
|
||||
notariusCfg *config.NotariusConfig,
|
||||
selectedArtifacts []string,
|
||||
) (*artifacts.ArtifactCatalog, error) {
|
||||
catalog := artifacts.NewArtifactCatalog()
|
||||
if err := catalog.RegisterBuiltIns(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
extractionDefinitions := artifacts.ExtractionDefinitionsFromConfig(notariusCfg)
|
||||
if err := catalog.RegisterExtractionArtifacts(extractionDefinitions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if notariusCfg != nil && notariusCfg.Enabled {
|
||||
catalog.HydrateExtractionArtifacts(paths, m, extractionDefinitions)
|
||||
}
|
||||
if scriptoriumCfg == nil {
|
||||
return catalog, nil
|
||||
}
|
||||
|
||||
@@ -5,12 +5,14 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
@@ -1358,6 +1360,149 @@ func TestAnalyzeSkipsWhenArtifactMapEmpty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzePassesOnlyExplicitExtractionInputsToSelectedArtifact(t *testing.T) {
|
||||
env, m, fake := setupAnalyzeEnv(t)
|
||||
paths := sessionPathsForEnv(env, m.SessionID)
|
||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "final.trimmed.json"), `{"segments":[]}`)
|
||||
extractionPaths := configureAnalyzeExtractionFixture(t, env, m)
|
||||
|
||||
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||
artifact.Inputs["encounters"] = config.ScriptoriumInputConfig{
|
||||
Source: artifacts.ExtractionArtifactSourceID("encounters"), Required: true,
|
||||
}
|
||||
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact
|
||||
env.SelectedArtifactKeys = []string{"session_recap"}
|
||||
tracker := &analyzeObjectStoreTracker{}
|
||||
env.ObjectStore = tracker
|
||||
|
||||
if _, err := (analyzeStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(fake.RunRequests) != 1 {
|
||||
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
|
||||
}
|
||||
inputs := fake.RunRequests[0].InputPaths
|
||||
if inputs["encounters"] != extractionPaths["encounters"] {
|
||||
t.Fatalf("encounters input = %q, want %q", inputs["encounters"], extractionPaths["encounters"])
|
||||
}
|
||||
if _, exists := inputs["characters"]; exists {
|
||||
t.Fatalf("unconfigured extraction input was added: %#v", inputs)
|
||||
}
|
||||
if len(inputs) != 2 {
|
||||
t.Fatalf("input paths = %#v, want transcript plus explicit encounters", inputs)
|
||||
}
|
||||
if tracker.called {
|
||||
t.Fatal("analyze extraction resolution called the object store")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeRequiredUnavailableExtractionFailsWithGuidance(t *testing.T) {
|
||||
env, m, fake := setupAnalyzeEnv(t)
|
||||
paths := sessionPathsForEnv(env, m.SessionID)
|
||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "final.trimmed.json"), `{"segments":[]}`)
|
||||
configureAnalyzeExtraction(t, env)
|
||||
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||
artifact.Inputs["encounters"] = config.ScriptoriumInputConfig{
|
||||
Source: artifacts.ExtractionArtifactSourceID("encounters"), Required: true,
|
||||
}
|
||||
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact
|
||||
|
||||
_, err := (analyzeStage{}).Run(context.Background(), env, m)
|
||||
if err == nil || !strings.Contains(err.Error(), "pipeline.notarius output \"encounters\"") || !strings.Contains(err.Error(), "rerun extract with --force") {
|
||||
t.Fatalf("Run() error = %v, want actionable extraction guidance", err)
|
||||
}
|
||||
if len(fake.RunRequests) != 0 {
|
||||
t.Fatalf("run requests = %d, want 0", len(fake.RunRequests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeOmitsOptionalUnavailableExtraction(t *testing.T) {
|
||||
env, m, fake := setupAnalyzeEnv(t)
|
||||
paths := sessionPathsForEnv(env, m.SessionID)
|
||||
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "final.trimmed.json"), `{"segments":[]}`)
|
||||
configureAnalyzeExtraction(t, env)
|
||||
artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"]
|
||||
artifact.Inputs["encounters"] = config.ScriptoriumInputConfig{
|
||||
Source: artifacts.ExtractionArtifactSourceID("encounters"), Required: false,
|
||||
}
|
||||
env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact
|
||||
|
||||
if _, err := (analyzeStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(fake.RunRequests) != 1 {
|
||||
t.Fatalf("run requests = %d, want 1", len(fake.RunRequests))
|
||||
}
|
||||
if _, exists := fake.RunRequests[0].InputPaths["encounters"]; exists {
|
||||
t.Fatalf("optional unavailable extraction was passed: %#v", fake.RunRequests[0].InputPaths)
|
||||
}
|
||||
}
|
||||
|
||||
func configureAnalyzeExtraction(t *testing.T, env *Env) {
|
||||
t.Helper()
|
||||
env.Config.Pipeline.Notarius = &config.NotariusConfig{
|
||||
Enabled: true, PipelineID: "campaign.extract",
|
||||
Outputs: map[string]config.NotariusOutputConfig{
|
||||
"characters": {LaneID: "characters", MediaType: "application/json", SchemaID: "characters", SchemaVersion: "1"},
|
||||
"encounters": {LaneID: "encounters", MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func configureAnalyzeExtractionFixture(t *testing.T, env *Env, m *manifest.Manifest) map[string]string {
|
||||
t.Helper()
|
||||
configureAnalyzeExtraction(t, env)
|
||||
paths := sessionPathsForEnv(env, m.SessionID)
|
||||
producerRunID := "extract-run-1"
|
||||
bundleRoot := filepath.Join(paths.ArtifactsDir, "notarius", producerRunID)
|
||||
keys := make([]string, 0, len(env.Config.Pipeline.Notarius.Outputs))
|
||||
for key := range env.Config.Pipeline.Notarius.Outputs {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
outputPaths := make(map[string]string, len(keys))
|
||||
outputs := make([]manifest.ArtifactRecord, 0, len(keys)+1)
|
||||
for _, key := range keys {
|
||||
definition := env.Config.Pipeline.Notarius.Outputs[key]
|
||||
outputPath := filepath.Join(bundleRoot, "lanes", key+".json")
|
||||
writeAnalyzeFile(t, outputPath, `{"items":[]}`)
|
||||
checksum, err := artifacts.SHA256File(outputPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outputPaths[key] = outputPath
|
||||
outputs = append(outputs, manifest.ArtifactRecord{
|
||||
Kind: "notarius_lane", SourceID: artifacts.ExtractionArtifactSourceID(key), LocalPath: outputPath,
|
||||
ProducerRunID: producerRunID, Checksum: checksum,
|
||||
Contract: &artifactmodel.ContractMetadata{
|
||||
MediaType: definition.MediaType, SchemaID: definition.SchemaID,
|
||||
SchemaVersion: definition.SchemaVersion, ModuleKey: definition.ModuleKey,
|
||||
},
|
||||
ExternalProvenance: &artifactmodel.ExternalProvenance{
|
||||
System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: definition.LaneID,
|
||||
},
|
||||
})
|
||||
}
|
||||
indexPath := filepath.Join(bundleRoot, "index.json")
|
||||
writeAnalyzeFile(t, indexPath, `{"lanes":[]}`)
|
||||
indexChecksum, err := artifacts.SHA256File(indexPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outputs = append(outputs, manifest.ArtifactRecord{
|
||||
Kind: "notarius_index", LocalPath: indexPath, ProducerRunID: producerRunID, Checksum: indexChecksum,
|
||||
})
|
||||
m.Stages["extract"] = &manifest.StageRecord{
|
||||
Name: "extract", Status: manifest.StatusSucceeded, Outputs: outputs,
|
||||
Metadata: map[string]any{
|
||||
"narratio_run_id": producerRunID, "bundle_root": bundleRoot,
|
||||
"receipt": map[string]any{"run_id": "notarius-run-1", "pipeline_id": "campaign.extract"},
|
||||
},
|
||||
}
|
||||
return outputPaths
|
||||
}
|
||||
|
||||
func setupAnalyzeEnv(t *testing.T) (*Env, *manifest.Manifest, *scriptorium.FakeRunner) {
|
||||
t.Helper()
|
||||
workspace := t.TempDir()
|
||||
|
||||
462
internal/stage/extract.go
Normal file
462
internal/stage/extract.go
Normal file
@@ -0,0 +1,462 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
extractSkipReason = "notarius_disabled"
|
||||
extractLaneOutputKind = "notarius_lane"
|
||||
extractIndexOutputKind = "notarius_index"
|
||||
maxDiagnosticSummaries = 100
|
||||
)
|
||||
|
||||
type extractStage struct{}
|
||||
|
||||
func (extractStage) Name() string { return "extract" }
|
||||
|
||||
func (extractStage) Declares() IODecl {
|
||||
return IODecl{
|
||||
Inputs: []artifacts.Ref{{
|
||||
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed,
|
||||
SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
|
||||
Category: "transcripts",
|
||||
RelativePath: artifactmodel.TranscriptPathFinalTrimmed,
|
||||
}},
|
||||
Outputs: []artifacts.Ref{
|
||||
{Kind: extractLaneOutputKind, Category: "artifacts", RelativePath: "artifacts/notarius/<run-id>/lanes/*.json"},
|
||||
{Kind: extractIndexOutputKind, Category: "artifacts", RelativePath: "artifacts/notarius/<run-id>/index.json"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (extractStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
|
||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
||||
return nil, fmt.Errorf("extract: resolved stage environment config is required")
|
||||
}
|
||||
notariusConfig := env.Config.Pipeline.Notarius
|
||||
if notariusConfig == nil || !notariusConfig.Enabled {
|
||||
return &StageResult{
|
||||
Disposition: StageDispositionSkipped,
|
||||
SkipReason: extractSkipReason,
|
||||
Metadata: map[string]any{
|
||||
"stage": "extract",
|
||||
"notarius_enabled": false,
|
||||
"reason": extractSkipReason,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
if env.ArtifactStore == nil {
|
||||
return nil, fmt.Errorf("extract: artifact store is required")
|
||||
}
|
||||
if env.Notarius == nil {
|
||||
return nil, fmt.Errorf("extract: notarius adapter is required")
|
||||
}
|
||||
if m == nil {
|
||||
return nil, fmt.Errorf("extract: session manifest is required")
|
||||
}
|
||||
|
||||
sessionID := strings.TrimSpace(m.SessionID)
|
||||
if sessionID == "" {
|
||||
sessionID = strings.TrimSpace(env.Config.Session.SessionID)
|
||||
}
|
||||
campaign := strings.TrimSpace(m.Campaign)
|
||||
if campaign == "" {
|
||||
campaign = strings.TrimSpace(env.Config.Session.Campaign)
|
||||
}
|
||||
runID := strings.TrimSpace(m.RunID)
|
||||
if sessionID == "" || campaign == "" || runID == "" {
|
||||
return nil, fmt.Errorf("extract: session id, campaign, and run id are required")
|
||||
}
|
||||
if !safePathSegment(runID) {
|
||||
return nil, fmt.Errorf("extract: run id %q is not a safe path segment", runID)
|
||||
}
|
||||
|
||||
paths := sessionPathsForEnv(env, sessionID)
|
||||
input, err := artifacts.ResolveSessionArtifact(paths, m, artifacts.ArtifactTranscriptFinalTrimmed)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve final-trimmed transcript: %w", err)
|
||||
}
|
||||
|
||||
timeout, err := time.ParseDuration(strings.TrimSpace(notariusConfig.Timeout))
|
||||
if err != nil || timeout <= 0 {
|
||||
return nil, fmt.Errorf("extract: invalid notarius timeout %q", notariusConfig.Timeout)
|
||||
}
|
||||
resolvedBinary, err := resolveExecutable(notariusConfig.Binary)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve notarius binary: %w", err)
|
||||
}
|
||||
configPath, err := absolutePath(notariusConfig.ConfigPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve notarius config path: %w", err)
|
||||
}
|
||||
inputPath, err := absolutePath(input.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve transcript input path: %w", err)
|
||||
}
|
||||
workingDirectory, err := absolutePath(notariusConfig.WorkingDirectory)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve notarius working directory: %w", err)
|
||||
}
|
||||
|
||||
workspaceRoot := strings.TrimSpace(paths.WorkspaceRoot)
|
||||
if workspaceRoot == "" {
|
||||
workspaceRoot = env.Config.Pipeline.Workspace.Root
|
||||
}
|
||||
receiptPath, err := absolutePath(artifacts.SessionRunNotariusReceiptPathForCampaign(workspaceRoot, campaign, sessionID, runID))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve receipt path: %w", err)
|
||||
}
|
||||
logPath, err := absolutePath(artifacts.SessionRunNotariusLogPathForCampaign(workspaceRoot, campaign, sessionID, runID))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve stderr path: %w", err)
|
||||
}
|
||||
outputRoot, err := absolutePath(artifacts.SessionRunNotariusOutputRootForCampaign(workspaceRoot, campaign, sessionID, runID))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve output root: %w", err)
|
||||
}
|
||||
durableBundle, err := absolutePath(artifacts.SessionNotariusBundleDirForCampaign(workspaceRoot, campaign, sessionID, runID))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve durable bundle path: %w", err)
|
||||
}
|
||||
for _, directory := range []string{filepath.Dir(receiptPath), outputRoot, filepath.Dir(durableBundle)} {
|
||||
if err := os.MkdirAll(directory, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("extract: create directory %q: %w", directory, err)
|
||||
}
|
||||
}
|
||||
|
||||
fingerprint, err := extractionFingerprint(resolvedBinary, configPath, notariusConfig, timeout, workingDirectory)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: build configuration fingerprint: %w", err)
|
||||
}
|
||||
request := notarius.RunRequest{
|
||||
Binary: resolvedBinary, ConfigPath: configPath, PipelineID: notariusConfig.PipelineID,
|
||||
InputPath: inputPath, OutputRoot: outputRoot, WorkingDirectory: workingDirectory,
|
||||
ReceiptPath: receiptPath, LogPath: logPath, Timeout: timeout,
|
||||
}
|
||||
adapterResult, err := env.Notarius.Run(ctx, request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: run notarius: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(adapterResult.BundleRoot) == "" || strings.TrimSpace(adapterResult.Index.Path) == "" {
|
||||
return nil, fmt.Errorf("extract: notarius result is missing bundle or index path")
|
||||
}
|
||||
if adapterResult.Receipt.RunID == "" || adapterResult.Receipt.PipelineID != notariusConfig.PipelineID {
|
||||
return nil, fmt.Errorf("extract: notarius receipt identity is missing or incompatible")
|
||||
}
|
||||
|
||||
selected, err := selectRequiredNotariusLanes(env.ArtifactStore, notariusConfig.Outputs, adapterResult)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
indexRelative, err := pathsafe.SlashRelativeFromRoot(adapterResult.BundleRoot, adapterResult.Index.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve staging index relative path: %w", err)
|
||||
}
|
||||
rejectionsRelative, err := pathsafe.SlashRelativeFromRoot(adapterResult.BundleRoot, adapterResult.Index.RejectedPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve staging rejections relative path: %w", err)
|
||||
}
|
||||
warningsRelative, err := pathsafe.SlashRelativeFromRoot(adapterResult.BundleRoot, adapterResult.Index.WarningsPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve staging warnings relative path: %w", err)
|
||||
}
|
||||
stagingIndexChecksum, err := checksumRegularFile(adapterResult.Index.Path, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: validate staging index: %w", err)
|
||||
}
|
||||
|
||||
if err := fileops.PromoteDirectory(adapterResult.BundleRoot, durableBundle); err != nil {
|
||||
return nil, fmt.Errorf("extract: promote notarius bundle: %w", err)
|
||||
}
|
||||
promotedIndexPath, err := pathsafe.JoinSlashRelativeUnderRoot(durableBundle, indexRelative)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve promoted index: %w", err)
|
||||
}
|
||||
promotedIndexChecksum, err := checksumRegularFile(promotedIndexPath, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: checksum promoted index: %w", err)
|
||||
}
|
||||
if promotedIndexChecksum != stagingIndexChecksum {
|
||||
return nil, fmt.Errorf("extract: promoted index checksum differs from staging index")
|
||||
}
|
||||
promotedRejectionsPath, err := pathsafe.JoinSlashRelativeUnderRoot(durableBundle, rejectionsRelative)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve promoted rejections: %w", err)
|
||||
}
|
||||
promotedWarningsPath, err := pathsafe.JoinSlashRelativeUnderRoot(durableBundle, warningsRelative)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve promoted warnings: %w", err)
|
||||
}
|
||||
|
||||
outputs := make([]artifacts.Ref, 0, len(selected)+1)
|
||||
for _, lane := range selected {
|
||||
promotedPath, err := pathsafe.JoinSlashRelativeUnderRoot(durableBundle, lane.RelativePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve promoted lane %q: %w", lane.Descriptor.LaneID, err)
|
||||
}
|
||||
checksum, err := checksumRegularFile(promotedPath, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: validate promoted lane %q: %w", lane.Descriptor.LaneID, err)
|
||||
}
|
||||
if checksum != lane.StagingChecksum {
|
||||
return nil, fmt.Errorf("extract: promoted lane %q checksum differs from staging payload", lane.Descriptor.LaneID)
|
||||
}
|
||||
relativePath, err := pathsafe.SlashRelativeFromRoot(paths.Root, promotedPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: derive lane %q session-relative path: %w", lane.Descriptor.LaneID, err)
|
||||
}
|
||||
outputs = append(outputs, artifacts.Ref{
|
||||
Kind: extractLaneOutputKind, SourceID: artifacts.ExtractionArtifactSourceID(lane.Key),
|
||||
Category: "artifacts", SessionID: sessionID, RelativePath: relativePath,
|
||||
AbsolutePath: promotedPath, Checksum: checksum,
|
||||
Contract: &artifactmodel.ContractMetadata{
|
||||
MediaType: lane.Descriptor.MediaType, SchemaID: lane.Descriptor.SchemaID,
|
||||
SchemaVersion: lane.Descriptor.SchemaVersion, ModuleKey: lane.Descriptor.ModuleKey,
|
||||
},
|
||||
ExternalProvenance: &artifactmodel.ExternalProvenance{
|
||||
System: "notarius", RunID: adapterResult.Receipt.RunID,
|
||||
PipelineID: adapterResult.Receipt.PipelineID, ArtifactID: lane.Descriptor.LaneID,
|
||||
},
|
||||
})
|
||||
}
|
||||
indexSessionRelative, err := pathsafe.SlashRelativeFromRoot(paths.Root, promotedIndexPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: derive index session-relative path: %w", err)
|
||||
}
|
||||
outputs = append(outputs, artifacts.Ref{
|
||||
Kind: extractIndexOutputKind, Category: "artifacts", SessionID: sessionID,
|
||||
RelativePath: indexSessionRelative, AbsolutePath: promotedIndexPath, Checksum: promotedIndexChecksum,
|
||||
})
|
||||
|
||||
metadata := map[string]any{
|
||||
"stage": "extract",
|
||||
"notarius_enabled": true,
|
||||
"bundle_root": durableBundle,
|
||||
"receipt_path": receiptPath,
|
||||
"diagnostic_path": logPath,
|
||||
"rejections_path": promotedRejectionsPath,
|
||||
"warnings_path": promotedWarningsPath,
|
||||
"narratio_run_id": runID,
|
||||
"configuration_fingerprint": fingerprint,
|
||||
"receipt": map[string]any{
|
||||
"run_id": adapterResult.Receipt.RunID, "pipeline_id": adapterResult.Receipt.PipelineID,
|
||||
"normalized_output_count": adapterResult.Receipt.NormalizedOutputCount,
|
||||
"rejected_output_count": adapterResult.Receipt.RejectedOutputCount,
|
||||
"warning_count": adapterResult.Receipt.WarningCount,
|
||||
"validation_status": adapterResult.Receipt.ValidationStatus,
|
||||
},
|
||||
"rejections": boundedRejectionMetadata(adapterResult.Rejections),
|
||||
"warnings": boundedWarningMetadata(adapterResult.Warnings),
|
||||
}
|
||||
return &StageResult{
|
||||
Outputs: outputs,
|
||||
Logs: []string{receiptPath, logPath},
|
||||
Metadata: metadata,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type selectedNotariusLane struct {
|
||||
Key string
|
||||
Descriptor notarius.LaneDescriptor
|
||||
RelativePath string
|
||||
StagingChecksum string
|
||||
}
|
||||
|
||||
func selectRequiredNotariusLanes(
|
||||
store artifacts.Store,
|
||||
required map[string]config.NotariusOutputConfig,
|
||||
result notarius.RunResult,
|
||||
) ([]selectedNotariusLane, error) {
|
||||
keys := make([]string, 0, len(required))
|
||||
for key := range required {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
selected := make([]selectedNotariusLane, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
expected := required[key]
|
||||
for _, rejection := range result.Rejections {
|
||||
if rejection.LaneID == expected.LaneID {
|
||||
return nil, fmt.Errorf("extract: required lane %q was rejected (reason_code=%q)", expected.LaneID, rejection.ReasonCode)
|
||||
}
|
||||
}
|
||||
matches := make([]notarius.LaneDescriptor, 0, 1)
|
||||
for _, descriptor := range result.Index.Lanes {
|
||||
if descriptor.LaneID == expected.LaneID {
|
||||
matches = append(matches, descriptor)
|
||||
}
|
||||
}
|
||||
if len(matches) != 1 {
|
||||
return nil, fmt.Errorf("extract: required lane %q has %d descriptors, want exactly one", expected.LaneID, len(matches))
|
||||
}
|
||||
descriptor := matches[0]
|
||||
if descriptor.MediaType != expected.MediaType || descriptor.SchemaID != expected.SchemaID ||
|
||||
descriptor.SchemaVersion != expected.SchemaVersion ||
|
||||
(expected.ModuleKey != "" && descriptor.ModuleKey != expected.ModuleKey) {
|
||||
return nil, fmt.Errorf("extract: required lane %q descriptor contract is incompatible", expected.LaneID)
|
||||
}
|
||||
checksum, err := checksumRegularFile(descriptor.Path, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: validate required lane %q: %w", expected.LaneID, err)
|
||||
}
|
||||
if store != nil {
|
||||
storeChecksum, err := store.Checksum(descriptor.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: checksum required lane %q: %w", expected.LaneID, err)
|
||||
}
|
||||
if storeChecksum != checksum {
|
||||
return nil, fmt.Errorf("extract: inconsistent staging checksum for required lane %q", expected.LaneID)
|
||||
}
|
||||
}
|
||||
relative, err := pathsafe.SlashRelativeFromRoot(result.BundleRoot, descriptor.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract: resolve required lane %q relative path: %w", expected.LaneID, err)
|
||||
}
|
||||
selected = append(selected, selectedNotariusLane{
|
||||
Key: key, Descriptor: descriptor, RelativePath: relative, StagingChecksum: checksum,
|
||||
})
|
||||
}
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
func checksumRegularFile(path string, requireJSON bool) (string, error) {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return "", fmt.Errorf("path %q must be a regular file without symlinks", path)
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
return "", fmt.Errorf("path %q must be non-empty", path)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if requireJSON && !json.Valid(data) {
|
||||
return "", fmt.Errorf("path %q is not valid JSON", path)
|
||||
}
|
||||
digest := sha256.Sum256(data)
|
||||
return hex.EncodeToString(digest[:]), nil
|
||||
}
|
||||
|
||||
type fingerprintOutput struct {
|
||||
Key string `json:"key"`
|
||||
LaneID string `json:"lane_id"`
|
||||
MediaType string `json:"media_type"`
|
||||
SchemaID string `json:"schema_id"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
}
|
||||
|
||||
type fingerprintDocument struct {
|
||||
Binary string `json:"binary"`
|
||||
ConfigPath string `json:"config_path"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
Timeout string `json:"timeout"`
|
||||
WorkingDirectory string `json:"working_directory"`
|
||||
Outputs []fingerprintOutput `json:"outputs"`
|
||||
}
|
||||
|
||||
func extractionFingerprint(
|
||||
binary, configPath string,
|
||||
cfg *config.NotariusConfig,
|
||||
timeout time.Duration,
|
||||
workingDirectory string,
|
||||
) (string, error) {
|
||||
keys := make([]string, 0, len(cfg.Outputs))
|
||||
for key := range cfg.Outputs {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
outputs := make([]fingerprintOutput, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
output := cfg.Outputs[key]
|
||||
outputs = append(outputs, fingerprintOutput{
|
||||
Key: key, LaneID: output.LaneID, MediaType: output.MediaType,
|
||||
SchemaID: output.SchemaID, SchemaVersion: output.SchemaVersion, ModuleKey: output.ModuleKey,
|
||||
})
|
||||
}
|
||||
payload, err := json.Marshal(fingerprintDocument{
|
||||
Binary: binary, ConfigPath: configPath, PipelineID: cfg.PipelineID,
|
||||
Timeout: timeout.String(), WorkingDirectory: workingDirectory, Outputs: outputs,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest := sha256.Sum256(payload)
|
||||
return hex.EncodeToString(digest[:]), nil
|
||||
}
|
||||
|
||||
func resolveExecutable(value string) (string, error) {
|
||||
resolved, err := exec.LookPath(strings.TrimSpace(value))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return absolutePath(resolved)
|
||||
}
|
||||
|
||||
func absolutePath(value string) (string, error) {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return "", fmt.Errorf("path is required")
|
||||
}
|
||||
resolved, err := filepath.Abs(trimmed)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Clean(resolved), nil
|
||||
}
|
||||
|
||||
func safePathSegment(value string) bool {
|
||||
return value != "" && value != "." && value != ".." && filepath.Base(value) == value &&
|
||||
!strings.ContainsAny(value, `/\\`)
|
||||
}
|
||||
|
||||
func boundedRejectionMetadata(values []notarius.RejectionSummary) []map[string]any {
|
||||
limit := len(values)
|
||||
if limit > maxDiagnosticSummaries {
|
||||
limit = maxDiagnosticSummaries
|
||||
}
|
||||
result := make([]map[string]any, 0, limit)
|
||||
for _, value := range values[:limit] {
|
||||
result = append(result, map[string]any{
|
||||
"stage": value.Stage, "step_id": value.StepID, "lane_id": value.LaneID,
|
||||
"module_key": value.ModuleKey, "chunk_id": value.ChunkID,
|
||||
"validator_name": value.ValidatorName, "reason_code": value.ReasonCode,
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func boundedWarningMetadata(values []notarius.WarningSummary) []map[string]any {
|
||||
limit := len(values)
|
||||
if limit > maxDiagnosticSummaries {
|
||||
limit = maxDiagnosticSummaries
|
||||
}
|
||||
result := make([]map[string]any, 0, limit)
|
||||
for _, value := range values[:limit] {
|
||||
result = append(result, map[string]any{"scope": value.Scope, "reason_code": value.ReasonCode})
|
||||
}
|
||||
return result
|
||||
}
|
||||
250
internal/stage/extract_resume.go
Normal file
250
internal/stage/extract_resume.go
Normal file
@@ -0,0 +1,250 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||
)
|
||||
|
||||
func (extractStage) ValidateResume(_ context.Context, env *Env, m *manifest.Manifest) (ResumeValidation, error) {
|
||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: resolved stage environment config is required")
|
||||
}
|
||||
cfg := env.Config.Pipeline.Notarius
|
||||
if cfg == nil || !cfg.Enabled {
|
||||
return NonResumable("Notarius extraction is disabled"), nil
|
||||
}
|
||||
if m == nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: session manifest is required")
|
||||
}
|
||||
record := m.Stages[(extractStage{}).Name()]
|
||||
if record == nil || record.Status != manifest.StatusSucceeded {
|
||||
return NonResumable("extract stage has no succeeded result"), nil
|
||||
}
|
||||
if record.Name != (extractStage{}).Name() {
|
||||
return NonResumable("extract stage record identity is inconsistent"), nil
|
||||
}
|
||||
|
||||
producerRunID := metadataString(record.Metadata, "narratio_run_id")
|
||||
if producerRunID == "" {
|
||||
return NonResumable("extract result is missing its producing run ID"), nil
|
||||
}
|
||||
if !safePathSegment(producerRunID) {
|
||||
return NonResumable("extract result has an invalid producing run ID"), nil
|
||||
}
|
||||
|
||||
timeout, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout))
|
||||
if err != nil || timeout <= 0 {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: invalid Notarius timeout %q", cfg.Timeout)
|
||||
}
|
||||
resolvedBinary, err := resolveExecutable(cfg.Binary)
|
||||
if err != nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: resolve Notarius binary: %w", err)
|
||||
}
|
||||
configPath, err := absolutePath(cfg.ConfigPath)
|
||||
if err != nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: resolve Notarius config path: %w", err)
|
||||
}
|
||||
workingDirectory, err := absolutePath(cfg.WorkingDirectory)
|
||||
if err != nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: resolve Notarius working directory: %w", err)
|
||||
}
|
||||
fingerprint, err := extractionFingerprint(resolvedBinary, configPath, cfg, timeout, workingDirectory)
|
||||
if err != nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: build configuration fingerprint: %w", err)
|
||||
}
|
||||
if metadataString(record.Metadata, "configuration_fingerprint") != fingerprint {
|
||||
return NonResumable("Notarius invocation contract changed"), nil
|
||||
}
|
||||
|
||||
sessionID := strings.TrimSpace(m.SessionID)
|
||||
if sessionID == "" {
|
||||
sessionID = strings.TrimSpace(env.Config.Session.SessionID)
|
||||
}
|
||||
campaign := strings.TrimSpace(m.Campaign)
|
||||
if campaign == "" {
|
||||
campaign = strings.TrimSpace(env.Config.Session.Campaign)
|
||||
}
|
||||
if sessionID == "" || campaign == "" {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: session ID and campaign are required")
|
||||
}
|
||||
paths := sessionPathsForEnv(env, sessionID)
|
||||
workspaceRoot := strings.TrimSpace(paths.WorkspaceRoot)
|
||||
if workspaceRoot == "" {
|
||||
workspaceRoot = env.Config.Pipeline.Workspace.Root
|
||||
}
|
||||
bundleRoot, err := absolutePath(artifacts.SessionNotariusBundleDirForCampaign(workspaceRoot, campaign, sessionID, producerRunID))
|
||||
if err != nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: resolve durable bundle path: %w", err)
|
||||
}
|
||||
storedBundleRoot := metadataString(record.Metadata, "bundle_root")
|
||||
if storedBundleRoot == "" || !filepath.IsAbs(storedBundleRoot) || filepath.Clean(storedBundleRoot) != bundleRoot {
|
||||
return NonResumable("extract result does not identify its canonical immutable bundle"), nil
|
||||
}
|
||||
bundleRelative, err := pathsafe.SlashRelativeFromRoot(paths.Root, bundleRoot)
|
||||
if err != nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: immutable bundle path is unsafe: %w", err)
|
||||
}
|
||||
if err := rejectSymlinkComponents(paths.Root, bundleRelative); err != nil {
|
||||
return ResumeValidation{}, err
|
||||
}
|
||||
bundleInfo, err := os.Lstat(bundleRoot)
|
||||
if os.IsNotExist(err) {
|
||||
return NonResumable("immutable Notarius bundle is missing"), nil
|
||||
}
|
||||
if err != nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: inspect immutable bundle: %w", err)
|
||||
}
|
||||
if bundleInfo.Mode()&os.ModeSymlink != 0 {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: immutable bundle must not be a symlink")
|
||||
}
|
||||
if !bundleInfo.IsDir() {
|
||||
return NonResumable("immutable Notarius bundle is not a directory"), nil
|
||||
}
|
||||
|
||||
receiptRunID, receiptPipelineID := receiptIdentity(record.Metadata)
|
||||
if receiptRunID == "" || receiptPipelineID != cfg.PipelineID {
|
||||
return NonResumable("extract result has incompatible Notarius receipt identity"), nil
|
||||
}
|
||||
|
||||
expectedSources := make(map[string]config.NotariusOutputConfig, len(cfg.Outputs))
|
||||
for key, output := range cfg.Outputs {
|
||||
expectedSources[artifacts.ExtractionArtifactSourceID(key)] = output
|
||||
}
|
||||
seenSources := make(map[string]struct{}, len(expectedSources))
|
||||
indexSeen := false
|
||||
for _, output := range record.Outputs {
|
||||
if output.ProducerRunID != producerRunID {
|
||||
return NonResumable("extract output producer identity is inconsistent"), nil
|
||||
}
|
||||
if output.SourceID == "" {
|
||||
if indexSeen || output.Kind != extractIndexOutputKind {
|
||||
return NonResumable("extract result has an unexpected non-selectable output"), nil
|
||||
}
|
||||
indexSeen = true
|
||||
expectedIndex := filepath.Join(bundleRoot, "index.json")
|
||||
if filepath.Clean(output.LocalPath) != expectedIndex {
|
||||
return NonResumable("extract index path is not canonical"), nil
|
||||
}
|
||||
validation, err := validateResumePayload(bundleRoot, output.LocalPath, output.Checksum)
|
||||
if err != nil || !validation.Resumable {
|
||||
return validation, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
expected, ok := expectedSources[output.SourceID]
|
||||
if !ok || output.Kind != extractLaneOutputKind {
|
||||
return NonResumable("extract result source set differs from current configuration"), nil
|
||||
}
|
||||
if _, duplicate := seenSources[output.SourceID]; duplicate {
|
||||
return NonResumable("extract result contains a duplicate configured source"), nil
|
||||
}
|
||||
seenSources[output.SourceID] = struct{}{}
|
||||
if !compatibleExtractionContract(output.Contract, expected) {
|
||||
return NonResumable("extract output contract is incompatible with current configuration"), nil
|
||||
}
|
||||
if !compatibleExtractionProvenance(output.ExternalProvenance, receiptRunID, receiptPipelineID, expected.LaneID) {
|
||||
return NonResumable("extract output has incompatible Notarius provenance"), nil
|
||||
}
|
||||
validation, err := validateResumePayload(bundleRoot, output.LocalPath, output.Checksum)
|
||||
if err != nil || !validation.Resumable {
|
||||
return validation, err
|
||||
}
|
||||
}
|
||||
if !indexSeen {
|
||||
return NonResumable("extract result is missing its canonical index"), nil
|
||||
}
|
||||
if len(seenSources) != len(expectedSources) || len(record.Outputs) != len(expectedSources)+1 {
|
||||
return NonResumable("extract result is missing configured sources"), nil
|
||||
}
|
||||
return Resumable(), nil
|
||||
}
|
||||
|
||||
func validateResumePayload(bundleRoot, path, checksum string) (ResumeValidation, error) {
|
||||
if !filepath.IsAbs(path) {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: persisted output path must be absolute")
|
||||
}
|
||||
relative, err := pathsafe.SlashRelativeFromRoot(bundleRoot, path)
|
||||
if err != nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: persisted output path is unsafe: %w", err)
|
||||
}
|
||||
if err := rejectSymlinkComponents(bundleRoot, relative); err != nil {
|
||||
return ResumeValidation{}, err
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if os.IsNotExist(err) {
|
||||
return NonResumable("extract output is missing"), nil
|
||||
}
|
||||
if err != nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: inspect output %q: %w", path, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: output %q must be a regular file without symlinks", path)
|
||||
}
|
||||
if strings.TrimSpace(checksum) == "" {
|
||||
return NonResumable("extract output is missing its checksum"), nil
|
||||
}
|
||||
actual, err := artifacts.SHA256File(path)
|
||||
if err != nil {
|
||||
return ResumeValidation{}, fmt.Errorf("extract resume: checksum output %q: %w", path, err)
|
||||
}
|
||||
if actual != checksum {
|
||||
return NonResumable("extract output checksum does not match durable bytes"), nil
|
||||
}
|
||||
return Resumable(), nil
|
||||
}
|
||||
|
||||
func rejectSymlinkComponents(root, slashRelative string) error {
|
||||
current := filepath.Clean(root)
|
||||
parts := strings.Split(filepath.FromSlash(slashRelative), string(filepath.Separator))
|
||||
for _, part := range parts[:len(parts)-1] {
|
||||
current = filepath.Join(current, part)
|
||||
info, err := os.Lstat(current)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("extract resume: inspect output directory %q: %w", current, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("extract resume: output directory %q must not be a symlink", current)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func compatibleExtractionContract(got *artifactmodel.ContractMetadata, want config.NotariusOutputConfig) bool {
|
||||
return got != nil && got.MediaType == want.MediaType && got.SchemaID == want.SchemaID &&
|
||||
got.SchemaVersion == want.SchemaVersion && (want.ModuleKey == "" || got.ModuleKey == want.ModuleKey)
|
||||
}
|
||||
|
||||
func compatibleExtractionProvenance(got *artifactmodel.ExternalProvenance, runID, pipelineID, laneID string) bool {
|
||||
return got != nil && got.System == "notarius" && got.RunID == runID &&
|
||||
got.PipelineID == pipelineID && got.ArtifactID == laneID
|
||||
}
|
||||
|
||||
func metadataString(metadata map[string]any, key string) string {
|
||||
if metadata == nil {
|
||||
return ""
|
||||
}
|
||||
value, _ := metadata[key].(string)
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func receiptIdentity(metadata map[string]any) (string, string) {
|
||||
if metadata == nil {
|
||||
return "", ""
|
||||
}
|
||||
receipt, _ := metadata["receipt"].(map[string]any)
|
||||
return metadataString(receipt, "run_id"), metadataString(receipt, "pipeline_id")
|
||||
}
|
||||
575
internal/stage/extract_test.go
Normal file
575
internal/stage/extract_test.go
Normal file
@@ -0,0 +1,575 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func TestExtractStageDisabledReturnsExplicitSkip(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
absent bool
|
||||
}{
|
||||
{name: "disabled"},
|
||||
{name: "absent", absent: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
env, m, fake := setupExtractEnv(t)
|
||||
if test.absent {
|
||||
env.Config.Pipeline.Notarius = nil
|
||||
} else {
|
||||
env.Config.Pipeline.Notarius.Enabled = false
|
||||
}
|
||||
|
||||
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if result.Disposition != StageDispositionSkipped || result.SkipReason != extractSkipReason {
|
||||
t.Fatalf("result disposition = %q reason = %q", result.Disposition, result.SkipReason)
|
||||
}
|
||||
if len(result.Outputs) != 0 || len(fake.Requests) != 0 {
|
||||
t.Fatalf("outputs = %#v; requests = %#v", result.Outputs, fake.Requests)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStageResolvesManifestInputAndBuildsExactRequest(t *testing.T) {
|
||||
env, m, fake := setupExtractEnv(t)
|
||||
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(fake.Requests) != 1 {
|
||||
t.Fatalf("requests = %d, want 1", len(fake.Requests))
|
||||
}
|
||||
req := fake.Requests[0]
|
||||
fixture := extractFixtureFromEnv(t, env, m)
|
||||
if req.InputPath != fixture.inputPath {
|
||||
t.Fatalf("input path = %q, want manifest path %q", req.InputPath, fixture.inputPath)
|
||||
}
|
||||
wantReceipt := artifacts.SessionRunNotariusReceiptPathForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID)
|
||||
wantLog := artifacts.SessionRunNotariusLogPathForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID)
|
||||
wantOutputRoot := artifacts.SessionRunNotariusOutputRootForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID)
|
||||
if req.ConfigPath != env.Config.Pipeline.Notarius.ConfigPath || req.PipelineID != "dnd-session" ||
|
||||
req.OutputRoot != wantOutputRoot || req.ReceiptPath != wantReceipt || req.LogPath != wantLog ||
|
||||
req.WorkingDirectory != env.Config.Pipeline.Notarius.WorkingDirectory || req.Timeout != 45*time.Minute {
|
||||
t.Fatalf("request = %#v", req)
|
||||
}
|
||||
if !filepath.IsAbs(req.Binary) {
|
||||
t.Fatalf("binary = %q, want absolute resolved path", req.Binary)
|
||||
}
|
||||
if len(result.Outputs) != 2 {
|
||||
t.Fatalf("outputs = %#v, want lane and index", result.Outputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStageProducesImmutableManifestReadyOutputs(t *testing.T) {
|
||||
env, m, fake := setupExtractEnv(t)
|
||||
fixture := extractFixtureFromEnv(t, env, m)
|
||||
stagingBundle := fake.Result.BundleRoot
|
||||
|
||||
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
durableBundle := artifacts.SessionNotariusBundleDirForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID)
|
||||
if result.Metadata["bundle_root"] != durableBundle || result.Metadata["narratio_run_id"] != fixture.runID {
|
||||
t.Fatalf("metadata = %#v", result.Metadata)
|
||||
}
|
||||
if fingerprint, _ := result.Metadata["configuration_fingerprint"].(string); len(fingerprint) != 64 {
|
||||
t.Fatalf("configuration fingerprint = %#v", result.Metadata["configuration_fingerprint"])
|
||||
}
|
||||
if result.Metadata["receipt_path"] != artifacts.SessionRunNotariusReceiptPathForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID) ||
|
||||
result.Metadata["diagnostic_path"] != artifacts.SessionRunNotariusLogPathForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID) ||
|
||||
result.Metadata["rejections_path"] != filepath.Join(durableBundle, "rejected.json") ||
|
||||
result.Metadata["warnings_path"] != filepath.Join(durableBundle, "warnings.json") {
|
||||
t.Fatalf("diagnostic metadata = %#v", result.Metadata)
|
||||
}
|
||||
if len(result.Outputs) != 2 {
|
||||
t.Fatalf("outputs = %#v", result.Outputs)
|
||||
}
|
||||
lane := result.Outputs[0]
|
||||
if lane.Kind != extractLaneOutputKind || lane.SourceID != "narratio.extraction.npc_registry" {
|
||||
t.Fatalf("lane identity = %#v", lane)
|
||||
}
|
||||
if lane.AbsolutePath != filepath.Join(durableBundle, "lanes", "npc.json") || lane.Checksum == "" {
|
||||
t.Fatalf("lane path/checksum = %#v", lane)
|
||||
}
|
||||
wantContract := &artifactmodel.ContractMetadata{
|
||||
MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry",
|
||||
SchemaVersion: "v1", ModuleKey: "dnd/npc-registry",
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Contract, wantContract) {
|
||||
t.Fatalf("lane contract = %#v, want %#v", lane.Contract, wantContract)
|
||||
}
|
||||
wantProvenance := &artifactmodel.ExternalProvenance{
|
||||
System: "notarius", RunID: "notarius-run-1", PipelineID: "dnd-session", ArtifactID: "npc-registry",
|
||||
}
|
||||
if !reflect.DeepEqual(lane.ExternalProvenance, wantProvenance) {
|
||||
t.Fatalf("lane provenance = %#v, want %#v", lane.ExternalProvenance, wantProvenance)
|
||||
}
|
||||
index := result.Outputs[1]
|
||||
if index.Kind != extractIndexOutputKind || index.SourceID != "" || index.AbsolutePath != filepath.Join(durableBundle, "index.json") || index.Checksum == "" {
|
||||
t.Fatalf("index output = %#v", index)
|
||||
}
|
||||
if !strings.HasPrefix(lane.AbsolutePath, durableBundle+string(filepath.Separator)) || strings.HasPrefix(lane.AbsolutePath, stagingBundle+string(filepath.Separator)) {
|
||||
t.Fatalf("lane path was not re-resolved after promotion: %q", lane.AbsolutePath)
|
||||
}
|
||||
for _, relative := range []string{
|
||||
"index.json", "manifest.json", "rejected.json", "warnings.json", "lanes/npc.json",
|
||||
"lanes/unconfigured.json", "chunk-map.json", "evidence-context.json", "unknown/private-debug.json",
|
||||
} {
|
||||
if _, err := os.Stat(filepath.Join(durableBundle, filepath.FromSlash(relative))); err != nil {
|
||||
t.Fatalf("promoted file %q missing: %v", relative, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(stagingBundle, "unknown", "private-debug.json")); err != nil {
|
||||
t.Fatalf("source bundle was not preserved: %v", err)
|
||||
}
|
||||
for _, output := range result.Outputs {
|
||||
if strings.Contains(output.AbsolutePath, "unconfigured") || strings.Contains(output.AbsolutePath, "private-debug") {
|
||||
t.Fatalf("unknown file registered as output: %#v", output)
|
||||
}
|
||||
}
|
||||
|
||||
rejections, _ := result.Metadata["rejections"].([]map[string]any)
|
||||
warnings, _ := result.Metadata["warnings"].([]map[string]any)
|
||||
if len(rejections) != 1 || rejections[0]["reason_code"] != "optional_rejected" || len(warnings) != 1 {
|
||||
t.Fatalf("diagnostic summaries = rejections %#v warnings %#v", rejections, warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStageRejectsMissingOrInvalidFinalTrimmedInputBeforeInvocation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
remove bool
|
||||
}{
|
||||
{name: "missing", remove: true},
|
||||
{name: "invalid json", content: "not-json"},
|
||||
{name: "missing segments", content: `{}`},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
env, m, fake := setupExtractEnv(t)
|
||||
fixture := extractFixtureFromEnv(t, env, m)
|
||||
if test.remove {
|
||||
if err := os.Remove(fixture.inputPath); err != nil {
|
||||
t.Fatalf("Remove(input) error = %v", err)
|
||||
}
|
||||
} else if err := os.WriteFile(fixture.inputPath, []byte(test.content), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(input) error = %v", err)
|
||||
}
|
||||
if _, err := (extractStage{}).Run(context.Background(), env, m); err == nil {
|
||||
t.Fatal("Run() error = nil")
|
||||
}
|
||||
if len(fake.Requests) != 0 {
|
||||
t.Fatalf("adapter requests = %d, want 0", len(fake.Requests))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStageEnforcesRequiredLanePolicy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*notarius.RunResult)
|
||||
want string
|
||||
}{
|
||||
{name: "missing", mutate: func(result *notarius.RunResult) { result.Index.Lanes = result.Index.Lanes[1:] }, want: "0 descriptors"},
|
||||
{name: "rejected", mutate: func(result *notarius.RunResult) {
|
||||
result.Rejections = append(result.Rejections, notarius.RejectionSummary{LaneID: "npc-registry", ReasonCode: "invalid_npc"})
|
||||
}, want: "was rejected"},
|
||||
{name: "duplicate", mutate: func(result *notarius.RunResult) {
|
||||
result.Index.Lanes = append(result.Index.Lanes, result.Index.Lanes[0])
|
||||
}, want: "2 descriptors"},
|
||||
{name: "media type", mutate: func(result *notarius.RunResult) { result.Index.Lanes[0].MediaType = "text/plain" }, want: "incompatible"},
|
||||
{name: "schema id", mutate: func(result *notarius.RunResult) { result.Index.Lanes[0].SchemaID = "other" }, want: "incompatible"},
|
||||
{name: "schema version", mutate: func(result *notarius.RunResult) { result.Index.Lanes[0].SchemaVersion = "v2" }, want: "incompatible"},
|
||||
{name: "module key", mutate: func(result *notarius.RunResult) { result.Index.Lanes[0].ModuleKey = "other" }, want: "incompatible"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
env, m, fake := setupExtractEnv(t)
|
||||
test.mutate(&fake.Result)
|
||||
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Run() result = %#v error = %v, want %q", result, err, test.want)
|
||||
}
|
||||
if result != nil && len(result.Outputs) != 0 {
|
||||
t.Fatalf("failure returned outputs: %#v", result.Outputs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStageValidatesSelectedLaneJSON(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
content string
|
||||
want string
|
||||
}{
|
||||
{name: "empty", content: "", want: "non-empty"},
|
||||
{name: "invalid", content: "not-json", want: "valid JSON"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
env, m, fake := setupExtractEnv(t)
|
||||
if err := os.WriteFile(fake.Result.Index.Lanes[0].Path, []byte(test.content), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(lane) error = %v", err)
|
||||
}
|
||||
if _, err := (extractStage{}).Run(context.Background(), env, m); err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Run() error = %v, want %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStageAdapterAndPromotionFailuresReturnNoOutputs(t *testing.T) {
|
||||
t.Run("adapter", func(t *testing.T) {
|
||||
env, m, fake := setupExtractEnv(t)
|
||||
fake.Err = errors.New("adapter failed")
|
||||
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||
if err == nil || !strings.Contains(err.Error(), "adapter failed") || result != nil {
|
||||
t.Fatalf("Run() result = %#v error = %v", result, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("promotion", func(t *testing.T) {
|
||||
env, m, fake := setupExtractEnv(t)
|
||||
fixture := extractFixtureFromEnv(t, env, m)
|
||||
destination := artifacts.SessionNotariusBundleDirForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID)
|
||||
if err := os.MkdirAll(destination, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(destination) error = %v", err)
|
||||
}
|
||||
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||
if err == nil || !strings.Contains(err.Error(), "promote") || result != nil {
|
||||
t.Fatalf("Run() result = %#v error = %v", result, err)
|
||||
}
|
||||
if _, err := os.Stat(fake.Result.BundleRoot); err != nil {
|
||||
t.Fatalf("failed promotion removed staging bundle: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestExtractionFingerprintIsIndependentOfOutputMapOrder(t *testing.T) {
|
||||
first := &config.NotariusConfig{PipelineID: "pipeline", Outputs: map[string]config.NotariusOutputConfig{
|
||||
"zeta": {LaneID: "z", MediaType: "application/json", SchemaID: "z", SchemaVersion: "v1"},
|
||||
"alpha": {LaneID: "a", MediaType: "application/json", SchemaID: "a", SchemaVersion: "v1"},
|
||||
}}
|
||||
second := &config.NotariusConfig{PipelineID: "pipeline", Outputs: map[string]config.NotariusOutputConfig{
|
||||
"alpha": first.Outputs["alpha"], "zeta": first.Outputs["zeta"],
|
||||
}}
|
||||
one, err := extractionFingerprint("/bin/notarius", "/etc/notarius.yml", first, time.Minute, "/work")
|
||||
if err != nil {
|
||||
t.Fatalf("extractionFingerprint(first) error = %v", err)
|
||||
}
|
||||
two, err := extractionFingerprint("/bin/notarius", "/etc/notarius.yml", second, time.Minute, "/work")
|
||||
if err != nil {
|
||||
t.Fatalf("extractionFingerprint(second) error = %v", err)
|
||||
}
|
||||
if one != two {
|
||||
t.Fatalf("fingerprints differ: %q != %q", one, two)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStageResumeValidationAcceptsCurrentImmutableResult(t *testing.T) {
|
||||
env, m, _ := setupExtractEnv(t)
|
||||
seedSucceededExtractResult(t, env, m)
|
||||
m.RunID = "20260810T020304Z-fedcba98"
|
||||
|
||||
validation, err := (extractStage{}).ValidateResume(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateResume() error = %v", err)
|
||||
}
|
||||
if !validation.Resumable || validation.Reason != "" {
|
||||
t.Fatalf("validation = %#v, want resumable", validation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStageAdapterResultIsImmediatelyReusableAndCatalogVisible(t *testing.T) {
|
||||
env, m, fake := setupExtractEnv(t)
|
||||
producerRunID := m.RunID
|
||||
receiptFixture := filepath.Join(t.TempDir(), "receipt.json")
|
||||
receipt, err := json.Marshal(map[string]any{
|
||||
"schema_version": notarius.ReceiptSchemaVersion,
|
||||
"run_id": "notarius-run-1", "pipeline_id": "dnd-session",
|
||||
"output_directory": fake.Result.BundleRoot, "index_file": "index.json",
|
||||
"normalized_output_count": 2, "rejected_output_count": 0, "warning_count": 0,
|
||||
"validation_status": "approved", "future_field": true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(receipt) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(receiptFixture, receipt, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(receipt fixture) error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(env.Config.Pipeline.Notarius.Binary, []byte("#!/bin/sh\ncat \"$NARRATIO_NOTARIUS_RECEIPT_FIXTURE\"\n"), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(notarius helper) error = %v", err)
|
||||
}
|
||||
t.Setenv("NARRATIO_NOTARIUS_RECEIPT_FIXTURE", receiptFixture)
|
||||
env.Notarius = notarius.NewSubprocessRunner()
|
||||
|
||||
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
durableIndex := filepath.Join(result.Metadata["bundle_root"].(string), "index.json")
|
||||
if len(result.Outputs) != 2 || result.Outputs[1].AbsolutePath != durableIndex {
|
||||
t.Fatalf("outputs = %#v, want canonical durable index %q", result.Outputs, durableIndex)
|
||||
}
|
||||
recordSucceededExtractResult(m, producerRunID, result)
|
||||
m.RunID = "20260810T020304Z-fedcba98"
|
||||
|
||||
validation, err := (extractStage{}).ValidateResume(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateResume() error = %v", err)
|
||||
}
|
||||
if !validation.Resumable {
|
||||
t.Fatalf("validation = %#v, want resumable", validation)
|
||||
}
|
||||
definition := env.Config.Pipeline.Notarius.Outputs["npc_registry"]
|
||||
definitions := map[string]artifacts.ExtractionArtifactDefinition{
|
||||
"npc_registry": {
|
||||
LaneID: definition.LaneID, PipelineID: env.Config.Pipeline.Notarius.PipelineID,
|
||||
MediaType: definition.MediaType, SchemaID: definition.SchemaID,
|
||||
SchemaVersion: definition.SchemaVersion, ModuleKey: definition.ModuleKey,
|
||||
},
|
||||
}
|
||||
catalog := artifacts.NewArtifactCatalog()
|
||||
if err := catalog.RegisterExtractionArtifacts(definitions); err != nil {
|
||||
t.Fatalf("RegisterExtractionArtifacts() error = %v", err)
|
||||
}
|
||||
catalog.HydrateExtractionArtifacts(sessionPathsForEnv(env, m.SessionID), m, definitions)
|
||||
entry, ok := catalog.Lookup(artifacts.ExtractionArtifactSourceID("npc_registry"))
|
||||
if !ok || !entry.Available || entry.Path != result.Outputs[0].AbsolutePath || entry.ProducerRunID != producerRunID {
|
||||
t.Fatalf("hydrated extraction entry = %#v, %v", entry, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStageResumeValidationRejectsObsoleteResults(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*testing.T, *Env, *manifest.Manifest)
|
||||
}{
|
||||
{name: "disabled", mutate: func(_ *testing.T, env *Env, _ *manifest.Manifest) {
|
||||
env.Config.Pipeline.Notarius.Enabled = false
|
||||
}},
|
||||
{name: "config changed", mutate: func(_ *testing.T, env *Env, _ *manifest.Manifest) {
|
||||
env.Config.Pipeline.Notarius.PipelineID = "changed"
|
||||
}},
|
||||
{name: "missing lane", mutate: func(t *testing.T, _ *Env, m *manifest.Manifest) {
|
||||
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
||||
t.Fatalf("Remove(lane) error = %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "tampered lane", mutate: func(t *testing.T, _ *Env, m *manifest.Manifest) {
|
||||
if err := os.WriteFile(m.Stages["extract"].Outputs[0].LocalPath, []byte(`{"tampered":true}`), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(lane) error = %v", err)
|
||||
}
|
||||
}},
|
||||
{name: "incompatible contract", mutate: func(_ *testing.T, _ *Env, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].Contract.SchemaVersion = "v2"
|
||||
}},
|
||||
{name: "missing source", mutate: func(_ *testing.T, _ *Env, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs = m.Stages["extract"].Outputs[1:]
|
||||
}},
|
||||
{name: "producer mismatch", mutate: func(_ *testing.T, _ *Env, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].ProducerRunID = "different-run"
|
||||
}},
|
||||
{name: "provenance mismatch", mutate: func(_ *testing.T, _ *Env, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].ExternalProvenance.RunID = "different-run"
|
||||
}},
|
||||
{name: "missing index", mutate: func(t *testing.T, _ *Env, m *manifest.Manifest) {
|
||||
index := m.Stages["extract"].Outputs[1]
|
||||
if err := os.Remove(index.LocalPath); err != nil {
|
||||
t.Fatalf("Remove(index) error = %v", err)
|
||||
}
|
||||
}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
env, m, _ := setupExtractEnv(t)
|
||||
seedSucceededExtractResult(t, env, m)
|
||||
test.mutate(t, env, m)
|
||||
|
||||
validation, err := (extractStage{}).ValidateResume(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateResume() error = %v", err)
|
||||
}
|
||||
if validation.Resumable || validation.Reason == "" || len(validation.Reason) > maxResumeReasonLength {
|
||||
t.Fatalf("validation = %#v, want bounded non-resumable result", validation)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStageResumeValidationRejectsUnsafePathWithError(t *testing.T) {
|
||||
env, m, _ := setupExtractEnv(t)
|
||||
seedSucceededExtractResult(t, env, m)
|
||||
prior := *m.Stages["extract"]
|
||||
m.Stages["extract"].Outputs[0].LocalPath = filepath.Join(env.Config.Pipeline.Workspace.Root, "outside.json")
|
||||
|
||||
if _, err := (extractStage{}).ValidateResume(context.Background(), env, m); err == nil || !strings.Contains(err.Error(), "unsafe") {
|
||||
t.Fatalf("ValidateResume() error = %v, want unsafe path error", err)
|
||||
}
|
||||
if m.Stages["extract"].Status != prior.Status || m.Stages["extract"].Error != prior.Error {
|
||||
t.Fatalf("validation mutated stage record: %#v", m.Stages["extract"])
|
||||
}
|
||||
}
|
||||
|
||||
func seedSucceededExtractResult(t *testing.T, env *Env, m *manifest.Manifest) {
|
||||
t.Helper()
|
||||
producerRunID := m.RunID
|
||||
result, err := (extractStage{}).Run(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
recordSucceededExtractResult(m, producerRunID, result)
|
||||
}
|
||||
|
||||
func recordSucceededExtractResult(m *manifest.Manifest, producerRunID string, result *StageResult) {
|
||||
records := make([]manifest.ArtifactRecord, 0, len(result.Outputs))
|
||||
for _, output := range result.Outputs {
|
||||
records = append(records, manifest.ArtifactRecord{
|
||||
Kind: output.Kind, SourceID: output.SourceID, LocalPath: output.AbsolutePath,
|
||||
Contract: output.Contract, ExternalProvenance: output.ExternalProvenance,
|
||||
ProducerRunID: producerRunID, Checksum: output.Checksum,
|
||||
})
|
||||
}
|
||||
m.MarkStageSucceeded("extract", time.Now().UTC(), records)
|
||||
m.Stages["extract"].Metadata = result.Metadata
|
||||
}
|
||||
|
||||
type extractFixture struct {
|
||||
workspace string
|
||||
campaign string
|
||||
sessionID string
|
||||
runID string
|
||||
inputPath string
|
||||
}
|
||||
|
||||
func extractFixtureFromEnv(t *testing.T, env *Env, m *manifest.Manifest) extractFixture {
|
||||
t.Helper()
|
||||
paths := sessionPathsForEnv(env, m.SessionID)
|
||||
return extractFixture{
|
||||
workspace: paths.WorkspaceRoot, campaign: m.Campaign, sessionID: m.SessionID,
|
||||
runID: m.RunID, inputPath: filepath.Join(paths.ArtifactsDir, "trimmed.from-manifest.json"),
|
||||
}
|
||||
}
|
||||
|
||||
func setupExtractEnv(t *testing.T) (*Env, *manifest.Manifest, *notarius.FakeRunner) {
|
||||
t.Helper()
|
||||
workspace := t.TempDir()
|
||||
campaign := "campaign-a"
|
||||
sessionID := "2026-08-09"
|
||||
runID := "20260809T010203Z-abcdef12"
|
||||
store := artifacts.NewLocalStore(workspace)
|
||||
paths, err := store.EnsureLayoutFor(campaign, sessionID)
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureLayoutFor() error = %v", err)
|
||||
}
|
||||
inputPath := filepath.Join(paths.ArtifactsDir, "trimmed.from-manifest.json")
|
||||
if err := os.WriteFile(inputPath, []byte(`{"segments":[{"id":1}]}`), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(input) error = %v", err)
|
||||
}
|
||||
configPath := filepath.Join(workspace, "notarius.yml")
|
||||
if err := os.WriteFile(configPath, []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(config) error = %v", err)
|
||||
}
|
||||
workingDirectory := filepath.Join(workspace, "notarius-work")
|
||||
if err := os.Mkdir(workingDirectory, 0o755); err != nil {
|
||||
t.Fatalf("Mkdir(working directory) error = %v", err)
|
||||
}
|
||||
binary := filepath.Join(workspace, "notarius")
|
||||
if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(binary) error = %v", err)
|
||||
}
|
||||
notariusConfig := &config.NotariusConfig{
|
||||
Enabled: true, Binary: binary, ConfigPath: configPath, PipelineID: "dnd-session",
|
||||
Timeout: "45m", WorkingDirectory: workingDirectory,
|
||||
Outputs: map[string]config.NotariusOutputConfig{
|
||||
"npc_registry": {
|
||||
LaneID: "npc-registry", MediaType: "application/json",
|
||||
SchemaID: "notarius.dnd.npc_registry", SchemaVersion: "v1", ModuleKey: "dnd/npc-registry",
|
||||
},
|
||||
},
|
||||
}
|
||||
m := manifest.New(sessionID, time.Now().UTC())
|
||||
m.Campaign = campaign
|
||||
m.RunID = runID
|
||||
m.MarkStageSucceeded("trim", time.Now().UTC(), []manifest.ArtifactRecord{{
|
||||
Kind: artifactmodel.TranscriptOutputKindFinalTrimmed, SourceID: artifactmodel.SourceTranscriptFinalTrimmed,
|
||||
LocalPath: inputPath,
|
||||
}})
|
||||
|
||||
outputRoot := artifacts.SessionRunNotariusOutputRootForCampaign(workspace, campaign, sessionID, runID)
|
||||
bundle := filepath.Join(outputRoot, "notarius-run-1")
|
||||
if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(bundle lanes) error = %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(bundle, "unknown"), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(bundle unknown) error = %v", err)
|
||||
}
|
||||
files := map[string]string{
|
||||
"index.json": `{"manifest_file":"manifest.json","output_files":[{"lane_id":"npc-registry","file":"lanes/npc.json","media_type":"application/json","module_key":"dnd/npc-registry","schema_id":"notarius.dnd.npc_registry","schema_version":"v1","future_field":true},{"lane_id":"unconfigured","file":"lanes/unconfigured.json"}],"rejected_file":"rejected.json","warnings_file":"warnings.json","future_field":true}`,
|
||||
"manifest.json": `{}`,
|
||||
"rejected.json": `{"rejected":[]}`,
|
||||
"warnings.json": `{"warnings":[]}`,
|
||||
"lanes/npc.json": `{"npcs":[]}`,
|
||||
"lanes/unconfigured.json": `{"spells":[]}`,
|
||||
"chunk-map.json": `{"chunks":[]}`,
|
||||
"evidence-context.json": `{"source_units":[]}`,
|
||||
"unknown/private-debug.json": `{"private":true}`,
|
||||
}
|
||||
for relative, content := range files {
|
||||
if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(relative)), []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) error = %v", relative, err)
|
||||
}
|
||||
}
|
||||
fake := ¬arius.FakeRunner{Result: notarius.RunResult{
|
||||
Receipt: notarius.Receipt{
|
||||
SchemaVersion: notarius.ReceiptSchemaVersion, RunID: "notarius-run-1", PipelineID: "dnd-session",
|
||||
OutputDirectory: bundle, IndexFile: "index.json", NormalizedOutputCount: 2,
|
||||
RejectedOutputCount: 1, WarningCount: 1, ValidationStatus: "rejected",
|
||||
},
|
||||
BundleRoot: bundle,
|
||||
Index: notarius.Index{
|
||||
Path: filepath.Join(bundle, "index.json"), RejectedPath: filepath.Join(bundle, "rejected.json"),
|
||||
WarningsPath: filepath.Join(bundle, "warnings.json"),
|
||||
Lanes: []notarius.LaneDescriptor{
|
||||
{
|
||||
LaneID: "npc-registry", File: "lanes/npc.json", Path: filepath.Join(bundle, "lanes", "npc.json"),
|
||||
MediaType: "application/json", ModuleKey: "dnd/npc-registry",
|
||||
SchemaID: "notarius.dnd.npc_registry", SchemaVersion: "v1",
|
||||
},
|
||||
{LaneID: "unconfigured", File: "lanes/unconfigured.json", Path: filepath.Join(bundle, "lanes", "unconfigured.json")},
|
||||
},
|
||||
},
|
||||
Rejections: []notarius.RejectionSummary{{LaneID: "optional", ReasonCode: "optional_rejected"}},
|
||||
Warnings: []notarius.WarningSummary{{Scope: "lane:npc-registry", ReasonCode: "normalized_name"}},
|
||||
}}
|
||||
env := &Env{
|
||||
Config: &config.Config{
|
||||
Pipeline: &config.PipelineConfig{Workspace: config.WorkspaceConfig{Root: workspace}, Notarius: notariusConfig},
|
||||
Session: &config.SessionConfig{SessionID: sessionID, Campaign: campaign},
|
||||
},
|
||||
ArtifactStore: store,
|
||||
Notarius: fake,
|
||||
}
|
||||
return env, m, fake
|
||||
}
|
||||
@@ -74,6 +74,7 @@ func All() []Stage {
|
||||
polishStage{},
|
||||
normalizeStage{},
|
||||
trimStage{},
|
||||
extractStage{},
|
||||
renderStage{},
|
||||
analyzeStage{},
|
||||
publishStage{},
|
||||
|
||||
@@ -113,7 +113,7 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
||||
}
|
||||
|
||||
m := manifest.New("2026-05-03", time.Now().UTC())
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze"} {
|
||||
m.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
}
|
||||
m.RunID = "20260516T000000Z-abcdef12"
|
||||
@@ -206,6 +206,12 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s.Name() == "extract" {
|
||||
if result.Disposition != StageDispositionSkipped || result.SkipReason != extractSkipReason {
|
||||
t.Fatalf("extract result = %#v, want disabled self-skip", result)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s.Name() == "render" {
|
||||
if result.Metadata["stage"] != "render" {
|
||||
t.Fatalf("render metadata = %#v, want stage=render", result.Metadata)
|
||||
|
||||
@@ -124,7 +124,12 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("publish: collect previous files: %w", err)
|
||||
}
|
||||
runtimeCatalog, err := buildPublishRuntimeArtifactCatalog(sessionPaths, env.Config.Pipeline.Scriptorium)
|
||||
runtimeCatalog, err := buildPublishRuntimeArtifactCatalog(
|
||||
sessionPaths,
|
||||
m,
|
||||
env.Config.Pipeline.Scriptorium,
|
||||
env.Config.Pipeline.Notarius,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("publish: build runtime artifact catalog: %w", err)
|
||||
}
|
||||
@@ -361,10 +366,11 @@ func resolvePublishOutputs(
|
||||
lockSet := publishLockSet(locks)
|
||||
selectedSet := publishSelectedArtifactSet(selectedArtifactKeys)
|
||||
configuredOutputs := configuredOutputPathMapFromCatalog(catalog)
|
||||
extractionOutputs := extractionOutputSetFromCatalog(catalog)
|
||||
for _, rule := range rules {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
required := rule.Required == nil || *rule.Required
|
||||
dest, err := resolvePublishOutputDest(rule, configuredOutputs)
|
||||
dest, err := resolvePublishOutputDest(rule, configuredOutputs, extractionOutputs)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
|
||||
}
|
||||
@@ -454,8 +460,8 @@ func publishLockSet(locks []config.PublishLockRule) map[string]config.PublishLoc
|
||||
return out
|
||||
}
|
||||
|
||||
func resolvePublishOutputDest(rule config.PublishOutputRule, configured map[string]string) (string, error) {
|
||||
return artifactpolicy.ResolvePublishedDestination(rule.Source, rule.Dest, configured)
|
||||
func resolvePublishOutputDest(rule config.PublishOutputRule, configured map[string]string, extractions map[string]struct{}) (string, error) {
|
||||
return artifactpolicy.ResolvePublishedDestinationWithExtractions(rule.Source, rule.Dest, configured, extractions)
|
||||
}
|
||||
|
||||
func configuredOutputPathMapFromCatalog(catalog *artifacts.ArtifactCatalog) map[string]string {
|
||||
@@ -472,14 +478,36 @@ func configuredOutputPathMapFromCatalog(catalog *artifacts.ArtifactCatalog) map[
|
||||
return out
|
||||
}
|
||||
|
||||
func extractionOutputSetFromCatalog(catalog *artifacts.ArtifactCatalog) map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
if catalog == nil {
|
||||
return out
|
||||
}
|
||||
for _, entry := range catalog.ListExtraction() {
|
||||
if strings.TrimSpace(entry.ExtractionKey) != "" {
|
||||
out[entry.ExtractionKey] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func buildPublishRuntimeArtifactCatalog(
|
||||
paths artifacts.SessionPaths,
|
||||
m *manifest.Manifest,
|
||||
scriptoriumCfg *config.ScriptoriumConfig,
|
||||
notariusCfg *config.NotariusConfig,
|
||||
) (*artifacts.ArtifactCatalog, error) {
|
||||
catalog := artifacts.NewArtifactCatalog()
|
||||
if err := catalog.RegisterBuiltIns(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
extractionDefinitions := artifacts.ExtractionDefinitionsFromConfig(notariusCfg)
|
||||
if err := catalog.RegisterExtractionArtifacts(extractionDefinitions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if notariusCfg != nil && notariusCfg.Enabled {
|
||||
catalog.HydrateExtractionArtifacts(paths, m, extractionDefinitions)
|
||||
}
|
||||
if scriptoriumCfg == nil {
|
||||
return catalog, nil
|
||||
}
|
||||
@@ -545,19 +573,7 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile,
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() {
|
||||
if path == runRoot {
|
||||
return nil
|
||||
}
|
||||
relDir, err := filepath.Rel(runRoot, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("relative dir from %q to %q: %w", runRoot, path, err)
|
||||
}
|
||||
relDir = filepath.ToSlash(relDir)
|
||||
// Preserve existing behavior: audio is not uploaded in publish run record.
|
||||
if relDir == "audio" || strings.HasPrefix(relDir, "audio/") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if path == runRoot {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(runRoot, path)
|
||||
@@ -565,6 +581,15 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile,
|
||||
return fmt.Errorf("relative path from %q to %q: %w", runRoot, path, err)
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if publishRunPathExcluded(rel) {
|
||||
if d.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
files = append(files, publishUploadFile{
|
||||
RelativePath: rel,
|
||||
LocalPath: path,
|
||||
@@ -604,6 +629,11 @@ func collectPublishRunFiles(runRoot, manifestPath string) ([]publishUploadFile,
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func publishRunPathExcluded(rel string) bool {
|
||||
return rel == "audio" || strings.HasPrefix(rel, "audio/") ||
|
||||
rel == "extract/notarius-output" || strings.HasPrefix(rel, "extract/notarius-output/")
|
||||
}
|
||||
|
||||
func collectPublishPreviousFiles(previousDir string) ([]publishUploadFile, error) {
|
||||
previousDir = filepath.Clean(strings.TrimSpace(previousDir))
|
||||
if previousDir == "" {
|
||||
|
||||
@@ -7,11 +7,13 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
@@ -63,8 +65,27 @@ func TestPublishFailsWhenPrerequisiteNotSucceeded(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
env, m, runRoot := publishFixture(t)
|
||||
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||
durableBundleRoot := filepath.Dir(filepath.Dir(lanePath))
|
||||
runBundleRoot := filepath.Join(runRoot, "extract", "notarius-output", "notarius-run-1")
|
||||
for rel, contents := range map[string]string{
|
||||
"index.json": `{"lanes":[]}`,
|
||||
"manifest.json": `{"run_id":"notarius-run-1"}`,
|
||||
"rejected.json": `[]`,
|
||||
"warnings.json": `[]`,
|
||||
"lanes/encounters.json": `{"encounters":[]}`,
|
||||
"pipeline/chunk-map.json": `{"chunks":[]}`,
|
||||
"pipeline/evidence-context.json": `{"evidence":[]}`,
|
||||
"unknown/notes.txt": "internal bundle note\n",
|
||||
} {
|
||||
writeStageTestFile(t, filepath.Join(runBundleRoot, filepath.FromSlash(rel)), contents)
|
||||
}
|
||||
writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius.receipt.json"), `{"run_id":"notarius-run-1"}`)
|
||||
writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius.stderr.log"), "notarius diagnostic\n")
|
||||
writeStageTestFile(t, filepath.Join(runRoot, "extract", "notarius-output-copy", "keep.json"), "{}\n")
|
||||
writeStageTestFile(t, filepath.Join(runRoot, "archive", "notarius-output", "keep.json"), "{}\n")
|
||||
|
||||
result, err := publishStage{}.Run(context.Background(), env, m)
|
||||
if err != nil {
|
||||
@@ -75,6 +96,10 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T)
|
||||
sessionPrefix := m.S3SessionPrefix
|
||||
wantRunUploads := []string{
|
||||
"analyze/outputs/artifacts/session_recap.md",
|
||||
"archive/notarius-output/keep.json",
|
||||
"extract/notarius-output-copy/keep.json",
|
||||
"extract/notarius.receipt.json",
|
||||
"extract/notarius.stderr.log",
|
||||
"merge/config/seriatim.generated.yml",
|
||||
"prepare/inputs/session.yml",
|
||||
"prepare/outputs/audio/speaker.flac",
|
||||
@@ -84,12 +109,24 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T)
|
||||
"transcribe/outputs/transcripts/raw/speaker.json",
|
||||
"trim/outputs/transcripts/final.trimmed.json",
|
||||
}
|
||||
sort.Strings(wantRunUploads)
|
||||
for _, rel := range wantRunUploads {
|
||||
key := runPrefix + rel
|
||||
if _, ok := fake.Objects[key]; !ok {
|
||||
t.Fatalf("missing run upload key %q", key)
|
||||
}
|
||||
}
|
||||
for key := range fake.Objects {
|
||||
if strings.HasPrefix(key, runPrefix+"extract/notarius-output/") {
|
||||
t.Fatalf("run-local Notarius bundle member was uploaded at %q", key)
|
||||
}
|
||||
}
|
||||
for _, upload := range fake.Uploads {
|
||||
localPath := filepath.Clean(upload.LocalPath)
|
||||
if localPath == durableBundleRoot || strings.HasPrefix(localPath, durableBundleRoot+string(filepath.Separator)) {
|
||||
t.Fatalf("durable Notarius bundle member was implicitly uploaded from %q", upload.LocalPath)
|
||||
}
|
||||
}
|
||||
|
||||
trimmedKey := sessionPrefix + "transcripts/final.trimmed.json"
|
||||
recapKey := sessionPrefix + "artifacts/session_recap.md"
|
||||
@@ -134,6 +171,12 @@ func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T)
|
||||
if result.Metadata["previous_files_uploaded"] != 0 {
|
||||
t.Fatalf("metadata previous_files_uploaded = %#v, want 0", result.Metadata["previous_files_uploaded"])
|
||||
}
|
||||
if result.Metadata["run_files_uploaded"] != len(wantRunUploads) {
|
||||
t.Fatalf("metadata run_files_uploaded = %#v, want %d", result.Metadata["run_files_uploaded"], len(wantRunUploads))
|
||||
}
|
||||
if got := result.Metadata["run_uploaded_paths"]; !reflect.DeepEqual(got, wantRunUploads) {
|
||||
t.Fatalf("metadata run_uploaded_paths = %#v, want %#v", got, wantRunUploads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishUploadsPreviousCacheWhenPresent(t *testing.T) {
|
||||
@@ -198,6 +241,126 @@ func TestPublishUsesCustomOutputRules(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishUploadsExplicitExtractionAndPreservesManifestMetadata(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||
env.SelectedArtifactKeys = []string{"session_recap"}
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||
}
|
||||
|
||||
result, err := publishStage{}.Run(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||
durableBundleRoot := filepath.Dir(filepath.Dir(lanePath))
|
||||
publishedKey := m.S3SessionPrefix + "artifacts/encounters.json"
|
||||
if got := string(fake.Objects[publishedKey].Data); got != `{"encounters":[]}` {
|
||||
t.Fatalf("published extraction = %q", got)
|
||||
}
|
||||
if _, ok := fake.Objects[m.S3SessionPrefix+"artifacts/notarius/index.json"]; ok {
|
||||
t.Fatal("Notarius index was implicitly published")
|
||||
}
|
||||
for key := range fake.Objects {
|
||||
if strings.Contains(key, "/notarius/") {
|
||||
t.Fatalf("Notarius bundle member was implicitly published at %q", key)
|
||||
}
|
||||
}
|
||||
for _, upload := range fake.Uploads {
|
||||
localPath := filepath.Clean(upload.LocalPath)
|
||||
if (localPath == durableBundleRoot || strings.HasPrefix(localPath, durableBundleRoot+string(filepath.Separator))) && localPath != filepath.Clean(lanePath) {
|
||||
t.Fatalf("durable Notarius bundle member other than selected lane was uploaded from %q", upload.LocalPath)
|
||||
}
|
||||
}
|
||||
if result.Metadata["published_files_uploaded"] != 1 {
|
||||
t.Fatalf("published_files_uploaded = %#v, want 1", result.Metadata["published_files_uploaded"])
|
||||
}
|
||||
if uploads := fake.Uploads; len(uploads) == 0 || uploads[len(uploads)-1].Key != m.S3SessionPrefix+"current/run_id.txt" {
|
||||
t.Fatalf("last upload = %#v, want current run pointer", uploads)
|
||||
}
|
||||
|
||||
var current manifest.Manifest
|
||||
if err := json.Unmarshal(fake.Objects[m.S3SessionPrefix+"current/manifest.json"].Data, ¤t); err != nil {
|
||||
t.Fatalf("unmarshal current manifest: %v", err)
|
||||
}
|
||||
extractRecord := current.Stages["extract"]
|
||||
if extractRecord == nil || len(extractRecord.Outputs) != 2 {
|
||||
t.Fatalf("extract record = %#v", extractRecord)
|
||||
}
|
||||
lane := extractRecord.Outputs[0]
|
||||
if lane.LocalPath != lanePath || lane.Contract == nil || lane.Contract.SchemaID != "encounters" ||
|
||||
lane.ExternalProvenance == nil || lane.ExternalProvenance.RunID != "notarius-run-1" {
|
||||
t.Fatalf("serialized extraction metadata = %#v", lane)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishRequiredInvalidExtractionFails(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||
}
|
||||
writeStageTestFile(t, lanePath, `{"tampered":true}`)
|
||||
|
||||
_, err := publishStage{}.Run(context.Background(), env, m)
|
||||
if err == nil || !strings.Contains(err.Error(), `required output source unavailable: "narratio.extraction.encounters"`) {
|
||||
t.Fatalf("Run() error = %v, want unavailable extraction failure", err)
|
||||
}
|
||||
if len(env.ObjectStore.(*storage.FakeBackend).Uploads) != 0 {
|
||||
t.Fatal("publish uploaded files after extraction validation failed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishDisabledExtractionConfigurationIsUnavailable(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
configurePublishExtractionFixture(t, env, m)
|
||||
env.Config.Pipeline.Notarius.Enabled = false
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||
}
|
||||
|
||||
_, err := publishStage{}.Run(context.Background(), env, m)
|
||||
if err == nil || !strings.Contains(err.Error(), `required output source unavailable: "narratio.extraction.encounters"`) {
|
||||
t.Fatalf("Run() error = %v, want unavailable disabled extraction", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishOptionalMissingExtractionIsSkipped(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(false)},
|
||||
}
|
||||
if err := os.Remove(lanePath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result, err := publishStage{}.Run(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if got := result.Metadata["skipped_optional_outputs"].([]string); !reflect.DeepEqual(got, []string{"artifacts/encounters.json"}) {
|
||||
t.Fatalf("skipped_optional_outputs = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishArtifactSelectionDoesNotFilterExtractionOutputs(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
configurePublishExtractionFixture(t, env, m)
|
||||
env.SelectedArtifactKeys = []string{"session_recap"}
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
{Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)},
|
||||
}
|
||||
|
||||
if _, err := (publishStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if _, ok := env.ObjectStore.(*storage.FakeBackend).Objects[m.S3SessionPrefix+"artifacts/encounters.json"]; !ok {
|
||||
t.Fatal("explicit extraction output was filtered by Scriptorium artifact selection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishSkipsOptionalMissingOutput(t *testing.T) {
|
||||
env, m, _ := publishFixture(t)
|
||||
env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{
|
||||
@@ -591,6 +754,56 @@ func publishFixture(t *testing.T) (*Env, *manifest.Manifest, string) {
|
||||
return env, m, runRoot
|
||||
}
|
||||
|
||||
func configurePublishExtractionFixture(t *testing.T, env *Env, m *manifest.Manifest) string {
|
||||
t.Helper()
|
||||
env.Config.Pipeline.Notarius = &config.NotariusConfig{
|
||||
Enabled: true, PipelineID: "campaign.extract",
|
||||
Outputs: map[string]config.NotariusOutputConfig{
|
||||
"encounters": {
|
||||
LaneID: "encounters", MediaType: "application/json", SchemaID: "encounters",
|
||||
SchemaVersion: "1", ModuleKey: "encounters",
|
||||
},
|
||||
},
|
||||
}
|
||||
paths := publishSessionPaths(env, m)
|
||||
producerRunID := "extract-run-1"
|
||||
bundleRoot := filepath.Join(paths.ArtifactsDir, "notarius", producerRunID)
|
||||
lanePath := filepath.Join(bundleRoot, "lanes", "encounters.json")
|
||||
indexPath := filepath.Join(bundleRoot, "index.json")
|
||||
writeStageTestFile(t, lanePath, `{"encounters":[]}`)
|
||||
writeStageTestFile(t, indexPath, `{"lanes":[]}`)
|
||||
writeStageTestFile(t, filepath.Join(bundleRoot, "manifest.json"), `{"run_id":"notarius-run-1"}`)
|
||||
writeStageTestFile(t, filepath.Join(bundleRoot, "rejected.json"), `[]`)
|
||||
writeStageTestFile(t, filepath.Join(bundleRoot, "warnings.json"), `[]`)
|
||||
writeStageTestFile(t, filepath.Join(bundleRoot, "pipeline", "chunk-map.json"), `{"chunks":[]}`)
|
||||
writeStageTestFile(t, filepath.Join(bundleRoot, "unknown", "notes.txt"), "internal bundle note\n")
|
||||
laneChecksum, err := artifacts.SHA256File(lanePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
indexChecksum, err := artifacts.SHA256File(indexPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Stages["extract"] = &manifest.StageRecord{
|
||||
Name: "extract", Status: manifest.StatusSucceeded,
|
||||
Metadata: map[string]any{
|
||||
"narratio_run_id": producerRunID, "bundle_root": bundleRoot,
|
||||
"receipt": map[string]any{"run_id": "notarius-run-1", "pipeline_id": "campaign.extract"},
|
||||
},
|
||||
Outputs: []manifest.ArtifactRecord{
|
||||
{
|
||||
Kind: "notarius_lane", SourceID: artifacts.ExtractionArtifactSourceID("encounters"), LocalPath: lanePath,
|
||||
ProducerRunID: producerRunID, Checksum: laneChecksum,
|
||||
Contract: &artifactmodel.ContractMetadata{MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters"},
|
||||
ExternalProvenance: &artifactmodel.ExternalProvenance{System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: "encounters"},
|
||||
},
|
||||
{Kind: "notarius_index", LocalPath: indexPath, ProducerRunID: producerRunID, Checksum: indexChecksum},
|
||||
},
|
||||
}
|
||||
return lanePath
|
||||
}
|
||||
|
||||
type publishedOutputFailingStore struct {
|
||||
delegate *storage.FakeBackend
|
||||
failKey string
|
||||
|
||||
@@ -3,8 +3,11 @@ package stage
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/audita"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notify"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
|
||||
@@ -26,6 +29,7 @@ type Env struct {
|
||||
WhisperX whisperx.Client
|
||||
Seriatim seriatim.Runner
|
||||
Audita audita.Runner
|
||||
Notarius notarius.Runner
|
||||
Scriptorium scriptorium.Runner
|
||||
ObjectStore storage.ObjectStore
|
||||
Notifier notify.Sender
|
||||
@@ -44,8 +48,61 @@ type Stage interface {
|
||||
Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error)
|
||||
}
|
||||
|
||||
const maxResumeReasonLength = 512
|
||||
|
||||
// ResumeValidation reports whether a previously succeeded stage can be reused.
|
||||
type ResumeValidation struct {
|
||||
Resumable bool
|
||||
Reason string
|
||||
}
|
||||
|
||||
// Normalized returns a result with a bounded reason and no reason on success.
|
||||
func (r ResumeValidation) Normalized() ResumeValidation {
|
||||
if r.Resumable {
|
||||
return Resumable()
|
||||
}
|
||||
return NonResumable(r.Reason)
|
||||
}
|
||||
|
||||
// ResumeValidator is implemented by stages that validate persisted success before reuse.
|
||||
type ResumeValidator interface {
|
||||
ValidateResume(ctx context.Context, env *Env, m *manifest.Manifest) (ResumeValidation, error)
|
||||
}
|
||||
|
||||
// Resumable reports a successful resume validation.
|
||||
func Resumable() ResumeValidation {
|
||||
return ResumeValidation{Resumable: true}
|
||||
}
|
||||
|
||||
// NonResumable reports a bounded reason that persisted success must be rerun.
|
||||
func NonResumable(reason string) ResumeValidation {
|
||||
reason = strings.TrimSpace(reason)
|
||||
if reason == "" {
|
||||
reason = "persisted stage result is not reusable"
|
||||
}
|
||||
if len(reason) > maxResumeReasonLength {
|
||||
cutoff := maxResumeReasonLength
|
||||
for cutoff > 0 && !utf8.ValidString(reason[:cutoff]) {
|
||||
cutoff--
|
||||
}
|
||||
reason = reason[:cutoff]
|
||||
}
|
||||
return ResumeValidation{Reason: reason}
|
||||
}
|
||||
|
||||
// StageDisposition describes the outcome of a stage that returned without an error.
|
||||
type StageDisposition string
|
||||
|
||||
const (
|
||||
// StageDispositionSucceeded is the zero value so existing stages remain successful.
|
||||
StageDispositionSucceeded StageDisposition = ""
|
||||
StageDispositionSkipped StageDisposition = "skipped"
|
||||
)
|
||||
|
||||
// StageResult is the declared output of a stage execution.
|
||||
type StageResult struct {
|
||||
Disposition StageDisposition
|
||||
SkipReason string
|
||||
Outputs []artifacts.Ref
|
||||
Logs []string
|
||||
GeneratedConfigs []string
|
||||
|
||||
Reference in New Issue
Block a user