Document and validate Notarius extraction workflows
This commit is contained in:
@@ -1,7 +1,8 @@
|
|||||||
# narratio
|
# narratio
|
||||||
|
|
||||||
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
Narratio is a stage-driven Go orchestrator for turning D&D session audio into
|
||||||
polished transcripts and generated artifacts.
|
polished transcripts, validated Notarius extraction lanes, and generated
|
||||||
|
artifacts.
|
||||||
|
|
||||||
It runs a deterministic workflow with manifest-driven continuation, remote
|
It runs a deterministic workflow with manifest-driven continuation, remote
|
||||||
publish, and restore support.
|
publish, and restore support.
|
||||||
|
|||||||
11
docs/cli.md
11
docs/cli.md
@@ -75,6 +75,8 @@ narratio run <session_id> [--force] [--artifacts <name[,name...]>] [...common co
|
|||||||
Behavior:
|
Behavior:
|
||||||
|
|
||||||
- evaluates full stage order;
|
- evaluates full stage order;
|
||||||
|
- runs `extract` between `trim` and `render`; an omitted or disabled Notarius
|
||||||
|
configuration makes extraction a no-op;
|
||||||
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
- skips already-succeeded stages unless `--force` is set or a stage-specific
|
||||||
resume check finds its durable result obsolete;
|
resume check finds its durable result obsolete;
|
||||||
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||||
@@ -220,7 +222,10 @@ default restore scope, report location, and conflict-handling workflow.
|
|||||||
narratio session artifacts <session_id> [--remote] [...common config flags]
|
narratio session artifacts <session_id> [--remote] [...common config flags]
|
||||||
```
|
```
|
||||||
|
|
||||||
Lists effective built-in and configured artifact sources, publish rules, lock state, and optional remote published-state availability.
|
Lists effective built-in, configured Scriptorium, and configured extraction
|
||||||
|
sources; reports planned, available, unavailable, and published state without
|
||||||
|
reading payload bodies; and includes publish rules, lock state, and optional
|
||||||
|
remote published-state availability.
|
||||||
|
|
||||||
### `session locks`
|
### `session locks`
|
||||||
|
|
||||||
@@ -250,7 +255,9 @@ Effects:
|
|||||||
|
|
||||||
- filters analyze execution to selected configured artifacts;
|
- filters analyze execution to selected configured artifacts;
|
||||||
- filters publish rules that source `narratio.artifact.<name>`;
|
- filters publish rules that source `narratio.artifact.<name>`;
|
||||||
- does not filter built-in transcript/bounds publish sources.
|
- does not filter built-in transcript/bounds or explicitly configured
|
||||||
|
`narratio.extraction.<name>` publish sources; and
|
||||||
|
- does not select or filter Notarius lanes.
|
||||||
|
|
||||||
## Common Workflows
|
## Common Workflows
|
||||||
|
|
||||||
|
|||||||
@@ -118,6 +118,9 @@ Rules:
|
|||||||
|
|
||||||
- `outputs[].source` is required.
|
- `outputs[].source` is required.
|
||||||
- `outputs[].dest` may be omitted when derivable from source.
|
- `outputs[].dest` may be omitted when derivable from source.
|
||||||
|
- extraction sources require an explicit `outputs[].dest` and publish only when
|
||||||
|
a rule names that source; the Notarius index and complete bundle are not
|
||||||
|
publish sources.
|
||||||
- `outputs[].required` defaults to `true`.
|
- `outputs[].required` defaults to `true`.
|
||||||
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
- static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates.
|
||||||
|
|
||||||
@@ -196,6 +199,13 @@ Rules:
|
|||||||
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
| `pipeline.trim.bounds.render_debug` | bool | No | `false` |
|
||||||
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
| `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true |
|
||||||
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
| `pipeline.trim.seriatim.report` | bool | No | `false` |
|
||||||
|
| `pipeline.notarius.enabled` | bool | No | `false` |
|
||||||
|
| `pipeline.notarius.binary` | string | No | `notarius` |
|
||||||
|
| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory |
|
||||||
|
| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled |
|
||||||
|
| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive |
|
||||||
|
| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory |
|
||||||
|
| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled |
|
||||||
| `pipeline.render.enabled` | bool | No | `true` |
|
| `pipeline.render.enabled` | bool | No | `true` |
|
||||||
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
|
| `pipeline.render.format` | string | No | `markdown` (only supported value) |
|
||||||
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
|
| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) |
|
||||||
@@ -211,6 +221,26 @@ Rules:
|
|||||||
| `pipeline.notification.recipient` | string | No | empty |
|
| `pipeline.notification.recipient` | string | No | empty |
|
||||||
| `pipeline.notification.timeout` | duration | No | empty |
|
| `pipeline.notification.timeout` | duration | No | empty |
|
||||||
|
|
||||||
|
### Notarius Output Entries
|
||||||
|
|
||||||
|
For each `pipeline.notarius.outputs.<name>`:
|
||||||
|
|
||||||
|
| Field | Type | Required | Rule |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `lane_id` | string | Yes | unique Notarius lane ID |
|
||||||
|
| `media_type` | string | Yes | exact accepted descriptor media type |
|
||||||
|
| `schema_id` | string | Yes | exact accepted descriptor schema ID |
|
||||||
|
| `schema_version` | string | Yes | exact accepted descriptor schema version |
|
||||||
|
| `module_key` | string | No | exact accepted module key when set |
|
||||||
|
|
||||||
|
Output names must match `^[a-z][a-z0-9_]*$` and become selectable sources named
|
||||||
|
`narratio.extraction.<name>`. Lane IDs must be unique. Every declared output is
|
||||||
|
required from a successful Notarius result; a missing, rejected, duplicate, or
|
||||||
|
contract-incompatible lane fails extraction. See the
|
||||||
|
[complete maintained example](../examples/pipeline.full.annotated.yml) for the
|
||||||
|
current ten-lane D&D mapping and the [Notarius contract](./integrations/notarius.md)
|
||||||
|
for compatibility ownership.
|
||||||
|
|
||||||
### Scriptorium Artifact Entries
|
### Scriptorium Artifact Entries
|
||||||
|
|
||||||
For each `pipeline.scriptorium.artifacts.<name>`:
|
For each `pipeline.scriptorium.artifacts.<name>`:
|
||||||
@@ -233,7 +263,7 @@ For each artifact input `pipeline.scriptorium.artifacts.<name>.inputs.<input_nam
|
|||||||
|
|
||||||
| Field | Type | Required | Rule |
|
| Field | Type | Required | Rule |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.extraction.<name>`, `narratio.artifact.<name>`, or `narratio.previous_session.artifact.<name>` |
|
||||||
| `artifact` | string | No | optional passthrough adapter field |
|
| `artifact` | string | No | optional passthrough adapter field |
|
||||||
| `path` | string | No | optional passthrough adapter field |
|
| `path` | string | No | optional passthrough adapter field |
|
||||||
| `required` | bool | No | optional input requirement |
|
| `required` | bool | No | optional input requirement |
|
||||||
|
|||||||
@@ -1,68 +1,81 @@
|
|||||||
# Integration: Notarius
|
# Notarius Integration Contract
|
||||||
|
|
||||||
## Purpose
|
## Boundary
|
||||||
|
|
||||||
Define the implemented Notarius subprocess and published-bundle discovery
|
Narratio uses Notarius as a subprocess to extract configured structured JSON
|
||||||
boundary. Extract-stage orchestration and required-lane policy are not yet
|
lanes from the final trimmed Seriatim transcript. Narratio owns invocation,
|
||||||
implemented.
|
safe bundle discovery, lane selection, and its own artifact metadata. Notarius
|
||||||
|
owns pipeline definitions, lane schemas, the receipt, and bundle formats.
|
||||||
|
|
||||||
## Invocation Contract
|
Canonical Notarius references:
|
||||||
|
|
||||||
The adapter invokes one complete pipeline with separate argument elements:
|
- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/subprocess.md)
|
||||||
|
- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/dnd-pipeline.md)
|
||||||
|
- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/run-result.md)
|
||||||
|
- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/json-output.md)
|
||||||
|
|
||||||
|
The [complete Narratio example](../../examples/pipeline.full.annotated.yml)
|
||||||
|
records the exact current constraints for all ten D&D lanes. Treat the linked
|
||||||
|
Notarius documents as canonical when changing those values; Narratio does not
|
||||||
|
duplicate the complete schemas.
|
||||||
|
|
||||||
|
## Invocation
|
||||||
|
|
||||||
|
When `pipeline.notarius.enabled` is true, Narratio resolves the executable,
|
||||||
|
configuration path, input path, output directory, and working directory to
|
||||||
|
absolute paths and invokes:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
notarius run <pipeline-id> --config <absolute-path> --input <absolute-path> --output-dir <absolute-path> --json
|
notarius run <pipeline_id> --config <config_path> --input <trimmed_json> --output-dir <staging_dir> --json
|
||||||
```
|
```
|
||||||
|
|
||||||
It does not pass `--session-id` or automatically execute
|
Standard output is reserved for the JSON receipt. Standard error is captured
|
||||||
`notarius config validate`. The configured working directory and inherited
|
separately as diagnostic output. Narratio applies the configured timeout and
|
||||||
parent environment apply to the process. Narratio's shared subprocess timeout
|
does not interpret stdout as a receipt unless the subprocess exits successfully.
|
||||||
and cancellation handling bounds execution.
|
It does not pass a Narratio session ID or run `notarius config validate`
|
||||||
|
automatically; the configured working directory and inherited environment
|
||||||
|
apply to the subprocess.
|
||||||
|
|
||||||
Standard output is written directly to the configured receipt path and
|
## Accepted Result
|
||||||
standard error to a separate log path. A nonzero exit, cancellation, or timeout
|
|
||||||
returns the subprocess failure without interpreting receipt bytes.
|
|
||||||
|
|
||||||
## Receipt Contract
|
Narratio currently accepts receipt schema `notarius.run-result.v1`. The receipt
|
||||||
|
must identify the configured pipeline and a bundle root containing a confined
|
||||||
|
`index.json`. All receipt, index, and lane paths must stay inside that bundle;
|
||||||
|
symlinks and non-regular lane payloads are rejected.
|
||||||
|
|
||||||
After exit status zero, the adapter accepts the tolerant
|
Supported receipt and index shapes tolerate unknown fields for forward
|
||||||
`notarius.run-result.v1` receipt. It requires run and pipeline identity, an
|
compatibility, while required identity, validation, count, manifest,
|
||||||
absolute run-specific output directory, the relative production index path,
|
rejection, warning, and lane-list fields remain mandatory. Narratio applies
|
||||||
non-negative output counts, and validation status. The receipt pipeline must
|
bounded reads to the receipt, index, rejection, and warning documents. Optional
|
||||||
match the requested pipeline.
|
chunk-map and evidence-context descriptors must carry their complete generic
|
||||||
|
contract metadata when present.
|
||||||
|
|
||||||
Receipt, index, rejection, and warning files have explicit read limits.
|
For every entry in `pipeline.notarius.outputs`, Narratio requires exactly one
|
||||||
Unknown fields are tolerated within supported document shapes.
|
index descriptor with the configured lane ID, media type, schema ID, schema
|
||||||
|
version, and, when configured, module key. Missing, duplicate, rejected, or
|
||||||
|
incompatible required lanes fail extraction even if Notarius exited zero.
|
||||||
|
Unconfigured lanes may remain in the preserved bundle but do not become
|
||||||
|
selectable Narratio sources.
|
||||||
|
|
||||||
## Bundle Discovery
|
Each accepted configured lane is registered as
|
||||||
|
`narratio.extraction.<output_key>`. The bundle index is retained for audit and
|
||||||
|
resume validation but is not selectable. Scriptorium and publish rules consume
|
||||||
|
only explicitly named lane sources; `--artifacts` never selects Notarius lanes.
|
||||||
|
|
||||||
The receipt's output directory must be strictly beneath the absolute output
|
## Failure And Compatibility Behavior
|
||||||
root supplied by Narratio. The adapter resolves the receipt index and every
|
|
||||||
index path beneath that bundle root.
|
|
||||||
|
|
||||||
The production index must provide manifest, rejection, warning, and lane-list
|
- Startup and nonzero-exit errors fail extraction and retain captured diagnostics.
|
||||||
fields. Lane descriptors require unique non-empty lane IDs and relative file
|
- Invalid receipt JSON or an unsupported receipt schema fails before bundle use.
|
||||||
paths. Optional chunk-map and evidence-context descriptors require their full
|
- Unsafe or incompatible index data and required-lane rejection fail before the
|
||||||
generic contract metadata when present.
|
staged bundle is promoted to durable storage.
|
||||||
|
- Contract and external provenance metadata are preserved on lane artifact
|
||||||
|
records and through explicit publication.
|
||||||
|
|
||||||
Every consumed path must remain lexically confined and identify a regular file
|
Rejection and warning summaries retain structured stage, scope, lane, and
|
||||||
through directory components that are not symlinks. Absolute logical paths,
|
reason-code fields for diagnostics without exposing free-form external messages
|
||||||
traversal, prefix-confusion escapes, symlinks, directories, and special files
|
or reading lane payload bodies.
|
||||||
are rejected.
|
|
||||||
|
|
||||||
The adapter returns lane and pipeline-wide descriptors without deciding which
|
Configuration fields and defaults are in [Configuration](../config.md).
|
||||||
lanes a caller requires or comparing their contracts to campaign policy. It
|
Operator paths, rerun procedures, and bundle retention are in
|
||||||
does not read lane payload bodies or interpret D&D data.
|
[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md)
|
||||||
|
for failure recovery.
|
||||||
## Diagnostic Summaries
|
|
||||||
|
|
||||||
`rejected.json` and `warnings.json` are decoded tolerantly after bounded reads.
|
|
||||||
The result retains structured fields such as stage, scope, lane ID, and reason
|
|
||||||
code. Free-form external messages are validated as required by the upstream
|
|
||||||
shape but are not returned in the summaries.
|
|
||||||
|
|
||||||
## Internal Implementation
|
|
||||||
|
|
||||||
The transport-neutral interface, production runner, and configurable fake live
|
|
||||||
in `internal/adapters/notarius`. Composition and adapter ownership are
|
|
||||||
documented in [Adapter Internals](../internal/adapters.md).
|
|
||||||
|
|||||||
@@ -41,11 +41,12 @@ Adapters do not own:
|
|||||||
- Seriatim subprocess runner.
|
- Seriatim subprocess runner.
|
||||||
- Audita subprocess runner.
|
- Audita subprocess runner.
|
||||||
- Scriptorium subprocess runner.
|
- Scriptorium subprocess runner.
|
||||||
|
- Notarius subprocess runner when extraction is enabled.
|
||||||
- Noop notifier (`notify.NoopSender`).
|
- Noop notifier (`notify.NoopSender`).
|
||||||
- Object store only when required by selected stages/config.
|
- Object store only when required by selected stages/config.
|
||||||
|
|
||||||
The Notarius subprocess adapter is implemented independently of stage policy;
|
Notarius is composed only when extraction is enabled; the extract stage owns
|
||||||
application composition and extract-stage wiring are not yet implemented.
|
receipt, bundle, and configured-lane policy rather than the adapter.
|
||||||
|
|
||||||
Object-store construction goes through `newCommandObjectStore`, which loads
|
Object-store construction goes through `newCommandObjectStore`, which loads
|
||||||
configured filesystem secrets before adapter initialization.
|
configured filesystem secrets before adapter initialization.
|
||||||
|
|||||||
@@ -24,12 +24,15 @@ Registry entries bind each ID to its producer, output kind, canonical fallback,
|
|||||||
and content validator. The focused stage documents own their input/output flow;
|
and content validator. The focused stage documents own their input/output flow;
|
||||||
[Configuration](../config.md) owns where operators may select these IDs.
|
[Configuration](../config.md) owns where operators may select these IDs.
|
||||||
|
|
||||||
## Configured and Previous-Session Sources
|
## Configured, Extraction, And Previous-Session Sources
|
||||||
|
|
||||||
- configured source ID format: `narratio.artifact.<artifact_key>`
|
- configured source ID format: `narratio.artifact.<artifact_key>`
|
||||||
|
- extraction source ID format: `narratio.extraction.<output_key>`
|
||||||
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
- previous-session source ID format: `narratio.previous_session.artifact.<artifact_key>`
|
||||||
|
|
||||||
Both formats are validated by strict source-policy rules.
|
All formats are validated by strict source-policy rules. Extraction sources are
|
||||||
|
registered only from `pipeline.notarius.outputs`; the Notarius index has no
|
||||||
|
selectable source ID.
|
||||||
|
|
||||||
## Runtime Catalog
|
## Runtime Catalog
|
||||||
|
|
||||||
@@ -58,6 +61,15 @@ Configured sources (`narratio.artifact.*`):
|
|||||||
|
|
||||||
- resolve only through runtime catalog availability.
|
- resolve only through runtime catalog availability.
|
||||||
|
|
||||||
|
Extraction sources (`narratio.extraction.*`):
|
||||||
|
|
||||||
|
- use the shared registration and manifest hydration path in
|
||||||
|
`extraction_catalog.go`;
|
||||||
|
- require a current successful extract record with the exact configured source,
|
||||||
|
compatible contract and Notarius provenance, a confined regular durable
|
||||||
|
payload, and matching checksum; and
|
||||||
|
- are never inferred by scanning the Notarius bundle directory.
|
||||||
|
|
||||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||||
|
|
||||||
- resolve only from local `previous/` cache state;
|
- resolve only from local `previous/` cache state;
|
||||||
@@ -127,19 +139,23 @@ physical layout.
|
|||||||
|
|
||||||
- source ID formats are stable contracts;
|
- source ID formats are stable contracts;
|
||||||
- artifact resolution is deterministic and manifest-aware;
|
- artifact resolution is deterministic and manifest-aware;
|
||||||
|
- extraction sources are available only from a compatible successful manifest
|
||||||
|
record;
|
||||||
- previous-session source resolution in `analyze` is local-only;
|
- previous-session source resolution in `analyze` is local-only;
|
||||||
- remote current-state key construction remains centralized in artifacts helpers.
|
- remote current-state key construction remains centralized in artifacts helpers.
|
||||||
|
|
||||||
## Implementation And Tests
|
## Implementation And Tests
|
||||||
|
|
||||||
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
||||||
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`
|
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`,
|
||||||
|
`internal/artifacts/extraction_catalog.go`
|
||||||
- Current state: `internal/artifacts/current_state.go`
|
- Current state: `internal/artifacts/current_state.go`
|
||||||
- Paths and keys: `internal/artifacts/paths.go`,
|
- Paths and keys: `internal/artifacts/paths.go`,
|
||||||
`internal/artifacts/s3_keys.go`
|
`internal/artifacts/s3_keys.go`
|
||||||
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
- Previous requirements: `internal/artifacts/previous_requirements.go`
|
||||||
- Tests: `internal/artifacts/artifact_resolver_test.go`,
|
- Tests: `internal/artifacts/artifact_resolver_test.go`,
|
||||||
`internal/artifacts/catalog_test.go`,
|
`internal/artifacts/catalog_test.go`,
|
||||||
|
`internal/artifacts/extraction_catalog_test.go`,
|
||||||
`internal/artifacts/current_state_test.go`,
|
`internal/artifacts/current_state_test.go`,
|
||||||
`internal/artifacts/paths_model_test.go`,
|
`internal/artifacts/paths_model_test.go`,
|
||||||
`internal/artifacts/previous_requirements_test.go`
|
`internal/artifacts/previous_requirements_test.go`
|
||||||
|
|||||||
@@ -50,13 +50,23 @@ The model admits these stage states:
|
|||||||
|
|
||||||
The application runner marks an executing stage running and then succeeded or
|
The application runner marks an executing stage running and then succeeded or
|
||||||
failed in both manifests, persisting each transition. On success it records
|
failed in both manifests, persisting each transition. On success it records
|
||||||
outputs, logs, generated configuration references, and metadata. A successful
|
outputs, logs, generated configuration references, and metadata. Artifact
|
||||||
forced rerun marks only succeeded downstream session-stage records stale.
|
records may include optional contract and external provenance objects; old
|
||||||
|
manifests remain compatible when those fields are absent. A successful forced
|
||||||
|
rerun marks only succeeded downstream session-stage records stale.
|
||||||
|
|
||||||
|
A stage may explicitly return a skipped disposition and stable reason. The
|
||||||
|
runner persists that outcome in both manifests, clears older outputs for the
|
||||||
|
session-stage record, and continues. This self-skip is distinct from deciding
|
||||||
|
not to execute an already-succeeded stage and is reconsidered on later runs.
|
||||||
|
Skipped results cannot contain outputs.
|
||||||
|
|
||||||
When an already-succeeded stage is skipped, the invocation run manifest records
|
When an already-succeeded stage is skipped, the invocation run manifest records
|
||||||
the `skip` action and reason. The session manifest deliberately retains its
|
the `skip` action and reason. The session manifest deliberately retains its
|
||||||
existing succeeded record because it remains the cross-invocation progress
|
existing succeeded record because it remains the cross-invocation progress
|
||||||
authority.
|
authority. Stages with a resume validator, currently extraction, may reject an
|
||||||
|
otherwise eligible skip when the recorded durable result is obsolete; the
|
||||||
|
runner marks it stale and executes it.
|
||||||
|
|
||||||
Session manifest is the authoritative stage-progress ledger across invocations.
|
Session manifest is the authoritative stage-progress ledger across invocations.
|
||||||
Run manifest is invocation-scoped audit state.
|
Run manifest is invocation-scoped audit state.
|
||||||
@@ -64,6 +74,7 @@ Run manifest is invocation-scoped audit state.
|
|||||||
## Invariants
|
## Invariants
|
||||||
|
|
||||||
- stage resume/skip decisions are session-manifest driven.
|
- stage resume/skip decisions are session-manifest driven.
|
||||||
|
- self-skipped stages do not retain stale outputs and are reconsidered.
|
||||||
- force reruns stale downstream succeeded stages.
|
- force reruns stale downstream succeeded stages.
|
||||||
- run manifest does not replace session manifest as progress authority.
|
- run manifest does not replace session manifest as progress authority.
|
||||||
|
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ The implemented canonical order is:
|
|||||||
4. [`polish`](stage-polish.md)
|
4. [`polish`](stage-polish.md)
|
||||||
5. [`normalize`](stage-normalize.md)
|
5. [`normalize`](stage-normalize.md)
|
||||||
6. [`trim`](stage-trim.md)
|
6. [`trim`](stage-trim.md)
|
||||||
7. `extract`
|
7. [`extract`](stage-extract.md)
|
||||||
8. [`render`](stage-render.md)
|
8. [`render`](stage-render.md)
|
||||||
9. [`analyze`](stage-analyze.md)
|
9. [`analyze`](stage-analyze.md)
|
||||||
10. [`publish`](stage-publish.md)
|
10. [`publish`](stage-publish.md)
|
||||||
@@ -84,6 +84,7 @@ and execution semantics.
|
|||||||
- [`polish`](stage-polish.md)
|
- [`polish`](stage-polish.md)
|
||||||
- [`normalize`](stage-normalize.md)
|
- [`normalize`](stage-normalize.md)
|
||||||
- [`trim`](stage-trim.md)
|
- [`trim`](stage-trim.md)
|
||||||
|
- [`extract`](stage-extract.md)
|
||||||
- [`render`](stage-render.md)
|
- [`render`](stage-render.md)
|
||||||
- [`analyze`](stage-analyze.md)
|
- [`analyze`](stage-analyze.md)
|
||||||
- [`publish`](stage-publish.md)
|
- [`publish`](stage-publish.md)
|
||||||
|
|||||||
76
docs/internal/stage-extract.md
Normal file
76
docs/internal/stage-extract.md
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
# Internal: Extract Stage
|
||||||
|
|
||||||
|
## Responsibility
|
||||||
|
|
||||||
|
`extract` runs after `trim` and before `render`. It converts the canonical
|
||||||
|
`narratio.transcript.final_trimmed` JSON into configured Notarius lane artifacts.
|
||||||
|
An omitted or disabled Notarius section makes the stage succeed as a no-op and
|
||||||
|
does not require a Notarius runner.
|
||||||
|
|
||||||
|
The external protocol is documented in the
|
||||||
|
[Notarius integration contract](../integrations/notarius.md). Configuration
|
||||||
|
fields belong in [Configuration](../config.md), and physical paths and force
|
||||||
|
procedures belong in [Operations](../operations.md).
|
||||||
|
|
||||||
|
## Lifecycle
|
||||||
|
|
||||||
|
`internal/stage/extract.go`:
|
||||||
|
|
||||||
|
1. resolves the final trimmed transcript from the shared artifact catalog;
|
||||||
|
2. resolves and fingerprints the Notarius invocation contract;
|
||||||
|
3. creates a run-local staging directory and invokes the injected
|
||||||
|
`notarius.Runner`;
|
||||||
|
4. validates the successful receipt, confined index, configured required lane
|
||||||
|
descriptors, and regular payload files;
|
||||||
|
5. atomically promotes the complete bundle to its immutable durable location;
|
||||||
|
6. records one non-selectable `notarius_index` output and one selectable
|
||||||
|
`notarius_lane` output per configured lane; and
|
||||||
|
7. registers each lane as `narratio.extraction.<output_key>` for downstream
|
||||||
|
Scriptorium and publish resolution.
|
||||||
|
|
||||||
|
Lane records retain checksum, contract, producer run ID, and Notarius system,
|
||||||
|
run, pipeline, and lane provenance. Stage metadata retains the durable bundle
|
||||||
|
root, receipt, diagnostic paths, rejection/warning summaries, producing
|
||||||
|
Narratio run ID, and invocation fingerprint. Validation completes before
|
||||||
|
promotion, so a rejected result cannot expose a partial durable bundle.
|
||||||
|
|
||||||
|
## Resume Validation
|
||||||
|
|
||||||
|
`internal/stage/extract_resume.go` permits a skip only when the existing stage
|
||||||
|
record succeeded and still matches the current invocation fingerprint. The
|
||||||
|
fingerprint covers the resolved executable and config paths, pipeline ID,
|
||||||
|
timeout, working directory, and sorted configured output contracts.
|
||||||
|
|
||||||
|
The validator then checks the producing run identity, canonical immutable
|
||||||
|
bundle root, path confinement and absence of symlink components, receipt
|
||||||
|
identity, exactly one canonical index, the exact configured source set,
|
||||||
|
contracts and provenance, regular-file status, and stored checksums. Missing or
|
||||||
|
obsolete results are non-resumable and run again; unsafe filesystem conditions
|
||||||
|
return an error rather than silently accepting or replacing data.
|
||||||
|
|
||||||
|
The fingerprint cannot observe files imported by Notarius configuration,
|
||||||
|
profile contents, prompt/module definitions, or other transitive inputs.
|
||||||
|
Operators must force extraction after changing any such input.
|
||||||
|
|
||||||
|
## Failure Behavior
|
||||||
|
|
||||||
|
Adapter startup, timeout, nonzero exit, receipt decoding, path confinement,
|
||||||
|
index compatibility, required-lane rejection, payload inspection, checksum, or
|
||||||
|
promotion errors fail the stage through ordinary manifest transition handling.
|
||||||
|
Stdout receipt and stderr diagnostics remain separate. Downstream stages are
|
||||||
|
not given selectable extraction sources unless the complete configured result
|
||||||
|
has passed validation and promotion.
|
||||||
|
|
||||||
|
## Implementation And Focused Tests
|
||||||
|
|
||||||
|
- Stage execution and selection: `internal/stage/extract.go`,
|
||||||
|
`internal/stage/extract_test.go`
|
||||||
|
- Resume validation: `internal/stage/extract_resume.go`,
|
||||||
|
`internal/stage/extract_resume_test.go`
|
||||||
|
- Subprocess boundary: `internal/adapters/notarius/subprocess.go`,
|
||||||
|
`internal/adapters/notarius/subprocess_test.go`
|
||||||
|
- Catalog hydration: `internal/artifacts/extraction_catalog.go`,
|
||||||
|
`internal/artifacts/extraction_catalog_test.go`
|
||||||
|
- Composition and downstream behavior: `internal/app/runner_test.go`,
|
||||||
|
`internal/stage/analyze_test.go`, `internal/stage/publish_test.go`
|
||||||
|
|
||||||
@@ -24,6 +24,14 @@ materialized into canonical session paths before stage success. Managed
|
|||||||
previous-session cache paths remain session-durable and are never redirected
|
previous-session cache paths remain session-durable and are never redirected
|
||||||
into run-local output space.
|
into run-local output space.
|
||||||
|
|
||||||
|
Extraction uses run-local receipt, stderr, and output-root helpers, then
|
||||||
|
promotes the validated external bundle to the unique immutable Notarius bundle
|
||||||
|
path supplied by `internal/artifacts`. `internal/fileops.PromoteDirectory`
|
||||||
|
copies only regular files and directories to a same-filesystem temporary
|
||||||
|
sibling and atomically renames it without following symlinks or replacing an
|
||||||
|
existing destination. Exact physical paths belong in
|
||||||
|
[Operations](../operations.md#extraction-workflow).
|
||||||
|
|
||||||
## Locking
|
## Locking
|
||||||
|
|
||||||
`artifacts.LocalStore` enforces the single-writer session lock via `.lock`
|
`artifacts.LocalStore` enforces the single-writer session lock via `.lock`
|
||||||
@@ -54,9 +62,11 @@ deletion scope belong in [CLI](../cli.md#clean) and
|
|||||||
- Path model and local store: `internal/artifacts/paths.go`,
|
- Path model and local store: `internal/artifacts/paths.go`,
|
||||||
`internal/artifacts/local.go`
|
`internal/artifacts/local.go`
|
||||||
- Run-local materialization: `internal/stage/run_local.go`
|
- Run-local materialization: `internal/stage/run_local.go`
|
||||||
|
- Immutable bundle promotion: `internal/fileops/directory.go`
|
||||||
- Cleanup confinement: `internal/app/cleanup_targets.go`,
|
- Cleanup confinement: `internal/app/cleanup_targets.go`,
|
||||||
`internal/app/post_publish_cleanup.go`
|
`internal/app/post_publish_cleanup.go`
|
||||||
- Tests: `internal/artifacts/paths_model_test.go`,
|
- Tests: `internal/artifacts/paths_model_test.go`,
|
||||||
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
`internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`,
|
||||||
|
`internal/fileops/directory_test.go`,
|
||||||
`internal/app/cleanup_targets_test.go`,
|
`internal/app/cleanup_targets_test.go`,
|
||||||
`internal/app/post_publish_cleanup_test.go`
|
`internal/app/post_publish_cleanup_test.go`
|
||||||
|
|||||||
@@ -75,10 +75,11 @@ Canonical stage order:
|
|||||||
4. `polish`
|
4. `polish`
|
||||||
5. `normalize`
|
5. `normalize`
|
||||||
6. `trim`
|
6. `trim`
|
||||||
7. `render`
|
7. `extract`
|
||||||
8. `analyze`
|
8. `render`
|
||||||
9. `publish`
|
9. `analyze`
|
||||||
10. `notify`
|
10. `publish`
|
||||||
|
11. `notify`
|
||||||
|
|
||||||
Execution rules:
|
Execution rules:
|
||||||
|
|
||||||
@@ -101,7 +102,40 @@ Selection behavior:
|
|||||||
- validates names against `pipeline.scriptorium.artifacts`;
|
- validates names against `pipeline.scriptorium.artifacts`;
|
||||||
- filters analyze execution to selected configured artifacts;
|
- filters analyze execution to selected configured artifacts;
|
||||||
- filters publish rules for `narratio.artifact.<name>` sources only;
|
- filters publish rules for `narratio.artifact.<name>` sources only;
|
||||||
- does not suppress built-in transcript or bounds publish sources.
|
- does not suppress built-in transcript, bounds, or explicitly configured
|
||||||
|
`narratio.extraction.<name>` publish sources; and
|
||||||
|
- never partially selects Notarius lanes.
|
||||||
|
|
||||||
|
## Extraction Workflow
|
||||||
|
|
||||||
|
When Notarius extraction is enabled, the stage consumes the final trimmed JSON
|
||||||
|
and preserves the complete validated Notarius bundle at:
|
||||||
|
|
||||||
|
- `artifacts/notarius/{narratio_run_id}/`
|
||||||
|
|
||||||
|
The directory is immutable once promoted. Configured lanes become
|
||||||
|
`narratio.extraction.<name>` sources for Scriptorium and explicit publish rules;
|
||||||
|
the bundle and `index.json` are retained for audit and resume validation but
|
||||||
|
are not selectable or published implicitly.
|
||||||
|
|
||||||
|
Run-local diagnostics are:
|
||||||
|
|
||||||
|
- `runs/{run_id}/extract/notarius.receipt.json`
|
||||||
|
- `runs/{run_id}/extract/notarius.stderr.log`
|
||||||
|
- `runs/{run_id}/extract/notarius-output/` before durable promotion
|
||||||
|
|
||||||
|
To intentionally replace the current extraction result, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage extract 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Narratio automatically reruns extraction when its recorded invocation contract
|
||||||
|
or durable output validation changes. It cannot fingerprint configuration
|
||||||
|
files, profiles, prompts, modules, or references loaded transitively by
|
||||||
|
Notarius. Force extraction after changing any of those inputs, even when the
|
||||||
|
top-level Narratio and Notarius config paths remain the same. A forced extract
|
||||||
|
marks successful downstream stages stale through normal continuation behavior.
|
||||||
|
|
||||||
## Publish Workflow
|
## Publish Workflow
|
||||||
|
|
||||||
@@ -198,6 +232,10 @@ Durable session paths:
|
|||||||
- `config/**`
|
- `config/**`
|
||||||
- `runs/**`
|
- `runs/**`
|
||||||
|
|
||||||
|
Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt,
|
||||||
|
stderr, and pre-promotion output remain in the producing run's `extract`
|
||||||
|
directory as described in [Extraction Workflow](#extraction-workflow).
|
||||||
|
|
||||||
Run-local layout:
|
Run-local layout:
|
||||||
|
|
||||||
- `runs/{run_id}/{stage}/outputs`
|
- `runs/{run_id}/{stage}/outputs`
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ their domains:
|
|||||||
|
|
||||||
- WhisperX performs transcription;
|
- WhisperX performs transcription;
|
||||||
- Seriatim performs deterministic transcript processing and rendering;
|
- Seriatim performs deterministic transcript processing and rendering;
|
||||||
- Audita performs transcript correction and polishing; and
|
- Audita performs transcript correction and polishing;
|
||||||
|
- Notarius extracts validated structured artifact bundles; and
|
||||||
- Scriptorium executes prompts and produces configured artifacts.
|
- Scriptorium executes prompts and produces configured artifacts.
|
||||||
|
|
||||||
Narratio owns orchestration, configuration resolution, session and run state,
|
Narratio owns orchestration, configuration resolution, session and run state,
|
||||||
@@ -51,9 +52,9 @@ HTTP, subprocess, notification, and object-storage mechanics, including command
|
|||||||
construction, transport behavior, provider response handling, and external
|
construction, transport behavior, provider response handling, and external
|
||||||
error adaptation. External dependency types must remain inside the adapter that
|
error adaptation. External dependency types must remain inside the adapter that
|
||||||
owns them unless that dependency is the adapter's explicit public contract.
|
owns them unless that dependency is the adapter's explicit public contract.
|
||||||
WhisperX HTTP behavior, Seriatim, Audita, and Scriptorium command construction,
|
WhisperX HTTP behavior, Seriatim, Audita, Notarius, and Scriptorium command
|
||||||
notification transport, and object-storage SDK details remain behind these
|
construction, notification transport, and object-storage SDK details remain
|
||||||
boundaries.
|
behind these boundaries.
|
||||||
|
|
||||||
State and path services must not infer stage policy. Storage implementations
|
State and path services must not infer stage policy. Storage implementations
|
||||||
receive explicit bucket-relative keys and do not infer campaign, session, run,
|
receive explicit bucket-relative keys and do not infer campaign, session, run,
|
||||||
@@ -89,6 +90,13 @@ should preserve enough local state and diagnostics for inspection, recovery,
|
|||||||
and resume. Forcing an upstream stage invalidates succeeded downstream work
|
and resume. Forcing an upstream stage invalidates succeeded downstream work
|
||||||
according to the canonical stage order.
|
according to the canonical stage order.
|
||||||
|
|
||||||
|
A stage may explicitly self-skip with a stable reason and no outputs. That
|
||||||
|
outcome is persisted, clears older outputs owned by the stage, and is
|
||||||
|
reconsidered on a later invocation. A stage may also validate whether an
|
||||||
|
otherwise successful recorded result is still resumable; an obsolete result
|
||||||
|
is staled and rerun, while an unsafe condition that prevents a sound decision
|
||||||
|
stops execution.
|
||||||
|
|
||||||
Shared behavior should live behind a narrow service or helper with one clear
|
Shared behavior should live behind a narrow service or helper with one clear
|
||||||
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
|
owner. Stages must not reach across boundaries or reproduce adapter, manifest,
|
||||||
artifact, or path policy ad hoc.
|
artifact, or path policy ad hoc.
|
||||||
@@ -141,6 +149,9 @@ not reconstruct canonical paths through scattered string concatenation.
|
|||||||
Artifact resolution is deterministic and manifest-aware. Producers materialize
|
Artifact resolution is deterministic and manifest-aware. Producers materialize
|
||||||
canonical outputs before reporting success, and consumers resolve declared
|
canonical outputs before reporting success, and consumers resolve declared
|
||||||
artifact identities rather than infer files from unrelated directory contents.
|
artifact identities rather than infer files from unrelated directory contents.
|
||||||
|
External artifact bundles become current only through validated immutable
|
||||||
|
promotion and manifest records; directory presence alone never establishes
|
||||||
|
availability.
|
||||||
|
|
||||||
Writes, moves, replacements, and deletions must use narrow, explicit,
|
Writes, moves, replacements, and deletions must use narrow, explicit,
|
||||||
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
root-confined destinations. Symlinks, traversal, broad roots, and ambiguous
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
## Status And Audience
|
## Status And Audience
|
||||||
|
|
||||||
Proposed and not started.
|
Complete.
|
||||||
|
|
||||||
This plan is written for a GPT-5.6 Terra coding agent. Implement the stages in
|
This plan is written for a GPT-5.6 Terra coding agent. Implement the stages in
|
||||||
strict numerical order. Do not skip ahead, combine stages merely to reduce the
|
strict numerical order. Do not skip ahead, combine stages merely to reduce the
|
||||||
@@ -24,7 +24,7 @@ implementation sequence.
|
|||||||
| Stage 6 | Complete |
|
| Stage 6 | Complete |
|
||||||
| Stage 7 | Complete |
|
| Stage 7 | Complete |
|
||||||
| Stage 8 | Complete |
|
| Stage 8 | Complete |
|
||||||
| Stage 9 | Not started |
|
| Stage 9 | Complete |
|
||||||
|
|
||||||
After completing and validating a stage, update only that stage's row to
|
After completing and validating a stage, update only that stage's row to
|
||||||
`Complete` and record any material deviation in the relevant stage section.
|
`Complete` and record any material deviation in the relevant stage section.
|
||||||
|
|||||||
@@ -2,12 +2,12 @@
|
|||||||
|
|
||||||
## Status
|
## Status
|
||||||
|
|
||||||
Proposed. Implementation has not started.
|
Complete.
|
||||||
|
|
||||||
The ordered implementation plan is maintained in
|
The completed implementation sequence is retained in
|
||||||
[implementation.md](implementation.md). Until that plan is complete, this
|
[implementation.md](implementation.md) as decision and delivery history.
|
||||||
document is the only Narratio documentation that describes the proposed
|
Current behavior is documented in the canonical configuration, CLI,
|
||||||
`extract` stage, Notarius configuration, or `narratio.extraction.*` sources.
|
operations, integration, and internal references linked from this roadmap.
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
|
|||||||
@@ -117,6 +117,118 @@ Safe fix:
|
|||||||
|
|
||||||
Relevant reference: [CLI artifact selection](./cli.md).
|
Relevant reference: [CLI artifact selection](./cli.md).
|
||||||
|
|
||||||
|
## Notarius executable missing
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction fails while resolving or starting the Notarius executable.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- `pipeline.notarius.binary` is not installed, executable, or on `PATH`;
|
||||||
|
- a configured executable path is wrong.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- install a compatible Notarius release or correct the binary setting, then
|
||||||
|
rerun extraction.
|
||||||
|
|
||||||
|
Relevant references: [Notarius configuration](./config.md#notarius-output-entries)
|
||||||
|
and [Notarius integration](./integrations/notarius.md).
|
||||||
|
|
||||||
|
## Notarius exits nonzero
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction reports a Notarius exit error instead of a receipt.
|
||||||
|
|
||||||
|
Diagnostics:
|
||||||
|
|
||||||
|
- inspect `runs/{run_id}/extract/notarius.stderr.log`; stdout is reserved for
|
||||||
|
the receipt and is not merged with diagnostics.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- correct the reported Notarius pipeline, input, provider, or configuration
|
||||||
|
failure and rerun extraction. Do not edit a staged output bundle into place.
|
||||||
|
|
||||||
|
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
|
## Notarius receipt or index incompatible
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction rejects the receipt schema, pipeline identity, bundle/index path,
|
||||||
|
lane descriptor, or payload path even though Notarius exited successfully.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- Narratio and Notarius versions disagree on their consumer contract;
|
||||||
|
- the configured pipeline or lane constraints are stale;
|
||||||
|
- output paths escape the bundle or traverse symlinks.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- compare installed Notarius output with the canonical Notarius contracts,
|
||||||
|
align `pipeline.notarius` constraints, and rerun. Do not bypass confinement or
|
||||||
|
schema checks.
|
||||||
|
|
||||||
|
Relevant reference: [Notarius integration](./integrations/notarius.md).
|
||||||
|
|
||||||
|
## Required Notarius lane rejected or missing
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- extraction fails because a configured lane is rejected, missing, duplicated,
|
||||||
|
or incompatible, including after a zero exit.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- inspect the Notarius diagnostic log and bundle rejection/warning information;
|
||||||
|
- correct the Notarius module or the exact declared lane contract;
|
||||||
|
- remove an output declaration only if downstream consumers genuinely no longer
|
||||||
|
require that source, then rerun extraction.
|
||||||
|
|
||||||
|
Every configured output is required. Narratio does not promote a partial result.
|
||||||
|
|
||||||
|
## Extraction resume invalidated
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- a previously successful extraction runs again during ordinary continuation.
|
||||||
|
|
||||||
|
Likely causes:
|
||||||
|
|
||||||
|
- the executable/config path, pipeline ID, timeout, working directory, or
|
||||||
|
configured output contracts changed;
|
||||||
|
- the durable bundle, index, lane set, provenance, regular-file status, or
|
||||||
|
checksum no longer validates.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
- allow the automatic rerun after verifying the current configuration. Treat
|
||||||
|
an unsafe path or symlink error as filesystem corruption or tampering and
|
||||||
|
investigate it rather than replacing files manually.
|
||||||
|
|
||||||
|
## Notarius transitive configuration changed
|
||||||
|
|
||||||
|
Symptom:
|
||||||
|
|
||||||
|
- Notarius profiles, prompts, modules, imported files, or references changed,
|
||||||
|
but Narratio still considers the previous extraction resumable.
|
||||||
|
|
||||||
|
Safe fix:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
narratio run-stage extract 2026-04-04 --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Narratio fingerprints its invocation contract, not the contents of transitive
|
||||||
|
Notarius inputs. Always force extraction after changing them; downstream
|
||||||
|
successful stages are then marked stale normally.
|
||||||
|
|
||||||
|
Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow).
|
||||||
|
|
||||||
## Previous-session artifact input missing
|
## Previous-session artifact input missing
|
||||||
|
|
||||||
Symptom:
|
Symptom:
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ in the [configuration reference](../docs/config.md).
|
|||||||
external tools, and configured Scriptorium artifacts.
|
external tools, and configured Scriptorium artifacts.
|
||||||
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
- [Full annotated pipeline](pipeline.full.annotated.yml): every implemented
|
||||||
pipeline section with explanatory comments.
|
pipeline section with explanatory comments.
|
||||||
|
- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused
|
||||||
|
Scriptorium artifact consuming only three declared Notarius lanes.
|
||||||
|
|
||||||
The existing `internal/config` example test loads and validates each pipeline
|
The existing `internal/config` example test loads and validates each pipeline
|
||||||
with the sample campaign and a compatible local- or S3-audio session.
|
with the sample campaign and a compatible local- or S3-audio session.
|
||||||
|
|||||||
55
examples/pipeline.extraction-subset.yml
Normal file
55
examples/pipeline.extraction-subset.yml
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
# Purpose-specific extraction example: a Scriptorium session brief consumes
|
||||||
|
# only the three Notarius lanes it needs.
|
||||||
|
|
||||||
|
campaigns:
|
||||||
|
root: /usr/local/share/narratio/campaigns
|
||||||
|
default_campaign_id: sample-campaign
|
||||||
|
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: "https://transcription.example.com/transcribe"
|
||||||
|
|
||||||
|
notarius:
|
||||||
|
enabled: true
|
||||||
|
binary: notarius
|
||||||
|
config_path: /usr/local/etc/notarius/config.yml
|
||||||
|
pipeline_id: dnd-session
|
||||||
|
timeout: 3h
|
||||||
|
outputs:
|
||||||
|
npc_registry:
|
||||||
|
lane_id: npc-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/npc-registry
|
||||||
|
location_registry:
|
||||||
|
lane_id: location-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.location_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/location-registry
|
||||||
|
scene_descriptions:
|
||||||
|
lane_id: scene-descriptions
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.scene_descriptions
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/scene-descriptions
|
||||||
|
|
||||||
|
scriptorium:
|
||||||
|
binary: scriptorium
|
||||||
|
config_path: /usr/local/etc/scriptorium/config.yml
|
||||||
|
artifacts:
|
||||||
|
session_brief:
|
||||||
|
enabled: true
|
||||||
|
prompt_id: dnd.session_brief
|
||||||
|
output_path: artifacts/session_brief.md
|
||||||
|
inputs:
|
||||||
|
npcs:
|
||||||
|
source: narratio.extraction.npc_registry
|
||||||
|
required: true
|
||||||
|
locations:
|
||||||
|
source: narratio.extraction.location_registry
|
||||||
|
required: true
|
||||||
|
scenes:
|
||||||
|
source: narratio.extraction.scene_descriptions
|
||||||
|
required: true
|
||||||
|
|
||||||
@@ -60,6 +60,11 @@ publish:
|
|||||||
- source: narratio.artifact.player_handout
|
- source: narratio.artifact.player_handout
|
||||||
dest: artifacts/player_handout.md
|
dest: artifacts/player_handout.md
|
||||||
required: false
|
required: false
|
||||||
|
# Extraction lanes publish only when named explicitly; the bundle and index
|
||||||
|
# are never implicit publish sources.
|
||||||
|
- source: narratio.extraction.npc_registry
|
||||||
|
dest: artifacts/extraction/npc-registry.json
|
||||||
|
required: true
|
||||||
|
|
||||||
whisperx:
|
whisperx:
|
||||||
# Required.
|
# Required.
|
||||||
@@ -123,6 +128,78 @@ trim:
|
|||||||
seriatim:
|
seriatim:
|
||||||
report: false
|
report: false
|
||||||
|
|
||||||
|
notarius:
|
||||||
|
# Optional structured extraction between trim and render.
|
||||||
|
enabled: true
|
||||||
|
binary: notarius
|
||||||
|
config_path: /usr/local/etc/notarius/config.yml
|
||||||
|
pipeline_id: dnd-session
|
||||||
|
timeout: 3h
|
||||||
|
working_directory: /usr/local/etc/notarius
|
||||||
|
# Each key creates source narratio.extraction.<key>. These constraints match
|
||||||
|
# the current Notarius D&D lane contracts; update them with Notarius.
|
||||||
|
outputs:
|
||||||
|
item_registry:
|
||||||
|
lane_id: item-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.item_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/item-registry
|
||||||
|
npc_registry:
|
||||||
|
lane_id: npc-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/npc-registry
|
||||||
|
location_registry:
|
||||||
|
lane_id: location-registry
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.location_registry
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/location-registry
|
||||||
|
scene_descriptions:
|
||||||
|
lane_id: scene-descriptions
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.scene_descriptions
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/scene-descriptions
|
||||||
|
item_occurrences:
|
||||||
|
lane_id: item-occurrences
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.item_occurrences
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/item-occurrences
|
||||||
|
spells:
|
||||||
|
lane_id: spells
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.spells
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/spells
|
||||||
|
combat_turns:
|
||||||
|
lane_id: combat-turns
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.combat_turns
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/combat-turns
|
||||||
|
npc_occurrences:
|
||||||
|
lane_id: npc-occurrences
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.npc_occurrences
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/npc-occurrences
|
||||||
|
location_occurrences:
|
||||||
|
lane_id: location-occurrences
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.location_occurrences
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/location-occurrences
|
||||||
|
enemy_events:
|
||||||
|
lane_id: enemy-events
|
||||||
|
media_type: application/json
|
||||||
|
schema_id: notarius.dnd.enemy_events
|
||||||
|
schema_version: v1
|
||||||
|
module_key: dnd/enemy-events
|
||||||
|
|
||||||
scriptorium:
|
scriptorium:
|
||||||
binary: scriptorium
|
binary: scriptorium
|
||||||
config_path: /usr/local/etc/scriptorium/config.yml
|
config_path: /usr/local/etc/scriptorium/config.yml
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package config
|
|||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
@@ -1049,6 +1050,11 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
|||||||
pipelineFile: "pipeline.full.annotated.yml",
|
pipelineFile: "pipeline.full.annotated.yml",
|
||||||
sessionFile: "session.local-audio.yml",
|
sessionFile: "session.local-audio.yml",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "extraction subset pipeline with local audio session",
|
||||||
|
pipelineFile: "pipeline.extraction-subset.yml",
|
||||||
|
sessionFile: "session.local-audio.yml",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -1068,6 +1074,47 @@ func TestExamplesLoadAndValidate(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaintainedExtractionExamplesPreservePublishedContracts(t *testing.T) {
|
||||||
|
examplesDir := filepath.Join("..", "..", "examples")
|
||||||
|
full, err := LoadPipeline(filepath.Join(examplesDir, "pipeline.full.annotated.yml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load full example error = %v", err)
|
||||||
|
}
|
||||||
|
want := map[string]NotariusOutputConfig{
|
||||||
|
"item_registry": {LaneID: "item-registry", MediaType: "application/json", SchemaID: "notarius.dnd.item_registry", SchemaVersion: "v1", ModuleKey: "dnd/item-registry"},
|
||||||
|
"npc_registry": {LaneID: "npc-registry", MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry", SchemaVersion: "v1", ModuleKey: "dnd/npc-registry"},
|
||||||
|
"location_registry": {LaneID: "location-registry", MediaType: "application/json", SchemaID: "notarius.dnd.location_registry", SchemaVersion: "v1", ModuleKey: "dnd/location-registry"},
|
||||||
|
"scene_descriptions": {LaneID: "scene-descriptions", MediaType: "application/json", SchemaID: "notarius.dnd.scene_descriptions", SchemaVersion: "v1", ModuleKey: "dnd/scene-descriptions"},
|
||||||
|
"item_occurrences": {LaneID: "item-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.item_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/item-occurrences"},
|
||||||
|
"spells": {LaneID: "spells", MediaType: "application/json", SchemaID: "notarius.dnd.spells", SchemaVersion: "v1", ModuleKey: "dnd/spells"},
|
||||||
|
"combat_turns": {LaneID: "combat-turns", MediaType: "application/json", SchemaID: "notarius.dnd.combat_turns", SchemaVersion: "v1", ModuleKey: "dnd/combat-turns"},
|
||||||
|
"npc_occurrences": {LaneID: "npc-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.npc_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/npc-occurrences"},
|
||||||
|
"location_occurrences": {LaneID: "location-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.location_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/location-occurrences"},
|
||||||
|
"enemy_events": {LaneID: "enemy-events", MediaType: "application/json", SchemaID: "notarius.dnd.enemy_events", SchemaVersion: "v1", ModuleKey: "dnd/enemy-events"},
|
||||||
|
}
|
||||||
|
if full.Notarius == nil || !reflect.DeepEqual(full.Notarius.Outputs, want) {
|
||||||
|
t.Fatalf("full example outputs = %#v, want %#v", full.Notarius, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
subset, err := LoadPipeline(filepath.Join(examplesDir, "pipeline.extraction-subset.yml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load subset example error = %v", err)
|
||||||
|
}
|
||||||
|
brief := subset.Scriptorium.Artifacts["session_brief"]
|
||||||
|
wantSources := map[string]string{
|
||||||
|
"npcs": "narratio.extraction.npc_registry",
|
||||||
|
"locations": "narratio.extraction.location_registry",
|
||||||
|
"scenes": "narratio.extraction.scene_descriptions",
|
||||||
|
}
|
||||||
|
gotSources := make(map[string]string, len(brief.Inputs))
|
||||||
|
for name, input := range brief.Inputs {
|
||||||
|
gotSources[name] = input.Source
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(gotSources, wantSources) {
|
||||||
|
t.Fatalf("subset example sources = %#v, want %#v", gotSources, wantSources)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, string) {
|
func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if !strings.Contains(pipelineYAML, "\naudita:") && !strings.HasPrefix(pipelineYAML, "audita:") {
|
if !strings.Contains(pipelineYAML, "\naudita:") && !strings.HasPrefix(pipelineYAML, "audita:") {
|
||||||
|
|||||||
Reference in New Issue
Block a user