From df40cbec6e33c7cace823a7ba5bcfd79666719af Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 10 Aug 2026 00:42:44 +0000 Subject: [PATCH] Document and validate Notarius extraction workflows --- README.md | 3 +- docs/cli.md | 11 ++- docs/config.md | 32 ++++++- docs/integrations/notarius.md | 115 +++++++++++++----------- docs/internal/adapters.md | 5 +- docs/internal/artifacts.md | 22 ++++- docs/internal/manifest.md | 17 +++- docs/internal/overview.md | 3 +- docs/internal/stage-extract.md | 76 ++++++++++++++++ docs/internal/workspace.md | 10 +++ docs/operations.md | 48 ++++++++-- docs/policy/architecture.md | 19 +++- docs/roadmap/implementation.md | 4 +- docs/roadmap/notarius-extract-stage.md | 10 +-- docs/troubleshooting.md | 112 +++++++++++++++++++++++ examples/README.md | 2 + examples/pipeline.extraction-subset.yml | 55 ++++++++++++ examples/pipeline.full.annotated.yml | 77 ++++++++++++++++ internal/config/load_validate_test.go | 47 ++++++++++ 19 files changed, 588 insertions(+), 80 deletions(-) create mode 100644 docs/internal/stage-extract.md create mode 100644 examples/pipeline.extraction-subset.yml diff --git a/README.md b/README.md index 8aa9d85..6f42cda 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # narratio Narratio is a stage-driven Go orchestrator for turning D&D session audio into -polished transcripts and generated artifacts. +polished transcripts, validated Notarius extraction lanes, and generated +artifacts. It runs a deterministic workflow with manifest-driven continuation, remote publish, and restore support. diff --git a/docs/cli.md b/docs/cli.md index 75447c7..5fc5ade 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -75,6 +75,8 @@ narratio run [--force] [--artifacts ] [...common co Behavior: - evaluates full stage order; +- runs `extract` between `trim` and `render`; an omitted or disabled Notarius + configuration makes extraction a no-op; - skips already-succeeded stages unless `--force` is set or a stage-specific resume check finds its durable result obsolete; - continues interrupted or partially completed sessions by running non-succeeded stages; @@ -220,7 +222,10 @@ default restore scope, report location, and conflict-handling workflow. narratio session artifacts [--remote] [...common config flags] ``` -Lists effective built-in and configured artifact sources, publish rules, lock state, and optional remote published-state availability. +Lists effective built-in, configured Scriptorium, and configured extraction +sources; reports planned, available, unavailable, and published state without +reading payload bodies; and includes publish rules, lock state, and optional +remote published-state availability. ### `session locks` @@ -250,7 +255,9 @@ Effects: - filters analyze execution to selected configured artifacts; - filters publish rules that source `narratio.artifact.`; -- does not filter built-in transcript/bounds publish sources. +- does not filter built-in transcript/bounds or explicitly configured + `narratio.extraction.` publish sources; and +- does not select or filter Notarius lanes. ## Common Workflows diff --git a/docs/config.md b/docs/config.md index bd39941..6c1db0d 100644 --- a/docs/config.md +++ b/docs/config.md @@ -118,6 +118,9 @@ Rules: - `outputs[].source` is required. - `outputs[].dest` may be omitted when derivable from source. +- extraction sources require an explicit `outputs[].dest` and publish only when + a rule names that source; the Notarius index and complete bundle are not + publish sources. - `outputs[].required` defaults to `true`. - static locks (`pipeline.publish.locks`) merge with remote locks (`{session_prefix}/locks.yml`), with static locks taking precedence on duplicates. @@ -196,6 +199,13 @@ Rules: | `pipeline.trim.bounds.render_debug` | bool | No | `false` | | `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true | | `pipeline.trim.seriatim.report` | bool | No | `false` | +| `pipeline.notarius.enabled` | bool | No | `false` | +| `pipeline.notarius.binary` | string | No | `notarius` | +| `pipeline.notarius.config_path` | string | Conditional | required when enabled; relative paths resolve from the pipeline file directory | +| `pipeline.notarius.pipeline_id` | string | Conditional | required when enabled | +| `pipeline.notarius.timeout` | duration | No | `3h`; must be positive | +| `pipeline.notarius.working_directory` | string | No | directory containing resolved `config_path`; relative paths resolve from the pipeline file directory | +| `pipeline.notarius.outputs` | map | Conditional | at least one entry when enabled | | `pipeline.render.enabled` | bool | No | `true` | | `pipeline.render.format` | string | No | `markdown` (only supported value) | | `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) | @@ -211,6 +221,26 @@ Rules: | `pipeline.notification.recipient` | string | No | empty | | `pipeline.notification.timeout` | duration | No | empty | +### Notarius Output Entries + +For each `pipeline.notarius.outputs.`: + +| Field | Type | Required | Rule | +| --- | --- | --- | --- | +| `lane_id` | string | Yes | unique Notarius lane ID | +| `media_type` | string | Yes | exact accepted descriptor media type | +| `schema_id` | string | Yes | exact accepted descriptor schema ID | +| `schema_version` | string | Yes | exact accepted descriptor schema version | +| `module_key` | string | No | exact accepted module key when set | + +Output names must match `^[a-z][a-z0-9_]*$` and become selectable sources named +`narratio.extraction.`. Lane IDs must be unique. Every declared output is +required from a successful Notarius result; a missing, rejected, duplicate, or +contract-incompatible lane fails extraction. See the +[complete maintained example](../examples/pipeline.full.annotated.yml) for the +current ten-lane D&D mapping and the [Notarius contract](./integrations/notarius.md) +for compatibility ownership. + ### Scriptorium Artifact Entries For each `pipeline.scriptorium.artifacts.`: @@ -233,7 +263,7 @@ For each artifact input `pipeline.scriptorium.artifacts..inputs.`, or `narratio.previous_session.artifact.` | +| `source` | string | Yes | built-in runtime source, prepared input source, `narratio.extraction.`, `narratio.artifact.`, or `narratio.previous_session.artifact.` | | `artifact` | string | No | optional passthrough adapter field | | `path` | string | No | optional passthrough adapter field | | `required` | bool | No | optional input requirement | diff --git a/docs/integrations/notarius.md b/docs/integrations/notarius.md index 2fc7f6b..fe99db2 100644 --- a/docs/integrations/notarius.md +++ b/docs/integrations/notarius.md @@ -1,68 +1,81 @@ -# Integration: Notarius +# Notarius Integration Contract -## Purpose +## Boundary -Define the implemented Notarius subprocess and published-bundle discovery -boundary. Extract-stage orchestration and required-lane policy are not yet -implemented. +Narratio uses Notarius as a subprocess to extract configured structured JSON +lanes from the final trimmed Seriatim transcript. Narratio owns invocation, +safe bundle discovery, lane selection, and its own artifact metadata. Notarius +owns pipeline definitions, lane schemas, the receipt, and bundle formats. -## Invocation Contract +Canonical Notarius references: -The adapter invokes one complete pipeline with separate argument elements: +- [Subprocess consumer contract](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/subprocess.md) +- [D&D pipeline and lane contracts](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/consumers/dnd-pipeline.md) +- [Run-result receipt](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/run-result.md) +- [JSON output bundle](https://gitea.maximumdirect.net/eric/notarius/src/branch/main/docs/integrations/json-output.md) + +The [complete Narratio example](../../examples/pipeline.full.annotated.yml) +records the exact current constraints for all ten D&D lanes. Treat the linked +Notarius documents as canonical when changing those values; Narratio does not +duplicate the complete schemas. + +## Invocation + +When `pipeline.notarius.enabled` is true, Narratio resolves the executable, +configuration path, input path, output directory, and working directory to +absolute paths and invokes: ```text -notarius run --config --input --output-dir --json +notarius run --config --input --output-dir --json ``` -It does not pass `--session-id` or automatically execute -`notarius config validate`. The configured working directory and inherited -parent environment apply to the process. Narratio's shared subprocess timeout -and cancellation handling bounds execution. +Standard output is reserved for the JSON receipt. Standard error is captured +separately as diagnostic output. Narratio applies the configured timeout and +does not interpret stdout as a receipt unless the subprocess exits successfully. +It does not pass a Narratio session ID or run `notarius config validate` +automatically; the configured working directory and inherited environment +apply to the subprocess. -Standard output is written directly to the configured receipt path and -standard error to a separate log path. A nonzero exit, cancellation, or timeout -returns the subprocess failure without interpreting receipt bytes. +## Accepted Result -## Receipt Contract +Narratio currently accepts receipt schema `notarius.run-result.v1`. The receipt +must identify the configured pipeline and a bundle root containing a confined +`index.json`. All receipt, index, and lane paths must stay inside that bundle; +symlinks and non-regular lane payloads are rejected. -After exit status zero, the adapter accepts the tolerant -`notarius.run-result.v1` receipt. It requires run and pipeline identity, an -absolute run-specific output directory, the relative production index path, -non-negative output counts, and validation status. The receipt pipeline must -match the requested pipeline. +Supported receipt and index shapes tolerate unknown fields for forward +compatibility, while required identity, validation, count, manifest, +rejection, warning, and lane-list fields remain mandatory. Narratio applies +bounded reads to the receipt, index, rejection, and warning documents. Optional +chunk-map and evidence-context descriptors must carry their complete generic +contract metadata when present. -Receipt, index, rejection, and warning files have explicit read limits. -Unknown fields are tolerated within supported document shapes. +For every entry in `pipeline.notarius.outputs`, Narratio requires exactly one +index descriptor with the configured lane ID, media type, schema ID, schema +version, and, when configured, module key. Missing, duplicate, rejected, or +incompatible required lanes fail extraction even if Notarius exited zero. +Unconfigured lanes may remain in the preserved bundle but do not become +selectable Narratio sources. -## Bundle Discovery +Each accepted configured lane is registered as +`narratio.extraction.`. The bundle index is retained for audit and +resume validation but is not selectable. Scriptorium and publish rules consume +only explicitly named lane sources; `--artifacts` never selects Notarius lanes. -The receipt's output directory must be strictly beneath the absolute output -root supplied by Narratio. The adapter resolves the receipt index and every -index path beneath that bundle root. +## Failure And Compatibility Behavior -The production index must provide manifest, rejection, warning, and lane-list -fields. Lane descriptors require unique non-empty lane IDs and relative file -paths. Optional chunk-map and evidence-context descriptors require their full -generic contract metadata when present. +- Startup and nonzero-exit errors fail extraction and retain captured diagnostics. +- Invalid receipt JSON or an unsupported receipt schema fails before bundle use. +- Unsafe or incompatible index data and required-lane rejection fail before the + staged bundle is promoted to durable storage. +- Contract and external provenance metadata are preserved on lane artifact + records and through explicit publication. -Every consumed path must remain lexically confined and identify a regular file -through directory components that are not symlinks. Absolute logical paths, -traversal, prefix-confusion escapes, symlinks, directories, and special files -are rejected. +Rejection and warning summaries retain structured stage, scope, lane, and +reason-code fields for diagnostics without exposing free-form external messages +or reading lane payload bodies. -The adapter returns lane and pipeline-wide descriptors without deciding which -lanes a caller requires or comparing their contracts to campaign policy. It -does not read lane payload bodies or interpret D&D data. - -## Diagnostic Summaries - -`rejected.json` and `warnings.json` are decoded tolerantly after bounded reads. -The result retains structured fields such as stage, scope, lane ID, and reason -code. Free-form external messages are validated as required by the upstream -shape but are not returned in the summaries. - -## Internal Implementation - -The transport-neutral interface, production runner, and configurable fake live -in `internal/adapters/notarius`. Composition and adapter ownership are -documented in [Adapter Internals](../internal/adapters.md). +Configuration fields and defaults are in [Configuration](../config.md). +Operator paths, rerun procedures, and bundle retention are in +[Operations](../operations.md). See [Troubleshooting](../troubleshooting.md) +for failure recovery. diff --git a/docs/internal/adapters.md b/docs/internal/adapters.md index 1e9f216..ce2a939 100644 --- a/docs/internal/adapters.md +++ b/docs/internal/adapters.md @@ -41,11 +41,12 @@ Adapters do not own: - Seriatim subprocess runner. - Audita subprocess runner. - Scriptorium subprocess runner. +- Notarius subprocess runner when extraction is enabled. - Noop notifier (`notify.NoopSender`). - Object store only when required by selected stages/config. -The Notarius subprocess adapter is implemented independently of stage policy; -application composition and extract-stage wiring are not yet implemented. +Notarius is composed only when extraction is enabled; the extract stage owns +receipt, bundle, and configured-lane policy rather than the adapter. Object-store construction goes through `newCommandObjectStore`, which loads configured filesystem secrets before adapter initialization. diff --git a/docs/internal/artifacts.md b/docs/internal/artifacts.md index 3e215a4..685777a 100644 --- a/docs/internal/artifacts.md +++ b/docs/internal/artifacts.md @@ -24,12 +24,15 @@ Registry entries bind each ID to its producer, output kind, canonical fallback, and content validator. The focused stage documents own their input/output flow; [Configuration](../config.md) owns where operators may select these IDs. -## Configured and Previous-Session Sources +## Configured, Extraction, And Previous-Session Sources - configured source ID format: `narratio.artifact.` +- extraction source ID format: `narratio.extraction.` - previous-session source ID format: `narratio.previous_session.artifact.` -Both formats are validated by strict source-policy rules. +All formats are validated by strict source-policy rules. Extraction sources are +registered only from `pipeline.notarius.outputs`; the Notarius index has no +selectable source ID. ## Runtime Catalog @@ -58,6 +61,15 @@ Configured sources (`narratio.artifact.*`): - resolve only through runtime catalog availability. +Extraction sources (`narratio.extraction.*`): + +- use the shared registration and manifest hydration path in + `extraction_catalog.go`; +- require a current successful extract record with the exact configured source, + compatible contract and Notarius provenance, a confined regular durable + payload, and matching checksum; and +- are never inferred by scanning the Notarius bundle directory. + Previous-session sources (`narratio.previous_session.artifact.*`): - resolve only from local `previous/` cache state; @@ -127,19 +139,23 @@ physical layout. - source ID formats are stable contracts; - artifact resolution is deterministic and manifest-aware; +- extraction sources are available only from a compatible successful manifest + record; - previous-session source resolution in `analyze` is local-only; - remote current-state key construction remains centralized in artifacts helpers. ## Implementation And Tests - Registry and resolution: `internal/artifacts/artifact_resolver.go`, - `internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go` + `internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`, + `internal/artifacts/extraction_catalog.go` - Current state: `internal/artifacts/current_state.go` - Paths and keys: `internal/artifacts/paths.go`, `internal/artifacts/s3_keys.go` - Previous requirements: `internal/artifacts/previous_requirements.go` - Tests: `internal/artifacts/artifact_resolver_test.go`, `internal/artifacts/catalog_test.go`, + `internal/artifacts/extraction_catalog_test.go`, `internal/artifacts/current_state_test.go`, `internal/artifacts/paths_model_test.go`, `internal/artifacts/previous_requirements_test.go` diff --git a/docs/internal/manifest.md b/docs/internal/manifest.md index 1506362..4c7d8f1 100644 --- a/docs/internal/manifest.md +++ b/docs/internal/manifest.md @@ -50,13 +50,23 @@ The model admits these stage states: The application runner marks an executing stage running and then succeeded or failed in both manifests, persisting each transition. On success it records -outputs, logs, generated configuration references, and metadata. A successful -forced rerun marks only succeeded downstream session-stage records stale. +outputs, logs, generated configuration references, and metadata. Artifact +records may include optional contract and external provenance objects; old +manifests remain compatible when those fields are absent. A successful forced +rerun marks only succeeded downstream session-stage records stale. + +A stage may explicitly return a skipped disposition and stable reason. The +runner persists that outcome in both manifests, clears older outputs for the +session-stage record, and continues. This self-skip is distinct from deciding +not to execute an already-succeeded stage and is reconsidered on later runs. +Skipped results cannot contain outputs. When an already-succeeded stage is skipped, the invocation run manifest records the `skip` action and reason. The session manifest deliberately retains its existing succeeded record because it remains the cross-invocation progress -authority. +authority. Stages with a resume validator, currently extraction, may reject an +otherwise eligible skip when the recorded durable result is obsolete; the +runner marks it stale and executes it. Session manifest is the authoritative stage-progress ledger across invocations. Run manifest is invocation-scoped audit state. @@ -64,6 +74,7 @@ Run manifest is invocation-scoped audit state. ## Invariants - stage resume/skip decisions are session-manifest driven. +- self-skipped stages do not retain stale outputs and are reconsidered. - force reruns stale downstream succeeded stages. - run manifest does not replace session manifest as progress authority. diff --git a/docs/internal/overview.md b/docs/internal/overview.md index 62b82ba..eb28dca 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -53,7 +53,7 @@ The implemented canonical order is: 4. [`polish`](stage-polish.md) 5. [`normalize`](stage-normalize.md) 6. [`trim`](stage-trim.md) -7. `extract` +7. [`extract`](stage-extract.md) 8. [`render`](stage-render.md) 9. [`analyze`](stage-analyze.md) 10. [`publish`](stage-publish.md) @@ -84,6 +84,7 @@ and execution semantics. - [`polish`](stage-polish.md) - [`normalize`](stage-normalize.md) - [`trim`](stage-trim.md) +- [`extract`](stage-extract.md) - [`render`](stage-render.md) - [`analyze`](stage-analyze.md) - [`publish`](stage-publish.md) diff --git a/docs/internal/stage-extract.md b/docs/internal/stage-extract.md new file mode 100644 index 0000000..312ec29 --- /dev/null +++ b/docs/internal/stage-extract.md @@ -0,0 +1,76 @@ +# Internal: Extract Stage + +## Responsibility + +`extract` runs after `trim` and before `render`. It converts the canonical +`narratio.transcript.final_trimmed` JSON into configured Notarius lane artifacts. +An omitted or disabled Notarius section makes the stage succeed as a no-op and +does not require a Notarius runner. + +The external protocol is documented in the +[Notarius integration contract](../integrations/notarius.md). Configuration +fields belong in [Configuration](../config.md), and physical paths and force +procedures belong in [Operations](../operations.md). + +## Lifecycle + +`internal/stage/extract.go`: + +1. resolves the final trimmed transcript from the shared artifact catalog; +2. resolves and fingerprints the Notarius invocation contract; +3. creates a run-local staging directory and invokes the injected + `notarius.Runner`; +4. validates the successful receipt, confined index, configured required lane + descriptors, and regular payload files; +5. atomically promotes the complete bundle to its immutable durable location; +6. records one non-selectable `notarius_index` output and one selectable + `notarius_lane` output per configured lane; and +7. registers each lane as `narratio.extraction.` for downstream + Scriptorium and publish resolution. + +Lane records retain checksum, contract, producer run ID, and Notarius system, +run, pipeline, and lane provenance. Stage metadata retains the durable bundle +root, receipt, diagnostic paths, rejection/warning summaries, producing +Narratio run ID, and invocation fingerprint. Validation completes before +promotion, so a rejected result cannot expose a partial durable bundle. + +## Resume Validation + +`internal/stage/extract_resume.go` permits a skip only when the existing stage +record succeeded and still matches the current invocation fingerprint. The +fingerprint covers the resolved executable and config paths, pipeline ID, +timeout, working directory, and sorted configured output contracts. + +The validator then checks the producing run identity, canonical immutable +bundle root, path confinement and absence of symlink components, receipt +identity, exactly one canonical index, the exact configured source set, +contracts and provenance, regular-file status, and stored checksums. Missing or +obsolete results are non-resumable and run again; unsafe filesystem conditions +return an error rather than silently accepting or replacing data. + +The fingerprint cannot observe files imported by Notarius configuration, +profile contents, prompt/module definitions, or other transitive inputs. +Operators must force extraction after changing any such input. + +## Failure Behavior + +Adapter startup, timeout, nonzero exit, receipt decoding, path confinement, +index compatibility, required-lane rejection, payload inspection, checksum, or +promotion errors fail the stage through ordinary manifest transition handling. +Stdout receipt and stderr diagnostics remain separate. Downstream stages are +not given selectable extraction sources unless the complete configured result +has passed validation and promotion. + +## Implementation And Focused Tests + +- Stage execution and selection: `internal/stage/extract.go`, + `internal/stage/extract_test.go` +- Resume validation: `internal/stage/extract_resume.go`, + `internal/stage/extract_resume_test.go` +- Subprocess boundary: `internal/adapters/notarius/subprocess.go`, + `internal/adapters/notarius/subprocess_test.go` +- Catalog hydration: `internal/artifacts/extraction_catalog.go`, + `internal/artifacts/extraction_catalog_test.go` +- Composition and downstream behavior: `internal/app/runner_test.go`, + `internal/stage/analyze_test.go`, `internal/stage/publish_test.go` + diff --git a/docs/internal/workspace.md b/docs/internal/workspace.md index 87882e7..e0ac124 100644 --- a/docs/internal/workspace.md +++ b/docs/internal/workspace.md @@ -24,6 +24,14 @@ materialized into canonical session paths before stage success. Managed previous-session cache paths remain session-durable and are never redirected into run-local output space. +Extraction uses run-local receipt, stderr, and output-root helpers, then +promotes the validated external bundle to the unique immutable Notarius bundle +path supplied by `internal/artifacts`. `internal/fileops.PromoteDirectory` +copies only regular files and directories to a same-filesystem temporary +sibling and atomically renames it without following symlinks or replacing an +existing destination. Exact physical paths belong in +[Operations](../operations.md#extraction-workflow). + ## Locking `artifacts.LocalStore` enforces the single-writer session lock via `.lock` @@ -54,9 +62,11 @@ deletion scope belong in [CLI](../cli.md#clean) and - Path model and local store: `internal/artifacts/paths.go`, `internal/artifacts/local.go` - Run-local materialization: `internal/stage/run_local.go` +- Immutable bundle promotion: `internal/fileops/directory.go` - Cleanup confinement: `internal/app/cleanup_targets.go`, `internal/app/post_publish_cleanup.go` - Tests: `internal/artifacts/paths_model_test.go`, `internal/artifacts/local_test.go`, `internal/stage/run_local_test.go`, + `internal/fileops/directory_test.go`, `internal/app/cleanup_targets_test.go`, `internal/app/post_publish_cleanup_test.go` diff --git a/docs/operations.md b/docs/operations.md index 6cefae3..722d4ef 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -75,10 +75,11 @@ Canonical stage order: 4. `polish` 5. `normalize` 6. `trim` -7. `render` -8. `analyze` -9. `publish` -10. `notify` +7. `extract` +8. `render` +9. `analyze` +10. `publish` +11. `notify` Execution rules: @@ -101,7 +102,40 @@ Selection behavior: - validates names against `pipeline.scriptorium.artifacts`; - filters analyze execution to selected configured artifacts; - filters publish rules for `narratio.artifact.` sources only; -- does not suppress built-in transcript or bounds publish sources. +- does not suppress built-in transcript, bounds, or explicitly configured + `narratio.extraction.` publish sources; and +- never partially selects Notarius lanes. + +## Extraction Workflow + +When Notarius extraction is enabled, the stage consumes the final trimmed JSON +and preserves the complete validated Notarius bundle at: + +- `artifacts/notarius/{narratio_run_id}/` + +The directory is immutable once promoted. Configured lanes become +`narratio.extraction.` sources for Scriptorium and explicit publish rules; +the bundle and `index.json` are retained for audit and resume validation but +are not selectable or published implicitly. + +Run-local diagnostics are: + +- `runs/{run_id}/extract/notarius.receipt.json` +- `runs/{run_id}/extract/notarius.stderr.log` +- `runs/{run_id}/extract/notarius-output/` before durable promotion + +To intentionally replace the current extraction result, run: + +```bash +narratio run-stage extract 2026-04-04 --force +``` + +Narratio automatically reruns extraction when its recorded invocation contract +or durable output validation changes. It cannot fingerprint configuration +files, profiles, prompts, modules, or references loaded transitively by +Notarius. Force extraction after changing any of those inputs, even when the +top-level Narratio and Notarius config paths remain the same. A forced extract +marks successful downstream stages stale through normal continuation behavior. ## Publish Workflow @@ -198,6 +232,10 @@ Durable session paths: - `config/**` - `runs/**` +Validated Notarius bundles live below `artifacts/notarius/{run_id}/`; receipt, +stderr, and pre-promotion output remain in the producing run's `extract` +directory as described in [Extraction Workflow](#extraction-workflow). + Run-local layout: - `runs/{run_id}/{stage}/outputs` diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index 74af1ff..0fbacff 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -17,7 +17,8 @@ their domains: - WhisperX performs transcription; - Seriatim performs deterministic transcript processing and rendering; -- Audita performs transcript correction and polishing; and +- Audita performs transcript correction and polishing; +- Notarius extracts validated structured artifact bundles; and - Scriptorium executes prompts and produces configured artifacts. Narratio owns orchestration, configuration resolution, session and run state, @@ -51,9 +52,9 @@ HTTP, subprocess, notification, and object-storage mechanics, including command construction, transport behavior, provider response handling, and external error adaptation. External dependency types must remain inside the adapter that owns them unless that dependency is the adapter's explicit public contract. -WhisperX HTTP behavior, Seriatim, Audita, and Scriptorium command construction, -notification transport, and object-storage SDK details remain behind these -boundaries. +WhisperX HTTP behavior, Seriatim, Audita, Notarius, and Scriptorium command +construction, notification transport, and object-storage SDK details remain +behind these boundaries. State and path services must not infer stage policy. Storage implementations receive explicit bucket-relative keys and do not infer campaign, session, run, @@ -89,6 +90,13 @@ should preserve enough local state and diagnostics for inspection, recovery, and resume. Forcing an upstream stage invalidates succeeded downstream work according to the canonical stage order. +A stage may explicitly self-skip with a stable reason and no outputs. That +outcome is persisted, clears older outputs owned by the stage, and is +reconsidered on a later invocation. A stage may also validate whether an +otherwise successful recorded result is still resumable; an obsolete result +is staled and rerun, while an unsafe condition that prevents a sound decision +stops execution. + Shared behavior should live behind a narrow service or helper with one clear owner. Stages must not reach across boundaries or reproduce adapter, manifest, artifact, or path policy ad hoc. @@ -141,6 +149,9 @@ not reconstruct canonical paths through scattered string concatenation. Artifact resolution is deterministic and manifest-aware. Producers materialize canonical outputs before reporting success, and consumers resolve declared artifact identities rather than infer files from unrelated directory contents. +External artifact bundles become current only through validated immutable +promotion and manifest records; directory presence alone never establishes +availability. Writes, moves, replacements, and deletions must use narrow, explicit, root-confined destinations. Symlinks, traversal, broad roots, and ambiguous diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index fe301e1..ea25462 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -2,7 +2,7 @@ ## Status And Audience -Proposed and not started. +Complete. This plan is written for a GPT-5.6 Terra coding agent. Implement the stages in strict numerical order. Do not skip ahead, combine stages merely to reduce the @@ -24,7 +24,7 @@ implementation sequence. | Stage 6 | Complete | | Stage 7 | Complete | | Stage 8 | Complete | -| Stage 9 | Not started | +| Stage 9 | Complete | After completing and validating a stage, update only that stage's row to `Complete` and record any material deviation in the relevant stage section. diff --git a/docs/roadmap/notarius-extract-stage.md b/docs/roadmap/notarius-extract-stage.md index 7a536af..fcd89d7 100644 --- a/docs/roadmap/notarius-extract-stage.md +++ b/docs/roadmap/notarius-extract-stage.md @@ -2,12 +2,12 @@ ## Status -Proposed. Implementation has not started. +Complete. -The ordered implementation plan is maintained in -[implementation.md](implementation.md). Until that plan is complete, this -document is the only Narratio documentation that describes the proposed -`extract` stage, Notarius configuration, or `narratio.extraction.*` sources. +The completed implementation sequence is retained in +[implementation.md](implementation.md) as decision and delivery history. +Current behavior is documented in the canonical configuration, CLI, +operations, integration, and internal references linked from this roadmap. ## Purpose diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 8c98caf..d58c089 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -117,6 +117,118 @@ Safe fix: Relevant reference: [CLI artifact selection](./cli.md). +## Notarius executable missing + +Symptom: + +- extraction fails while resolving or starting the Notarius executable. + +Likely causes: + +- `pipeline.notarius.binary` is not installed, executable, or on `PATH`; +- a configured executable path is wrong. + +Safe fix: + +- install a compatible Notarius release or correct the binary setting, then + rerun extraction. + +Relevant references: [Notarius configuration](./config.md#notarius-output-entries) +and [Notarius integration](./integrations/notarius.md). + +## Notarius exits nonzero + +Symptom: + +- extraction reports a Notarius exit error instead of a receipt. + +Diagnostics: + +- inspect `runs/{run_id}/extract/notarius.stderr.log`; stdout is reserved for + the receipt and is not merged with diagnostics. + +Safe fix: + +- correct the reported Notarius pipeline, input, provider, or configuration + failure and rerun extraction. Do not edit a staged output bundle into place. + +Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow). + +## Notarius receipt or index incompatible + +Symptom: + +- extraction rejects the receipt schema, pipeline identity, bundle/index path, + lane descriptor, or payload path even though Notarius exited successfully. + +Likely causes: + +- Narratio and Notarius versions disagree on their consumer contract; +- the configured pipeline or lane constraints are stale; +- output paths escape the bundle or traverse symlinks. + +Safe fix: + +- compare installed Notarius output with the canonical Notarius contracts, + align `pipeline.notarius` constraints, and rerun. Do not bypass confinement or + schema checks. + +Relevant reference: [Notarius integration](./integrations/notarius.md). + +## Required Notarius lane rejected or missing + +Symptom: + +- extraction fails because a configured lane is rejected, missing, duplicated, + or incompatible, including after a zero exit. + +Safe fix: + +- inspect the Notarius diagnostic log and bundle rejection/warning information; +- correct the Notarius module or the exact declared lane contract; +- remove an output declaration only if downstream consumers genuinely no longer + require that source, then rerun extraction. + +Every configured output is required. Narratio does not promote a partial result. + +## Extraction resume invalidated + +Symptom: + +- a previously successful extraction runs again during ordinary continuation. + +Likely causes: + +- the executable/config path, pipeline ID, timeout, working directory, or + configured output contracts changed; +- the durable bundle, index, lane set, provenance, regular-file status, or + checksum no longer validates. + +Safe fix: + +- allow the automatic rerun after verifying the current configuration. Treat + an unsafe path or symlink error as filesystem corruption or tampering and + investigate it rather than replacing files manually. + +## Notarius transitive configuration changed + +Symptom: + +- Notarius profiles, prompts, modules, imported files, or references changed, + but Narratio still considers the previous extraction resumable. + +Safe fix: + +```bash +narratio run-stage extract 2026-04-04 --force +``` + +Narratio fingerprints its invocation contract, not the contents of transitive +Notarius inputs. Always force extraction after changing them; downstream +successful stages are then marked stale normally. + +Relevant reference: [Operations: Extraction Workflow](./operations.md#extraction-workflow). + ## Previous-session artifact input missing Symptom: diff --git a/examples/README.md b/examples/README.md index ec2b52f..db05f54 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,6 +13,8 @@ in the [configuration reference](../docs/config.md). external tools, and configured Scriptorium artifacts. - [Full annotated pipeline](pipeline.full.annotated.yml): every implemented pipeline section with explanatory comments. +- [Extraction subset pipeline](pipeline.extraction-subset.yml): a focused + Scriptorium artifact consuming only three declared Notarius lanes. The existing `internal/config` example test loads and validates each pipeline with the sample campaign and a compatible local- or S3-audio session. diff --git a/examples/pipeline.extraction-subset.yml b/examples/pipeline.extraction-subset.yml new file mode 100644 index 0000000..b341b23 --- /dev/null +++ b/examples/pipeline.extraction-subset.yml @@ -0,0 +1,55 @@ +# Purpose-specific extraction example: a Scriptorium session brief consumes +# only the three Notarius lanes it needs. + +campaigns: + root: /usr/local/share/narratio/campaigns + default_campaign_id: sample-campaign + +whisperx: + transcribe_url: "https://transcription.example.com/transcribe" + +notarius: + enabled: true + binary: notarius + config_path: /usr/local/etc/notarius/config.yml + pipeline_id: dnd-session + timeout: 3h + outputs: + npc_registry: + lane_id: npc-registry + media_type: application/json + schema_id: notarius.dnd.npc_registry + schema_version: v1 + module_key: dnd/npc-registry + location_registry: + lane_id: location-registry + media_type: application/json + schema_id: notarius.dnd.location_registry + schema_version: v1 + module_key: dnd/location-registry + scene_descriptions: + lane_id: scene-descriptions + media_type: application/json + schema_id: notarius.dnd.scene_descriptions + schema_version: v1 + module_key: dnd/scene-descriptions + +scriptorium: + binary: scriptorium + config_path: /usr/local/etc/scriptorium/config.yml + artifacts: + session_brief: + enabled: true + prompt_id: dnd.session_brief + output_path: artifacts/session_brief.md + inputs: + npcs: + source: narratio.extraction.npc_registry + required: true + locations: + source: narratio.extraction.location_registry + required: true + scenes: + source: narratio.extraction.scene_descriptions + required: true + diff --git a/examples/pipeline.full.annotated.yml b/examples/pipeline.full.annotated.yml index bed9db4..618899c 100644 --- a/examples/pipeline.full.annotated.yml +++ b/examples/pipeline.full.annotated.yml @@ -60,6 +60,11 @@ publish: - source: narratio.artifact.player_handout dest: artifacts/player_handout.md required: false + # Extraction lanes publish only when named explicitly; the bundle and index + # are never implicit publish sources. + - source: narratio.extraction.npc_registry + dest: artifacts/extraction/npc-registry.json + required: true whisperx: # Required. @@ -123,6 +128,78 @@ trim: seriatim: report: false +notarius: + # Optional structured extraction between trim and render. + enabled: true + binary: notarius + config_path: /usr/local/etc/notarius/config.yml + pipeline_id: dnd-session + timeout: 3h + working_directory: /usr/local/etc/notarius + # Each key creates source narratio.extraction.. These constraints match + # the current Notarius D&D lane contracts; update them with Notarius. + outputs: + item_registry: + lane_id: item-registry + media_type: application/json + schema_id: notarius.dnd.item_registry + schema_version: v1 + module_key: dnd/item-registry + npc_registry: + lane_id: npc-registry + media_type: application/json + schema_id: notarius.dnd.npc_registry + schema_version: v1 + module_key: dnd/npc-registry + location_registry: + lane_id: location-registry + media_type: application/json + schema_id: notarius.dnd.location_registry + schema_version: v1 + module_key: dnd/location-registry + scene_descriptions: + lane_id: scene-descriptions + media_type: application/json + schema_id: notarius.dnd.scene_descriptions + schema_version: v1 + module_key: dnd/scene-descriptions + item_occurrences: + lane_id: item-occurrences + media_type: application/json + schema_id: notarius.dnd.item_occurrences + schema_version: v1 + module_key: dnd/item-occurrences + spells: + lane_id: spells + media_type: application/json + schema_id: notarius.dnd.spells + schema_version: v1 + module_key: dnd/spells + combat_turns: + lane_id: combat-turns + media_type: application/json + schema_id: notarius.dnd.combat_turns + schema_version: v1 + module_key: dnd/combat-turns + npc_occurrences: + lane_id: npc-occurrences + media_type: application/json + schema_id: notarius.dnd.npc_occurrences + schema_version: v1 + module_key: dnd/npc-occurrences + location_occurrences: + lane_id: location-occurrences + media_type: application/json + schema_id: notarius.dnd.location_occurrences + schema_version: v1 + module_key: dnd/location-occurrences + enemy_events: + lane_id: enemy-events + media_type: application/json + schema_id: notarius.dnd.enemy_events + schema_version: v1 + module_key: dnd/enemy-events + scriptorium: binary: scriptorium config_path: /usr/local/etc/scriptorium/config.yml diff --git a/internal/config/load_validate_test.go b/internal/config/load_validate_test.go index 91ca10e..0190783 100644 --- a/internal/config/load_validate_test.go +++ b/internal/config/load_validate_test.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "reflect" "strings" "testing" ) @@ -1049,6 +1050,11 @@ func TestExamplesLoadAndValidate(t *testing.T) { pipelineFile: "pipeline.full.annotated.yml", sessionFile: "session.local-audio.yml", }, + { + name: "extraction subset pipeline with local audio session", + pipelineFile: "pipeline.extraction-subset.yml", + sessionFile: "session.local-audio.yml", + }, } for _, tt := range tests { @@ -1068,6 +1074,47 @@ func TestExamplesLoadAndValidate(t *testing.T) { } } +func TestMaintainedExtractionExamplesPreservePublishedContracts(t *testing.T) { + examplesDir := filepath.Join("..", "..", "examples") + full, err := LoadPipeline(filepath.Join(examplesDir, "pipeline.full.annotated.yml")) + if err != nil { + t.Fatalf("load full example error = %v", err) + } + want := map[string]NotariusOutputConfig{ + "item_registry": {LaneID: "item-registry", MediaType: "application/json", SchemaID: "notarius.dnd.item_registry", SchemaVersion: "v1", ModuleKey: "dnd/item-registry"}, + "npc_registry": {LaneID: "npc-registry", MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry", SchemaVersion: "v1", ModuleKey: "dnd/npc-registry"}, + "location_registry": {LaneID: "location-registry", MediaType: "application/json", SchemaID: "notarius.dnd.location_registry", SchemaVersion: "v1", ModuleKey: "dnd/location-registry"}, + "scene_descriptions": {LaneID: "scene-descriptions", MediaType: "application/json", SchemaID: "notarius.dnd.scene_descriptions", SchemaVersion: "v1", ModuleKey: "dnd/scene-descriptions"}, + "item_occurrences": {LaneID: "item-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.item_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/item-occurrences"}, + "spells": {LaneID: "spells", MediaType: "application/json", SchemaID: "notarius.dnd.spells", SchemaVersion: "v1", ModuleKey: "dnd/spells"}, + "combat_turns": {LaneID: "combat-turns", MediaType: "application/json", SchemaID: "notarius.dnd.combat_turns", SchemaVersion: "v1", ModuleKey: "dnd/combat-turns"}, + "npc_occurrences": {LaneID: "npc-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.npc_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/npc-occurrences"}, + "location_occurrences": {LaneID: "location-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.location_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/location-occurrences"}, + "enemy_events": {LaneID: "enemy-events", MediaType: "application/json", SchemaID: "notarius.dnd.enemy_events", SchemaVersion: "v1", ModuleKey: "dnd/enemy-events"}, + } + if full.Notarius == nil || !reflect.DeepEqual(full.Notarius.Outputs, want) { + t.Fatalf("full example outputs = %#v, want %#v", full.Notarius, want) + } + + subset, err := LoadPipeline(filepath.Join(examplesDir, "pipeline.extraction-subset.yml")) + if err != nil { + t.Fatalf("load subset example error = %v", err) + } + brief := subset.Scriptorium.Artifacts["session_brief"] + wantSources := map[string]string{ + "npcs": "narratio.extraction.npc_registry", + "locations": "narratio.extraction.location_registry", + "scenes": "narratio.extraction.scene_descriptions", + } + gotSources := make(map[string]string, len(brief.Inputs)) + for name, input := range brief.Inputs { + gotSources[name] = input.Source + } + if !reflect.DeepEqual(gotSources, wantSources) { + t.Fatalf("subset example sources = %#v, want %#v", gotSources, wantSources) + } +} + func writeConfigFiles(t *testing.T, pipelineYAML, sessionYAML string) (string, string) { t.Helper() if !strings.Contains(pipelineYAML, "\naudita:") && !strings.HasPrefix(pipelineYAML, "audita:") {