Close the completed roadmap documents
This commit is contained in:
@@ -1,349 +0,0 @@
|
|||||||
# Notarius Extraction Implementation Plan
|
|
||||||
|
|
||||||
## Status And Audience
|
|
||||||
|
|
||||||
Remediation required. Stages 1 through 14 are complete. Stages 15 through 18
|
|
||||||
address the remaining security, lifecycle-state, and native-platform validation
|
|
||||||
gaps identified by the second post-implementation review.
|
|
||||||
|
|
||||||
This plan is written for a GPT-5.6 Terra coding agent. Implement incomplete
|
|
||||||
stages in strict numerical order. Do not skip ahead, combine stages merely to
|
|
||||||
reduce the number of prompts, or describe unfinished behavior as current
|
|
||||||
functionality.
|
|
||||||
|
|
||||||
The target state and acceptance criteria are owned by
|
|
||||||
[notarius-extract-stage.md](notarius-extract-stage.md). This document owns the
|
|
||||||
implementation sequence.
|
|
||||||
|
|
||||||
## Progress
|
|
||||||
|
|
||||||
| Stage | Status |
|
|
||||||
| --- | --- |
|
|
||||||
| Stage 1 | Complete |
|
|
||||||
| Stage 2 | Complete |
|
|
||||||
| Stage 3 | Complete |
|
|
||||||
| Stage 4 | Complete |
|
|
||||||
| Stage 5 | Complete |
|
|
||||||
| Stage 6 | Complete |
|
|
||||||
| Stage 7 | Complete |
|
|
||||||
| Stage 8 | Complete |
|
|
||||||
| Stage 9 | Complete |
|
|
||||||
| Stage 10 | Complete |
|
|
||||||
| Stage 11 | Complete |
|
|
||||||
| Stage 12 | Complete |
|
|
||||||
| Stage 13 | Complete |
|
|
||||||
| Stage 14 | Complete |
|
|
||||||
| Stage 15 | Complete |
|
|
||||||
| Stage 16 | Complete |
|
|
||||||
| Stage 17 | Pending |
|
|
||||||
| Stage 18 | Pending |
|
|
||||||
|
|
||||||
After completing and validating a stage, update only that stage's row to
|
|
||||||
`Complete` and record any material deviation in the relevant stage section.
|
|
||||||
Do not mark a stage complete while required tests or exit criteria remain.
|
|
||||||
|
|
||||||
## Working Rules
|
|
||||||
|
|
||||||
Before Stage 15, read:
|
|
||||||
|
|
||||||
- `docs/development.md` and its task-specific references;
|
|
||||||
- `docs/roadmap/notarius-extract-stage.md` completely; and
|
|
||||||
- the focused Narratio documents, production code, and tests named by the
|
|
||||||
current stage.
|
|
||||||
|
|
||||||
For every incomplete stage:
|
|
||||||
|
|
||||||
1. Inspect the current implementation before editing. Prefer the codebase
|
|
||||||
knowledge graph for code discovery as required by `AGENTS.md`.
|
|
||||||
2. Preserve unrelated worktree changes.
|
|
||||||
3. Implement the complete stage scope, including focused regression tests. Do
|
|
||||||
not leave TODO implementations or defer a known part of the stage.
|
|
||||||
4. Keep tests offline and independent of real Notarius, PromptKit, LLM
|
|
||||||
providers, credentials, network services, and mutable external state.
|
|
||||||
5. Run the focused package tests listed for the stage, then `go test ./...`.
|
|
||||||
6. Run `go vet ./...` and build `cmd/narratio` whenever shared contracts,
|
|
||||||
composition, portability, or documentation examples change. Direct build
|
|
||||||
output to a temporary directory so validation does not leave an ignored
|
|
||||||
binary in the repository root.
|
|
||||||
7. Do not broaden the feature beyond the target architecture while fixing a
|
|
||||||
remediation. Prefer a general manifest or filesystem invariant when the
|
|
||||||
defect is general, but avoid unrelated cleanup.
|
|
||||||
|
|
||||||
## Completed Stages 1-14 Summary
|
|
||||||
|
|
||||||
1. **Stage 1 — Shared contracts:** Added explicit stage self-skip semantics,
|
|
||||||
artifact identities, contract metadata, provenance, and compatible manifest
|
|
||||||
serialization.
|
|
||||||
2. **Stage 2 — Configuration and source policy:** Added strict optional
|
|
||||||
Notarius configuration, deterministic path resolution, lane contracts, and
|
|
||||||
the `narratio.extraction.<key>` source family.
|
|
||||||
3. **Stage 3 — Immutable promotion:** Added regular-file-only bundle copying,
|
|
||||||
static symlink and special-file rejection, syncing, cleanup, and atomic
|
|
||||||
no-replace installation on Linux.
|
|
||||||
4. **Stage 4 — Notarius adapter:** Added exact subprocess invocation, separate
|
|
||||||
receipt and diagnostic streams, bounded tolerant decoding, confinement, and
|
|
||||||
receipt/index discovery.
|
|
||||||
5. **Stage 5 — Extract execution:** Added transcript consumption, required-lane
|
|
||||||
validation, immutable promotion, checksums, outputs, provenance,
|
|
||||||
diagnostics, and invocation fingerprinting.
|
|
||||||
6. **Stage 6 — Lifecycle and resume:** Registered `extract` between `trim` and
|
|
||||||
`render`, added CLI selection, resume validation, force behavior, and
|
|
||||||
old-manifest compatibility.
|
|
||||||
7. **Stage 7 — Analyze integration:** Added shared manifest-backed extraction
|
|
||||||
catalog hydration and explicit required/optional Scriptorium input
|
|
||||||
resolution.
|
|
||||||
8. **Stage 8 — Publish and inspection integration:** Added explicit extraction
|
|
||||||
publish rules, metadata round-tripping, restore compatibility, and operator
|
|
||||||
availability reporting.
|
|
||||||
9. **Stage 9 — Documentation and examples:** Added maintained complete and
|
|
||||||
subset examples, integration and maintainer documentation, operational
|
|
||||||
guidance, and repository-wide validation.
|
|
||||||
10. **Stage 10 — Replacement invalidation:** Generalized runner semantics so
|
|
||||||
forced, failed, skipped, changed, and non-resumable upstream executions
|
|
||||||
invalidate succeeded downstream results without causing perpetual reruns
|
|
||||||
for identical self-skips.
|
|
||||||
11. **Stage 11 — Explicit-only publication:** Excluded the run-local Notarius
|
|
||||||
bundle from run-record uploads while retaining diagnostics and explicit
|
|
||||||
configured-lane publication.
|
|
||||||
12. **Stage 12 — Canonical management files:** Required the production receipt
|
|
||||||
and index to use the canonical index, manifest, rejection, and warning
|
|
||||||
filenames assumed by extraction, resume, and catalog code.
|
|
||||||
13. **Stage 13 — Platform promotion contract:** Added atomic no-replace
|
|
||||||
implementations for Linux, macOS, and Windows, explicit early failure on
|
|
||||||
unsupported systems, and platform-specific directory syncing.
|
|
||||||
14. **Stage 14 — Documentation reconciliation:** Aligned current-behavior,
|
|
||||||
operations, troubleshooting, integration, internal, and roadmap
|
|
||||||
documentation with the implemented lifecycle and publication semantics.
|
|
||||||
|
|
||||||
## Stage 15: Make Bundle Promotion Race-Safe Against Symlink Replacement
|
|
||||||
|
|
||||||
Close the gap between the documented no-symlink promotion invariant and the
|
|
||||||
current path-based recursive copy. A source directory that is replaced after
|
|
||||||
inspection must never cause Narratio to traverse a symlink or copy data from
|
|
||||||
outside the originally opened source tree.
|
|
||||||
|
|
||||||
Implementation:
|
|
||||||
|
|
||||||
1. Refactor `internal/fileops` source traversal to anchor the copy to an opened
|
|
||||||
source root for the entire operation. Use Go's `os.Root` APIs, available at
|
|
||||||
the module's declared Go version, instead of recursively reopening source
|
|
||||||
directories through unconstrained absolute paths.
|
|
||||||
2. Reject a source root that is itself a symlink. After opening the root,
|
|
||||||
compare the opened root identity with the identity inspected before opening;
|
|
||||||
fail if the source changed during that transition.
|
|
||||||
3. For every source entry:
|
|
||||||
- inspect it relative to the already opened parent root;
|
|
||||||
- reject symlinks, link-like Windows reparse points exposed as links, and
|
|
||||||
every non-regular, non-directory file type;
|
|
||||||
- open regular files relative to that root, compare the opened file identity
|
|
||||||
with the inspected identity, and copy only from the verified handle; and
|
|
||||||
- open child directories relative to the parent root, verify the opened
|
|
||||||
directory identity against the inspected identity, and recurse through the
|
|
||||||
opened child root rather than its pathname.
|
|
||||||
4. Treat any identity mismatch, disappearing entry, replacement, or unsupported
|
|
||||||
file type as a clean promotion failure. Preserve the source, remove the
|
|
||||||
temporary destination tree, and never install a partial destination.
|
|
||||||
5. Keep the destination-side behavior unchanged: deterministic traversal,
|
|
||||||
normalized permissions, file and directory syncing, sibling temporary tree,
|
|
||||||
and atomic no-replace installation.
|
|
||||||
6. Keep support bounded to Linux, macOS, and Windows. Do not weaken the strict
|
|
||||||
no-symlink rule to permit links that happen to remain inside the source root.
|
|
||||||
7. Keep test-only orchestration narrow. If deterministic replacement testing
|
|
||||||
requires an internal seam between inspection and opening, inject only that
|
|
||||||
boundary; do not expose it publicly or turn file copying into a generic
|
|
||||||
filesystem framework.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- retain the existing nested-copy, source-preservation, static symlink,
|
|
||||||
special-file, cleanup, existing-destination, and install-collision tests;
|
|
||||||
- deterministically replace an inspected child directory with a symlink before
|
|
||||||
it is opened, and prove promotion fails without copying an outside sentinel;
|
|
||||||
- cover source-root replacement between initial inspection and root opening;
|
|
||||||
- cover regular-file and directory identity mismatches without timing-based
|
|
||||||
sleeps or probabilistic race loops;
|
|
||||||
- prove every failure leaves the source and any concurrent destination intact
|
|
||||||
and removes the temporary sibling; and
|
|
||||||
- run the focused tests with the race detector.
|
|
||||||
|
|
||||||
Exit criteria:
|
|
||||||
|
|
||||||
- Recursive promotion cannot escape or switch away from the originally opened
|
|
||||||
source tree through a symlink replacement race.
|
|
||||||
- Static and concurrent symlink replacement are both protected by durable,
|
|
||||||
deterministic tests.
|
|
||||||
- `go test -count=1 ./internal/fileops`,
|
|
||||||
`go test -race -count=1 ./internal/fileops`, `go test ./...`, and
|
|
||||||
`go vet ./...` pass on the development platform.
|
|
||||||
|
|
||||||
## Stage 16: Clear Superseded Session-Stage Result Payloads
|
|
||||||
|
|
||||||
Make the session manifest describe only the current stage outcome. Historical
|
|
||||||
successful result details belong to their immutable run manifests and must not
|
|
||||||
remain attached to a current stage record that is running, failed, skipped, or
|
|
||||||
newly succeeded without corresponding details.
|
|
||||||
|
|
||||||
Implementation:
|
|
||||||
|
|
||||||
1. Centralize clearing of result-bearing `manifest.StageRecord` fields:
|
|
||||||
`Outputs`, `Logs`, `GeneratedConfigs`, and `Metadata`.
|
|
||||||
2. Clear those fields when `MarkStageRunning` begins a replacement attempt.
|
|
||||||
The runner already captures the prior outcome before this transition; do not
|
|
||||||
change self-skip comparison or downstream invalidation decisions.
|
|
||||||
3. Also make `MarkStageFailed` and `MarkStageSkipped` enforce the empty-result
|
|
||||||
invariant themselves so callers cannot construct a failed or skipped record
|
|
||||||
with inherited successful output details. Avoid runner-only cleanup that
|
|
||||||
leaves direct manifest transitions inconsistent.
|
|
||||||
4. Do not clear result details merely when `MarkStageStale` is called. Resume
|
|
||||||
validation must still be able to inspect the prior result before deciding to
|
|
||||||
rerun it, and stale records remain useful diagnostic state until execution
|
|
||||||
actually begins.
|
|
||||||
5. Ensure `MarkStageSucceeded` followed by result application cannot inherit
|
|
||||||
old logs, generated configurations, or metadata when the new successful
|
|
||||||
result omits those fields. Clearing at the running transition should own this
|
|
||||||
invariant; do not add scattered empty-map special cases.
|
|
||||||
6. Preserve immutable prior run manifests and durable Notarius bundles. This
|
|
||||||
stage changes the current session-stage record, not historical files or
|
|
||||||
retention policy.
|
|
||||||
7. Consolidate the existing runner self-skip cleanup helper if it becomes
|
|
||||||
redundant after manifest transitions own the invariant.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- add manifest transition tests proving running, failed, and skipped records
|
|
||||||
cannot retain prior outputs, logs, generated configurations, or metadata;
|
|
||||||
- prove `MarkStageStale` retains the prior details required for resume
|
|
||||||
validation;
|
|
||||||
- change the forced-extraction-failure lifecycle fixture to first complete a
|
|
||||||
successful extraction, then force a failing replacement, and assert the
|
|
||||||
current extract record is failed with no inherited result payload;
|
|
||||||
- retain the assertion that succeeded downstream stages become stale;
|
|
||||||
- prove a successful replacement whose new result omits optional details does
|
|
||||||
not inherit details from its predecessor; and
|
|
||||||
- retain resume-error, repeated-self-skip, and ordinary-retry coverage.
|
|
||||||
|
|
||||||
Exit criteria:
|
|
||||||
|
|
||||||
- A current running, failed, or skipped session-stage record never advertises
|
|
||||||
result payload belonging to an earlier success.
|
|
||||||
- Historical run manifests and immutable extraction bundles remain available
|
|
||||||
for audit and recovery.
|
|
||||||
- `go test -count=1 ./internal/manifest ./internal/app ./internal/stage`,
|
|
||||||
focused lifecycle race tests, `go test ./...`, and `go vet ./...` pass.
|
|
||||||
|
|
||||||
## Stage 17: Establish Native Promotion Validation On Every Supported Platform
|
|
||||||
|
|
||||||
Turn the Linux, macOS, and Windows promotion claim into a continuously verified
|
|
||||||
contract. Cross-compilation remains useful but cannot substitute for executing
|
|
||||||
filesystem operations on each native platform.
|
|
||||||
|
|
||||||
Implementation:
|
|
||||||
|
|
||||||
1. Resolve the CI substrate and native runner labels in the open question below
|
|
||||||
before editing workflow files. Follow the repository host's established
|
|
||||||
workflow location and syntax; do not invent runner labels that cannot run.
|
|
||||||
2. Add checked-in CI jobs that execute on native Linux, macOS, and Windows
|
|
||||||
runners. Each native job must run at least:
|
|
||||||
- `go test -count=1 ./internal/fileops`;
|
|
||||||
- the platform-specific no-replace test; and
|
|
||||||
- the portable promotion conformance tests, including successful promotion,
|
|
||||||
collision preservation, cleanup, and the Stage 15 race-safe traversal
|
|
||||||
cases that apply on that platform.
|
|
||||||
3. Ensure the jobs exercise the real platform implementation rather than a
|
|
||||||
mocked rename function. A destination created before installation must win,
|
|
||||||
and the source and destination contents must prove that no replacement
|
|
||||||
occurred.
|
|
||||||
4. On Windows, ensure successful promotion exercises both `MoveFileEx` and the
|
|
||||||
directory-sync path. Do not treat permission or ordinary I/O failures as
|
|
||||||
unsupported-operation success.
|
|
||||||
5. Retain deterministic cross-compilation of the fileops test binary for Linux,
|
|
||||||
Darwin, and Windows. Write compiled test binaries to a temporary or CI
|
|
||||||
artifact directory, never the repository root.
|
|
||||||
6. Keep unsupported-platform compilation coverage, but do not misrepresent an
|
|
||||||
unexecuted cross-compiled test as native validation.
|
|
||||||
7. Record any required runner-specific limitation in this stage and in the
|
|
||||||
canonical contributor/CI documentation. Do not weaken production semantics
|
|
||||||
merely to accommodate an inadequately provisioned runner.
|
|
||||||
|
|
||||||
Validation:
|
|
||||||
|
|
||||||
- obtain a successful native Linux job;
|
|
||||||
- obtain a successful native macOS job;
|
|
||||||
- obtain a successful native Windows job;
|
|
||||||
- retain successful Linux, Darwin, and Windows cross-compilation; and
|
|
||||||
- confirm CI does not leave generated test binaries or application binaries in
|
|
||||||
the worktree or commit them as artifacts of the source tree.
|
|
||||||
|
|
||||||
Exit criteria:
|
|
||||||
|
|
||||||
- Atomic no-replace promotion and its directory-sync behavior execute
|
|
||||||
successfully on native Linux, macOS, and Windows.
|
|
||||||
- The checked-in CI configuration will rerun those tests on future changes.
|
|
||||||
- Stage 17 is not marked complete based solely on Linux execution and
|
|
||||||
cross-compilation.
|
|
||||||
|
|
||||||
## Stage 18: Reconcile Security Documentation And Perform Final Validation
|
|
||||||
|
|
||||||
Close the remediation only after the corrected filesystem and manifest
|
|
||||||
invariants are implemented and native platform evidence exists.
|
|
||||||
|
|
||||||
Implementation:
|
|
||||||
|
|
||||||
1. Reconcile `docs/roadmap/notarius-extract-stage.md` with the Stage 15 source
|
|
||||||
traversal mechanism and Stage 16 current-result semantics. Keep normative
|
|
||||||
security and manifest guarantees in their existing canonical owners.
|
|
||||||
2. Update focused internal or contributor documentation only where the new
|
|
||||||
mechanism or CI workflow changes maintained implementation guidance. Do not
|
|
||||||
duplicate volatile platform commands across multiple documents.
|
|
||||||
3. Recheck the extraction, workspace, manifest, operations, troubleshooting,
|
|
||||||
integration, and testing documentation for contradictions introduced by the
|
|
||||||
remediation.
|
|
||||||
4. Keep completed roadmap history concise. Mark the target feature roadmap and
|
|
||||||
this implementation plan complete only after all Stage 15 through 17 exit
|
|
||||||
criteria are satisfied.
|
|
||||||
5. Confirm that no secrets, private campaign material, generated binaries,
|
|
||||||
platform test binaries, or temporary bundle trees were added to the
|
|
||||||
repository.
|
|
||||||
|
|
||||||
Validation:
|
|
||||||
|
|
||||||
- run focused fileops, manifest, runner lifecycle, extraction, adapter,
|
|
||||||
artifact-catalog, analyze, publish, restore, and operator tests;
|
|
||||||
- run focused race tests covering fileops, manifest transitions, runner
|
|
||||||
lifecycle, extraction, artifacts, and publication;
|
|
||||||
- run `go test -count=1 ./...`;
|
|
||||||
- run `go vet ./...`;
|
|
||||||
- build `cmd/narratio` with `-o` targeting a temporary directory;
|
|
||||||
- rerun Linux, Darwin, and Windows fileops test cross-compilation;
|
|
||||||
- confirm the native CI jobs from Stage 17 are successful;
|
|
||||||
- run `git diff --check`; and
|
|
||||||
- verify the worktree contains no generated validation artifacts.
|
|
||||||
|
|
||||||
Exit criteria:
|
|
||||||
|
|
||||||
- The source traversal and current-manifest replacement gaps are closed by
|
|
||||||
focused regression tests.
|
|
||||||
- Native filesystem behavior substantiates the documented Linux, macOS, and
|
|
||||||
Windows support boundary.
|
|
||||||
- Current documentation matches the remediated implementation.
|
|
||||||
- Every progress row is `Complete`, the repository-wide validation suite
|
|
||||||
passes, and the target feature roadmap can truthfully remain complete.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
### Which CI system and native runner labels are authoritative for this repository?
|
|
||||||
|
|
||||||
**Recommended approach:** Use Gitea Actions with a checked-in workflow under
|
|
||||||
`.gitea/workflows/`, provided the repository owner supplies or confirms native
|
|
||||||
Linux, macOS, and Windows runner labels. The repository is hosted on Gitea, so
|
|
||||||
keeping the workflow with the authoritative repository minimizes mirrored
|
|
||||||
configuration and makes the platform contract visible beside the code. Stage
|
|
||||||
17 must remain pending until all three native jobs have actually run.
|
|
||||||
|
|
||||||
**Viable alternative:** Use an existing external CI service with real native
|
|
||||||
capacity, such as a maintained GitHub Actions mirror or Buildkite installation,
|
|
||||||
and check its workflow or pipeline definition into the conventional repository
|
|
||||||
location. This is appropriate when that service already owns release gating or
|
|
||||||
when native macOS and Windows runners are unavailable in Gitea. The alternative
|
|
||||||
must still produce repeatable native results for all three platforms;
|
|
||||||
cross-compilation or an undocumented one-time manual run is not sufficient.
|
|
||||||
@@ -1,645 +0,0 @@
|
|||||||
# Notarius Extraction Stage
|
|
||||||
|
|
||||||
## Status
|
|
||||||
|
|
||||||
Pending native promotion validation.
|
|
||||||
|
|
||||||
The implementation sequence is retained in
|
|
||||||
[implementation.md](implementation.md) as decision and delivery history.
|
|
||||||
Current behavior is documented in the canonical configuration, CLI,
|
|
||||||
operations, integration, and internal references linked from this roadmap.
|
|
||||||
The extraction feature is implemented, but completion remains pending until
|
|
||||||
atomic promotion has run successfully in native Linux, macOS, and Windows CI.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Add a first-class Narratio `extract` stage that runs Notarius against the
|
|
||||||
session's final trimmed transcript and makes validated structured artifacts
|
|
||||||
available to later analysis and publish work.
|
|
||||||
|
|
||||||
Notarius remains responsible for its D&D extraction pipeline, prompts,
|
|
||||||
references, LLM profiles, retries, validation, normalization, and published
|
|
||||||
schemas. Narratio owns invocation, required-output policy, safe bundle
|
|
||||||
ingestion, artifact identity, manifest state, resume, and downstream
|
|
||||||
availability.
|
|
||||||
|
|
||||||
The integration must preserve Narratio's explicit stage model. It must not
|
|
||||||
turn Narratio into a generic workflow engine or reproduce Notarius's
|
|
||||||
configuration language.
|
|
||||||
|
|
||||||
## User Outcome
|
|
||||||
|
|
||||||
An operator can enable one Notarius pipeline for a Narratio campaign. During a
|
|
||||||
normal run, Narratio will:
|
|
||||||
|
|
||||||
1. produce the final trimmed Seriatim JSON transcript;
|
|
||||||
2. invoke Notarius once with that transcript;
|
|
||||||
3. discover the exact run bundle through Notarius's machine-readable receipt;
|
|
||||||
4. validate every output contract Narratio is configured to require;
|
|
||||||
5. promote the complete validated bundle into immutable session storage;
|
|
||||||
6. record exact artifact paths, checksums, contracts, and external provenance;
|
|
||||||
7. make configured lanes available as `narratio.extraction.<key>` sources; and
|
|
||||||
8. allow individual Scriptorium artifacts and publish rules to select those
|
|
||||||
sources explicitly.
|
|
||||||
|
|
||||||
The maintained complete D&D example will require all ten lanes published by
|
|
||||||
Notarius's `dnd-session` pipeline. Ordinary deployments may configure a
|
|
||||||
narrower required set.
|
|
||||||
|
|
||||||
## Chosen Architecture
|
|
||||||
|
|
||||||
### First-Class Stage
|
|
||||||
|
|
||||||
Extraction is an independently observable and resumable pipeline stage. It is
|
|
||||||
not part of `analyze`, a Scriptorium artifact producer, or an implicit external
|
|
||||||
preprocessing requirement.
|
|
||||||
|
|
||||||
This boundary is required because Notarius work is expensive, produces
|
|
||||||
multiple durable outputs, has its own compatibility and diagnostic contracts,
|
|
||||||
and may be consumed by both `analyze` and `publish`.
|
|
||||||
|
|
||||||
### Canonical Order
|
|
||||||
|
|
||||||
The canonical stage order becomes:
|
|
||||||
|
|
||||||
```text
|
|
||||||
prepare -> transcribe -> merge -> polish -> normalize -> trim
|
|
||||||
-> extract -> render -> analyze -> publish -> notify
|
|
||||||
```
|
|
||||||
|
|
||||||
`extract` consumes `narratio.transcript.final_trimmed`, produced by `trim`. It
|
|
||||||
does not consume rendered Markdown.
|
|
||||||
|
|
||||||
Narratio invalidates succeeded stages by canonical downstream order whenever a
|
|
||||||
forced run or a different executed upstream outcome replaces current state.
|
|
||||||
Placing `extract` before `render` means replacing extraction may rerun the less
|
|
||||||
expensive deterministic render stage, while replacing render does not rerun the
|
|
||||||
more expensive Notarius pipeline. This is preferable to placing extraction
|
|
||||||
after render and does not require dependency-aware scheduling or a DAG.
|
|
||||||
|
|
||||||
Adding the stage must update every canonical-stage inventory, full-plan and
|
|
||||||
single-stage selection, prerequisite validation, downstream invalidation,
|
|
||||||
resume behavior, run manifests, CLI validation and help, and focused tests.
|
|
||||||
|
|
||||||
### Stage Boundary
|
|
||||||
|
|
||||||
The stage owns Narratio policy:
|
|
||||||
|
|
||||||
- resolve the final trimmed transcript through the manifest-aware artifact
|
|
||||||
resolver;
|
|
||||||
- allocate run-local receipt, log, and output-root paths;
|
|
||||||
- build a transport-neutral Notarius request from resolved configuration;
|
|
||||||
- call the configured Notarius adapter once;
|
|
||||||
- enforce Narratio's configured required-output policy;
|
|
||||||
- validate selected payloads as regular, non-empty, syntactically valid JSON;
|
|
||||||
- promote the validated bundle to immutable session artifact storage;
|
|
||||||
- return explicit source IDs, checksums, contracts, and external provenance;
|
|
||||||
and
|
|
||||||
- report an explicit skipped outcome when Notarius is disabled.
|
|
||||||
|
|
||||||
The stage must not construct subprocess arguments, guess Notarius filenames,
|
|
||||||
parse interactive output, decode D&D payload structures, or reproduce
|
|
||||||
Notarius pipeline configuration.
|
|
||||||
|
|
||||||
### Adapter Boundary
|
|
||||||
|
|
||||||
Add `internal/adapters/notarius` with a narrow runner interface, production
|
|
||||||
subprocess implementation, and small fake.
|
|
||||||
|
|
||||||
The request contains only:
|
|
||||||
|
|
||||||
- resolved executable path or name;
|
|
||||||
- absolute Notarius configuration path;
|
|
||||||
- pipeline ID;
|
|
||||||
- absolute transcript path;
|
|
||||||
- absolute output root;
|
|
||||||
- working directory;
|
|
||||||
- timeout; and
|
|
||||||
- stdout receipt and stderr log destinations.
|
|
||||||
|
|
||||||
The adapter owns:
|
|
||||||
|
|
||||||
- exact `notarius run ... --json` argument construction;
|
|
||||||
- stdout and stderr separation;
|
|
||||||
- context cancellation and timeout through Narratio's shared subprocess
|
|
||||||
boundary;
|
|
||||||
- environment inheritance;
|
|
||||||
- exit-status handling;
|
|
||||||
- bounded receipt loading after exit status zero;
|
|
||||||
- tolerant decoding of supported `notarius.run-result.v1` documents;
|
|
||||||
- validation of required receipt fields;
|
|
||||||
- confinement of the receipt's absolute `output_directory` beneath the
|
|
||||||
absolute output root Narratio supplied for this invocation;
|
|
||||||
- requiring receipt `index_file` to be exactly `index.json` beneath the
|
|
||||||
receipt's absolute `output_directory`;
|
|
||||||
- tolerant decoding of the supported `index.json` contract;
|
|
||||||
- confinement of every index descriptor path beneath the bundle root; and
|
|
||||||
- bounded tolerant decoding of `rejected.json` and `warnings.json` into
|
|
||||||
transport-neutral summaries without reading lane payload bodies; summaries
|
|
||||||
persisted by Narratio contain structured scope, lane, and reason fields but
|
|
||||||
not unbounded free-form external messages; and
|
|
||||||
- returning transport-neutral receipt, descriptor, diagnostic, and bundle
|
|
||||||
information.
|
|
||||||
|
|
||||||
Only exit status zero permits receipt decoding. Unsupported schema versions,
|
|
||||||
malformed documents, missing required fields, absolute logical paths, path
|
|
||||||
escapes, symlinks at consumed paths, and incompatible structural metadata are
|
|
||||||
integration failures.
|
|
||||||
|
|
||||||
The production runner will not execute `notarius config validate` before every
|
|
||||||
session. Notarius run-time validation remains authoritative, while operators
|
|
||||||
may use the separate validation command as deployment preflight. A second
|
|
||||||
automatic subprocess can be added later only if operational evidence warrants
|
|
||||||
it.
|
|
||||||
|
|
||||||
The adapter does not write Narratio manifests, decide required lanes, choose
|
|
||||||
analysis inputs, or interpret D&D payloads.
|
|
||||||
|
|
||||||
## External Contract Baseline
|
|
||||||
|
|
||||||
The integration consumes the contracts documented by Notarius in
|
|
||||||
`../notarius/docs/consumers/dnd-pipeline.md` and its linked canonical
|
|
||||||
integration documents.
|
|
||||||
|
|
||||||
The initial compatibility baseline is:
|
|
||||||
|
|
||||||
- `notarius run <pipeline-id> --config ... --input ... --output-dir ... --json`;
|
|
||||||
- successful receipt schema `notarius.run-result.v1`;
|
|
||||||
- an absolute receipt `output_directory`;
|
|
||||||
- receipt `index_file` exactly `index.json` beneath that directory;
|
|
||||||
- the production JSON `index.json` descriptor model with management files
|
|
||||||
exactly `manifest.json`, `rejected.json`, and `warnings.json`; and
|
|
||||||
- the exact media type and schema identity configured for each required lane.
|
|
||||||
|
|
||||||
Compatibility is decided from these published contracts, not by parsing
|
|
||||||
`notarius --version`. Unknown fields in supported receipt and index versions
|
|
||||||
are tolerated. Unsupported versions or incompatible required descriptors fail
|
|
||||||
before any artifact becomes current in Narratio.
|
|
||||||
|
|
||||||
## Configuration Contract
|
|
||||||
|
|
||||||
Add a strict optional `pipeline.notarius` section:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
notarius:
|
|
||||||
enabled: true
|
|
||||||
binary: notarius
|
|
||||||
config_path: /absolute/path/to/notarius.yml
|
|
||||||
pipeline_id: dnd-session
|
|
||||||
timeout: 3h
|
|
||||||
working_directory: /absolute/path/to/deployment
|
|
||||||
outputs:
|
|
||||||
npc_registry:
|
|
||||||
lane_id: npc-registry
|
|
||||||
media_type: application/json
|
|
||||||
schema_id: <published-schema-id>
|
|
||||||
schema_version: <published-schema-version>
|
|
||||||
```
|
|
||||||
|
|
||||||
Fields:
|
|
||||||
|
|
||||||
- `enabled` is an explicit opt-in and defaults to false;
|
|
||||||
- `binary` defaults to `notarius`;
|
|
||||||
- `config_path` is required when enabled;
|
|
||||||
- `pipeline_id` is required when enabled;
|
|
||||||
- `timeout` defaults to `3h` and must be positive;
|
|
||||||
- `working_directory` is optional and defaults to the directory containing
|
|
||||||
`config_path`; and
|
|
||||||
- `outputs` maps stable Narratio extraction keys to required Notarius lane
|
|
||||||
contracts.
|
|
||||||
|
|
||||||
All configured paths become absolute during configuration resolution. Notarius
|
|
||||||
reference paths continue to follow Notarius's configuration-relative rules,
|
|
||||||
while PromptKit profile paths remain relative to the chosen process working
|
|
||||||
directory where Notarius permits that behavior.
|
|
||||||
|
|
||||||
Each output entry contains:
|
|
||||||
|
|
||||||
- exact `lane_id`;
|
|
||||||
- exact `media_type`;
|
|
||||||
- exact `schema_id`;
|
|
||||||
- exact `schema_version`; and
|
|
||||||
- optional `module_key`.
|
|
||||||
|
|
||||||
The map key produces `narratio.extraction.<key>`. Keys use Narratio's existing
|
|
||||||
path-safe configured-artifact key grammar. Keys, normalized source IDs, and
|
|
||||||
lane IDs must be non-empty and unique. Extraction source IDs must not collide
|
|
||||||
with built-ins or configured Scriptorium sources.
|
|
||||||
|
|
||||||
Every configured output is required for extraction-stage success. Operators
|
|
||||||
who need only a subset configure only that subset. Narratio does not add a
|
|
||||||
second `required_lanes` list or a runtime lane-selection flag.
|
|
||||||
|
|
||||||
Narratio does not configure Notarius lane topology, references, prompts, LLM
|
|
||||||
profiles, model settings, concurrency, retry behavior, or session IDs. One
|
|
||||||
stage execution runs the configured Notarius pipeline as a unit. Narratio does
|
|
||||||
not pass Notarius's `--session-id` override.
|
|
||||||
|
|
||||||
## Stage Outcomes And Resume
|
|
||||||
|
|
||||||
### Explicit Self-Skip
|
|
||||||
|
|
||||||
Extend the stage-result contract with an explicit disposition whose zero value
|
|
||||||
remains successful for backward compatibility. A stage may return:
|
|
||||||
|
|
||||||
- succeeded; or
|
|
||||||
- skipped with a stable reason.
|
|
||||||
|
|
||||||
When Notarius is absent or disabled, `extract` returns skipped with reason
|
|
||||||
`notarius_disabled`, no outputs, and bounded metadata. The runner records a
|
|
||||||
skipped session-stage and run-stage outcome rather than a successful empty
|
|
||||||
stage. Skipped stages are reconsidered on later invocations, so subsequently
|
|
||||||
enabling Notarius causes extraction to run without requiring force.
|
|
||||||
|
|
||||||
A genuine self-skip clears any older outputs, logs, generated configuration
|
|
||||||
references, and metadata before persisting the new skipped state and its own
|
|
||||||
bounded metadata. Downstream consumers cannot resolve artifacts retained from
|
|
||||||
an earlier extraction after the stage is disabled.
|
|
||||||
|
|
||||||
### Resume Validation
|
|
||||||
|
|
||||||
Add a small optional resume-validation interface implemented by `extract`.
|
|
||||||
Before skipping an already-succeeded extract stage, the runner asks it whether
|
|
||||||
the recorded result remains resumable.
|
|
||||||
|
|
||||||
The resume validator confirms:
|
|
||||||
|
|
||||||
- Notarius is still enabled;
|
|
||||||
- recorded configuration identity matches the current config path, pipeline
|
|
||||||
ID, and output-contract map;
|
|
||||||
- the immutable bundle and canonical index still exist;
|
|
||||||
- every configured source is present in the succeeded extract record;
|
|
||||||
- lane files remain confined regular files;
|
|
||||||
- stored checksums still match; and
|
|
||||||
- stored descriptor contracts and external provenance remain compatible.
|
|
||||||
|
|
||||||
An ordinary contract mismatch returns a non-resumable decision with a bounded
|
|
||||||
reason. The runner marks the stage stale and executes it. An environmental
|
|
||||||
error that prevents making a safe decision returns an error and stops the run.
|
|
||||||
|
|
||||||
Narratio does not recursively interpret Notarius configuration, PromptKit
|
|
||||||
profiles, or reference files. Changes to those external inputs are therefore
|
|
||||||
not automatically detectable. Operator documentation must require
|
|
||||||
`--force extract` after changing them.
|
|
||||||
|
|
||||||
The configuration fingerprint covers the resolved binary, config path,
|
|
||||||
pipeline ID, timeout, working directory, and the deterministically sorted
|
|
||||||
output-contract map. It identifies Narratio's invocation contract, not the
|
|
||||||
transitive content of files owned by Notarius.
|
|
||||||
|
|
||||||
### Force And Invalidation
|
|
||||||
|
|
||||||
- Forcing `trim` or an earlier stage stales succeeded `extract` and all later
|
|
||||||
stages.
|
|
||||||
- Forcing `extract` stales succeeded `render`, `analyze`, `publish`, and
|
|
||||||
`notify` under the canonical-order rule, even if extraction later skips or
|
|
||||||
fails.
|
|
||||||
- Forcing `render` does not stale `extract` because extraction precedes it.
|
|
||||||
- A non-forced extraction failure, changed self-skip, or success replacing a
|
|
||||||
different effective outcome stales succeeded downstream stages. Repeating
|
|
||||||
the same `notarius_disabled` self-skip with no outputs does not stale them
|
|
||||||
again.
|
|
||||||
- Reusing a resumable succeeded result does not invalidate downstream stages;
|
|
||||||
a resume-validation error stops without mutating either result.
|
|
||||||
- A failed, skipped, stale, or interrupted extract stage never supplies current
|
|
||||||
extraction sources.
|
|
||||||
|
|
||||||
When a replacement attempt starts, the current session-stage record drops the
|
|
||||||
prior result payload before it is persisted as running. Failed and skipped
|
|
||||||
transitions enforce the same clearing invariant directly, and a later success
|
|
||||||
contains only details produced by that attempt. Merely marking a record stale
|
|
||||||
retains its prior details for resume validation and diagnosis until execution
|
|
||||||
actually begins. Immutable invocation run manifests and previously promoted
|
|
||||||
bundles preserve the historical successful result.
|
|
||||||
|
|
||||||
## Bundle Storage And Commit
|
|
||||||
|
|
||||||
### Run-Local Execution
|
|
||||||
|
|
||||||
Notarius runs against a run-local output root beneath the Narratio run's
|
|
||||||
`extract` directory. Receipt bytes and stderr remain run-local diagnostics.
|
|
||||||
Failed and malformed bundles remain outside durable artifact storage for
|
|
||||||
inspection and never become current merely because files exist.
|
|
||||||
|
|
||||||
### Immutable Durable Bundles
|
|
||||||
|
|
||||||
After complete validation, promote the exact Notarius bundle into:
|
|
||||||
|
|
||||||
```text
|
|
||||||
artifacts/notarius/<narratio-run-id>/
|
|
||||||
```
|
|
||||||
|
|
||||||
The destination is unique and must not already exist. Promotion uses a sibling
|
|
||||||
temporary directory on the same filesystem, traverses the source through
|
|
||||||
confined directory handles, recursively copies only regular files and
|
|
||||||
directories, rejects symlinks and special files, and preserves relative
|
|
||||||
layout. It verifies that each opened root, directory, and file is the same
|
|
||||||
object that was inspected, rejecting path replacement during traversal. The
|
|
||||||
completed temporary tree is then renamed into place without replacing a
|
|
||||||
destination created by another writer.
|
|
||||||
|
|
||||||
Atomic no-replace promotion is implemented on Linux, macOS, and Windows. On
|
|
||||||
other operating systems extraction reports an unsupported-capability error
|
|
||||||
before creating the sibling temporary tree. This limitation is confined to
|
|
||||||
extraction bundle promotion and does not define a broader platform guarantee.
|
|
||||||
|
|
||||||
The promoted tree preserves `index.json`, `manifest.json`, `rejected.json`,
|
|
||||||
`warnings.json`, `lanes/`, and any emitted pipeline-wide artifacts such as
|
|
||||||
`chunk-map.json` and `evidence-context.json`. Unknown regular files may be
|
|
||||||
preserved because the complete bundle is provenance, but no unknown file is
|
|
||||||
registered as a stable Narratio source.
|
|
||||||
|
|
||||||
There is no mutable filesystem `current` directory or symlink. The atomically
|
|
||||||
saved Narratio session manifest selects the current successful bundle. Older
|
|
||||||
successful bundles remain immutable until explicit cleanup policy removes
|
|
||||||
them.
|
|
||||||
|
|
||||||
The raw Notarius receipt is retained without rewriting its original
|
|
||||||
`output_directory`. Narratio's manifest records promoted durable paths.
|
|
||||||
|
|
||||||
## Artifact And Manifest Model
|
|
||||||
|
|
||||||
### Explicit Source Identity
|
|
||||||
|
|
||||||
Extend `artifacts.Ref` with an optional explicit `SourceID`. The application
|
|
||||||
runner prefers that value and retains existing stage-specific inference only
|
|
||||||
as a backward-compatible fallback. `extract` must not require another
|
|
||||||
stage-name special case in output mapping.
|
|
||||||
|
|
||||||
Add neutral optional artifact metadata models under `internal/artifactmodel`:
|
|
||||||
|
|
||||||
- contract metadata: media type, schema ID, schema version, and optional
|
|
||||||
module key; and
|
|
||||||
- external provenance: system, external run ID, pipeline ID, and external
|
|
||||||
artifact ID.
|
|
||||||
|
|
||||||
Both `artifacts.Ref` and `manifest.ArtifactRecord` carry these nested models.
|
|
||||||
Existing manifests remain readable because the fields are optional and use
|
|
||||||
`omitempty` encoding.
|
|
||||||
|
|
||||||
For a Notarius lane, external provenance uses:
|
|
||||||
|
|
||||||
- system `notarius`;
|
|
||||||
- receipt run ID;
|
|
||||||
- receipt pipeline ID; and
|
|
||||||
- lane ID as the external artifact ID.
|
|
||||||
|
|
||||||
### Registered Sources
|
|
||||||
|
|
||||||
For each configured output, `extract` records one artifact with:
|
|
||||||
|
|
||||||
- source ID `narratio.extraction.<key>`;
|
|
||||||
- durable lane path discovered through the promoted canonical index;
|
|
||||||
- producer stage and Narratio run ID;
|
|
||||||
- SHA-256 checksum;
|
|
||||||
- configured and observed contract metadata; and
|
|
||||||
- Notarius external provenance.
|
|
||||||
|
|
||||||
The canonical promoted `index.json` is also a stage output with kind
|
|
||||||
`notarius_index`, but it is not a selectable extraction source. Extract-stage
|
|
||||||
metadata records the durable bundle root, receipt path, Notarius validation
|
|
||||||
status, counts, warnings/rejections paths and summaries, the producing Narratio
|
|
||||||
run ID, and the normalized configured-contract fingerprint used by resume
|
|
||||||
validation.
|
|
||||||
|
|
||||||
The producing Narratio run ID belongs to the successful extract result. It is
|
|
||||||
not compared with the session manifest's top-level `run_id`, which advances on
|
|
||||||
later invocations even when extraction is validly resumed. Resume and catalog
|
|
||||||
checks instead require the extract outputs, immutable bundle path, and stored
|
|
||||||
extract-stage producer identity to agree with one another.
|
|
||||||
|
|
||||||
## Artifact Catalog And Resolution
|
|
||||||
|
|
||||||
Add extraction as a first-class artifact-policy and runtime-catalog family:
|
|
||||||
|
|
||||||
- source kind `extraction`;
|
|
||||||
- canonical prefix `narratio.extraction.`;
|
|
||||||
- registration from `pipeline.notarius.outputs`; and
|
|
||||||
- manifest-backed availability from the current successful `extract` record.
|
|
||||||
|
|
||||||
Extraction availability is never inferred by scanning
|
|
||||||
`artifacts/notarius/`. A source is available only when:
|
|
||||||
|
|
||||||
- it is declared in current configuration;
|
|
||||||
- the session manifest records `extract` as succeeded and not stale;
|
|
||||||
- the exact matching source output is present;
|
|
||||||
- its durable path is confined and valid;
|
|
||||||
- its checksum matches; and
|
|
||||||
- its recorded contract and external provenance are compatible.
|
|
||||||
|
|
||||||
The catalog should expose one shared registration and hydration path used by
|
|
||||||
both `analyze` and `publish`. Avoid parallel extract-specific resolution logic
|
|
||||||
inside each stage.
|
|
||||||
|
|
||||||
## Analyze Integration
|
|
||||||
|
|
||||||
An enabled Scriptorium artifact may select an extraction source through the
|
|
||||||
existing input contract:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
inputs:
|
|
||||||
npcs:
|
|
||||||
source: narratio.extraction.npc_registry
|
|
||||||
required: true
|
|
||||||
```
|
|
||||||
|
|
||||||
Required missing extraction inputs fail with guidance to enable/configure or
|
|
||||||
rerun `extract`. Optional missing inputs follow the existing Scriptorium input
|
|
||||||
contract.
|
|
||||||
|
|
||||||
Narratio never injects every extraction output into every analysis. Each
|
|
||||||
Scriptorium artifact chooses the smallest useful set. This limits context
|
|
||||||
size, cost, and the risk of treating derived claims as transcript authority.
|
|
||||||
Analysis prompts should continue to treat the transcript as authoritative and
|
|
||||||
Notarius artifacts as structured, cited, derived evidence.
|
|
||||||
|
|
||||||
The existing `--artifacts` selection remains scoped to Scriptorium artifacts.
|
|
||||||
It does not select Notarius lanes or partially run the Notarius pipeline.
|
|
||||||
|
|
||||||
## Publish Integration
|
|
||||||
|
|
||||||
Publish source validation and resolution accept configured
|
|
||||||
`narratio.extraction.<key>` sources through the shared runtime artifact
|
|
||||||
catalog. Operators may publish individual structured lanes without manually
|
|
||||||
copying files.
|
|
||||||
|
|
||||||
The run-record upload excludes the run-local
|
|
||||||
`extract/notarius-output/**` staging bundle while retaining its receipt and
|
|
||||||
stderr diagnostics as eligible run files. Durable Notarius bundles are not
|
|
||||||
scanned or automatically published wholesale. Only lanes named by explicit
|
|
||||||
configured publish rules are uploaded. Existing publish locking, destination
|
|
||||||
safety, commit ordering, and required/unselected artifact behavior remain
|
|
||||||
unchanged.
|
|
||||||
|
|
||||||
## Failure And Diagnostic Semantics
|
|
||||||
|
|
||||||
The stage fails before invoking Notarius when enabled configuration or the
|
|
||||||
final trimmed transcript is invalid.
|
|
||||||
|
|
||||||
The stage fails after invocation for:
|
|
||||||
|
|
||||||
- cancellation or timeout;
|
|
||||||
- nonzero process exit;
|
|
||||||
- malformed, oversized, or unsupported receipt data;
|
|
||||||
- malformed or unsupported index data;
|
|
||||||
- unsafe receipt, descriptor, or filesystem paths;
|
|
||||||
- symlinks or special files in the promoted bundle;
|
|
||||||
- mismatched receipt pipeline identity;
|
|
||||||
- missing configured lanes;
|
|
||||||
- duplicate lane descriptors;
|
|
||||||
- rejected required lanes;
|
|
||||||
- incompatible media type, schema identity/version, or module key;
|
|
||||||
- empty or syntactically invalid required JSON payloads;
|
|
||||||
- checksum or promotion failure; or
|
|
||||||
- manifest persistence failure.
|
|
||||||
|
|
||||||
Process success alone does not establish consumer success. Notarius may exit
|
|
||||||
zero while omitting or rejecting a lane, and Narratio's configured required
|
|
||||||
set remains authoritative.
|
|
||||||
|
|
||||||
Failure retains bounded receipt and stderr diagnostics plus the run-local
|
|
||||||
bundle where available. Durable artifact storage and extraction source records
|
|
||||||
are updated only after complete validation and promotion. A promoted unique
|
|
||||||
bundle whose later manifest save fails is unreferenced and may be reclaimed by
|
|
||||||
explicit cleanup; it is never inferred as current.
|
|
||||||
|
|
||||||
## Security And Privacy
|
|
||||||
|
|
||||||
Transcripts, Notarius lanes, evidence context, manifests, receipts, warnings,
|
|
||||||
rejections, debug data, and logs are private campaign material.
|
|
||||||
|
|
||||||
- Secrets do not appear in command arguments, generated configuration,
|
|
||||||
artifact metadata, logs, examples, or documentation.
|
|
||||||
- Credentials continue to enter through Notarius and PromptKit's documented
|
|
||||||
environment or secret mechanisms.
|
|
||||||
- Receipt and index paths are untrusted external input until confined.
|
|
||||||
- Recursive promotion never follows symlinks or copies special files.
|
|
||||||
- Diagnostics remain bounded and do not echo payload bodies.
|
|
||||||
- Automatic cleanup follows Narratio's existing post-publish gates and path
|
|
||||||
safety rules; it does not silently remove immutable extraction bundles
|
|
||||||
outside an explicit covered policy.
|
|
||||||
|
|
||||||
## Maintained Complete D&D Example
|
|
||||||
|
|
||||||
Add a maintained example that enables Notarius's complete `dnd-session`
|
|
||||||
pipeline and maps these required lanes:
|
|
||||||
|
|
||||||
| Extraction key | Notarius lane ID |
|
|
||||||
| --- | --- |
|
|
||||||
| `item_registry` | `item-registry` |
|
|
||||||
| `npc_registry` | `npc-registry` |
|
|
||||||
| `location_registry` | `location-registry` |
|
|
||||||
| `scene_descriptions` | `scene-descriptions` |
|
|
||||||
| `item_occurrences` | `item-occurrences` |
|
|
||||||
| `spells` | `spells` |
|
|
||||||
| `combat_turns` | `combat-turns` |
|
|
||||||
| `npc_occurrences` | `npc-occurrences` |
|
|
||||||
| `location_occurrences` | `location-occurrences` |
|
|
||||||
| `enemy_events` | `enemy-events` |
|
|
||||||
|
|
||||||
The example obtains exact media types and schema identities from Notarius's
|
|
||||||
published contracts at implementation time. It demonstrates at least one
|
|
||||||
Scriptorium artifact consuming a small, purpose-specific subset of extraction
|
|
||||||
sources. It uses only placeholders and repository-relative example paths,
|
|
||||||
contains no credentials, and passes maintained example validation.
|
|
||||||
|
|
||||||
## Documentation Deliverables
|
|
||||||
|
|
||||||
When implementation lands, update current-behavior documentation in the same
|
|
||||||
change:
|
|
||||||
|
|
||||||
- add `docs/integrations/notarius.md` for the consumed CLI, receipt, index, and
|
|
||||||
compatibility contract, linking to Notarius's canonical documentation;
|
|
||||||
- add `docs/internal/stage-extract.md` for stage flow, collaborators, state,
|
|
||||||
failures, resume validation, and focused tests;
|
|
||||||
- update `docs/internal/overview.md`, `docs/internal/adapters.md`,
|
|
||||||
`docs/internal/artifacts.md`, `docs/internal/manifest.md`, and
|
|
||||||
`docs/internal/workspace.md` within their canonical scopes;
|
|
||||||
- update `docs/policy/architecture.md` for the Notarius boundary and explicit
|
|
||||||
skipped/resume-validation contracts;
|
|
||||||
- update `docs/config.md`, `docs/cli.md`, `docs/operations.md`,
|
|
||||||
`docs/troubleshooting.md`, `README.md`, and maintained examples only within
|
|
||||||
their canonical scopes; and
|
|
||||||
- update `docs/development.md` only if its contributor routing changes.
|
|
||||||
|
|
||||||
Outside this roadmap, do not describe the feature as implemented until its
|
|
||||||
code and documentation are complete.
|
|
||||||
|
|
||||||
## Testing Expectations
|
|
||||||
|
|
||||||
Tests should protect contracts and meaningful risks rather than private helper
|
|
||||||
structure. The implementation plan assigns detailed ownership, with coverage
|
|
||||||
for:
|
|
||||||
|
|
||||||
- strict configuration, defaults, normalization, cross-source validation, and
|
|
||||||
maintained examples;
|
|
||||||
- stage result dispositions and clearing of superseded session-result payloads
|
|
||||||
when execution starts, fails, or skips while retaining stale diagnostics;
|
|
||||||
- backward-compatible artifact metadata serialization;
|
|
||||||
- exact adapter arguments, streams, cancellation, timeout, exit behavior,
|
|
||||||
receipt/index compatibility, and every path-confinement boundary;
|
|
||||||
- recursive promotion safety, source-replacement detection, atomic visibility,
|
|
||||||
cleanup on failure, symlink rejection, and immutable destination behavior;
|
|
||||||
- required-lane policy, descriptor compatibility, JSON syntax, checksums, and
|
|
||||||
provenance;
|
|
||||||
- canonical order, single-stage selection, force, staleness, resume
|
|
||||||
validation, and enable-after-skip behavior;
|
|
||||||
- manifest-backed extraction catalog resolution for required, optional,
|
|
||||||
missing, stale, skipped, incompatible, and tampered artifacts;
|
|
||||||
- analyze and publish integration without automatic lane injection; and
|
|
||||||
- representative assembled execution with a fake Notarius runner and no live
|
|
||||||
LLM, credentials, or external subprocess in the ordinary test suite.
|
|
||||||
|
|
||||||
Repository-wide tests, vet, build, and maintained-example validation are
|
|
||||||
required after focused tests pass.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
- `extract` is a first-class stage between `trim` and `render` everywhere
|
|
||||||
Narratio models stage order and lifecycle.
|
|
||||||
- Narratio invokes Notarius only through a narrow tested adapter.
|
|
||||||
- Extraction consumes the manifest-resolved final trimmed Seriatim JSON.
|
|
||||||
- Disabled extraction is recorded as skipped and runs normally if later
|
|
||||||
enabled.
|
|
||||||
- Different executed extraction outcomes stale succeeded downstream stages,
|
|
||||||
while an identical repeated disabled self-skip remains stable.
|
|
||||||
- Successful resume requires valid manifest-recorded immutable outputs rather
|
|
||||||
than filesystem presence alone.
|
|
||||||
- The complete bundle is promoted to a unique immutable directory without
|
|
||||||
following symlinks or exposing a partial destination.
|
|
||||||
- Atomic no-replace bundle promotion is supported on Linux, macOS, and Windows;
|
|
||||||
unsupported operating systems fail before a temporary promotion tree is
|
|
||||||
created.
|
|
||||||
- Every configured lane is discovered by lane ID, contract-checked, checksummed,
|
|
||||||
and recorded with explicit source identity and external provenance.
|
|
||||||
- Analysis and publish resolve extraction sources only from a current
|
|
||||||
successful extract manifest record.
|
|
||||||
- Run-record uploads exclude the staged Notarius bundle, and only explicitly
|
|
||||||
configured extraction lanes are published; receipt and stderr diagnostics
|
|
||||||
remain eligible run files.
|
|
||||||
- Failed, skipped, stale, partial, rejected, unsafe, incompatible, or tampered
|
|
||||||
output never becomes a current input.
|
|
||||||
- The complete D&D example maps all ten current lanes and demonstrates curated
|
|
||||||
analysis inputs.
|
|
||||||
- Tests remain deterministic, offline, and independent of real Notarius,
|
|
||||||
PromptKit, LLM providers, and credentials.
|
|
||||||
- Current-behavior documentation is updated only as implementation becomes
|
|
||||||
complete.
|
|
||||||
|
|
||||||
## Non-Goals
|
|
||||||
|
|
||||||
- Reimplementing Notarius extraction, configuration, schemas, prompts,
|
|
||||||
references, retries, profiles, validation, or lane orchestration.
|
|
||||||
- Supporting arbitrary extractor programs or multiple Notarius pipelines in
|
|
||||||
one Narratio run.
|
|
||||||
- Turning canonical stage execution into a DAG or generic workflow engine.
|
|
||||||
- Folding extraction into `analyze` or Scriptorium.
|
|
||||||
- Automatically injecting all structured outputs into every prompt.
|
|
||||||
- Selecting Notarius lanes through Narratio's `--artifacts` flag.
|
|
||||||
- Decoding D&D payload bodies in the generic adapter or stage.
|
|
||||||
- Automatically detecting changes throughout Notarius's referenced config,
|
|
||||||
PromptKit profile, and campaign-reference closure.
|
|
||||||
- Supporting previous-session extraction sources in the initial feature.
|
|
||||||
- Automatically publishing the complete Notarius bundle.
|
|
||||||
- Requiring live Notarius, PromptKit, an LLM provider, or external services in
|
|
||||||
the ordinary test suite.
|
|
||||||
@@ -205,6 +205,32 @@ func TestPromoteDirectoryRejectsSourceRootReplacementBeforeOpen(t *testing.T) {
|
|||||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsIdentityPreservingSourceRootSymlinkReplacement(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
preserved := filepath.Join(root, "source-preserved")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("original"), 0o644)
|
||||||
|
|
||||||
|
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||||
|
afterRootInspect: func() {
|
||||||
|
if err := os.Rename(src, preserved); err != nil {
|
||||||
|
t.Fatalf("Rename(inspected source) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(filepath.Base(preserved), src); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("promoteDirectoryWithHooks() error = nil, want source-root symlink replacement failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||||
|
assertPathIsSymlink(t, src)
|
||||||
|
assertPathMissing(t, dst)
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromoteDirectoryRejectsInspectedDirectorySymlinkReplacement(t *testing.T) {
|
func TestPromoteDirectoryRejectsInspectedDirectorySymlinkReplacement(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
src := filepath.Join(root, "source")
|
src := filepath.Join(root, "source")
|
||||||
@@ -274,6 +300,38 @@ func TestPromoteDirectoryRejectsInspectedFileIdentityMismatch(t *testing.T) {
|
|||||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsIdentityPreservingFileSymlinkReplacement(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
file := filepath.Join(src, "value.txt")
|
||||||
|
preserved := filepath.Join(src, "value-preserved.txt")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, file, []byte("original"), 0o644)
|
||||||
|
|
||||||
|
replaced := false
|
||||||
|
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||||
|
afterEntryInspect: func(path string) {
|
||||||
|
if replaced || path != file {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
replaced = true
|
||||||
|
if err := os.Rename(file, preserved); err != nil {
|
||||||
|
t.Fatalf("Rename(inspected file) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(filepath.Base(preserved), file); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("promoteDirectoryWithHooks() error = nil, want file symlink replacement failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, preserved, []byte("original"))
|
||||||
|
assertPathIsSymlink(t, file)
|
||||||
|
assertPathMissing(t, dst)
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromoteDirectoryRejectsInspectedDirectoryIdentityMismatch(t *testing.T) {
|
func TestPromoteDirectoryRejectsInspectedDirectoryIdentityMismatch(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
src := filepath.Join(root, "source")
|
src := filepath.Join(root, "source")
|
||||||
@@ -304,6 +362,38 @@ func TestPromoteDirectoryRejectsInspectedDirectoryIdentityMismatch(t *testing.T)
|
|||||||
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromoteDirectoryRejectsIdentityPreservingDirectorySymlinkReplacement(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
src := filepath.Join(root, "source")
|
||||||
|
child := filepath.Join(src, "child")
|
||||||
|
preserved := filepath.Join(src, "child-preserved")
|
||||||
|
dst := filepath.Join(root, "destination")
|
||||||
|
mustWriteFile(t, filepath.Join(child, "value.txt"), []byte("original"), 0o644)
|
||||||
|
|
||||||
|
replaced := false
|
||||||
|
err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{
|
||||||
|
afterEntryInspect: func(path string) {
|
||||||
|
if replaced || path != child {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
replaced = true
|
||||||
|
if err := os.Rename(child, preserved); err != nil {
|
||||||
|
t.Fatalf("Rename(inspected directory) error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(filepath.Base(preserved), child); err != nil {
|
||||||
|
t.Skipf("Symlink() unavailable: %v", err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("promoteDirectoryWithHooks() error = nil, want directory symlink replacement failure")
|
||||||
|
}
|
||||||
|
assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original"))
|
||||||
|
assertPathIsSymlink(t, child)
|
||||||
|
assertPathMissing(t, dst)
|
||||||
|
assertNoMatchingTempDirectories(t, root, ".destination.tmp-")
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromoteDirectoryDoesNotReplaceDestinationCreatedBeforeInstall(t *testing.T) {
|
func TestPromoteDirectoryDoesNotReplaceDestinationCreatedBeforeInstall(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
src := filepath.Join(root, "source")
|
src := filepath.Join(root, "source")
|
||||||
@@ -369,6 +459,14 @@ func assertPathMissing(t *testing.T, path string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assertPathIsSymlink(t *testing.T, path string) {
|
||||||
|
t.Helper()
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil || info.Mode()&os.ModeSymlink == 0 {
|
||||||
|
t.Fatalf("Lstat(%q) info = %v, error = %v, want symlink", path, info, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func treeLayout(t *testing.T, root string) []string {
|
func treeLayout(t *testing.T, root string) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
var layout []string
|
var layout []string
|
||||||
|
|||||||
Reference in New Issue
Block a user