From 74e2d21de5fb2ada0be5ef3fe9333e0d48ac7fb3 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 10 Aug 2026 03:24:49 +0000 Subject: [PATCH] Close the completed roadmap documents --- docs/roadmap/implementation.md | 349 ------------- docs/roadmap/notarius-extract-stage.md | 645 ------------------------- internal/fileops/directory_test.go | 98 ++++ 3 files changed, 98 insertions(+), 994 deletions(-) delete mode 100644 docs/roadmap/implementation.md delete mode 100644 docs/roadmap/notarius-extract-stage.md diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md deleted file mode 100644 index 1807268..0000000 --- a/docs/roadmap/implementation.md +++ /dev/null @@ -1,349 +0,0 @@ -# Notarius Extraction Implementation Plan - -## Status And Audience - -Remediation required. Stages 1 through 14 are complete. Stages 15 through 18 -address the remaining security, lifecycle-state, and native-platform validation -gaps identified by the second post-implementation review. - -This plan is written for a GPT-5.6 Terra coding agent. Implement incomplete -stages in strict numerical order. Do not skip ahead, combine stages merely to -reduce the number of prompts, or describe unfinished behavior as current -functionality. - -The target state and acceptance criteria are owned by -[notarius-extract-stage.md](notarius-extract-stage.md). This document owns the -implementation sequence. - -## Progress - -| Stage | Status | -| --- | --- | -| Stage 1 | Complete | -| Stage 2 | Complete | -| Stage 3 | Complete | -| Stage 4 | Complete | -| Stage 5 | Complete | -| Stage 6 | Complete | -| Stage 7 | Complete | -| Stage 8 | Complete | -| Stage 9 | Complete | -| Stage 10 | Complete | -| Stage 11 | Complete | -| Stage 12 | Complete | -| Stage 13 | Complete | -| Stage 14 | Complete | -| Stage 15 | Complete | -| Stage 16 | Complete | -| Stage 17 | Pending | -| Stage 18 | Pending | - -After completing and validating a stage, update only that stage's row to -`Complete` and record any material deviation in the relevant stage section. -Do not mark a stage complete while required tests or exit criteria remain. - -## Working Rules - -Before Stage 15, read: - -- `docs/development.md` and its task-specific references; -- `docs/roadmap/notarius-extract-stage.md` completely; and -- the focused Narratio documents, production code, and tests named by the - current stage. - -For every incomplete stage: - -1. Inspect the current implementation before editing. Prefer the codebase - knowledge graph for code discovery as required by `AGENTS.md`. -2. Preserve unrelated worktree changes. -3. Implement the complete stage scope, including focused regression tests. Do - not leave TODO implementations or defer a known part of the stage. -4. Keep tests offline and independent of real Notarius, PromptKit, LLM - providers, credentials, network services, and mutable external state. -5. Run the focused package tests listed for the stage, then `go test ./...`. -6. Run `go vet ./...` and build `cmd/narratio` whenever shared contracts, - composition, portability, or documentation examples change. Direct build - output to a temporary directory so validation does not leave an ignored - binary in the repository root. -7. Do not broaden the feature beyond the target architecture while fixing a - remediation. Prefer a general manifest or filesystem invariant when the - defect is general, but avoid unrelated cleanup. - -## Completed Stages 1-14 Summary - -1. **Stage 1 — Shared contracts:** Added explicit stage self-skip semantics, - artifact identities, contract metadata, provenance, and compatible manifest - serialization. -2. **Stage 2 — Configuration and source policy:** Added strict optional - Notarius configuration, deterministic path resolution, lane contracts, and - the `narratio.extraction.` source family. -3. **Stage 3 — Immutable promotion:** Added regular-file-only bundle copying, - static symlink and special-file rejection, syncing, cleanup, and atomic - no-replace installation on Linux. -4. **Stage 4 — Notarius adapter:** Added exact subprocess invocation, separate - receipt and diagnostic streams, bounded tolerant decoding, confinement, and - receipt/index discovery. -5. **Stage 5 — Extract execution:** Added transcript consumption, required-lane - validation, immutable promotion, checksums, outputs, provenance, - diagnostics, and invocation fingerprinting. -6. **Stage 6 — Lifecycle and resume:** Registered `extract` between `trim` and - `render`, added CLI selection, resume validation, force behavior, and - old-manifest compatibility. -7. **Stage 7 — Analyze integration:** Added shared manifest-backed extraction - catalog hydration and explicit required/optional Scriptorium input - resolution. -8. **Stage 8 — Publish and inspection integration:** Added explicit extraction - publish rules, metadata round-tripping, restore compatibility, and operator - availability reporting. -9. **Stage 9 — Documentation and examples:** Added maintained complete and - subset examples, integration and maintainer documentation, operational - guidance, and repository-wide validation. -10. **Stage 10 — Replacement invalidation:** Generalized runner semantics so - forced, failed, skipped, changed, and non-resumable upstream executions - invalidate succeeded downstream results without causing perpetual reruns - for identical self-skips. -11. **Stage 11 — Explicit-only publication:** Excluded the run-local Notarius - bundle from run-record uploads while retaining diagnostics and explicit - configured-lane publication. -12. **Stage 12 — Canonical management files:** Required the production receipt - and index to use the canonical index, manifest, rejection, and warning - filenames assumed by extraction, resume, and catalog code. -13. **Stage 13 — Platform promotion contract:** Added atomic no-replace - implementations for Linux, macOS, and Windows, explicit early failure on - unsupported systems, and platform-specific directory syncing. -14. **Stage 14 — Documentation reconciliation:** Aligned current-behavior, - operations, troubleshooting, integration, internal, and roadmap - documentation with the implemented lifecycle and publication semantics. - -## Stage 15: Make Bundle Promotion Race-Safe Against Symlink Replacement - -Close the gap between the documented no-symlink promotion invariant and the -current path-based recursive copy. A source directory that is replaced after -inspection must never cause Narratio to traverse a symlink or copy data from -outside the originally opened source tree. - -Implementation: - -1. Refactor `internal/fileops` source traversal to anchor the copy to an opened - source root for the entire operation. Use Go's `os.Root` APIs, available at - the module's declared Go version, instead of recursively reopening source - directories through unconstrained absolute paths. -2. Reject a source root that is itself a symlink. After opening the root, - compare the opened root identity with the identity inspected before opening; - fail if the source changed during that transition. -3. For every source entry: - - inspect it relative to the already opened parent root; - - reject symlinks, link-like Windows reparse points exposed as links, and - every non-regular, non-directory file type; - - open regular files relative to that root, compare the opened file identity - with the inspected identity, and copy only from the verified handle; and - - open child directories relative to the parent root, verify the opened - directory identity against the inspected identity, and recurse through the - opened child root rather than its pathname. -4. Treat any identity mismatch, disappearing entry, replacement, or unsupported - file type as a clean promotion failure. Preserve the source, remove the - temporary destination tree, and never install a partial destination. -5. Keep the destination-side behavior unchanged: deterministic traversal, - normalized permissions, file and directory syncing, sibling temporary tree, - and atomic no-replace installation. -6. Keep support bounded to Linux, macOS, and Windows. Do not weaken the strict - no-symlink rule to permit links that happen to remain inside the source root. -7. Keep test-only orchestration narrow. If deterministic replacement testing - requires an internal seam between inspection and opening, inject only that - boundary; do not expose it publicly or turn file copying into a generic - filesystem framework. - -Tests: - -- retain the existing nested-copy, source-preservation, static symlink, - special-file, cleanup, existing-destination, and install-collision tests; -- deterministically replace an inspected child directory with a symlink before - it is opened, and prove promotion fails without copying an outside sentinel; -- cover source-root replacement between initial inspection and root opening; -- cover regular-file and directory identity mismatches without timing-based - sleeps or probabilistic race loops; -- prove every failure leaves the source and any concurrent destination intact - and removes the temporary sibling; and -- run the focused tests with the race detector. - -Exit criteria: - -- Recursive promotion cannot escape or switch away from the originally opened - source tree through a symlink replacement race. -- Static and concurrent symlink replacement are both protected by durable, - deterministic tests. -- `go test -count=1 ./internal/fileops`, - `go test -race -count=1 ./internal/fileops`, `go test ./...`, and - `go vet ./...` pass on the development platform. - -## Stage 16: Clear Superseded Session-Stage Result Payloads - -Make the session manifest describe only the current stage outcome. Historical -successful result details belong to their immutable run manifests and must not -remain attached to a current stage record that is running, failed, skipped, or -newly succeeded without corresponding details. - -Implementation: - -1. Centralize clearing of result-bearing `manifest.StageRecord` fields: - `Outputs`, `Logs`, `GeneratedConfigs`, and `Metadata`. -2. Clear those fields when `MarkStageRunning` begins a replacement attempt. - The runner already captures the prior outcome before this transition; do not - change self-skip comparison or downstream invalidation decisions. -3. Also make `MarkStageFailed` and `MarkStageSkipped` enforce the empty-result - invariant themselves so callers cannot construct a failed or skipped record - with inherited successful output details. Avoid runner-only cleanup that - leaves direct manifest transitions inconsistent. -4. Do not clear result details merely when `MarkStageStale` is called. Resume - validation must still be able to inspect the prior result before deciding to - rerun it, and stale records remain useful diagnostic state until execution - actually begins. -5. Ensure `MarkStageSucceeded` followed by result application cannot inherit - old logs, generated configurations, or metadata when the new successful - result omits those fields. Clearing at the running transition should own this - invariant; do not add scattered empty-map special cases. -6. Preserve immutable prior run manifests and durable Notarius bundles. This - stage changes the current session-stage record, not historical files or - retention policy. -7. Consolidate the existing runner self-skip cleanup helper if it becomes - redundant after manifest transitions own the invariant. - -Tests: - -- add manifest transition tests proving running, failed, and skipped records - cannot retain prior outputs, logs, generated configurations, or metadata; -- prove `MarkStageStale` retains the prior details required for resume - validation; -- change the forced-extraction-failure lifecycle fixture to first complete a - successful extraction, then force a failing replacement, and assert the - current extract record is failed with no inherited result payload; -- retain the assertion that succeeded downstream stages become stale; -- prove a successful replacement whose new result omits optional details does - not inherit details from its predecessor; and -- retain resume-error, repeated-self-skip, and ordinary-retry coverage. - -Exit criteria: - -- A current running, failed, or skipped session-stage record never advertises - result payload belonging to an earlier success. -- Historical run manifests and immutable extraction bundles remain available - for audit and recovery. -- `go test -count=1 ./internal/manifest ./internal/app ./internal/stage`, - focused lifecycle race tests, `go test ./...`, and `go vet ./...` pass. - -## Stage 17: Establish Native Promotion Validation On Every Supported Platform - -Turn the Linux, macOS, and Windows promotion claim into a continuously verified -contract. Cross-compilation remains useful but cannot substitute for executing -filesystem operations on each native platform. - -Implementation: - -1. Resolve the CI substrate and native runner labels in the open question below - before editing workflow files. Follow the repository host's established - workflow location and syntax; do not invent runner labels that cannot run. -2. Add checked-in CI jobs that execute on native Linux, macOS, and Windows - runners. Each native job must run at least: - - `go test -count=1 ./internal/fileops`; - - the platform-specific no-replace test; and - - the portable promotion conformance tests, including successful promotion, - collision preservation, cleanup, and the Stage 15 race-safe traversal - cases that apply on that platform. -3. Ensure the jobs exercise the real platform implementation rather than a - mocked rename function. A destination created before installation must win, - and the source and destination contents must prove that no replacement - occurred. -4. On Windows, ensure successful promotion exercises both `MoveFileEx` and the - directory-sync path. Do not treat permission or ordinary I/O failures as - unsupported-operation success. -5. Retain deterministic cross-compilation of the fileops test binary for Linux, - Darwin, and Windows. Write compiled test binaries to a temporary or CI - artifact directory, never the repository root. -6. Keep unsupported-platform compilation coverage, but do not misrepresent an - unexecuted cross-compiled test as native validation. -7. Record any required runner-specific limitation in this stage and in the - canonical contributor/CI documentation. Do not weaken production semantics - merely to accommodate an inadequately provisioned runner. - -Validation: - -- obtain a successful native Linux job; -- obtain a successful native macOS job; -- obtain a successful native Windows job; -- retain successful Linux, Darwin, and Windows cross-compilation; and -- confirm CI does not leave generated test binaries or application binaries in - the worktree or commit them as artifacts of the source tree. - -Exit criteria: - -- Atomic no-replace promotion and its directory-sync behavior execute - successfully on native Linux, macOS, and Windows. -- The checked-in CI configuration will rerun those tests on future changes. -- Stage 17 is not marked complete based solely on Linux execution and - cross-compilation. - -## Stage 18: Reconcile Security Documentation And Perform Final Validation - -Close the remediation only after the corrected filesystem and manifest -invariants are implemented and native platform evidence exists. - -Implementation: - -1. Reconcile `docs/roadmap/notarius-extract-stage.md` with the Stage 15 source - traversal mechanism and Stage 16 current-result semantics. Keep normative - security and manifest guarantees in their existing canonical owners. -2. Update focused internal or contributor documentation only where the new - mechanism or CI workflow changes maintained implementation guidance. Do not - duplicate volatile platform commands across multiple documents. -3. Recheck the extraction, workspace, manifest, operations, troubleshooting, - integration, and testing documentation for contradictions introduced by the - remediation. -4. Keep completed roadmap history concise. Mark the target feature roadmap and - this implementation plan complete only after all Stage 15 through 17 exit - criteria are satisfied. -5. Confirm that no secrets, private campaign material, generated binaries, - platform test binaries, or temporary bundle trees were added to the - repository. - -Validation: - -- run focused fileops, manifest, runner lifecycle, extraction, adapter, - artifact-catalog, analyze, publish, restore, and operator tests; -- run focused race tests covering fileops, manifest transitions, runner - lifecycle, extraction, artifacts, and publication; -- run `go test -count=1 ./...`; -- run `go vet ./...`; -- build `cmd/narratio` with `-o` targeting a temporary directory; -- rerun Linux, Darwin, and Windows fileops test cross-compilation; -- confirm the native CI jobs from Stage 17 are successful; -- run `git diff --check`; and -- verify the worktree contains no generated validation artifacts. - -Exit criteria: - -- The source traversal and current-manifest replacement gaps are closed by - focused regression tests. -- Native filesystem behavior substantiates the documented Linux, macOS, and - Windows support boundary. -- Current documentation matches the remediated implementation. -- Every progress row is `Complete`, the repository-wide validation suite - passes, and the target feature roadmap can truthfully remain complete. - -## Open Questions - -### Which CI system and native runner labels are authoritative for this repository? - -**Recommended approach:** Use Gitea Actions with a checked-in workflow under -`.gitea/workflows/`, provided the repository owner supplies or confirms native -Linux, macOS, and Windows runner labels. The repository is hosted on Gitea, so -keeping the workflow with the authoritative repository minimizes mirrored -configuration and makes the platform contract visible beside the code. Stage -17 must remain pending until all three native jobs have actually run. - -**Viable alternative:** Use an existing external CI service with real native -capacity, such as a maintained GitHub Actions mirror or Buildkite installation, -and check its workflow or pipeline definition into the conventional repository -location. This is appropriate when that service already owns release gating or -when native macOS and Windows runners are unavailable in Gitea. The alternative -must still produce repeatable native results for all three platforms; -cross-compilation or an undocumented one-time manual run is not sufficient. diff --git a/docs/roadmap/notarius-extract-stage.md b/docs/roadmap/notarius-extract-stage.md deleted file mode 100644 index d0597a0..0000000 --- a/docs/roadmap/notarius-extract-stage.md +++ /dev/null @@ -1,645 +0,0 @@ -# Notarius Extraction Stage - -## Status - -Pending native promotion validation. - -The implementation sequence is retained in -[implementation.md](implementation.md) as decision and delivery history. -Current behavior is documented in the canonical configuration, CLI, -operations, integration, and internal references linked from this roadmap. -The extraction feature is implemented, but completion remains pending until -atomic promotion has run successfully in native Linux, macOS, and Windows CI. - -## Purpose - -Add a first-class Narratio `extract` stage that runs Notarius against the -session's final trimmed transcript and makes validated structured artifacts -available to later analysis and publish work. - -Notarius remains responsible for its D&D extraction pipeline, prompts, -references, LLM profiles, retries, validation, normalization, and published -schemas. Narratio owns invocation, required-output policy, safe bundle -ingestion, artifact identity, manifest state, resume, and downstream -availability. - -The integration must preserve Narratio's explicit stage model. It must not -turn Narratio into a generic workflow engine or reproduce Notarius's -configuration language. - -## User Outcome - -An operator can enable one Notarius pipeline for a Narratio campaign. During a -normal run, Narratio will: - -1. produce the final trimmed Seriatim JSON transcript; -2. invoke Notarius once with that transcript; -3. discover the exact run bundle through Notarius's machine-readable receipt; -4. validate every output contract Narratio is configured to require; -5. promote the complete validated bundle into immutable session storage; -6. record exact artifact paths, checksums, contracts, and external provenance; -7. make configured lanes available as `narratio.extraction.` sources; and -8. allow individual Scriptorium artifacts and publish rules to select those - sources explicitly. - -The maintained complete D&D example will require all ten lanes published by -Notarius's `dnd-session` pipeline. Ordinary deployments may configure a -narrower required set. - -## Chosen Architecture - -### First-Class Stage - -Extraction is an independently observable and resumable pipeline stage. It is -not part of `analyze`, a Scriptorium artifact producer, or an implicit external -preprocessing requirement. - -This boundary is required because Notarius work is expensive, produces -multiple durable outputs, has its own compatibility and diagnostic contracts, -and may be consumed by both `analyze` and `publish`. - -### Canonical Order - -The canonical stage order becomes: - -```text -prepare -> transcribe -> merge -> polish -> normalize -> trim - -> extract -> render -> analyze -> publish -> notify -``` - -`extract` consumes `narratio.transcript.final_trimmed`, produced by `trim`. It -does not consume rendered Markdown. - -Narratio invalidates succeeded stages by canonical downstream order whenever a -forced run or a different executed upstream outcome replaces current state. -Placing `extract` before `render` means replacing extraction may rerun the less -expensive deterministic render stage, while replacing render does not rerun the -more expensive Notarius pipeline. This is preferable to placing extraction -after render and does not require dependency-aware scheduling or a DAG. - -Adding the stage must update every canonical-stage inventory, full-plan and -single-stage selection, prerequisite validation, downstream invalidation, -resume behavior, run manifests, CLI validation and help, and focused tests. - -### Stage Boundary - -The stage owns Narratio policy: - -- resolve the final trimmed transcript through the manifest-aware artifact - resolver; -- allocate run-local receipt, log, and output-root paths; -- build a transport-neutral Notarius request from resolved configuration; -- call the configured Notarius adapter once; -- enforce Narratio's configured required-output policy; -- validate selected payloads as regular, non-empty, syntactically valid JSON; -- promote the validated bundle to immutable session artifact storage; -- return explicit source IDs, checksums, contracts, and external provenance; - and -- report an explicit skipped outcome when Notarius is disabled. - -The stage must not construct subprocess arguments, guess Notarius filenames, -parse interactive output, decode D&D payload structures, or reproduce -Notarius pipeline configuration. - -### Adapter Boundary - -Add `internal/adapters/notarius` with a narrow runner interface, production -subprocess implementation, and small fake. - -The request contains only: - -- resolved executable path or name; -- absolute Notarius configuration path; -- pipeline ID; -- absolute transcript path; -- absolute output root; -- working directory; -- timeout; and -- stdout receipt and stderr log destinations. - -The adapter owns: - -- exact `notarius run ... --json` argument construction; -- stdout and stderr separation; -- context cancellation and timeout through Narratio's shared subprocess - boundary; -- environment inheritance; -- exit-status handling; -- bounded receipt loading after exit status zero; -- tolerant decoding of supported `notarius.run-result.v1` documents; -- validation of required receipt fields; -- confinement of the receipt's absolute `output_directory` beneath the - absolute output root Narratio supplied for this invocation; -- requiring receipt `index_file` to be exactly `index.json` beneath the - receipt's absolute `output_directory`; -- tolerant decoding of the supported `index.json` contract; -- confinement of every index descriptor path beneath the bundle root; and -- bounded tolerant decoding of `rejected.json` and `warnings.json` into - transport-neutral summaries without reading lane payload bodies; summaries - persisted by Narratio contain structured scope, lane, and reason fields but - not unbounded free-form external messages; and -- returning transport-neutral receipt, descriptor, diagnostic, and bundle - information. - -Only exit status zero permits receipt decoding. Unsupported schema versions, -malformed documents, missing required fields, absolute logical paths, path -escapes, symlinks at consumed paths, and incompatible structural metadata are -integration failures. - -The production runner will not execute `notarius config validate` before every -session. Notarius run-time validation remains authoritative, while operators -may use the separate validation command as deployment preflight. A second -automatic subprocess can be added later only if operational evidence warrants -it. - -The adapter does not write Narratio manifests, decide required lanes, choose -analysis inputs, or interpret D&D payloads. - -## External Contract Baseline - -The integration consumes the contracts documented by Notarius in -`../notarius/docs/consumers/dnd-pipeline.md` and its linked canonical -integration documents. - -The initial compatibility baseline is: - -- `notarius run --config ... --input ... --output-dir ... --json`; -- successful receipt schema `notarius.run-result.v1`; -- an absolute receipt `output_directory`; -- receipt `index_file` exactly `index.json` beneath that directory; -- the production JSON `index.json` descriptor model with management files - exactly `manifest.json`, `rejected.json`, and `warnings.json`; and -- the exact media type and schema identity configured for each required lane. - -Compatibility is decided from these published contracts, not by parsing -`notarius --version`. Unknown fields in supported receipt and index versions -are tolerated. Unsupported versions or incompatible required descriptors fail -before any artifact becomes current in Narratio. - -## Configuration Contract - -Add a strict optional `pipeline.notarius` section: - -```yaml -notarius: - enabled: true - binary: notarius - config_path: /absolute/path/to/notarius.yml - pipeline_id: dnd-session - timeout: 3h - working_directory: /absolute/path/to/deployment - outputs: - npc_registry: - lane_id: npc-registry - media_type: application/json - schema_id: - schema_version: -``` - -Fields: - -- `enabled` is an explicit opt-in and defaults to false; -- `binary` defaults to `notarius`; -- `config_path` is required when enabled; -- `pipeline_id` is required when enabled; -- `timeout` defaults to `3h` and must be positive; -- `working_directory` is optional and defaults to the directory containing - `config_path`; and -- `outputs` maps stable Narratio extraction keys to required Notarius lane - contracts. - -All configured paths become absolute during configuration resolution. Notarius -reference paths continue to follow Notarius's configuration-relative rules, -while PromptKit profile paths remain relative to the chosen process working -directory where Notarius permits that behavior. - -Each output entry contains: - -- exact `lane_id`; -- exact `media_type`; -- exact `schema_id`; -- exact `schema_version`; and -- optional `module_key`. - -The map key produces `narratio.extraction.`. Keys use Narratio's existing -path-safe configured-artifact key grammar. Keys, normalized source IDs, and -lane IDs must be non-empty and unique. Extraction source IDs must not collide -with built-ins or configured Scriptorium sources. - -Every configured output is required for extraction-stage success. Operators -who need only a subset configure only that subset. Narratio does not add a -second `required_lanes` list or a runtime lane-selection flag. - -Narratio does not configure Notarius lane topology, references, prompts, LLM -profiles, model settings, concurrency, retry behavior, or session IDs. One -stage execution runs the configured Notarius pipeline as a unit. Narratio does -not pass Notarius's `--session-id` override. - -## Stage Outcomes And Resume - -### Explicit Self-Skip - -Extend the stage-result contract with an explicit disposition whose zero value -remains successful for backward compatibility. A stage may return: - -- succeeded; or -- skipped with a stable reason. - -When Notarius is absent or disabled, `extract` returns skipped with reason -`notarius_disabled`, no outputs, and bounded metadata. The runner records a -skipped session-stage and run-stage outcome rather than a successful empty -stage. Skipped stages are reconsidered on later invocations, so subsequently -enabling Notarius causes extraction to run without requiring force. - -A genuine self-skip clears any older outputs, logs, generated configuration -references, and metadata before persisting the new skipped state and its own -bounded metadata. Downstream consumers cannot resolve artifacts retained from -an earlier extraction after the stage is disabled. - -### Resume Validation - -Add a small optional resume-validation interface implemented by `extract`. -Before skipping an already-succeeded extract stage, the runner asks it whether -the recorded result remains resumable. - -The resume validator confirms: - -- Notarius is still enabled; -- recorded configuration identity matches the current config path, pipeline - ID, and output-contract map; -- the immutable bundle and canonical index still exist; -- every configured source is present in the succeeded extract record; -- lane files remain confined regular files; -- stored checksums still match; and -- stored descriptor contracts and external provenance remain compatible. - -An ordinary contract mismatch returns a non-resumable decision with a bounded -reason. The runner marks the stage stale and executes it. An environmental -error that prevents making a safe decision returns an error and stops the run. - -Narratio does not recursively interpret Notarius configuration, PromptKit -profiles, or reference files. Changes to those external inputs are therefore -not automatically detectable. Operator documentation must require -`--force extract` after changing them. - -The configuration fingerprint covers the resolved binary, config path, -pipeline ID, timeout, working directory, and the deterministically sorted -output-contract map. It identifies Narratio's invocation contract, not the -transitive content of files owned by Notarius. - -### Force And Invalidation - -- Forcing `trim` or an earlier stage stales succeeded `extract` and all later - stages. -- Forcing `extract` stales succeeded `render`, `analyze`, `publish`, and - `notify` under the canonical-order rule, even if extraction later skips or - fails. -- Forcing `render` does not stale `extract` because extraction precedes it. -- A non-forced extraction failure, changed self-skip, or success replacing a - different effective outcome stales succeeded downstream stages. Repeating - the same `notarius_disabled` self-skip with no outputs does not stale them - again. -- Reusing a resumable succeeded result does not invalidate downstream stages; - a resume-validation error stops without mutating either result. -- A failed, skipped, stale, or interrupted extract stage never supplies current - extraction sources. - -When a replacement attempt starts, the current session-stage record drops the -prior result payload before it is persisted as running. Failed and skipped -transitions enforce the same clearing invariant directly, and a later success -contains only details produced by that attempt. Merely marking a record stale -retains its prior details for resume validation and diagnosis until execution -actually begins. Immutable invocation run manifests and previously promoted -bundles preserve the historical successful result. - -## Bundle Storage And Commit - -### Run-Local Execution - -Notarius runs against a run-local output root beneath the Narratio run's -`extract` directory. Receipt bytes and stderr remain run-local diagnostics. -Failed and malformed bundles remain outside durable artifact storage for -inspection and never become current merely because files exist. - -### Immutable Durable Bundles - -After complete validation, promote the exact Notarius bundle into: - -```text -artifacts/notarius// -``` - -The destination is unique and must not already exist. Promotion uses a sibling -temporary directory on the same filesystem, traverses the source through -confined directory handles, recursively copies only regular files and -directories, rejects symlinks and special files, and preserves relative -layout. It verifies that each opened root, directory, and file is the same -object that was inspected, rejecting path replacement during traversal. The -completed temporary tree is then renamed into place without replacing a -destination created by another writer. - -Atomic no-replace promotion is implemented on Linux, macOS, and Windows. On -other operating systems extraction reports an unsupported-capability error -before creating the sibling temporary tree. This limitation is confined to -extraction bundle promotion and does not define a broader platform guarantee. - -The promoted tree preserves `index.json`, `manifest.json`, `rejected.json`, -`warnings.json`, `lanes/`, and any emitted pipeline-wide artifacts such as -`chunk-map.json` and `evidence-context.json`. Unknown regular files may be -preserved because the complete bundle is provenance, but no unknown file is -registered as a stable Narratio source. - -There is no mutable filesystem `current` directory or symlink. The atomically -saved Narratio session manifest selects the current successful bundle. Older -successful bundles remain immutable until explicit cleanup policy removes -them. - -The raw Notarius receipt is retained without rewriting its original -`output_directory`. Narratio's manifest records promoted durable paths. - -## Artifact And Manifest Model - -### Explicit Source Identity - -Extend `artifacts.Ref` with an optional explicit `SourceID`. The application -runner prefers that value and retains existing stage-specific inference only -as a backward-compatible fallback. `extract` must not require another -stage-name special case in output mapping. - -Add neutral optional artifact metadata models under `internal/artifactmodel`: - -- contract metadata: media type, schema ID, schema version, and optional - module key; and -- external provenance: system, external run ID, pipeline ID, and external - artifact ID. - -Both `artifacts.Ref` and `manifest.ArtifactRecord` carry these nested models. -Existing manifests remain readable because the fields are optional and use -`omitempty` encoding. - -For a Notarius lane, external provenance uses: - -- system `notarius`; -- receipt run ID; -- receipt pipeline ID; and -- lane ID as the external artifact ID. - -### Registered Sources - -For each configured output, `extract` records one artifact with: - -- source ID `narratio.extraction.`; -- durable lane path discovered through the promoted canonical index; -- producer stage and Narratio run ID; -- SHA-256 checksum; -- configured and observed contract metadata; and -- Notarius external provenance. - -The canonical promoted `index.json` is also a stage output with kind -`notarius_index`, but it is not a selectable extraction source. Extract-stage -metadata records the durable bundle root, receipt path, Notarius validation -status, counts, warnings/rejections paths and summaries, the producing Narratio -run ID, and the normalized configured-contract fingerprint used by resume -validation. - -The producing Narratio run ID belongs to the successful extract result. It is -not compared with the session manifest's top-level `run_id`, which advances on -later invocations even when extraction is validly resumed. Resume and catalog -checks instead require the extract outputs, immutable bundle path, and stored -extract-stage producer identity to agree with one another. - -## Artifact Catalog And Resolution - -Add extraction as a first-class artifact-policy and runtime-catalog family: - -- source kind `extraction`; -- canonical prefix `narratio.extraction.`; -- registration from `pipeline.notarius.outputs`; and -- manifest-backed availability from the current successful `extract` record. - -Extraction availability is never inferred by scanning -`artifacts/notarius/`. A source is available only when: - -- it is declared in current configuration; -- the session manifest records `extract` as succeeded and not stale; -- the exact matching source output is present; -- its durable path is confined and valid; -- its checksum matches; and -- its recorded contract and external provenance are compatible. - -The catalog should expose one shared registration and hydration path used by -both `analyze` and `publish`. Avoid parallel extract-specific resolution logic -inside each stage. - -## Analyze Integration - -An enabled Scriptorium artifact may select an extraction source through the -existing input contract: - -```yaml -inputs: - npcs: - source: narratio.extraction.npc_registry - required: true -``` - -Required missing extraction inputs fail with guidance to enable/configure or -rerun `extract`. Optional missing inputs follow the existing Scriptorium input -contract. - -Narratio never injects every extraction output into every analysis. Each -Scriptorium artifact chooses the smallest useful set. This limits context -size, cost, and the risk of treating derived claims as transcript authority. -Analysis prompts should continue to treat the transcript as authoritative and -Notarius artifacts as structured, cited, derived evidence. - -The existing `--artifacts` selection remains scoped to Scriptorium artifacts. -It does not select Notarius lanes or partially run the Notarius pipeline. - -## Publish Integration - -Publish source validation and resolution accept configured -`narratio.extraction.` sources through the shared runtime artifact -catalog. Operators may publish individual structured lanes without manually -copying files. - -The run-record upload excludes the run-local -`extract/notarius-output/**` staging bundle while retaining its receipt and -stderr diagnostics as eligible run files. Durable Notarius bundles are not -scanned or automatically published wholesale. Only lanes named by explicit -configured publish rules are uploaded. Existing publish locking, destination -safety, commit ordering, and required/unselected artifact behavior remain -unchanged. - -## Failure And Diagnostic Semantics - -The stage fails before invoking Notarius when enabled configuration or the -final trimmed transcript is invalid. - -The stage fails after invocation for: - -- cancellation or timeout; -- nonzero process exit; -- malformed, oversized, or unsupported receipt data; -- malformed or unsupported index data; -- unsafe receipt, descriptor, or filesystem paths; -- symlinks or special files in the promoted bundle; -- mismatched receipt pipeline identity; -- missing configured lanes; -- duplicate lane descriptors; -- rejected required lanes; -- incompatible media type, schema identity/version, or module key; -- empty or syntactically invalid required JSON payloads; -- checksum or promotion failure; or -- manifest persistence failure. - -Process success alone does not establish consumer success. Notarius may exit -zero while omitting or rejecting a lane, and Narratio's configured required -set remains authoritative. - -Failure retains bounded receipt and stderr diagnostics plus the run-local -bundle where available. Durable artifact storage and extraction source records -are updated only after complete validation and promotion. A promoted unique -bundle whose later manifest save fails is unreferenced and may be reclaimed by -explicit cleanup; it is never inferred as current. - -## Security And Privacy - -Transcripts, Notarius lanes, evidence context, manifests, receipts, warnings, -rejections, debug data, and logs are private campaign material. - -- Secrets do not appear in command arguments, generated configuration, - artifact metadata, logs, examples, or documentation. -- Credentials continue to enter through Notarius and PromptKit's documented - environment or secret mechanisms. -- Receipt and index paths are untrusted external input until confined. -- Recursive promotion never follows symlinks or copies special files. -- Diagnostics remain bounded and do not echo payload bodies. -- Automatic cleanup follows Narratio's existing post-publish gates and path - safety rules; it does not silently remove immutable extraction bundles - outside an explicit covered policy. - -## Maintained Complete D&D Example - -Add a maintained example that enables Notarius's complete `dnd-session` -pipeline and maps these required lanes: - -| Extraction key | Notarius lane ID | -| --- | --- | -| `item_registry` | `item-registry` | -| `npc_registry` | `npc-registry` | -| `location_registry` | `location-registry` | -| `scene_descriptions` | `scene-descriptions` | -| `item_occurrences` | `item-occurrences` | -| `spells` | `spells` | -| `combat_turns` | `combat-turns` | -| `npc_occurrences` | `npc-occurrences` | -| `location_occurrences` | `location-occurrences` | -| `enemy_events` | `enemy-events` | - -The example obtains exact media types and schema identities from Notarius's -published contracts at implementation time. It demonstrates at least one -Scriptorium artifact consuming a small, purpose-specific subset of extraction -sources. It uses only placeholders and repository-relative example paths, -contains no credentials, and passes maintained example validation. - -## Documentation Deliverables - -When implementation lands, update current-behavior documentation in the same -change: - -- add `docs/integrations/notarius.md` for the consumed CLI, receipt, index, and - compatibility contract, linking to Notarius's canonical documentation; -- add `docs/internal/stage-extract.md` for stage flow, collaborators, state, - failures, resume validation, and focused tests; -- update `docs/internal/overview.md`, `docs/internal/adapters.md`, - `docs/internal/artifacts.md`, `docs/internal/manifest.md`, and - `docs/internal/workspace.md` within their canonical scopes; -- update `docs/policy/architecture.md` for the Notarius boundary and explicit - skipped/resume-validation contracts; -- update `docs/config.md`, `docs/cli.md`, `docs/operations.md`, - `docs/troubleshooting.md`, `README.md`, and maintained examples only within - their canonical scopes; and -- update `docs/development.md` only if its contributor routing changes. - -Outside this roadmap, do not describe the feature as implemented until its -code and documentation are complete. - -## Testing Expectations - -Tests should protect contracts and meaningful risks rather than private helper -structure. The implementation plan assigns detailed ownership, with coverage -for: - -- strict configuration, defaults, normalization, cross-source validation, and - maintained examples; -- stage result dispositions and clearing of superseded session-result payloads - when execution starts, fails, or skips while retaining stale diagnostics; -- backward-compatible artifact metadata serialization; -- exact adapter arguments, streams, cancellation, timeout, exit behavior, - receipt/index compatibility, and every path-confinement boundary; -- recursive promotion safety, source-replacement detection, atomic visibility, - cleanup on failure, symlink rejection, and immutable destination behavior; -- required-lane policy, descriptor compatibility, JSON syntax, checksums, and - provenance; -- canonical order, single-stage selection, force, staleness, resume - validation, and enable-after-skip behavior; -- manifest-backed extraction catalog resolution for required, optional, - missing, stale, skipped, incompatible, and tampered artifacts; -- analyze and publish integration without automatic lane injection; and -- representative assembled execution with a fake Notarius runner and no live - LLM, credentials, or external subprocess in the ordinary test suite. - -Repository-wide tests, vet, build, and maintained-example validation are -required after focused tests pass. - -## Acceptance Criteria - -- `extract` is a first-class stage between `trim` and `render` everywhere - Narratio models stage order and lifecycle. -- Narratio invokes Notarius only through a narrow tested adapter. -- Extraction consumes the manifest-resolved final trimmed Seriatim JSON. -- Disabled extraction is recorded as skipped and runs normally if later - enabled. -- Different executed extraction outcomes stale succeeded downstream stages, - while an identical repeated disabled self-skip remains stable. -- Successful resume requires valid manifest-recorded immutable outputs rather - than filesystem presence alone. -- The complete bundle is promoted to a unique immutable directory without - following symlinks or exposing a partial destination. -- Atomic no-replace bundle promotion is supported on Linux, macOS, and Windows; - unsupported operating systems fail before a temporary promotion tree is - created. -- Every configured lane is discovered by lane ID, contract-checked, checksummed, - and recorded with explicit source identity and external provenance. -- Analysis and publish resolve extraction sources only from a current - successful extract manifest record. -- Run-record uploads exclude the staged Notarius bundle, and only explicitly - configured extraction lanes are published; receipt and stderr diagnostics - remain eligible run files. -- Failed, skipped, stale, partial, rejected, unsafe, incompatible, or tampered - output never becomes a current input. -- The complete D&D example maps all ten current lanes and demonstrates curated - analysis inputs. -- Tests remain deterministic, offline, and independent of real Notarius, - PromptKit, LLM providers, and credentials. -- Current-behavior documentation is updated only as implementation becomes - complete. - -## Non-Goals - -- Reimplementing Notarius extraction, configuration, schemas, prompts, - references, retries, profiles, validation, or lane orchestration. -- Supporting arbitrary extractor programs or multiple Notarius pipelines in - one Narratio run. -- Turning canonical stage execution into a DAG or generic workflow engine. -- Folding extraction into `analyze` or Scriptorium. -- Automatically injecting all structured outputs into every prompt. -- Selecting Notarius lanes through Narratio's `--artifacts` flag. -- Decoding D&D payload bodies in the generic adapter or stage. -- Automatically detecting changes throughout Notarius's referenced config, - PromptKit profile, and campaign-reference closure. -- Supporting previous-session extraction sources in the initial feature. -- Automatically publishing the complete Notarius bundle. -- Requiring live Notarius, PromptKit, an LLM provider, or external services in - the ordinary test suite. diff --git a/internal/fileops/directory_test.go b/internal/fileops/directory_test.go index 5d95673..baa1ee2 100644 --- a/internal/fileops/directory_test.go +++ b/internal/fileops/directory_test.go @@ -205,6 +205,32 @@ func TestPromoteDirectoryRejectsSourceRootReplacementBeforeOpen(t *testing.T) { assertNoMatchingTempDirectories(t, root, ".destination.tmp-") } +func TestPromoteDirectoryRejectsIdentityPreservingSourceRootSymlinkReplacement(t *testing.T) { + root := t.TempDir() + src := filepath.Join(root, "source") + preserved := filepath.Join(root, "source-preserved") + dst := filepath.Join(root, "destination") + mustWriteFile(t, filepath.Join(src, "value.txt"), []byte("original"), 0o644) + + err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{ + afterRootInspect: func() { + if err := os.Rename(src, preserved); err != nil { + t.Fatalf("Rename(inspected source) error = %v", err) + } + if err := os.Symlink(filepath.Base(preserved), src); err != nil { + t.Skipf("Symlink() unavailable: %v", err) + } + }, + }) + if err == nil { + t.Fatal("promoteDirectoryWithHooks() error = nil, want source-root symlink replacement failure") + } + assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original")) + assertPathIsSymlink(t, src) + assertPathMissing(t, dst) + assertNoMatchingTempDirectories(t, root, ".destination.tmp-") +} + func TestPromoteDirectoryRejectsInspectedDirectorySymlinkReplacement(t *testing.T) { root := t.TempDir() src := filepath.Join(root, "source") @@ -274,6 +300,38 @@ func TestPromoteDirectoryRejectsInspectedFileIdentityMismatch(t *testing.T) { assertNoMatchingTempDirectories(t, root, ".destination.tmp-") } +func TestPromoteDirectoryRejectsIdentityPreservingFileSymlinkReplacement(t *testing.T) { + root := t.TempDir() + src := filepath.Join(root, "source") + file := filepath.Join(src, "value.txt") + preserved := filepath.Join(src, "value-preserved.txt") + dst := filepath.Join(root, "destination") + mustWriteFile(t, file, []byte("original"), 0o644) + + replaced := false + err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{ + afterEntryInspect: func(path string) { + if replaced || path != file { + return + } + replaced = true + if err := os.Rename(file, preserved); err != nil { + t.Fatalf("Rename(inspected file) error = %v", err) + } + if err := os.Symlink(filepath.Base(preserved), file); err != nil { + t.Skipf("Symlink() unavailable: %v", err) + } + }, + }) + if err == nil { + t.Fatal("promoteDirectoryWithHooks() error = nil, want file symlink replacement failure") + } + assertFileBytes(t, preserved, []byte("original")) + assertPathIsSymlink(t, file) + assertPathMissing(t, dst) + assertNoMatchingTempDirectories(t, root, ".destination.tmp-") +} + func TestPromoteDirectoryRejectsInspectedDirectoryIdentityMismatch(t *testing.T) { root := t.TempDir() src := filepath.Join(root, "source") @@ -304,6 +362,38 @@ func TestPromoteDirectoryRejectsInspectedDirectoryIdentityMismatch(t *testing.T) assertNoMatchingTempDirectories(t, root, ".destination.tmp-") } +func TestPromoteDirectoryRejectsIdentityPreservingDirectorySymlinkReplacement(t *testing.T) { + root := t.TempDir() + src := filepath.Join(root, "source") + child := filepath.Join(src, "child") + preserved := filepath.Join(src, "child-preserved") + dst := filepath.Join(root, "destination") + mustWriteFile(t, filepath.Join(child, "value.txt"), []byte("original"), 0o644) + + replaced := false + err := promoteDirectoryWithHooks(src, dst, renameDirectoryNoReplace, sourceTraversalHooks{ + afterEntryInspect: func(path string) { + if replaced || path != child { + return + } + replaced = true + if err := os.Rename(child, preserved); err != nil { + t.Fatalf("Rename(inspected directory) error = %v", err) + } + if err := os.Symlink(filepath.Base(preserved), child); err != nil { + t.Skipf("Symlink() unavailable: %v", err) + } + }, + }) + if err == nil { + t.Fatal("promoteDirectoryWithHooks() error = nil, want directory symlink replacement failure") + } + assertFileBytes(t, filepath.Join(preserved, "value.txt"), []byte("original")) + assertPathIsSymlink(t, child) + assertPathMissing(t, dst) + assertNoMatchingTempDirectories(t, root, ".destination.tmp-") +} + func TestPromoteDirectoryDoesNotReplaceDestinationCreatedBeforeInstall(t *testing.T) { root := t.TempDir() src := filepath.Join(root, "source") @@ -369,6 +459,14 @@ func assertPathMissing(t *testing.T, path string) { } } +func assertPathIsSymlink(t *testing.T, path string) { + t.Helper() + info, err := os.Lstat(path) + if err != nil || info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("Lstat(%q) info = %v, error = %v, want symlink", path, info, err) + } +} + func treeLayout(t *testing.T, root string) []string { t.Helper() var layout []string