From 42ed81cbe1e4282c226d67513650926a62a93f65 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 29 Aug 2026 14:41:22 +0000 Subject: [PATCH] Update Notarius integration and plan references --- docs/integrations/notarius.md | 31 +- docs/internal/stage-extract.md | 11 +- docs/roadmap/implementation.md | 546 ++++++++++++++++++ docs/roadmap/notarius-v0.6-cli-references.md | 274 +++++++++ docs/troubleshooting.md | 2 +- internal/adapters/notarius/runner.go | 70 ++- internal/adapters/notarius/subprocess.go | 346 +++++++++-- internal/adapters/notarius/subprocess_test.go | 79 ++- internal/app/extract_lifecycle_test.go | 16 +- internal/stage/extract.go | 79 ++- internal/stage/extract_test.go | 110 +++- 11 files changed, 1450 insertions(+), 114 deletions(-) create mode 100644 docs/roadmap/implementation.md create mode 100644 docs/roadmap/notarius-v0.6-cli-references.md diff --git a/docs/integrations/notarius.md b/docs/integrations/notarius.md index a65673e..c03e544 100644 --- a/docs/integrations/notarius.md +++ b/docs/integrations/notarius.md @@ -38,24 +38,30 @@ environment apply to the subprocess. ## Accepted Result -Narratio currently accepts receipt schema `notarius.run-result.v1`. The receipt +Narratio accepts the Notarius v0.5.0 receipt schema +`notarius.run-result.v2`. The receipt must identify the configured pipeline, and its `index_file` must be exactly `index.json` beneath the reported bundle root. The production index must name -the management files exactly as `manifest.json`, `rejected.json`, and -`warnings.json`. All receipt, index, and lane paths must stay inside that -bundle; symlinks and non-regular lane payloads are rejected. +the management files exactly as `manifest.json`, `rejected.json`, +`warnings.json`, and `diagnostics.json`. All receipt, index, and lane paths must +stay inside that bundle; symlinks and non-regular lane payloads are rejected. Supported receipt and index shapes tolerate unknown fields for forward compatibility, while required identity, validation, count, manifest, -rejection, warning, and lane-list fields remain mandatory. Narratio applies -bounded reads to the receipt, index, rejection, and warning documents. Optional -chunk-map and evidence-context descriptors must carry their complete generic -contract metadata when present. +rejection, warning, diagnostic, and lane-list fields remain mandatory. +Narratio applies bounded reads to the receipt, index, rejection, warning, and +diagnostic documents. Warning and diagnostic envelopes, group counts, +occurrence counts, truncation state, framework-owned origins, and +receipt-to-bundle counts must be internally consistent. Optional chunk-map and +evidence-context descriptors must carry their complete generic contract +metadata when present. For every entry in `pipeline.notarius.outputs`, Narratio requires exactly one index descriptor with the configured lane ID, media type, schema ID, schema version, and, when configured, module key. Missing, duplicate, rejected, or -incompatible required lanes fail extraction even if Notarius exited zero. +incompatible required lanes fail extraction even if Notarius exited zero. A +configured lane whose v2 validation summary is `rejected` or `incomplete` also +fails extraction. Unconfigured lanes may remain in the preserved bundle but do not become selectable Narratio sources. @@ -73,9 +79,10 @@ only explicitly named lane sources; `--artifacts` never selects Notarius lanes. - Contract and external provenance metadata are preserved on lane artifact records and through explicit publication. -Rejection and warning summaries retain structured stage, scope, lane, and -reason-code fields for diagnostics without exposing free-form external messages -or reading lane payload bodies. +Rejection, validation, warning, and diagnostic summaries retain bounded stable +identity, category, origin, reason-code, status, and occurrence fields without +copying free-form external messages into Narratio manifest metadata or reading +lane payload bodies. Configuration fields and defaults are in [Configuration](../config.md). Operator paths, rerun procedures, and bundle retention are in diff --git a/docs/internal/stage-extract.md b/docs/internal/stage-extract.md index 6913862..4d78554 100644 --- a/docs/internal/stage-extract.md +++ b/docs/internal/stage-extract.md @@ -20,8 +20,9 @@ procedures belong in [Operations](../operations.md). 2. resolves and fingerprints the Notarius invocation contract; 3. creates a run-local staging directory and invokes the injected `notarius.Runner`; -4. validates the successful receipt, confined index, configured required lane - descriptors, and regular payload files; +4. validates the v2 successful receipt, confined index, management documents, + configured required lane descriptors, validation summaries, and regular + payload files; 5. atomically promotes the complete bundle to its immutable durable location; 6. records one non-selectable `notarius_index` output and one selectable `notarius_lane` output per configured lane; and @@ -65,8 +66,10 @@ Operators must force extraction after changing any such input. ## Failure Behavior Adapter startup, timeout, nonzero exit, receipt decoding, path confinement, -index compatibility, required-lane rejection, payload inspection, checksum, or -promotion errors fail the stage through ordinary manifest transition handling. +index compatibility, inconsistent warning or diagnostic envelopes, +required-lane rejection or incomplete validation, payload inspection, +checksum, or promotion errors fail the stage through ordinary manifest +transition handling. Stdout receipt and stderr diagnostics remain separate. Downstream stages are not given selectable extraction sources unless the complete configured result has passed validation and promotion. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md new file mode 100644 index 0000000..118b15d --- /dev/null +++ b/docs/roadmap/implementation.md @@ -0,0 +1,546 @@ +# Notarius v0.6 CLI References Implementation Plan + +## Purpose And Status + +This is the executable implementation plan for the accepted target state in +[Notarius v0.6 CLI Reference Integration](notarius-v0.6-cli-references.md). It +is written for a `gpt-5.6-terra` coding agent that will implement exactly one +pending stage per prompt, in order. + +The feature roadmap owns user intent, architectural boundaries, settled policy, +and the target end state. This document owns delivery order, concrete changes, +test allocation, and implementation status. Do not restate or change a roadmap +decision here during implementation; if current Notarius v0.6.0 evidence +contradicts the roadmap, stop and record the conflict instead of inventing a +different contract. + +| Stage | Outcome | Status | +| ---: | --- | --- | +| 1 | Add the reference-selector and configuration vocabulary, including optional spell-catalog inputs. | Pending | +| 2 | Materialize and inventory the optional spell catalog through the prepare and operator lifecycle. | Pending | +| 3 | Centralize manifest-authoritative prepared-input resolution and migrate analyze to it. | Pending | +| 4 | Add deterministic Notarius v0.6 reference arguments at the subprocess adapter boundary. | Pending | +| 5 | Resolve references in extract and bind fingerprints, resume, and metadata to their identities. | Pending | +| 6 | Prove assembled extraction lifecycle and downstream invalidation behavior. | Pending | +| 7 | Update canonical documentation and maintained examples for the completed feature. | Pending | +| 8 | Perform compatibility, quality, and repository-wide closure validation. | Pending | + +## Governing Decisions + +The following requirements are settled and are not questions for the +implementing agent: + +1. `pipeline.notarius.references` is a map whose key is a Notarius v0.6 CLI + reference selector and whose value is a prepared Narratio source ID. It is + not a map of paths. +2. Every configured binding is required. There is no per-entry `required` + field. An optional Notarius reference is omitted by omitting the map entry. + An empty or omitted map remains valid for custom pipelines and backward + compatibility. + The map is limited by the centrally declared configuration constant + `MaxNotariusReferenceBindings = 256`, which is far above the four-entry + maintained D&D case while bounding argv and manifest growth. +3. The supported prepared reference sources are + `narratio.input.party`, `narratio.input.players`, + `narratio.input.glossary`, and `narratio.input.spell_catalog`. + Arbitrary Notarius slot names and qualified selectors may bind those sources; + direct paths and later-stage artifacts may not. +4. Campaign and session `spell_catalog_file` are optional. A session value + overrides the campaign value; an empty session value inherits the campaign + value. A `narratio.input.spell_catalog` reference binding requires an + effective configured file. +5. Prepared sources are authoritative only when the current session manifest + records the matching canonical input and checksum. Consumers do not fall + back to campaign/session source paths or accept an incidental workspace file. +6. Narratio passes absolute prepared-file paths to Notarius. Fingerprints and + metadata use the canonical workspace-relative path identity together with + selector, source ID, SHA-256 checksum, and size so workspace relocation does + not become the only identity signal. +7. External CLI references are sorted by normalized selector. They override + matching external references in the Notarius configuration. Narratio never + emits `--without-reference`, the deprecated `roster` alias, or CLI bindings + for generated D&D artifact handoffs. +8. Notarius remains authoritative for whether a selected target declares a + slot, accepted reference media types and sizes, generated-handoff collisions, + pipeline topology, and D&D payload schemas. Narratio validates selector + structure and its own source contract only. +9. Notarius v0.6.0 is the minimum supported CLI contract when references are + configured. Do not add version-string parsing or an automatic per-session + `notarius config validate` subprocess. +10. The existing receipt-v2, bundle-confinement, diagnostic, ten-lane selection, + immutable promotion, and analysis-source behavior must remain intact. +11. The default test suite remains offline, deterministic, and independent of + a sibling checkout or installed Notarius binary. A real v0.6.0 smoke run is + useful supplementary evidence when locally available, not a default-suite + dependency. +12. Add no external Go dependency for this feature. Use narrow owner-specific + types and existing file, path, artifact, manifest, adapter, and stage + facilities. + +## Instructions For Every Stage + +For each implementation prompt, the coding agent must: + +1. Read `docs/development.md`, all three files under `docs/policy/`, the feature + roadmap, this plan, and the stage-specific documents and source named below. + Inspect the current tree because earlier stages may have changed names or + ownership boundaries. +2. Use the repository knowledge graph first for code discovery and call tracing; + use text search for documentation, configuration, examples, string literals, + and evidence the graph cannot supply. +3. Confirm the worktree state and preserve unrelated changes. Implement only the + current stage. Do not begin a later stage merely because an adjacent file is + open. +4. Keep production code, focused tests, fakes, and fixtures consistent within + the stage. Remove superseded helpers when their final caller migrates. Do not + retain two competing source maps, path resolvers, fingerprint paths, or + subprocess argument builders. +5. Follow the testing policy's ownership rule. Parser/config tests own selector + and configuration cases; artifact tests own prepared-file identity and + integrity; adapter tests own exact arguments; stage tests own orchestration and + resume; application tests own lifecycle invalidation. Do not repeat every + lower-level case at higher levels. +6. Keep errors actionable and content-free. They may identify a selector, + Narratio source ID, canonical path, or checksum mismatch, but must not include + reference contents. Preserve ordinary group-workspace permissions and + restrictive API-key handling. +7. Run `gofmt` on changed Go files and focused tests while iterating. Before + marking any stage complete, run at minimum: + + ```sh + go test ./... + go test -race ./... + go vet ./... + go build ./... + go test ./internal/doccheck + go test ./internal/config -run '^TestExamplesLoadAndValidate$' + ``` + + Default tests must not contact live services or require credentials. +8. Compare the final diff against the stage goal and exit criteria. Update only + the current stage's status row from `Pending` to `Completed`. Do not mark a + stage complete while a required check fails or required behavior is absent. + Intermediate commits are implementation-branch state and must not be released + before Stage 7 has reconciled current-behavior documentation. + +## Stage 1 — Reference And Configuration Vocabulary + +**Read first:** `docs/config.md`, `docs/integrations/notarius.md`, +`internal/config/config.go`, `internal/config/defaults.go`, +`internal/config/load.go`, `internal/config/validate.go`, +`internal/config/notarius_test.go`, `internal/config/campaign_config_test.go`, +and `internal/artifactpolicy/policy.go` and its tests. Read the tagged Notarius +v0.6.0 `docs/cli.md` reference-selector section from `../notarius` when that +checkout is available; otherwise use the canonical link from the feature +roadmap. + +**Depends on:** None. + +**Goal:** Establish one normalized reference-selector grammar and the strict +configuration model needed by later stages, without adding path-valued Notarius +configuration or making spell catalogs mandatory for every campaign. + +**Work:** + +- Add a small dependency-free `internal/notariusref` package as the contract + owner for Notarius reference selector normalization. Its exported normalizer + must trim the selector and each component, reject empty components and `=`, + and accept only the v0.6 forms `slot`, + `chunk.slot`, `lane.slot`, `lane.extract.slot`, `lane.merge.slot`, and + `lane.normalize.slot`. A three-component selector accepts only `extract`, + `merge`, or `normalize` in its middle component. Do not check the selector + against a Notarius module or lane registry. +- Add `References map[string]string` with YAML key `references` to + `config.NotariusConfig`. During enabled Notarius validation, sort raw keys, + normalize each selector through the shared contract, trim each source value, + reject empty values and normalized-selector collisions, require each value to + be one of the four prepared reference sources, and replace the config map with + its normalized form. Enforce the named, centrally discoverable + `config.MaxNotariusReferenceBindings` limit of 256 entries with an error that + identifies the field and limit. Keep a nil/empty map valid. Rely on strict YAML + decoding to reject duplicate identical keys, but explicitly reject distinct + raw keys that normalize to one selector. +- Add `SpellCatalogFile string` with YAML key `spell_catalog_file` to campaign + inputs and session inputs, plus `SpellCatalogFile ResolvedInputFile` to + resolved stable inputs. Merge it with the existing session-over-campaign + helper. It is not part of the campaign-required input set. Reject a non-empty + configured scalar that becomes empty after trimming. +- Add `artifactpolicy.SourceInputSpellCatalog` and make the artifact-policy + owner describe all four prepared reference sources, including their canonical + manifest kind and filename. Use that owner for stable-source recognition + instead of adding a second switch in configuration validation. Preserve the + existing three source IDs and their behavior. +- Extend cross-configuration validation so a normalized reference to + `narratio.input.spell_catalog` requires a non-empty effective resolved + `spell_catalog_file`. Existing campaign requirements already guarantee party, + players, and glossary declarations. Do not check filesystem existence during + configuration validation. +- If analyze's current private filename switch must change to keep the tree + behaviorally coherent, make it delegate to the artifact-policy descriptor and + recognize spell catalog; Stage 3 will replace the filesystem-only resolver. + +**Tests and exit criteria:** At the contract/config owners, cover every accepted +selector shape; zero, empty, excess, invalid-stage, and `=` forms; whitespace +normalization; normalized collisions; unsupported and empty source IDs; nil and +empty maps; exactly the configured binding limit and limit plus one; strict +unknown fields; campaign inheritance and session override; optional omission; +and the cross-config missing-spell-catalog failure. Prefer table-driven parser +and validator tests over assertions against private helper structure. Existing +configuration and example tests must still pass without adding spell catalogs +to every campaign. The codebase has one selector grammar owner and one +prepared-source descriptor owner. + +## Stage 2 — Spell Catalog Prepare And Operator Lifecycle + +**Read first:** `docs/internal/stage-prepare.md`, `docs/internal/workspace.md`, +`docs/operations.md`, `internal/stage/prepare.go` and its tests, +`internal/app/operator_inspection.go`, `internal/app/operator_findings.go` and +their focused tests, `internal/manifest/manifest.go`, and the relevant confined +file-operation helpers. + +**Depends on:** Stage 1. + +**Goal:** Make the effective optional spell catalog a normal prepared session +input with canonical storage, checksum/provenance, safe stale-file handling, and +operator visibility. + +**Work:** + +- Resolve `StableInputs.SpellCatalogFile` with the same origin-preserving + campaign/session behavior as the five existing stable inputs. When configured, + require a regular readable source, copy it atomically to + `inputs/spell_catalog.json`, preserve ordinary workspace permissions, and add + one manifest input record with kind `spell_catalog`, canonical destination, + checksum, and `campaign_config` or `session_config` source provenance. +- Treat the input as optional when no effective path is configured. Do not call + the required-input path resolver with an empty value and do not create a + manifest record. Remove an obsolete canonical `inputs/spell_catalog.json` + without following it when a forced prepare transitions from configured to + absent; refuse to recursively remove a directory or other ambiguous object at + that exact file path. +- Include a configured spell catalog in operator inspection and validation + findings. Omission is not an error unless Stage 1 cross-configuration policy + says the enabled Notarius reference requires it. Reuse the common resolved + stable-input enumeration where practical instead of extending parallel + hand-written lists in several functions. +- Adjust input-slice capacity, deterministic ordering, test fixtures, and any + manifest assumptions affected by the optional sixth stable file. Do not parse + or schema-validate the JSON payload in Narratio; Notarius owns that contract. + +**Tests and exit criteria:** Through prepare and operator package behavior, cover +campaign and session source provenance, canonical destination bytes and +checksum, optional omission, missing configured source, replacement after source +change, safe removal when configuration is removed, rejection of an ambiguous +destination object, deterministic manifest ordering, and operator reporting. +Do not duplicate selector-validation cases from Stage 1. Existing sessions with +no spell catalog remain valid and produce no stale manifest entry. + +## Stage 3 — Manifest-Authoritative Prepared Input Resolution + +**Read first:** `docs/internal/artifacts.md`, `docs/internal/manifest.md`, +`docs/internal/stage-analyze.md`, `internal/artifacts/artifact_resolver.go`, +`internal/artifacts/resolve.go`, `internal/artifacts/checksum.go`, their tests, +and the prepared stable-input resolution path in `internal/stage/analyze.go` and +`internal/stage/analyze_test.go`. + +**Depends on:** Stage 2. + +**Goal:** Give analyze and extract one integrity-checked resolver for prepared +stable sources so neither stage trusts incidental files or reconstructs its own +source-to-filename table. + +**Work:** + +- Add an artifacts-owned `PreparedInputIdentity` contract containing source ID, + manifest kind, absolute canonical path, slash-separated path relative to the + session root, SHA-256 checksum, and byte size. Add one resolver that accepts + session paths, the current session manifest, and a stable source ID. Provide a + typed or sentinel absence classification so callers can distinguish no current + manifest record from corrupt or unsafe recorded evidence. +- Derive kind and filename exclusively from the artifact-policy descriptor. The + resolver must require exactly one current manifest input record with the + expected kind and canonical path; resolve/rebase recorded local paths through + existing session-local path safety helpers; require the result to equal the + canonical file below `inputs/`; reject escapes, symlinks, non-regular files, + empty files, missing checksums, duplicate records, and checksum mismatches; and + calculate size without loading the complete file into memory. Do not fall back + to the configured campaign/session path or accept canonical file presence + without manifest evidence. Zero matching manifest records is the typed absent + case; once a record exists, a missing or invalid file is an integrity error, + not optional absence. +- Return owner-neutral errors from `internal/artifacts`. At stage boundaries, + wrap unavailable or stale prepared inputs with the source ID and actionable + `narratio run-stage prepare --force` guidance. Do not include file + contents. +- Replace analyze's private prepared-source filename switch and filesystem-only + resolver with the shared artifact resolver. Preserve required-versus-optional + Scriptorium input behavior: a typed absent optional source is omitted, an + absent required source fails with prepare guidance, and invalid recorded + evidence fails regardless of optionality. Make `narratio.input.spell_catalog` + usable wherever another prepared Scriptorium source is accepted. + +**Tests and exit criteria:** Artifact-package tests own valid resolution and the +missing-record, duplicate-record, wrong-kind/path, traversal/rebase, symlink, +non-regular, empty, missing-checksum, and checksum-mismatch boundaries. Analyze +tests need only prove required/optional stage behavior and successful use of the +shared source, including spell catalog; do not repeat the artifact resolver's +full matrix. Remove the old filename/path resolver after its final caller moves. + +## Stage 4 — Notarius Adapter Reference Arguments + +**Read first:** `docs/internal/adapters.md`, `docs/integrations/notarius.md`, +`internal/adapters/notarius/runner.go`, `fake.go`, `subprocess.go`, and focused +adapter tests. Re-read the Notarius v0.6.0 subprocess and CLI reference-selector +contracts from the tagged sibling checkout when available. + +**Depends on:** Stage 1. + +**Goal:** Extend the transport-neutral Notarius request and production adapter +to emit safe, exact, repeatable v0.6 `--reference` arguments without changing +receipt or bundle ingestion. + +**Work:** + +- Add a transport-neutral reference binding containing normalized selector and + absolute path, and add an ordered slice of those bindings to `RunRequest`. + Keep source IDs and manifest identities out of the adapter contract; those are + stage policy. +- Validate each adapter binding before process launch: normalize/validate the + selector through the shared Stage 1 contract, require a non-empty absolute + path, reject duplicate normalized selectors, and avoid mutating the caller's + slice. Do not open or parse the reference file in the adapter. +- Build arguments as repeated pairs `--reference`, + `=` after `--output-dir` and before + `--json`. Preserve one argument for the combined selector/path value so spaces, + additional `=` characters within the path portion, and platform separators do + not involve shell interpretation. The request order is authoritative; Stage 5 + will supply sorted bindings. +- Preserve current executable, environment, timeout, cancellation, diagnostic, + receipt-v2, bounded-read, confinement, and bundle-discovery behavior. Do not + add `--without-reference`, generated reference arguments, version probing, or + configuration preflight. +- Update the fake only as required to retain and expose the extended request. + +**Tests and exit criteria:** Adapter tests own exact argv with zero and multiple +references, position before `--json`, spaces and `=` in paths, selector +normalization, duplicate/invalid selector rejection, relative/empty path +rejection, and no subprocess start after request-validation failure. Existing +receipt-v2 and bundle fixture tests must remain unchanged in meaning and pass. +Do not assert stage-level source sorting here beyond preserving the request +order. + +## Stage 5 — Extract Reference Identity, Invocation, And Resume + +**Read first:** `docs/internal/stage-extract.md`, +`docs/integrations/notarius.md`, `docs/internal/manifest.md`, +`internal/stage/extract.go`, `internal/stage/extract_resume.go`, their focused +tests, the Stage 3 prepared-input identity contract, and the Stage 4 Notarius +request contract. + +**Depends on:** Stages 3 and 4. + +**Goal:** Make configured references part of the actual extraction invocation +and durable reuse contract, using one resolution path for initial execution and +resume validation. + +**Work:** + +- Add one extract-owned reference-resolution helper used by both `Run` and + `ValidateResume`. Iterate normalized config bindings in lexical selector + order, resolve each source through the Stage 3 manifest-authoritative resolver, + and produce both adapter bindings and immutable reference identities. Resolve + every reference before creating run-local receipt, log, output, or promotion + directories and before invoking the adapter. +- Define the fingerprint/metadata identity as normalized selector, source ID, + canonical session-relative slash path, SHA-256 checksum, and byte size. Do not + include contents or original campaign/session absolute paths. Pass only + selector and absolute prepared path to the adapter. +- Extend the extraction fingerprint document with the sorted reference + identities. Keep all existing binary, config path, pipeline, timeout, working + directory, trimmed-transcript identity, and required-output identities. The + result must be independent of YAML map iteration order and must change for a + selector, source, relative path, checksum, or size change. +- Persist `reference_count` and a bounded deterministic `references` metadata + list on successful extraction. Each entry contains exactly `selector`, + `source_id`, `path`, `checksum`, and `size_bytes`. Empty bindings produce count + zero and an empty list. Do not duplicate Notarius reference payloads or + downstream error messages. +- Make resume recompute current reference identities through the same helper + before comparing the configuration fingerprint. A valid changed prepared + input yields a fingerprint mismatch and a non-resumable result so extraction + reruns. Missing, unsafe, or checksum-inconsistent current input is an error + with prepare-force guidance because immediately rerunning extract cannot + succeed. Do not silently reuse the old bundle. +- Preserve explicit disabled-stage skip without resolving references. Preserve + required lane selection, immutable promotion, receipt identity, and bundle + evidence behavior. + +**Tests and exit criteria:** Stage tests own sorted request construction for all +four D&D bindings, zero bindings, failure before adapter invocation for an +unavailable source, content-free metadata, and fingerprint changes for each +identity field while remaining stable across map order. Resume tests must prove +reuse with unchanged references, non-reuse after a valid prepared-reference +change, hard failure for missing or checksum-invalid current evidence, and no +reference resolution when disabled. Use the fake adapter; do not duplicate exact +subprocess argv cases from Stage 4. + +## Stage 6 — Assembled Lifecycle And Invalidation Coverage + +**Read first:** `docs/internal/overview.md`, `docs/internal/manifest.md`, +`docs/internal/stage-extract.md`, `docs/internal/stage-prepare.md`, +`internal/app/runner.go`, +`internal/app/extract_lifecycle_test.go`, and representative full pipeline and +stage fixtures. Inspect existing downstream invalidation tests before adding +new cases. + +**Depends on:** Stage 5. + +**Goal:** Prove at the application boundary that prepared campaign context +reaches Notarius and that reference changes cannot leave extraction or later +analysis falsely current. + +**Work:** + +- Extend the smallest existing assembled runner fixture to execute prepare and + extract with party, players, glossary, and spell catalog bindings. Assert that + the fake Notarius request receives the four canonical prepared absolute paths, + not the original campaign/session source paths, and that the successful + manifest records bounded reference identity. +- Add one lifecycle regression covering a valid reference-content change: + rerun/force prepare so the manifest and prepared checksum change, then verify + extract resume is rejected, Notarius runs again, and succeeded canonical + downstream stages are invalidated according to the existing stage-order + policy. Assert outcomes, not private runner call choreography. +- Add one representative session override case to prove the overridden prepared + bytes/checksum reach extraction. Do not repeat all four configuration merge + cases or artifact-integrity failures already owned by earlier stages. +- Confirm an empty reference map preserves the pre-v0.6 invocation behavior and + that all ten configured D&D lanes remain registered as the same + `narratio.extraction.` sources available to analyze. +- Fix production integration defects exposed by these assembled tests without + broadening the feature or adding a DAG, generic reference workflow, or direct + Notarius payload parsing. + +**Tests and exit criteria:** The application-level tests must be deterministic, +offline, and fake only the external Notarius boundary. They must credibly fail +if Narratio passes original paths, omits one configured reference, reuses stale +extraction, or loses a configured lane, while remaining insensitive to private +helper structure and exact non-contractual diagnostics. Earlier focused suites +and the repository baseline remain green. + +## Stage 7 — Canonical Documentation And Maintained Examples + +**Read first:** `docs/policy/documentation.md`, `docs/config.md`, +`docs/operations.md`, `docs/troubleshooting.md`, +`docs/integrations/notarius.md`, `docs/internal/overview.md`, +`docs/internal/adapters.md`, `docs/internal/artifacts.md`, +`docs/internal/stage-prepare.md`, `docs/internal/stage-extract.md`, +`docs/internal/stage-analyze.md`, `examples/README.md`, and all maintained +pipeline, campaign, and session examples affected by the new fields. + +**Depends on:** Stage 6. + +**Goal:** Move the completed behavior from roadmap-only future state into its +canonical current-behavior owners and provide valid copyable D&D examples +without duplicating volatile Notarius contracts. + +**Work:** + +- Update `docs/config.md` with `pipeline.notarius.references`, its selector-to- + source shape, normalization/validation rules, required-by-presence behavior, + supported stable source IDs, and campaign/session `spell_catalog_file` + precedence and optionality. Keep complete copyable YAML in `examples/`. +- Update `docs/integrations/notarius.md` to the v0.6.0 baseline and exact + repeatable-reference invocation boundary. Explain absolute CLI paths, + precedence over configured external paths, the four maintained external D&D + slots, the generated-handoff exclusion, and unchanged receipt-v2/ten-lane + output compatibility. Link to Notarius's canonical v0.6 CLI and D&D consumer + docs instead of copying its target/module matrix. +- Update operations and troubleshooting with prepared input location, + fingerprint/rerun consequences, operator inspection, missing-reference + diagnosis, and Notarius undeclared-slot/generated-collision failures. Update + internal component documents only with implemented ownership and flow; do not + duplicate configuration field definitions there. +- Add a valid, secret-free sample spell catalog following Notarius v0.6's + published overlay schema, add `spell_catalog_file` to the sample campaign, and + configure all four external reference bindings in the complete annotated D&D + pipeline. Add the same bindings to other Notarius-enabled maintained examples + only when their selected pipeline declares them; do not add a Notarius section + to examples that intentionally omit extraction. +- Ensure the maintained command snippets place repeated `--reference` arguments + before `--json`, use `party` rather than `roster`, and never show generated + handoffs on the CLI. Remove stale v0.5 compatibility wording where it refers to + the supported invocation baseline. + +**Tests and exit criteria:** Run documentation-link checks and the example +loader explicitly. Verify every changed example is accepted by strict config +validation, contains no credentials or private infrastructure values, and has +one canonical owner for each volatile fact. Search current-behavior docs and +examples for stale v0.5 invocation wording, deprecated `roster` emission, and +generated D&D CLI handoff examples. Do not mark the roadmap itself implemented; +its status remains target-state context until the implementation sprint is +reviewed and closed. + +## Stage 8 — Compatibility And Quality Closure + +**Read first:** The feature roadmap, every completed stage diff, the final +current-behavior docs, `.woodpecker/verify.yml`, `.woodpecker/release.yml`, and +`.woodpecker/shuffle.yml`. Re-read the tagged Notarius v0.6.0 +`docs/consumers/dnd-pipeline.md`, `docs/cli.md`, and linked spell-catalog overlay +contract when the sibling checkout is available. + +**Depends on:** Stage 7. + +**Goal:** Verify the delivered code matches the accepted boundary, remains +compatible with all ten default D&D artifacts, and is ready for review without +dead compatibility paths or duplicated policy. + +**Work:** + +- Audit the final diff against every target-state and out-of-scope statement in + the feature roadmap. Confirm only four external prepared sources are exposed, + custom selectors remain possible, every configured binding is required, and + no Notarius pipeline topology or generated-handoff logic moved into Narratio. +- Trace initial extract and resume paths to confirm both use the same prepared + identity and reference resolution, the adapter is the sole argv builder, and + artifact policy is the sole source-to-kind/filename vocabulary. Remove dead + helpers, redundant switches, stale fixtures, and low-value duplicate tests + found during this review. +- Confirm the complete D&D example still declares and validates the exact ten + output lanes and that analyze can consume those sources after reference- + enabled extraction. Confirm empty-reference custom pipelines remain supported. +- If a local Notarius v0.6.0 binary and its required offline/test configuration + are already available, perform a non-credentialed smoke invocation with all + four reference flags and record the result in the implementation handoff. Do + not download tools, contact paid providers, add a default test dependency, or + block completion solely because this supplementary environment is absent. +- Run the repository baseline plus the scheduled shuffled suite and release + cross-build commands: + + ```sh + go test ./... + go test -race ./... + go test -race -shuffle=on -count=3 ./... + go vet ./... + go build ./... + go test ./internal/doccheck + go test ./internal/config -run '^TestExamplesLoadAndValidate$' + narratio_cross_dir="$(mktemp -d)" + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "$narratio_cross_dir/narratio-linux-amd64" ./cmd/narratio + CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o "$narratio_cross_dir/narratio-darwin-amd64" ./cmd/narratio + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o "$narratio_cross_dir/narratio-windows-amd64.exe" ./cmd/narratio + ``` + + Cross-builds are compilation evidence only; do not claim native macOS or + Windows runtime validation. + +**Tests and exit criteria:** Every required command passes, `git diff --check` +is clean, the worktree contains no unintended generated test artifacts, and the +implementation is traceably complete against the roadmap. Summarize any +unavailable supplementary smoke evidence without treating it as a product +question or silently weakening the default suite. + +## Open Questions + +None. The feature roadmap and governing decisions above are sufficient to +implement the plan without additional product or architecture choices. diff --git a/docs/roadmap/notarius-v0.6-cli-references.md b/docs/roadmap/notarius-v0.6-cli-references.md new file mode 100644 index 0000000..789c67c --- /dev/null +++ b/docs/roadmap/notarius-v0.6-cli-references.md @@ -0,0 +1,274 @@ +# Notarius v0.6 CLI Reference Integration + +## Status + +Accepted target state. Delivery sequencing and implementation status are owned +by [implementation.md](implementation.md). + +## Purpose + +Upgrade Narratio's extraction boundary to the Notarius v0.6.0 subprocess +contract and supply session reference documents explicitly with repeatable +`--reference selector=path` arguments. + +The maintained D&D integration must make the prepared party roster, player +context, glossary, and optional spell catalog available to every compatible +Notarius target. Notarius must continue to own pipeline topology, reference-slot +compatibility, generated artifact handoffs, prompts, and D&D schemas. Narratio +owns selection and preparation of its external reference files, exact CLI +invocation, provenance, and extraction reuse correctness. + +## Current State And Gap + +Narratio currently invokes Notarius as: + +```text +notarius run --config --input --output-dir --json +``` + +The `prepare` stage already materializes campaign/session party, players, and +glossary files under the session `inputs/` directory, but `extract` does not +pass them to Notarius. Narratio also has no stable spell-catalog input. As a +result, a Notarius deployment must duplicate these paths in its own +configuration, cannot reliably receive session overrides, and may extract +without the same campaign context supplied to Narratio's analysis stage. + +Notarius v0.6.0 makes an unqualified CLI selector pipeline-scoped. For example, +`--reference party=/absolute/path/party.yml` supplies the file to every +selected target that declares `party`. Scoped selectors remain available for +exceptional overrides. CLI paths are resolved from the Notarius process working +directory, so subprocess callers are expected to provide absolute paths. + +The v0.6.0 receipt, index, warning, diagnostic, and ten-lane D&D artifact +contracts remain compatible with Narratio's current v0.5 integration. This +feature changes the invocation and input-provenance contract rather than the +accepted output inventory. + +## User Outcome + +With the maintained complete D&D configuration, an operator can declare the +campaign reference sources once in Narratio. For each extraction Narratio will: + +1. materialize the effective campaign/session files during `prepare`; +2. resolve those prepared files by stable Narratio source ID; +3. pass absolute paths for `party`, `players`, `glossary`, and, when configured, + `spell_catalog` to Notarius through repeatable CLI arguments; +4. fail before launching Notarius when a configured reference is unavailable; +5. rerun extraction when a selector, source binding, or reference file changes; + and +6. retain bounded reference identities and checksums for diagnosis and + provenance without copying reference contents into manifest metadata. + +Session-level stable-input overrides must flow through the same mechanism. A +custom Notarius pipeline may bind different external slots without requiring a +Narratio code change. + +## Chosen Architecture + +### Explicit Reference Bindings + +Extend `pipeline.notarius` with an explicit map from a Notarius CLI selector to +a prepared Narratio input source: + +```yaml +notarius: + enabled: true + binary: notarius + config_path: /usr/local/etc/notarius/config.yml + pipeline_id: dnd-session + working_directory: /usr/local/etc/notarius + references: + party: narratio.input.party + players: narratio.input.players + glossary: narratio.input.glossary + spell_catalog: narratio.input.spell_catalog + outputs: + # Existing required lane contracts remain unchanged. +``` + +Each configured binding is required. An operator who does not maintain an +optional Notarius reference, such as a spell catalog, omits that binding. This +keeps missing-input behavior explicit and avoids a second required/optional +policy inside each entry. + +The maintained complete D&D example will show all four external reference +slots. The three existing campaign context bindings use the canonical `party`, +`players`, and `glossary` spellings. Narratio will not emit the deprecated +`roster` alias. + +The binding is deliberately source-based rather than path-based. Pipeline +configuration should not reconstruct session workspace paths or bypass +`prepare`; it names the stable input whose effective campaign/session value is +already owned by Narratio. The map also avoids hard-coded behavior keyed to the +literal `dnd-session` pipeline ID, preserving custom-pipeline support. + +Narratio accepts the selector forms published by Notarius v0.6.0: + +- `slot`; +- `chunk.slot`; +- `lane.slot`; and +- `lane.extract.slot`, `lane.merge.slot`, or `lane.normalize.slot`. + +Configuration validation will reject empty or structurally invalid selectors, +selectors containing `=`, unsupported source IDs, and duplicate YAML keys. +Notarius remains authoritative for whether a selected target actually declares +the slot and whether a file satisfies that slot's media type and size contract. +Narratio will not duplicate the Notarius module registry. + +### Stable Reference Inputs + +Continue to use the existing prepared sources and canonical files: + +| Narratio source | Prepared file | Notarius slot | +| --- | --- | --- | +| `narratio.input.party` | `inputs/party.yml` | `party` | +| `narratio.input.players` | `inputs/players.yml` | `players` | +| `narratio.input.glossary` | `inputs/glossary.yml` | `glossary` | +| `narratio.input.spell_catalog` | `inputs/spell_catalog.json` | `spell_catalog` | + +Add optional `spell_catalog_file` fields to campaign and session inputs, with +the existing campaign-default/session-override resolution behavior. When +provided, `prepare` copies it into the session input area and records its +origin and checksum consistently with the other stable inputs. The prepared +filename remains JSON so Notarius can apply its published spell-catalog media +contract. + +The new source must be added everywhere stable inputs are enumerated: strict +configuration decoding and merging, validation, prepare materialization, +artifact policy/source descriptions, operator inspection, manifest input +records, examples, and canonical documentation. It remains optional at the +campaign level; a configured Notarius binding makes it mandatory for that +extraction. + +Extract and analyze should use one shared prepared-input source resolver rather +than maintain separate source-to-filename tables. The resolver must return an +absolute, regular, non-empty file beneath the current session workspace and +produce actionable `prepare --force` guidance when a configured source is +missing. It must not fall back to the original campaign path after preparation. + +### Adapter Request And CLI Construction + +Extend the transport-neutral Notarius run request with an ordered collection of +resolved reference bindings. Each binding contains only its selector and +absolute prepared-file path. The extraction stage resolves source IDs and file +identity; the subprocess adapter validates and serializes the request. + +The production command becomes: + +```text +notarius run + --config + --input + --output-dir + --reference party= + --reference players= + --reference glossary= + --reference spell_catalog= + --json +``` + +Only configured bindings are emitted. Selectors are sorted before request +construction so argument order, tests, logs, and fingerprints are deterministic. +Arguments are passed directly to the subprocess without shell interpretation; +paths containing spaces or platform-specific separators remain one argument. + +CLI bindings intentionally override matching external paths in the deployed +Notarius configuration. Narratio must not pass `--without-reference` and must +not synthesize CLI bindings for `location_registry`, `item_registry`, +`npc_registry`, `scene_descriptions`, `combat_turns`, or `npc_occurrences`. +Those are generated same-run artifact handoffs in the complete D&D pipeline and +remain entirely under Notarius configuration and execution control. A custom +configuration that collides an external CLI binding with a generated handoff is +expected to fail with Notarius's normal resolution error. + +### Fingerprints, Resume, And Provenance + +Reference identity is part of the extraction input contract. The extraction +fingerprint and resume validator must include, in deterministic selector order: + +- the selector; +- the configured Narratio source ID; +- the resolved prepared path identity; and +- the prepared file's content checksum and size. + +This is required even though Notarius generates a prompt session ID from the +input module and transcript bytes: Notarius intentionally does not include +references in that identifier. Narratio must therefore prevent an old +extraction from being reused after a roster, player list, glossary, spell +catalog, selector, or source mapping changes. + +A changed reference makes the prior `extract` result non-reusable and follows +Narratio's normal downstream invalidation rules. A failed reference-resolution +or checksum check also prevents reuse; it must not silently accept the prior +bundle. + +Successful extract metadata should record a bounded, deterministic list of +selector, source ID, workspace-relative path, checksum, and size. It must not +record reference contents, original absolute operator paths, or values from the +files. Existing receipt and bundle provenance behavior remains unchanged. + +### Error And Compatibility Behavior + +Narratio's documented minimum supported Notarius version becomes v0.6.0 for an +enabled reference binding. Compatibility remains contract-based rather than +dependent on parsing `notarius --version`: an older or incompatible executable +will fail at the CLI boundary with captured diagnostics. + +Errors must identify the responsible selector and Narratio source without +including file contents. Configuration errors are reported before pipeline +execution. Missing, empty, non-regular, unsafe, or unreadable prepared files +fail extraction before the Notarius subprocess starts. Notarius continues to +report undeclared slots, media incompatibility, size limits, required-slot +failures, and generated-handoff collisions. + +When Notarius is disabled, extraction retains its current explicit skip +behavior and does not resolve reference inputs. Receipt v2 ingestion, bundle +confinement, ten-lane selection, and downstream artifact source IDs are not +otherwise changed by this feature. + +## Target End State + +Narratio and Notarius have a clear orchestration boundary: + +- `prepare` owns the effective, immutable session copies of external campaign + context; +- `extract` maps configured stable source IDs to Notarius v0.6 CLI selectors, + supplies absolute file paths, and owns reuse/provenance policy; +- the Notarius adapter owns exact subprocess serialization and supported result + decoding; +- Notarius owns slot compatibility, reference precedence within its pipeline, + generated artifact handoffs, and output schemas; and +- `analyze` consumes the resulting ten structured lane artifacts exactly as it + does today. + +The maintained complete D&D workflow passes party, players, glossary, and spell +catalog context from the same prepared session inputs used elsewhere in +Narratio. Updating any of those documents deterministically causes fresh +extraction, and operators can diagnose the effective bindings without exposing +file contents. + +## Out Of Scope + +- Reproducing Notarius pipeline, lane, binding, or media-type validation in + Narratio. +- Passing or overriding Notarius generated artifact handoffs. +- Adding `--without-reference`, Notarius resume/recompute controls, lane + selection, model selection, profile selection, or session-ID overrides. +- Changing the ten accepted D&D lane contracts or the Scriptorium analysis + design. +- Reading reference payloads into Narratio manifests or logs. +- Automatically running `notarius config validate` for every session. + +## Settled Policy Choices + +The implementation must preserve these choices unless implementation evidence +shows a contract conflict: + +- explicit selector-to-source mappings are preferred over pipeline-ID-specific + defaults; +- every configured mapping is required; +- `spell_catalog_file` is optional until a mapping requests its prepared + source; +- the complete D&D example demonstrates all four external references; and +- Notarius v0.6.0 is the minimum supported CLI contract for reference-enabled + extraction. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 3806e66..1d393eb 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -198,7 +198,7 @@ Safe fix: - compare installed Notarius output with the canonical Notarius contracts, including receipt `index_file: index.json` and index management names - `manifest.json`, `rejected.json`, and `warnings.json`; align + `manifest.json`, `rejected.json`, `warnings.json`, and `diagnostics.json`; align `pipeline.notarius` constraints and rerun. Do not bypass confinement or schema checks. diff --git a/internal/adapters/notarius/runner.go b/internal/adapters/notarius/runner.go index 165119c..faa156d 100644 --- a/internal/adapters/notarius/runner.go +++ b/internal/adapters/notarius/runner.go @@ -6,7 +6,7 @@ import ( "time" ) -const ReceiptSchemaVersion = "notarius.run-result.v1" +const ReceiptSchemaVersion = "notarius.run-result.v2" // Runner is the adapter boundary for a complete Notarius pipeline invocation. type Runner interface { @@ -28,16 +28,36 @@ type RunRequest struct { // Receipt is the transport-neutral successful run receipt. type Receipt struct { - SchemaVersion string - RunID string - PipelineID string - OutputDirectory string - IndexFile string - NormalizedOutputCount int - RejectedOutputCount int - WarningCount int - ValidationStatus string - DebugDirectory string + SchemaVersion string + RunID string + PipelineID string + OutputDirectory string + IndexFile string + NormalizedOutputCount int + RejectedOutputCount int + WarningGroupCount int + WarningOccurrenceCount int + DiagnosticGroupCount int + DiagnosticOccurrenceCount int + DiagnosticsTruncated bool + ValidationStatus string + ValidationSummaries []ValidationSummary + DebugDirectory string +} + +// ValidationSummary retains the bounded outcome of one Notarius producer result. +type ValidationSummary struct { + Stage string + StepID string + LaneID string + ModuleKey string + ChunkID string + Status string + RejectingValidators []string + ReasonCodes []string + IncompleteValidators []string + ProducerAttemptCount int + TerminalAction string } // LaneDescriptor identifies one normalized lane payload discovered through the index. @@ -72,6 +92,8 @@ type Index struct { RejectedPath string WarningsFile string WarningsPath string + DiagnosticsFile string + DiagnosticsPath string Lanes []LaneDescriptor ChunkMap *PipelineDescriptor EvidenceContext *PipelineDescriptor @@ -90,8 +112,29 @@ type RejectionSummary struct { // WarningSummary retains structured warning identity without free-form messages. type WarningSummary struct { - Scope string - ReasonCode string + Disposition string + Category string + ReasonCode string + Origin DiagnosticOrigin + OccurrenceCount int +} + +// DiagnosticOrigin identifies the framework-owned pipeline location of a finding. +type DiagnosticOrigin struct { + Stage string + StepID string + LaneID string + ModuleKey string + ValidatorKey string +} + +// DiagnosticSummary retains bounded advisory or observation group metadata. +type DiagnosticSummary struct { + Disposition string + Category string + ReasonCode string + Origin DiagnosticOrigin + OccurrenceCount int } // RunResult describes a successfully decoded and validated Notarius bundle. @@ -105,4 +148,5 @@ type RunResult struct { Duration time.Duration Rejections []RejectionSummary Warnings []WarningSummary + Diagnostics []DiagnosticSummary } diff --git a/internal/adapters/notarius/subprocess.go b/internal/adapters/notarius/subprocess.go index 5e1447a..940f085 100644 --- a/internal/adapters/notarius/subprocess.go +++ b/internal/adapters/notarius/subprocess.go @@ -15,13 +15,19 @@ import ( ) const ( - maxReceiptBytes = 1 << 20 - maxIndexBytes = 4 << 20 - maxSummaryBytes = 4 << 20 - canonicalIndexFile = "index.json" - canonicalManifestFile = "manifest.json" - canonicalRejectedFile = "rejected.json" - canonicalWarningsFile = "warnings.json" + maxReceiptBytes = 1 << 20 + maxIndexBytes = 4 << 20 + maxSummaryBytes = 4 << 20 + canonicalIndexFile = "index.json" + canonicalManifestFile = "manifest.json" + canonicalRejectedFile = "rejected.json" + canonicalWarningsFile = "warnings.json" + canonicalDiagnosticsFile = "diagnostics.json" + warningsSchemaVersion = "notarius.warnings.v2" + diagnosticsSchemaVersion = "notarius.diagnostics.v1" + maxWarningGroups = 128 + maxDiagnosticGroups = 256 + maxFindingSamples = 3 ) type subprocessRun func(context.Context, subprocess.RunRequest) (subprocess.RunResult, error) @@ -95,12 +101,30 @@ func (r *SubprocessRunner) Run(ctx context.Context, req RunRequest) (RunResult, if err != nil { return baseResult, err } + diagnostics, diagnosticOccurrences, diagnosticsTruncated, err := loadDiagnostics(index.DiagnosticsPath) + if err != nil { + return baseResult, err + } + if receipt.NormalizedOutputCount != len(index.Lanes) || receipt.RejectedOutputCount != len(rejections) || + receipt.WarningGroupCount != len(warnings) || receipt.DiagnosticGroupCount != len(diagnostics) { + return baseResult, fmt.Errorf("notarius receipt counts do not match published bundle") + } + warningOccurrences, err := sumWarningOccurrences(warnings) + if err != nil { + return baseResult, err + } + if receipt.WarningOccurrenceCount != warningOccurrences || + receipt.DiagnosticOccurrenceCount != diagnosticOccurrences || + receipt.DiagnosticsTruncated != diagnosticsTruncated { + return baseResult, fmt.Errorf("notarius receipt occurrence counts do not match published bundle") + } baseResult.Receipt = receipt baseResult.Index = index baseResult.BundleRoot = bundleRoot baseResult.Rejections = rejections baseResult.Warnings = warnings + baseResult.Diagnostics = diagnostics return baseResult, nil } @@ -154,16 +178,74 @@ func validateRunRequest(req RunRequest) error { } type receiptDocument struct { - SchemaVersion string `json:"schema_version"` - RunID string `json:"run_id"` - PipelineID string `json:"pipeline_id"` - OutputDirectory string `json:"output_directory"` - IndexFile string `json:"index_file"` - NormalizedOutputCount *int `json:"normalized_output_count"` - RejectedOutputCount *int `json:"rejected_output_count"` - WarningCount *int `json:"warning_count"` - ValidationStatus string `json:"validation_status"` - DebugDirectory string `json:"debug_directory"` + SchemaVersion string `json:"schema_version"` + RunID string `json:"run_id"` + PipelineID string `json:"pipeline_id"` + OutputDirectory string `json:"output_directory"` + IndexFile string `json:"index_file"` + NormalizedOutputCount *int `json:"normalized_output_count"` + RejectedOutputCount *int `json:"rejected_output_count"` + WarningGroupCount *int `json:"warning_group_count"` + WarningOccurrenceCount *int `json:"warning_occurrence_count"` + DiagnosticGroupCount *int `json:"diagnostic_group_count"` + DiagnosticOccurrenceCount *int `json:"diagnostic_occurrence_count"` + DiagnosticsTruncated *bool `json:"diagnostics_truncated"` + ValidationStatus string `json:"validation_status"` + ValidationSummaries []validationSummaryDocument `json:"validation_summaries"` + DebugDirectory string `json:"debug_directory"` +} + +type validationSummaryDocument struct { + Stage string `json:"stage"` + StepID string `json:"step_id"` + LaneID string `json:"lane_id"` + ModuleKey string `json:"module_key"` + ChunkID string `json:"chunk_id"` + Status string `json:"status"` + RejectingValidators []string `json:"rejecting_validators"` + ReasonCodes []string `json:"reason_codes"` + IncompleteValidators []string `json:"incomplete_validators"` + ProducerAttemptCount *int `json:"producer_attempt_count"` + TerminalAction string `json:"terminal_action"` +} + +func validValidationStatus(value string) bool { + switch value { + case "approved", "rejected", "incomplete": + return true + default: + return false + } +} + +func validateValidationSummaries(documents []validationSummaryDocument) ([]ValidationSummary, error) { + summaries := make([]ValidationSummary, 0, len(documents)) + for _, document := range documents { + if document.Status != "complete" && document.Status != "rejected" && document.Status != "incomplete" { + return nil, fmt.Errorf("notarius validation summary status %q is invalid", document.Status) + } + if document.ProducerAttemptCount == nil || *document.ProducerAttemptCount <= 0 || !validTerminalAction(document.TerminalAction) { + return nil, fmt.Errorf("notarius validation summary is missing required fields") + } + summaries = append(summaries, ValidationSummary{ + Stage: document.Stage, StepID: document.StepID, LaneID: document.LaneID, + ModuleKey: document.ModuleKey, ChunkID: document.ChunkID, Status: document.Status, + RejectingValidators: append([]string(nil), document.RejectingValidators...), + ReasonCodes: append([]string(nil), document.ReasonCodes...), + IncompleteValidators: append([]string(nil), document.IncompleteValidators...), + ProducerAttemptCount: *document.ProducerAttemptCount, TerminalAction: document.TerminalAction, + }) + } + return summaries, nil +} + +func validTerminalAction(value string) bool { + switch value { + case "accepted", "reject_output", "warn_continue", "fail_run": + return true + default: + return false + } } func loadReceipt(path, pipelineID string) (Receipt, error) { @@ -177,7 +259,9 @@ func loadReceipt(path, pipelineID string) (Receipt, error) { if strings.TrimSpace(document.RunID) == "" || strings.TrimSpace(document.PipelineID) == "" || strings.TrimSpace(document.OutputDirectory) == "" || strings.TrimSpace(document.ValidationStatus) == "" || document.NormalizedOutputCount == nil || - document.RejectedOutputCount == nil || document.WarningCount == nil { + document.RejectedOutputCount == nil || document.WarningGroupCount == nil || + document.WarningOccurrenceCount == nil || document.DiagnosticGroupCount == nil || + document.DiagnosticOccurrenceCount == nil || document.DiagnosticsTruncated == nil { return Receipt{}, fmt.Errorf("notarius receipt is missing required fields") } if document.IndexFile != canonicalIndexFile { @@ -186,9 +270,18 @@ func loadReceipt(path, pipelineID string) (Receipt, error) { if document.PipelineID != pipelineID { return Receipt{}, fmt.Errorf("notarius receipt pipeline id %q does not match requested pipeline %q", document.PipelineID, pipelineID) } - if *document.NormalizedOutputCount < 0 || *document.RejectedOutputCount < 0 || *document.WarningCount < 0 { + if *document.NormalizedOutputCount < 0 || *document.RejectedOutputCount < 0 || + *document.WarningGroupCount < 0 || *document.WarningOccurrenceCount < 0 || + *document.DiagnosticGroupCount < 0 || *document.DiagnosticOccurrenceCount < 0 { return Receipt{}, fmt.Errorf("notarius receipt counts must be non-negative") } + if !validValidationStatus(document.ValidationStatus) { + return Receipt{}, fmt.Errorf("notarius receipt validation_status %q is invalid", document.ValidationStatus) + } + validationSummaries, err := validateValidationSummaries(document.ValidationSummaries) + if err != nil { + return Receipt{}, err + } if !filepath.IsAbs(document.OutputDirectory) { return Receipt{}, fmt.Errorf("notarius receipt output directory must be absolute") } @@ -196,16 +289,21 @@ func loadReceipt(path, pipelineID string) (Receipt, error) { return Receipt{}, fmt.Errorf("notarius receipt debug directory must be absolute when present") } return Receipt{ - SchemaVersion: document.SchemaVersion, - RunID: document.RunID, - PipelineID: document.PipelineID, - OutputDirectory: filepath.Clean(document.OutputDirectory), - IndexFile: document.IndexFile, - NormalizedOutputCount: *document.NormalizedOutputCount, - RejectedOutputCount: *document.RejectedOutputCount, - WarningCount: *document.WarningCount, - ValidationStatus: document.ValidationStatus, - DebugDirectory: document.DebugDirectory, + SchemaVersion: document.SchemaVersion, + RunID: document.RunID, + PipelineID: document.PipelineID, + OutputDirectory: filepath.Clean(document.OutputDirectory), + IndexFile: document.IndexFile, + NormalizedOutputCount: *document.NormalizedOutputCount, + RejectedOutputCount: *document.RejectedOutputCount, + WarningGroupCount: *document.WarningGroupCount, + WarningOccurrenceCount: *document.WarningOccurrenceCount, + DiagnosticGroupCount: *document.DiagnosticGroupCount, + DiagnosticOccurrenceCount: *document.DiagnosticOccurrenceCount, + DiagnosticsTruncated: *document.DiagnosticsTruncated, + ValidationStatus: document.ValidationStatus, + ValidationSummaries: validationSummaries, + DebugDirectory: document.DebugDirectory, }, nil } @@ -214,6 +312,7 @@ type indexDocument struct { OutputFiles *[]laneDocument `json:"output_files"` RejectedFile string `json:"rejected_file"` WarningsFile string `json:"warnings_file"` + DiagnosticsFile string `json:"diagnostics_file"` ChunkMap *pipelineDocument `json:"chunk_map"` EvidenceContext *pipelineDocument `json:"evidence_context"` } @@ -250,6 +349,7 @@ func loadIndex(bundleRoot, indexPath string) (Index, error) { {name: "manifest_file", got: document.ManifestFile, want: canonicalManifestFile}, {name: "rejected_file", got: document.RejectedFile, want: canonicalRejectedFile}, {name: "warnings_file", got: document.WarningsFile, want: canonicalWarningsFile}, + {name: "diagnostics_file", got: document.DiagnosticsFile, want: canonicalDiagnosticsFile}, } { if field.got != field.want { return Index{}, fmt.Errorf("notarius index %s %q is incompatible; want %q", field.name, field.got, field.want) @@ -260,10 +360,11 @@ func loadIndex(bundleRoot, indexPath string) (Index, error) { } index := Index{ - Path: indexPath, - ManifestFile: document.ManifestFile, - RejectedFile: document.RejectedFile, - WarningsFile: document.WarningsFile, + Path: indexPath, + ManifestFile: document.ManifestFile, + RejectedFile: document.RejectedFile, + WarningsFile: document.WarningsFile, + DiagnosticsFile: document.DiagnosticsFile, } var err error if index.ManifestPath, err = resolveRegularFile(bundleRoot, index.ManifestFile); err != nil { @@ -275,6 +376,9 @@ func loadIndex(bundleRoot, indexPath string) (Index, error) { if index.WarningsPath, err = resolveRegularFile(bundleRoot, index.WarningsFile); err != nil { return Index{}, fmt.Errorf("resolve notarius warning file: %w", err) } + if index.DiagnosticsPath, err = resolveRegularFile(bundleRoot, index.DiagnosticsFile); err != nil { + return Index{}, fmt.Errorf("resolve notarius diagnostics file: %w", err) + } seenLanes := make(map[string]struct{}, len(*document.OutputFiles)) for _, lane := range *document.OutputFiles { @@ -362,12 +466,34 @@ func loadRejections(path string) ([]RejectionSummary, error) { return summaries, nil } -type warningDocument struct { - Warnings *[]struct { +type findingGroupDocument struct { + Disposition string `json:"disposition"` + Category string `json:"category"` + ReasonCode string `json:"reason_code"` + Origin diagnosticOriginDocument `json:"origin"` + OccurrenceCount *int `json:"occurrence_count"` + Samples *[]struct { Scope string `json:"scope"` - ReasonCode string `json:"reason_code"` Message string `json:"message"` - } `json:"warnings"` + ChunkID string `json:"chunk_id"` + ChunkIndex *int `json:"chunk_index"` + } `json:"samples"` + OmittedSampleCount *int `json:"omitted_sample_count"` +} + +type diagnosticOriginDocument struct { + Stage string `json:"stage"` + StepID string `json:"step_id"` + LaneID string `json:"lane_id"` + ModuleKey string `json:"module_key"` + ValidatorKey string `json:"validator_key"` +} + +type warningDocument struct { + SchemaVersion string `json:"schema_version"` + GroupCount *int `json:"group_count"` + OccurrenceCount *int `json:"occurrence_count"` + Groups *[]findingGroupDocument `json:"groups"` } func loadWarnings(path string) ([]WarningSummary, error) { @@ -375,19 +501,151 @@ func loadWarnings(path string) ([]WarningSummary, error) { if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil { return nil, fmt.Errorf("decode notarius warnings: %w", err) } - if document.Warnings == nil { - return nil, fmt.Errorf("notarius warning document is missing warnings array") + if document.SchemaVersion != warningsSchemaVersion || document.GroupCount == nil || + document.OccurrenceCount == nil || document.Groups == nil { + return nil, fmt.Errorf("notarius warning document is missing or incompatible required fields") } - summaries := make([]WarningSummary, 0, len(*document.Warnings)) - for _, item := range *document.Warnings { - if strings.TrimSpace(item.ReasonCode) == "" || strings.TrimSpace(item.Message) == "" { - return nil, fmt.Errorf("notarius warning entries require reason_code and message") + if *document.GroupCount < 0 || *document.GroupCount > maxWarningGroups || *document.OccurrenceCount < 0 || + *document.GroupCount != len(*document.Groups) { + return nil, fmt.Errorf("notarius warning document counts are inconsistent") + } + summaries := make([]WarningSummary, 0, len(*document.Groups)) + occurrences := 0 + for _, group := range *document.Groups { + if err := validateFindingGroup(group); err != nil { + return nil, fmt.Errorf("notarius warning group: %w", err) } - summaries = append(summaries, WarningSummary{Scope: item.Scope, ReasonCode: item.ReasonCode}) + if group.Disposition != "warning" { + return nil, fmt.Errorf("notarius warning group disposition %q is invalid", group.Disposition) + } + if *group.OccurrenceCount > int(^uint(0)>>1)-occurrences { + return nil, fmt.Errorf("notarius warning occurrence count overflows") + } + occurrences += *group.OccurrenceCount + summaries = append(summaries, WarningSummary{ + Disposition: group.Disposition, Category: group.Category, ReasonCode: group.ReasonCode, + Origin: diagnosticOrigin(group.Origin), OccurrenceCount: *group.OccurrenceCount, + }) + } + if occurrences != *document.OccurrenceCount { + return nil, fmt.Errorf("notarius warning document occurrence count is inconsistent") } return summaries, nil } +type diagnosticDocument struct { + SchemaVersion string `json:"schema_version"` + GroupCount *int `json:"group_count"` + OccurrenceCount *int `json:"occurrence_count"` + Truncated *bool `json:"truncated"` + UnrepresentedOccurrenceCount *int `json:"unrepresented_occurrence_count"` + Groups *[]findingGroupDocument `json:"groups"` +} + +func loadDiagnostics(path string) ([]DiagnosticSummary, int, bool, error) { + var document diagnosticDocument + if err := decodeBoundedJSON(path, maxSummaryBytes, &document); err != nil { + return nil, 0, false, fmt.Errorf("decode notarius diagnostics: %w", err) + } + if document.SchemaVersion != diagnosticsSchemaVersion || document.GroupCount == nil || + document.OccurrenceCount == nil || document.Truncated == nil || + document.UnrepresentedOccurrenceCount == nil || document.Groups == nil { + return nil, 0, false, fmt.Errorf("notarius diagnostics document is missing or incompatible required fields") + } + if *document.GroupCount < 0 || *document.GroupCount > maxDiagnosticGroups || *document.OccurrenceCount < 0 || + *document.UnrepresentedOccurrenceCount < 0 || *document.GroupCount != len(*document.Groups) { + return nil, 0, false, fmt.Errorf("notarius diagnostics document counts are inconsistent") + } + if !*document.Truncated && *document.UnrepresentedOccurrenceCount != 0 { + return nil, 0, false, fmt.Errorf("notarius diagnostics document has unrepresented occurrences without truncation") + } + summaries := make([]DiagnosticSummary, 0, len(*document.Groups)) + representedOccurrences := 0 + for _, group := range *document.Groups { + if err := validateFindingGroup(group); err != nil { + return nil, 0, false, fmt.Errorf("notarius diagnostic group: %w", err) + } + if group.Disposition != "advisory" && group.Disposition != "observation" { + return nil, 0, false, fmt.Errorf("notarius diagnostic group disposition %q is invalid", group.Disposition) + } + if *group.OccurrenceCount > int(^uint(0)>>1)-representedOccurrences { + return nil, 0, false, fmt.Errorf("notarius diagnostic occurrence count overflows") + } + representedOccurrences += *group.OccurrenceCount + summaries = append(summaries, DiagnosticSummary{ + Disposition: group.Disposition, Category: group.Category, ReasonCode: group.ReasonCode, + Origin: diagnosticOrigin(group.Origin), OccurrenceCount: *group.OccurrenceCount, + }) + } + if *document.UnrepresentedOccurrenceCount > int(^uint(0)>>1)-representedOccurrences || + representedOccurrences+*document.UnrepresentedOccurrenceCount != *document.OccurrenceCount { + return nil, 0, false, fmt.Errorf("notarius diagnostics document occurrence count is inconsistent") + } + return summaries, *document.OccurrenceCount, *document.Truncated, nil +} + +func validateFindingGroup(group findingGroupDocument) error { + if strings.TrimSpace(group.Disposition) == "" || strings.TrimSpace(group.Category) == "" || + strings.TrimSpace(group.ReasonCode) == "" || !validDiagnosticOriginStage(group.Origin.Stage) || + !validDiagnosticCategory(group.Disposition, group.Category) || + group.OccurrenceCount == nil || *group.OccurrenceCount <= 0 || group.Samples == nil || + group.OmittedSampleCount == nil || *group.OmittedSampleCount < 0 { + return fmt.Errorf("missing required fields") + } + if len(*group.Samples) == 0 || len(*group.Samples) > maxFindingSamples || + *group.OmittedSampleCount != *group.OccurrenceCount-len(*group.Samples) { + return fmt.Errorf("sample counts are inconsistent") + } + for _, sample := range *group.Samples { + if strings.TrimSpace(sample.Scope) == "" || strings.TrimSpace(sample.Message) == "" || + (sample.ChunkIndex != nil && *sample.ChunkIndex < 0) { + return fmt.Errorf("samples require scope and message") + } + } + return nil +} + +func validDiagnosticCategory(disposition, category string) bool { + switch disposition { + case "warning": + return category == "configuration" || category == "degradation" || + category == "validation_incomplete" || category == "fallback" + case "advisory": + return category == "data_quality" + case "observation": + return category == "normalization" + default: + return false + } +} + +func validDiagnosticOriginStage(stage string) bool { + switch stage { + case "references", "chunk", "extract", "merge", "normalize": + return true + default: + return false + } +} + +func diagnosticOrigin(document diagnosticOriginDocument) DiagnosticOrigin { + return DiagnosticOrigin{ + Stage: document.Stage, StepID: document.StepID, LaneID: document.LaneID, + ModuleKey: document.ModuleKey, ValidatorKey: document.ValidatorKey, + } +} + +func sumWarningOccurrences(values []WarningSummary) (int, error) { + total := 0 + for _, value := range values { + if value.OccurrenceCount > int(^uint(0)>>1)-total { + return 0, fmt.Errorf("notarius warning occurrence count overflows") + } + total += value.OccurrenceCount + } + return total, nil +} + func decodeBoundedJSON(path string, limit int64, destination any) error { data, err := fileops.ReadRegularFile(path, limit) if err != nil { diff --git a/internal/adapters/notarius/subprocess_test.go b/internal/adapters/notarius/subprocess_test.go index c86a086..f0627b7 100644 --- a/internal/adapters/notarius/subprocess_test.go +++ b/internal/adapters/notarius/subprocess_test.go @@ -52,6 +52,10 @@ func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) { if result.Receipt.SchemaVersion != ReceiptSchemaVersion || result.Receipt.RunID != "notarius-run-1" { t.Fatalf("receipt = %#v", result.Receipt) } + if len(result.Receipt.ValidationSummaries) != 1 || result.Receipt.ValidationSummaries[0].LaneID != "npc-registry" || + result.Receipt.ValidationSummaries[0].Status != "complete" { + t.Fatalf("validation summaries = %#v", result.Receipt.ValidationSummaries) + } if len(result.Index.Lanes) != 1 || result.Index.Lanes[0].LaneID != "npc-registry" { t.Fatalf("lanes = %#v", result.Index.Lanes) } @@ -64,9 +68,12 @@ func TestSubprocessRunnerBuildsExactInvocationAndDiscoversBundle(t *testing.T) { if len(result.Rejections) != 1 || result.Rejections[0].LaneID != "spells" || result.Rejections[0].ReasonCode != "invalid_spell" { t.Fatalf("rejections = %#v", result.Rejections) } - if len(result.Warnings) != 1 || result.Warnings[0].Scope != "lane:npc-registry" || result.Warnings[0].ReasonCode != "normalized_name" { + if len(result.Warnings) != 1 || result.Warnings[0].Category != "degradation" || result.Warnings[0].ReasonCode != "normalized_name" { t.Fatalf("warnings = %#v", result.Warnings) } + if len(result.Diagnostics) != 1 || result.Diagnostics[0].Category != "data_quality" || result.Diagnostics[0].ReasonCode != "low_confidence" { + t.Fatalf("diagnostics = %#v", result.Diagnostics) + } } func TestSubprocessRunnerUsesMinimalEnvironmentAndSeparatesStreams(t *testing.T) { @@ -171,8 +178,10 @@ func TestLoadReceiptValidation(t *testing.T) { valid := map[string]any{ "schema_version": ReceiptSchemaVersion, "run_id": "run-1", "pipeline_id": "pipeline-1", "output_directory": filepath.Join(root, "outputs", "run-1"), "index_file": "index.json", - "normalized_output_count": 1, "rejected_output_count": 0, "warning_count": 0, - "validation_status": "approved", "future_field": true, + "normalized_output_count": 1, "rejected_output_count": 0, + "warning_group_count": 0, "warning_occurrence_count": 0, + "diagnostic_group_count": 0, "diagnostic_occurrence_count": 0, + "diagnostics_truncated": false, "validation_status": "approved", "future_field": true, } tests := []struct { name string @@ -183,11 +192,12 @@ func TestLoadReceiptValidation(t *testing.T) { }{ {name: "unknown fields tolerated", wantOK: true}, {name: "malformed", raw: []byte("{")}, - {name: "unsupported version", mutate: func(v map[string]any) { v["schema_version"] = "notarius.run-result.v2" }}, + {name: "unsupported version", mutate: func(v map[string]any) { v["schema_version"] = "notarius.run-result.v1" }}, {name: "missing field", mutate: func(v map[string]any) { delete(v, "run_id") }}, {name: "pipeline mismatch", mutate: func(v map[string]any) { v["pipeline_id"] = "other" }}, {name: "relative output", mutate: func(v map[string]any) { v["output_directory"] = "run-1" }}, - {name: "negative count", mutate: func(v map[string]any) { v["warning_count"] = -1 }}, + {name: "negative count", mutate: func(v map[string]any) { v["warning_group_count"] = -1 }}, + {name: "invalid validation status", mutate: func(v map[string]any) { v["validation_status"] = "valid" }}, { name: "nested index", mutate: func(v map[string]any) { v["index_file"] = "nested/index.json" }, wantError: `index_file "nested/index.json"`, @@ -369,22 +379,26 @@ func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testin root := t.TempDir() rejectedPath := filepath.Join(root, "rejected.json") warningsPath := filepath.Join(root, "warnings.json") + diagnosticsPath := filepath.Join(root, "diagnostics.json") writeJSONFile(t, rejectedPath, map[string]any{"rejected": []any{map[string]any{ "stage": "validate", "lane_id": "spells", "reason_code": "invalid", "message": "do not retain this", "future": true, }}, "future": true}) - writeJSONFile(t, warningsPath, map[string]any{"warnings": []any{map[string]any{ - "scope": "lane:spells", "reason_code": "bounded", "message": "do not retain this", "future": true, - }}, "future": true}) + writeJSONFile(t, warningsPath, findingEnvelope(warningsSchemaVersion, []any{findingGroup("warning", "degradation", "bounded", "normalize", 2)}, 2, false, 0)) + writeJSONFile(t, diagnosticsPath, findingEnvelope(diagnosticsSchemaVersion, []any{findingGroup("advisory", "data_quality", "low_confidence", "normalize", 3)}, 4, true, 1)) rejections, err := loadRejections(rejectedPath) if err != nil || len(rejections) != 1 || rejections[0].ReasonCode != "invalid" { t.Fatalf("loadRejections() = %#v, %v", rejections, err) } warnings, err := loadWarnings(warningsPath) - if err != nil || len(warnings) != 1 || warnings[0].Scope != "lane:spells" { + if err != nil || len(warnings) != 1 || warnings[0].Category != "degradation" || warnings[0].OccurrenceCount != 2 { t.Fatalf("loadWarnings() = %#v, %v", warnings, err) } + diagnostics, occurrences, truncated, err := loadDiagnostics(diagnosticsPath) + if err != nil || len(diagnostics) != 1 || occurrences != 4 || !truncated || diagnostics[0].Category != "data_quality" { + t.Fatalf("loadDiagnostics() = %#v, %d, %t, %v", diagnostics, occurrences, truncated, err) + } - for name, path := range map[string]string{"rejections": rejectedPath, "warnings": warningsPath} { + for name, path := range map[string]string{"rejections": rejectedPath, "warnings": warningsPath, "diagnostics": diagnosticsPath} { t.Run("malformed "+name, func(t *testing.T) { if err := os.WriteFile(path, []byte("{"), 0o644); err != nil { t.Fatalf("WriteFile() error = %v", err) @@ -392,8 +406,10 @@ func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testin var err error if name == "rejections" { _, err = loadRejections(path) - } else { + } else if name == "warnings" { _, err = loadWarnings(path) + } else { + _, _, _, err = loadDiagnostics(path) } if err == nil { t.Fatal("summary decoder error = nil") @@ -410,6 +426,9 @@ func TestLoadDiagnosticSummariesValidateBoundsAndTolerateUnknownFields(t *testin if _, err := loadRejections(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") { t.Fatalf("loadRejections(oversized) error = %v", err) } + if _, _, _, err := loadDiagnostics(oversized); err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("loadDiagnostics(oversized) error = %v", err) + } } func TestFakeRunnerCapturesRequestsAndHonorsContextAndError(t *testing.T) { @@ -478,13 +497,12 @@ func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown boo } } rejection := map[string]any{"stage": "validate", "lane_id": "spells", "reason_code": "invalid_spell", "message": strings.Repeat("external detail", 20)} - warning := map[string]any{"scope": "lane:npc-registry", "reason_code": "normalized_name", "message": strings.Repeat("external warning", 20)} if includeUnknown { rejection["future"] = true - warning["future"] = true } writeJSONFile(t, filepath.Join(bundle, "rejected.json"), map[string]any{"rejected": []any{rejection}, "future": true}) - writeJSONFile(t, filepath.Join(bundle, "warnings.json"), map[string]any{"warnings": []any{warning}, "future": true}) + writeJSONFile(t, filepath.Join(bundle, "warnings.json"), findingEnvelope(warningsSchemaVersion, []any{findingGroup("warning", "degradation", "normalized_name", "normalize", 2)}, 2, false, 0)) + writeJSONFile(t, filepath.Join(bundle, "diagnostics.json"), findingEnvelope(diagnosticsSchemaVersion, []any{findingGroup("advisory", "data_quality", "low_confidence", "normalize", 3)}, 4, true, 1)) index := validIndexValue([]any{map[string]any{ "lane_id": "npc-registry", "file": "lanes/npc.json", "media_type": "application/json", "module_key": "dnd/npc-registry", "schema_id": "notarius.dnd.npc_registry", @@ -503,7 +521,13 @@ func writeValidBundleAndReceipt(t *testing.T, req RunRequest, includeUnknown boo receipt := map[string]any{ "schema_version": ReceiptSchemaVersion, "run_id": "notarius-run-1", "pipeline_id": req.PipelineID, "output_directory": bundle, "index_file": "index.json", "normalized_output_count": 1, - "rejected_output_count": 1, "warning_count": 1, "validation_status": "rejected", + "rejected_output_count": 1, "warning_group_count": 1, "warning_occurrence_count": 2, + "diagnostic_group_count": 1, "diagnostic_occurrence_count": 4, + "diagnostics_truncated": true, "validation_status": "rejected", + "validation_summaries": []any{map[string]any{ + "stage": "normalize", "lane_id": "npc-registry", "status": "complete", + "producer_attempt_count": 1, "terminal_action": "accepted", + }}, } if includeUnknown { receipt["future"] = true @@ -517,7 +541,7 @@ func createBundleSkeleton(t *testing.T) string { if err := os.MkdirAll(filepath.Join(bundle, "lanes"), 0o755); err != nil { t.Fatalf("MkdirAll(bundle) error = %v", err) } - for _, name := range []string{"manifest.json", "rejected.json", "warnings.json", "lanes/npc.json", "chunk-map.json"} { + for _, name := range []string{"manifest.json", "rejected.json", "warnings.json", "diagnostics.json", "lanes/npc.json", "chunk-map.json"} { if err := os.WriteFile(filepath.Join(bundle, filepath.FromSlash(name)), []byte("{}"), 0o644); err != nil { t.Fatalf("WriteFile(%q) error = %v", name, err) } @@ -528,10 +552,31 @@ func createBundleSkeleton(t *testing.T) string { func validIndexValue(lanes []any) map[string]any { return map[string]any{ "manifest_file": "manifest.json", "output_files": lanes, - "rejected_file": "rejected.json", "warnings_file": "warnings.json", + "rejected_file": "rejected.json", "warnings_file": "warnings.json", "diagnostics_file": "diagnostics.json", } } +func findingGroup(disposition, category, reasonCode, origin string, occurrences int) map[string]any { + return map[string]any{ + "disposition": disposition, "category": category, "reason_code": reasonCode, + "origin": map[string]any{"stage": origin, "lane_id": "npc-registry"}, "occurrence_count": occurrences, + "samples": []any{map[string]any{"scope": "lane:npc-registry", "message": "external detail"}}, + "omitted_sample_count": occurrences - 1, + } +} + +func findingEnvelope(schema string, groups []any, occurrences int, truncated bool, unrepresented int) map[string]any { + value := map[string]any{ + "schema_version": schema, "group_count": len(groups), "occurrence_count": occurrences, + "groups": groups, + } + if schema == diagnosticsSchemaVersion { + value["truncated"] = truncated + value["unrepresented_occurrence_count"] = unrepresented + } + return value +} + func writeJSONFile(t *testing.T, path string, value any) { t.Helper() data, err := json.Marshal(value) diff --git a/internal/app/extract_lifecycle_test.go b/internal/app/extract_lifecycle_test.go index 52fbbdb..fa55dfb 100644 --- a/internal/app/extract_lifecycle_test.go +++ b/internal/app/extract_lifecycle_test.go @@ -38,11 +38,12 @@ func (r *materializingNotariusRunner) Run(_ context.Context, req notarius.RunReq return notarius.RunResult{}, err } for path, content := range map[string]string{ - filepath.Join(bundle, "index.json"): `{"manifest_file":"manifest.json"}`, - filepath.Join(bundle, "manifest.json"): `{}`, - filepath.Join(bundle, "rejected.json"): `{"rejected":[]}`, - filepath.Join(bundle, "warnings.json"): `{"warnings":[]}`, - filepath.Join(lanesDir, "npcs.json"): `{"npcs":[]}`, + filepath.Join(bundle, "index.json"): `{"manifest_file":"manifest.json"}`, + filepath.Join(bundle, "manifest.json"): `{}`, + filepath.Join(bundle, "rejected.json"): `{"rejected":[]}`, + filepath.Join(bundle, "warnings.json"): `{"schema_version":"notarius.warnings.v2","group_count":0,"occurrence_count":0,"groups":[]}`, + filepath.Join(bundle, "diagnostics.json"): `{"schema_version":"notarius.diagnostics.v1","group_count":0,"occurrence_count":0,"truncated":false,"unrepresented_occurrence_count":0,"groups":[]}`, + filepath.Join(lanesDir, "npcs.json"): `{"npcs":[]}`, } { if err := os.WriteFile(path, []byte(content), 0o644); err != nil { return notarius.RunResult{}, err @@ -53,12 +54,13 @@ func (r *materializingNotariusRunner) Run(_ context.Context, req notarius.RunReq Receipt: notarius.Receipt{ SchemaVersion: notarius.ReceiptSchemaVersion, RunID: externalRunID, PipelineID: req.PipelineID, OutputDirectory: bundle, IndexFile: "index.json", - NormalizedOutputCount: 1, ValidationStatus: "valid", + NormalizedOutputCount: 1, ValidationStatus: "approved", }, BundleRoot: bundle, Index: notarius.Index{ Path: filepath.Join(bundle, "index.json"), RejectedPath: filepath.Join(bundle, "rejected.json"), - WarningsPath: filepath.Join(bundle, "warnings.json"), + WarningsPath: filepath.Join(bundle, "warnings.json"), + DiagnosticsPath: filepath.Join(bundle, "diagnostics.json"), Lanes: []notarius.LaneDescriptor{{ LaneID: output.LaneID, File: "lanes/npcs.json", Path: filepath.Join(lanesDir, "npcs.json"), MediaType: output.MediaType, SchemaID: output.SchemaID, diff --git a/internal/stage/extract.go b/internal/stage/extract.go index d76cb2e..cd6651c 100644 --- a/internal/stage/extract.go +++ b/internal/stage/extract.go @@ -161,6 +161,10 @@ func (extractStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S if err != nil { return nil, fmt.Errorf("extract: resolve staging warnings relative path: %w", err) } + diagnosticsRelative, err := pathsafe.SlashRelativeFromRoot(adapterResult.BundleRoot, adapterResult.Index.DiagnosticsPath) + if err != nil { + return nil, fmt.Errorf("extract: resolve staging diagnostics relative path: %w", err) + } stagingIndexChecksum, err := checksumRegularFile(adapterResult.Index.Path, false) if err != nil { return nil, fmt.Errorf("extract: validate staging index: %w", err) @@ -188,6 +192,10 @@ func (extractStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S if err != nil { return nil, fmt.Errorf("extract: resolve promoted warnings: %w", err) } + promotedDiagnosticsPath, err := pathsafe.JoinSlashRelativeUnderRoot(durableBundle, diagnosticsRelative) + if err != nil { + return nil, fmt.Errorf("extract: resolve promoted diagnostics: %w", err) + } outputs := make([]artifacts.Ref, 0, len(selected)+1) for _, lane := range selected { @@ -237,18 +245,25 @@ func (extractStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S "diagnostic_path": logPath, "rejections_path": promotedRejectionsPath, "warnings_path": promotedWarningsPath, + "diagnostics_path": promotedDiagnosticsPath, "narratio_run_id": runID, "configuration_fingerprint": fingerprint, "direct_input": input.Metadata(), "receipt": map[string]any{ "run_id": adapterResult.Receipt.RunID, "pipeline_id": adapterResult.Receipt.PipelineID, - "normalized_output_count": adapterResult.Receipt.NormalizedOutputCount, - "rejected_output_count": adapterResult.Receipt.RejectedOutputCount, - "warning_count": adapterResult.Receipt.WarningCount, - "validation_status": adapterResult.Receipt.ValidationStatus, + "normalized_output_count": adapterResult.Receipt.NormalizedOutputCount, + "rejected_output_count": adapterResult.Receipt.RejectedOutputCount, + "warning_group_count": adapterResult.Receipt.WarningGroupCount, + "warning_occurrence_count": adapterResult.Receipt.WarningOccurrenceCount, + "diagnostic_group_count": adapterResult.Receipt.DiagnosticGroupCount, + "diagnostic_occurrence_count": adapterResult.Receipt.DiagnosticOccurrenceCount, + "diagnostics_truncated": adapterResult.Receipt.DiagnosticsTruncated, + "validation_status": adapterResult.Receipt.ValidationStatus, }, - "rejections": boundedRejectionMetadata(adapterResult.Rejections), - "warnings": boundedWarningMetadata(adapterResult.Warnings), + "rejections": boundedRejectionMetadata(adapterResult.Rejections), + "warnings": boundedWarningMetadata(adapterResult.Warnings), + "diagnostics": boundedDiagnosticMetadata(adapterResult.Diagnostics), + "validation_summaries": boundedValidationMetadata(adapterResult.Receipt.ValidationSummaries), } return &StageResult{ Outputs: outputs, @@ -282,6 +297,11 @@ func selectRequiredNotariusLanes( return nil, fmt.Errorf("extract: required lane %q was rejected (reason_code=%q)", expected.LaneID, rejection.ReasonCode) } } + for _, validation := range result.Receipt.ValidationSummaries { + if validation.LaneID == expected.LaneID && validation.Status != "complete" { + return nil, fmt.Errorf("extract: required lane %q validation is %q", expected.LaneID, validation.Status) + } + } matches := make([]notarius.LaneDescriptor, 0, 1) for _, descriptor := range result.Index.Lanes { if descriptor.LaneID == expected.LaneID { @@ -441,7 +461,52 @@ func boundedWarningMetadata(values []notarius.WarningSummary) []map[string]any { } result := make([]map[string]any, 0, limit) for _, value := range values[:limit] { - result = append(result, map[string]any{"scope": value.Scope, "reason_code": value.ReasonCode}) + result = append(result, map[string]any{ + "disposition": value.Disposition, "category": value.Category, + "reason_code": value.ReasonCode, "origin": diagnosticOriginMetadata(value.Origin), + "occurrence_count": value.OccurrenceCount, + }) } return result } + +func boundedDiagnosticMetadata(values []notarius.DiagnosticSummary) []map[string]any { + limit := len(values) + if limit > maxDiagnosticSummaries { + limit = maxDiagnosticSummaries + } + result := make([]map[string]any, 0, limit) + for _, value := range values[:limit] { + result = append(result, map[string]any{ + "disposition": value.Disposition, "category": value.Category, + "reason_code": value.ReasonCode, "origin": diagnosticOriginMetadata(value.Origin), + "occurrence_count": value.OccurrenceCount, + }) + } + return result +} + +func boundedValidationMetadata(values []notarius.ValidationSummary) []map[string]any { + limit := len(values) + if limit > maxDiagnosticSummaries { + limit = maxDiagnosticSummaries + } + result := make([]map[string]any, 0, limit) + for _, value := range values[:limit] { + result = append(result, map[string]any{ + "stage": value.Stage, "step_id": value.StepID, "lane_id": value.LaneID, + "module_key": value.ModuleKey, "chunk_id": value.ChunkID, "status": value.Status, + "rejecting_validators": value.RejectingValidators, "reason_codes": value.ReasonCodes, + "incomplete_validators": value.IncompleteValidators, + "producer_attempt_count": value.ProducerAttemptCount, "terminal_action": value.TerminalAction, + }) + } + return result +} + +func diagnosticOriginMetadata(value notarius.DiagnosticOrigin) map[string]any { + return map[string]any{ + "stage": value.Stage, "step_id": value.StepID, "lane_id": value.LaneID, + "module_key": value.ModuleKey, "validator_key": value.ValidatorKey, + } +} diff --git a/internal/stage/extract_test.go b/internal/stage/extract_test.go index 01bce55..903d9c8 100644 --- a/internal/stage/extract_test.go +++ b/internal/stage/extract_test.go @@ -99,7 +99,8 @@ func TestExtractStageProducesImmutableManifestReadyOutputs(t *testing.T) { if result.Metadata["receipt_path"] != artifacts.SessionRunNotariusReceiptPathForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID) || result.Metadata["diagnostic_path"] != artifacts.SessionRunNotariusLogPathForCampaign(fixture.workspace, fixture.campaign, fixture.sessionID, fixture.runID) || result.Metadata["rejections_path"] != filepath.Join(durableBundle, "rejected.json") || - result.Metadata["warnings_path"] != filepath.Join(durableBundle, "warnings.json") { + result.Metadata["warnings_path"] != filepath.Join(durableBundle, "warnings.json") || + result.Metadata["diagnostics_path"] != filepath.Join(durableBundle, "diagnostics.json") { t.Fatalf("diagnostic metadata = %#v", result.Metadata) } if len(result.Outputs) != 2 { @@ -133,7 +134,7 @@ func TestExtractStageProducesImmutableManifestReadyOutputs(t *testing.T) { t.Fatalf("lane path was not re-resolved after promotion: %q", lane.AbsolutePath) } for _, relative := range []string{ - "index.json", "manifest.json", "rejected.json", "warnings.json", "lanes/npc.json", + "index.json", "manifest.json", "rejected.json", "warnings.json", "diagnostics.json", "lanes/npc.json", "lanes/unconfigured.json", "chunk-map.json", "evidence-context.json", "unknown/private-debug.json", } { if _, err := os.Stat(filepath.Join(durableBundle, filepath.FromSlash(relative))); err != nil { @@ -156,6 +157,81 @@ func TestExtractStageProducesImmutableManifestReadyOutputs(t *testing.T) { } } +func TestExtractStageConsumesAllDefaultDndPipelineArtifacts(t *testing.T) { + env, m, fake := setupExtractEnv(t) + + contracts := map[string]config.NotariusOutputConfig{ + "item_registry": {LaneID: "item-registry", MediaType: "application/json", SchemaID: "notarius.dnd.item_registry", SchemaVersion: "v1", ModuleKey: "dnd/item-registry"}, + "npc_registry": {LaneID: "npc-registry", MediaType: "application/json", SchemaID: "notarius.dnd.npc_registry", SchemaVersion: "v1", ModuleKey: "dnd/npc-registry"}, + "location_registry": {LaneID: "location-registry", MediaType: "application/json", SchemaID: "notarius.dnd.location_registry", SchemaVersion: "v1", ModuleKey: "dnd/location-registry"}, + "scene_descriptions": {LaneID: "scene-descriptions", MediaType: "application/json", SchemaID: "notarius.dnd.scene_descriptions", SchemaVersion: "v1", ModuleKey: "dnd/scene-descriptions"}, + "item_occurrences": {LaneID: "item-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.item_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/item-occurrences"}, + "spells": {LaneID: "spells", MediaType: "application/json", SchemaID: "notarius.dnd.spells", SchemaVersion: "v1", ModuleKey: "dnd/spells"}, + "combat_turns": {LaneID: "combat-turns", MediaType: "application/json", SchemaID: "notarius.dnd.combat_turns", SchemaVersion: "v1", ModuleKey: "dnd/combat-turns"}, + "npc_occurrences": {LaneID: "npc-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.npc_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/npc-occurrences"}, + "location_occurrences": {LaneID: "location-occurrences", MediaType: "application/json", SchemaID: "notarius.dnd.location_occurrences", SchemaVersion: "v1", ModuleKey: "dnd/location-occurrences"}, + "enemy_events": {LaneID: "enemy-events", MediaType: "application/json", SchemaID: "notarius.dnd.enemy_events", SchemaVersion: "v1", ModuleKey: "dnd/enemy-events"}, + } + env.Config.Pipeline.Notarius.Outputs = contracts + fake.Result.Index.Lanes = nil + fake.Result.Rejections = nil + fake.Result.Warnings = nil + fake.Result.Diagnostics = nil + fake.Result.Receipt.NormalizedOutputCount = len(contracts) + fake.Result.Receipt.RejectedOutputCount = 0 + fake.Result.Receipt.WarningGroupCount = 0 + fake.Result.Receipt.WarningOccurrenceCount = 0 + fake.Result.Receipt.DiagnosticGroupCount = 0 + fake.Result.Receipt.DiagnosticOccurrenceCount = 0 + fake.Result.Receipt.ValidationStatus = "approved" + indexDescriptors := make([]map[string]any, 0, len(contracts)) + for _, contract := range contracts { + relative := "lanes/" + contract.LaneID + ".json" + path := filepath.Join(fake.Result.BundleRoot, filepath.FromSlash(relative)) + if err := os.WriteFile(path, []byte(`{"records":[]}`), 0o644); err != nil { + t.Fatalf("WriteFile(%q) error = %v", path, err) + } + fake.Result.Index.Lanes = append(fake.Result.Index.Lanes, notarius.LaneDescriptor{ + LaneID: contract.LaneID, File: relative, Path: path, MediaType: contract.MediaType, + ModuleKey: contract.ModuleKey, SchemaID: contract.SchemaID, SchemaVersion: contract.SchemaVersion, + }) + indexDescriptors = append(indexDescriptors, map[string]any{ + "lane_id": contract.LaneID, "file": relative, "media_type": contract.MediaType, + "module_key": contract.ModuleKey, "schema_id": contract.SchemaID, + "schema_version": contract.SchemaVersion, + }) + } + if err := writeJSONFile(fake.Result.Index.Path, map[string]any{ + "manifest_file": "manifest.json", "output_files": indexDescriptors, + "rejected_file": "rejected.json", "warnings_file": "warnings.json", + "diagnostics_file": "diagnostics.json", + }); err != nil { + t.Fatalf("write complete D&D index: %v", err) + } + + result, err := (extractStage{}).Run(context.Background(), env, m) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if len(result.Outputs) != len(contracts)+1 { + t.Fatalf("output count = %d, want %d", len(result.Outputs), len(contracts)+1) + } + got := make(map[string]*artifactmodel.ContractMetadata, len(contracts)) + for _, output := range result.Outputs { + if output.Kind == extractLaneOutputKind { + got[output.SourceID] = output.Contract + } + } + for key, contract := range contracts { + sourceID := artifacts.ExtractionArtifactSourceID(key) + actual := got[sourceID] + if actual == nil || actual.MediaType != contract.MediaType || actual.SchemaID != contract.SchemaID || + actual.SchemaVersion != contract.SchemaVersion || actual.ModuleKey != contract.ModuleKey { + t.Fatalf("source %q contract = %#v, want %#v", sourceID, actual, contract) + } + } +} + func TestExtractStageRejectsMissingOrInvalidFinalTrimmedInputBeforeInvocation(t *testing.T) { tests := []struct { name string @@ -197,6 +273,9 @@ func TestExtractStageEnforcesRequiredLanePolicy(t *testing.T) { {name: "rejected", mutate: func(result *notarius.RunResult) { result.Rejections = append(result.Rejections, notarius.RejectionSummary{LaneID: "npc-registry", ReasonCode: "invalid_npc"}) }, want: "was rejected"}, + {name: "incomplete validation", mutate: func(result *notarius.RunResult) { + result.Receipt.ValidationSummaries = []notarius.ValidationSummary{{LaneID: "npc-registry", Status: "incomplete"}} + }, want: `validation is "incomplete"`}, {name: "duplicate", mutate: func(result *notarius.RunResult) { result.Index.Lanes = append(result.Index.Lanes, result.Index.Lanes[0]) }, want: "2 descriptors"}, @@ -338,8 +417,11 @@ func TestExtractStageAdapterResultIsImmediatelyReusableAndCatalogVisible(t *test "schema_version": notarius.ReceiptSchemaVersion, "run_id": "notarius-run-1", "pipeline_id": "dnd-session", "output_directory": fake.Result.BundleRoot, "index_file": "index.json", - "normalized_output_count": 2, "rejected_output_count": 0, "warning_count": 0, - "validation_status": "approved", "future_field": true, + "normalized_output_count": 2, "rejected_output_count": 0, + "warning_group_count": 0, "warning_occurrence_count": 0, + "diagnostic_group_count": 0, "diagnostic_occurrence_count": 0, + "diagnostics_truncated": false, + "validation_status": "approved", "future_field": true, }) if err != nil { t.Fatalf("json.Marshal(receipt) error = %v", err) @@ -594,10 +676,11 @@ func setupExtractEnv(t *testing.T) (*Env, *manifest.Manifest, *notarius.FakeRunn t.Fatalf("MkdirAll(bundle unknown) error = %v", err) } files := map[string]string{ - "index.json": `{"manifest_file":"manifest.json","output_files":[{"lane_id":"npc-registry","file":"lanes/npc.json","media_type":"application/json","module_key":"dnd/npc-registry","schema_id":"notarius.dnd.npc_registry","schema_version":"v1","future_field":true},{"lane_id":"unconfigured","file":"lanes/unconfigured.json"}],"rejected_file":"rejected.json","warnings_file":"warnings.json","future_field":true}`, + "index.json": `{"manifest_file":"manifest.json","output_files":[{"lane_id":"npc-registry","file":"lanes/npc.json","media_type":"application/json","module_key":"dnd/npc-registry","schema_id":"notarius.dnd.npc_registry","schema_version":"v1","future_field":true},{"lane_id":"unconfigured","file":"lanes/unconfigured.json"}],"rejected_file":"rejected.json","warnings_file":"warnings.json","diagnostics_file":"diagnostics.json","future_field":true}`, "manifest.json": `{}`, "rejected.json": `{"rejected":[]}`, - "warnings.json": `{"warnings":[]}`, + "warnings.json": `{"schema_version":"notarius.warnings.v2","group_count":0,"occurrence_count":0,"groups":[]}`, + "diagnostics.json": `{"schema_version":"notarius.diagnostics.v1","group_count":0,"occurrence_count":0,"truncated":false,"unrepresented_occurrence_count":0,"groups":[]}`, "lanes/npc.json": `{"npcs":[]}`, "lanes/unconfigured.json": `{"spells":[]}`, "chunk-map.json": `{"chunks":[]}`, @@ -613,12 +696,14 @@ func setupExtractEnv(t *testing.T) (*Env, *manifest.Manifest, *notarius.FakeRunn Receipt: notarius.Receipt{ SchemaVersion: notarius.ReceiptSchemaVersion, RunID: "notarius-run-1", PipelineID: "dnd-session", OutputDirectory: bundle, IndexFile: "index.json", NormalizedOutputCount: 2, - RejectedOutputCount: 1, WarningCount: 1, ValidationStatus: "rejected", + RejectedOutputCount: 1, WarningGroupCount: 1, WarningOccurrenceCount: 1, + DiagnosticGroupCount: 1, DiagnosticOccurrenceCount: 1, ValidationStatus: "rejected", }, BundleRoot: bundle, Index: notarius.Index{ Path: filepath.Join(bundle, "index.json"), RejectedPath: filepath.Join(bundle, "rejected.json"), - WarningsPath: filepath.Join(bundle, "warnings.json"), + WarningsPath: filepath.Join(bundle, "warnings.json"), + DiagnosticsPath: filepath.Join(bundle, "diagnostics.json"), Lanes: []notarius.LaneDescriptor{ { LaneID: "npc-registry", File: "lanes/npc.json", Path: filepath.Join(bundle, "lanes", "npc.json"), @@ -629,7 +714,14 @@ func setupExtractEnv(t *testing.T) (*Env, *manifest.Manifest, *notarius.FakeRunn }, }, Rejections: []notarius.RejectionSummary{{LaneID: "optional", ReasonCode: "optional_rejected"}}, - Warnings: []notarius.WarningSummary{{Scope: "lane:npc-registry", ReasonCode: "normalized_name"}}, + Warnings: []notarius.WarningSummary{{ + Disposition: "warning", Category: "degradation", ReasonCode: "normalized_name", + Origin: notarius.DiagnosticOrigin{Stage: "normalize", LaneID: "npc-registry"}, OccurrenceCount: 1, + }}, + Diagnostics: []notarius.DiagnosticSummary{{ + Disposition: "advisory", Category: "data_quality", ReasonCode: "low_confidence", + Origin: notarius.DiagnosticOrigin{Stage: "normalize", ValidatorKey: "validator"}, OccurrenceCount: 1, + }}, }} env := &Env{ Config: &config.Config{