From 7071102ab74c66508330dc67f4b52c1af0d7b7cf Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 21 Jul 2026 22:25:11 +0000 Subject: [PATCH] Document ordered pipeline operations and retire sequential examples --- README.md | 1 + docs/cli.md | 62 ++++--- docs/config.md | 159 +++++++++++++----- .../integrations/dnd-combat-turn-artifacts.md | 30 ++-- docs/integrations/dnd-npc-artifacts.md | 46 +++-- docs/integrations/dnd-spell-artifacts.md | 35 ++-- docs/internal/modules.md | 32 ++-- docs/internal/overview.md | 33 ++-- docs/internal/pipeline.md | 138 ++++++++++----- docs/internal/state.md | 16 ++ docs/operations.md | 97 ++++++----- examples/dnd-npc-combat-sequential.config.yml | 31 ---- examples/dnd-npc-spell-sequential.config.yml | 28 --- internal/cli/npc_registry_contract_test.go | 6 +- internal/cli/production_contract_test.go | 2 - .../integration/dnd_combat_runner_test.go | 86 ---------- .../dnd_npc_spell_sequential_test.go | 130 -------------- .../dnd_npc_spell_sequential_pipeline.yml | 25 --- 18 files changed, 431 insertions(+), 526 deletions(-) delete mode 100644 examples/dnd-npc-combat-sequential.config.yml delete mode 100644 examples/dnd-npc-spell-sequential.config.yml delete mode 100644 internal/modules/integration/dnd_npc_spell_sequential_test.go delete mode 100644 internal/modules/integration/testdata/dnd_npc_spell_sequential_pipeline.yml diff --git a/README.md b/README.md index 91b50c2..bf109b0 100644 --- a/README.md +++ b/README.md @@ -30,4 +30,5 @@ Useful references: - [Developer guide](docs/development.md) - [Internal implementation docs](docs/internal/overview.md) - [Maintained example config](examples/dnd-spells.config.yml) +- [NPC-grounded example config](examples/dnd-npc-grounded.config.yml) - [Maintained example input](examples/seriatim-minimal-transcript.json) diff --git a/docs/cli.md b/docs/cli.md index 53d9cbc..32e360a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -9,7 +9,7 @@ For the minimal end-to-end invocation, see the [README](../README.md). ```text notarius help -notarius run --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--chunk_cache auto|bypass|refresh] [--output-dir path] [--resume] [--debug [--debug-dir path]] [--llm-profile id] [--session-id id] [--reference selector=path] [--without-reference selector] +notarius run --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--chunk_cache auto|bypass|refresh] [--output-dir path] [--resume] [--recompute-step step-id] [--debug [--debug-dir path]] [--llm-profile id] [--session-id id] [--reference selector=path] [--without-reference selector] notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b] notarius pipelines list [--config path/to/config.yml] [--json] ``` @@ -28,11 +28,17 @@ Flags: - `--config path`: config file path. If omitted, Notarius uses the discovery rules in [Configuration](config.md#discovery). - `--only lane-a,lane-b`: run only the named artifact lanes. Values are - comma-separated and must be non-empty. + comma-separated and must be non-empty. This retains its existing behavior for + implicit single-step pipelines; explicit multi-step pipelines reject it + rather than inferring dependency closure. - `--resume`: request checkpoint reuse for this invocation. Checkpoint recording must be enabled in configuration. See [Operations](operations.md#checkpoint-cache) for prerequisites and reuse behavior. +- `--recompute-step step-id`: with `--resume` and checkpoint recording enabled, + force the named ordered step and every transitive dependent lane to execute. + Compatible required predecessors and unrelated lanes remain reusable. The + value must identify an explicit step; it cannot be combined with `--only`. - `--chunk_cache auto|bypass|refresh`: select chunk-plan reuse for this invocation. `auto` reuses a valid plan by canonical source digest, `bypass` performs no plan-cache I/O, and `refresh` regenerates and replaces a valid @@ -58,8 +64,10 @@ rejected output counts, and the output directory. A debug-enabled run also prints `debug=`. If the run completes with warnings, the warning count is printed to stderr. -Reference flags are resolved against selected chunk, extractor, merger, and -normalizer targets before the run starts. Flat slot names are accepted only +Reference flags are external file bindings resolved against selected chunk, +extractor, merger, and normalizer targets before the run starts. Generated +artifact bindings are configured in ordered steps and cannot be introduced by a +CLI path flag. Flat slot names are accepted only when exactly one selected target declares that slot. For configured reference bindings, precedence, path resolution, and validation, see [Configuration](config.md#pipelines). @@ -94,30 +102,21 @@ go run ./cmd/notarius run dnd-session \ --reference spells.extract.glossary=./campaign-glossary.txt ``` -For the operator-driven NPC-to-spell workflow, bind the normalized NPC lane -payload from the completed NPC run to the spell extractor: +For the maintained NPC-grounded workflow, use the explicit ordered pipeline. +The first step produces the normalized NPC artifact; the second step receives +it in memory and fans it out to spell extraction, combat extraction, and combat +normalization: ```sh -go run ./cmd/notarius run dnd-spells \ - --config examples/dnd-npc-spell-sequential.config.yml \ +go run ./cmd/notarius run dnd-npc-grounded \ + --config examples/dnd-npc-grounded.config.yml \ --input examples/seriatim-minimal-transcript.json \ - --reference spells.extract.npcs=./npc-run/lanes/npcs.json + --output-dir ./npc-grounded-output ``` -For the independent NPC-to-combat workflow, bind the same completed NPC lane -to both combat stages explicitly: - -```sh -go run ./cmd/notarius run dnd-combat \ - --config examples/dnd-npc-combat-sequential.config.yml \ - --input examples/seriatim-minimal-transcript.json \ - --reference combat.extract.npcs=./npc-run/lanes/npcs.json \ - --reference combat.normalize.npcs=./npc-run/lanes/npcs.json -``` - -The two selectors are independent stage-local bindings. Binding extraction -does not implicitly bind normalization, and Notarius does not discover or -schedule the preceding NPC run. +The generated NPC content remains contextual grounding, not spell or combat +evidence. It is represented in manifests and debug summaries by bounded +identity and producer provenance, not by payload content or a filesystem path. The same grammar can target chunk, merge, and normalize slots when the configured modules declare them: @@ -161,6 +160,23 @@ go run ./cmd/notarius run dnd-session \ --resume ``` +To selectively rerun one ordered step and its dependent lanes, use the step ID +from the configuration. The selected step and dependents are reported as +`forced_recompute`; reusable predecessors are reported as `reused`: + +```sh +go run ./cmd/notarius run dnd-npc-grounded \ + --config examples/dnd-npc-grounded.config.yml \ + --input examples/seriatim-minimal-transcript.json \ + --resume --recompute-step grounded-events +``` + +Checkpoint decisions use these categories: `reused`, `executed`, +`forced_recompute`, and `dependency_invalidated`. The reason code and bounded +detail identify the decision without exposing reference content, local paths, +or secrets. `--recompute-step` requires checkpoint recording and `--resume`; +unknown step IDs, empty values, and combinations with `--only` are rejected. + Use `--debug` to retain the redacted summary and trace bundle for one run. The bundle is allocated before pipeline resolution; once allocated, its path is also printed to stderr if the command fails. Debug-write failures cause exit diff --git a/docs/config.md b/docs/config.md index 959432b..7e257b4 100644 --- a/docs/config.md +++ b/docs/config.md @@ -22,9 +22,8 @@ The explicit-path option is defined in the [CLI reference](cli.md). - [Minimal D&D spell configuration](../examples/dnd-spells.config.yml) - [Production-oriented D&D spell configuration](../examples/dnd-spells-production.config.yml) - [D&D NPC configuration](../examples/dnd-npcs.config.yml) -- [Sequential D&D NPC and spell configuration](../examples/dnd-npc-spell-sequential.config.yml) - [D&D combat-turn configuration](../examples/dnd-combat-turns.config.yml) -- [Sequential D&D NPC and combat-turn configuration](../examples/dnd-npc-combat-sequential.config.yml) +- [D&D NPC-grounded spell and combat configuration](../examples/dnd-npc-grounded.config.yml) All are complete version 3 files. The fragments below illustrate individual fields and are not alternate complete configurations. @@ -140,13 +139,64 @@ Pipeline fields: - `input`: required module binding. - `chunk`: optional module binding. Default module is `generic`. -- `artifacts`: required for pipeline resolution. It maps artifact lane IDs to - lane definitions. +- `artifacts`: the artifact lane map for a single-step pipeline. It is treated + as an implicit step with the stable ID `default`. +- `steps`: an ordered, non-empty list of step definitions. A pipeline may use + `steps` or `artifacts`, but not both. Step IDs must be unique after trimming. - `output`: optional module binding. Default module is `json`. - `references`: optional map of reference slot names to reference paths. These bindings are defaults for eligible pipeline targets that declare the matching slot. +Each explicit step contains an `id`, an optional `references` map, and a +non-empty `artifacts` map. Steps share the pipeline input, chunk plan, worker +budget, output encoder, manifest, and failure boundary. Lanes within a step +retain the fixed extract, validate, merge, validate, normalize, and validate +workflow; the next step starts only after the current step is terminal. + +Generated references use the structured `artifact` source form to identify one +accepted normalized lane from an earlier step: + +```yaml +pipelines: + dnd-npc-grounded: + input: seriatim + steps: + - id: identify-npcs + artifacts: + npcs: + extract: dnd/npcs + normalize: dnd/npcs + - id: grounded-events + references: + npcs: + artifact: + step: identify-npcs + lane: npcs + artifacts: + spells: + extract: dnd/spells + normalize: dnd/spells + combat: + extract: dnd/combat-turns + normalize: dnd/combat-turns +``` + +The generated binding is explicit and typed; it is not inferred from module, +lane, or slot names. It may be declared at step scope, applying to every +selected target in that step that declares the slot, or at one target's +`references` map. A producer may fan out to compatible target slots, but a +slot accepts only one producer. A producer must be in an earlier step, and a +configured generated dependency is required even when the consumer slot is +otherwise optional. Aggregating several producer artifacts is unsupported. + +The producer codec supplies the artifact kind, complete schema identity, media +type, canonical content digest, and size used for compatibility and checkpoint +dependency checks. Only one accepted normalized artifact crosses the boundary; +raw extraction results, rejected output, intermediate values, and validator +diagnostics do not. Generated content is supplied in memory and is never +represented by a filesystem path. + Artifact lane fields: - `extract`: required module binding. @@ -164,19 +214,23 @@ See [CLI Reference](cli.md) for command syntax. Reference bindings are validated against reference slots declared by eligible chunk, extract, merge, and normalize targets during pipeline resolution. Required slots must be bound after config defaults, target-local references, lane-level -compatibility bindings, and command-line reference overrides are applied. -Config-relative paths are resolved relative to the config file; command-line -reference paths are resolved relative to the current working directory. Bound -files must be UTF-8 text. Reference media types are inferred from file -extensions and checked when a module restricts accepted types; unknown -extensions use `application/octet-stream`. See [CLI Reference](cli.md#run) for -command-line selectors and [Operations](operations.md) for recorded provenance -and sensitive-data handling. +compatibility bindings, step-local references, and command-line reference +overrides are applied. Config-relative paths are resolved relative to the +config file; command-line reference paths are resolved relative to the current +working directory. Bound files must be UTF-8 text. Reference media types are +inferred from file extensions and checked when a module restricts accepted +types; unknown extensions use `application/octet-stream`. See [CLI Reference](cli.md#run) +for command-line selectors and [Operations](operations.md) for recorded +provenance and sensitive-data handling. Pipeline-level `references` are defaults. They are valid when at least one eligible target in the full configured pipeline declares the slot, including chunk, extractor, merger, and normalizer targets. During a run, they apply only -to the selected targets that declare the slot: +to the selected targets that declare the slot. For external bindings, a +step-local binding overrides a pipeline-level default and a target-local +binding retains the existing most-specific precedence. Generated and external +bindings may not resolve to the same target slot, and a step-scoped generated +binding may not be duplicated by a target-local generated binding. ```yaml pipelines: @@ -226,6 +280,18 @@ Target-local reference fields use the same map shape at: Each binding is valid only when that target module declares the slot. +Reference source forms are: + +- a scalar string, which is an external file path; or +- an object with only `artifact`, containing trimmed `step` and `lane` IDs for + an earlier producer lane. + +Pipeline-level references accept only external paths. Generated references are +valid at step scope or on a target-local `references` map. Their producer's +registered codec is authoritative for schema, media type, and canonical +content identity; an external file is not treated as generated merely because +its bytes decode as the same artifact. + ## Module Bindings Every module binding may use shorthand: @@ -400,15 +466,17 @@ The extractor uses campaign references only as supporting disambiguation material; spell casts still must be present in the source transcript. It also declares an optional `npcs` slot for a normalized NPC artifact. The -slot accepts exactly one `application/json` file no larger than 1 MiB. During -extractor preparation Notarius strictly decodes and identity-validates the -artifact, then gives the model canonical JSON for caster-name grounding. -Registry source references may belong to the NPC-producing session and are -provenance only; they are not spell evidence. The bound registry contributes a -semantic digest and NPC count to extractor metadata and checkpoint identity, -while its names, aliases, content, and path do not appear there. When absent, -the prompt receives the exact empty value `{"npcs":[]}` and no registry -provenance or fingerprint is recorded. +slot accepts exactly one `application/json` artifact no larger than 1 MiB. An +external file is decoded and identity-validated during preparation. A +generated binding is validated at the step handoff and is provided to the +operation through the same reference contract. In both cases, the model +receives canonical JSON for caster-name grounding. Registry source references +may belong to the NPC-producing session and are provenance only; they are not +spell evidence. Generated reference identity and bounded producer provenance +are recorded by the framework; NPC names, aliases, content, and paths are not +copied into manifests or checkpoint decisions. When absent, the prompt receives +the exact empty value `{"npcs":[]}` and no registry provenance or fingerprint +is recorded. The `dnd/spells` normalizer declares the same optional `spell_catalog` slot. When an overlay is used, bind it independently under @@ -419,29 +487,34 @@ bound. The `dnd/npcs` extractor declares the same optional campaign slots as the spell extractor, but it does not declare the `npcs` registry slot. Its normalizer -accepts no references. To pass an NPC result to a later spell run, bind the -normalized payload explicitly at runtime; the maintained sequential example -documents that operator workflow. +accepts no references. The maintained +[NPC-grounded example](../examples/dnd-npc-grounded.config.yml) binds its +accepted normalized output to the later spell and combat targets through an +explicit ordered step. The `dnd/combat-turns` extractor declares the optional campaign slots and the structured `npcs` slot. Campaign references guide only the LLM extraction -stage. The deterministic normalizer declares only `npcs`, whose prepared -immutable registry supports the same actor and target canonicalization. Each -`npcs` slot accepts exactly one UTF-8 `application/json` file no larger than 1 -MiB. The registry's source ranges remain provenance for the reference and never -become combat evidence. Binding `npcs` to extraction and normalization is -stage-local, so an operator-driven combat run uses two explicit selectors: +stage. The deterministic normalizer declares only `npcs`, whose operation-time +registry supports the same actor and target canonicalization. Each `npcs` slot +accepts exactly one UTF-8 `application/json` artifact no larger than 1 MiB. The +registry's source ranges remain provenance for the reference and never become +combat evidence. An ordered step binding fans the same generated NPC artifact +out to extraction and normalization: -```text -combat.extract.npcs=/lanes/npcs.json -combat.normalize.npcs=/lanes/npcs.json +```yaml +references: + npcs: + artifact: + step: identify-npcs + lane: npcs ``` -When bound, the combat extractor and normalizer record only the registry's -semantic digest and count in their metadata and checkpoint fingerprints; names, -aliases, content, and paths are not recorded there. When absent, the combat -prompt receives the exact empty registry value `{"npcs":[]}` and no registry -provenance or fingerprint is recorded. +When bound, the combat extractor and normalizer receive the generated registry +at operation time. Framework provenance and checkpoint dependencies contain its +kind, schema identity, media type, canonical digest, size, and bounded producer +identity; names, aliases, content, and paths are not recorded there. When +absent, the combat prompt receives the exact empty registry value +`{"npcs":[]}` and no registry provenance or fingerprint is recorded. ## State Surfaces @@ -572,8 +645,11 @@ Configuration validation checks: Pipeline resolution additionally checks: - the pipeline ID exists; -- at least one artifact lane is declared and selected; +- at least one artifact lane is declared and selected in each explicit step; +- `artifacts` and `steps` are mutually exclusive, explicit steps are non-empty, + and step IDs are unique after trimming; - lanes selected through the CLI exist in the resolved pipeline; +- lane IDs are globally unique across ordered steps; - required module keys are present; - module keys are registered for the expected slot; - module capability requirements are satisfied; @@ -583,4 +659,7 @@ Pipeline resolution additionally checks: Scriptorium profile IDs; - bound reference slots are declared by selected chunk, extractor, merger, or normalizer targets; +- generated references identify one lane in an earlier step, use a declared + compatible artifact kind, and do not conflict with external or target-local + generated bindings; - required reference slots are bound for selected targets. diff --git a/docs/integrations/dnd-combat-turn-artifacts.md b/docs/integrations/dnd-combat-turn-artifacts.md index ff15cf7..807ccc3 100644 --- a/docs/integrations/dnd-combat-turn-artifacts.md +++ b/docs/integrations/dnd-combat-turn-artifacts.md @@ -92,6 +92,9 @@ chunk-scoped transcript plus the existing `players`, `party`, and `glossary` inputs, and optionally the deprecated `roster` reference through the shared party mapping. The optional `npcs` reference is an approved normalized NPC artifact used only for identity grounding; it never supplies combat evidence. +An external file is validated during preparation. In an ordered pipeline, the +same slot may receive the producer's canonical generated artifact at the step +handoff. The private response envelope has the same fields and JSON types as the durable turn/action shape except that source references contain only `start_unit_id` @@ -130,8 +133,10 @@ schema, combat shape, source references, then source relatedness. The standalone normalizer uses key `dnd/combat-turns`, requires `merged`, provides `normalized`, accepts no options, and accepts only the optional structured `npcs` reference. Campaign references are LLM extraction context and -are not normalizer inputs. The NPC registry is resolved during preparation; -runtime normalization uses that immutable prepared view. +are not normalizer inputs. For an external file, the NPC registry is resolved +during preparation; for a generated binding, it is resolved at the operation- +time handoff. Runtime normalization uses that immutable prepared or handed-off +view. Normalization policy is `dnd.combat_turns.normalize.v1`. It display-normalizes actor, summary, declarations, targets, and non-null resolutions; canonicalizes @@ -143,9 +148,11 @@ merging its actions or prose. Invalid evidence is never eligible for duplicate collapse. Every mutation and collapse emits a bounded warning using the merged input index in its scope. -The normalizer reports `normalization_policy` and `identity_policy` metadata and -fingerprints, plus `npc_registry_digest`, `npc_count`, and `npc_registry` only -when a registry is bound. The normalized-invariants validator is +The normalizer reports `normalization_policy` and `identity_policy` metadata +and fingerprints. An external registry may additionally contribute +`npc_registry_digest` and `npc_count`; generated registry identity is retained +in framework handoff provenance and dependency fingerprints. The +normalized-invariants validator is `normalize/dnd/combat-turns/invariants`; it defers shape and source-reference failures, then checks display normalization, target identity uniqueness, canonical evidence ordering, chronology, and duplicate identity. It rejects @@ -162,9 +169,10 @@ merge validator chain. The selectable lane uses extractor and normalizer key `dnd/combat-turns`, `appendorder` for the typed merger, and the durable codec above. A bound `npcs` -reference contributes raw-file provenance to the run manifest. Prepared combat -extractor and normalizer metadata and checkpoint fingerprints contain only the -NPC registry's semantic digest and count; the registry content, path, and NPC -source ranges are not copied into combat output. The normalized lane is emitted -as `lanes/.json` by the JSON output module, and warnings and rejection -summaries remain in their shared companion files. +reference contributes raw-file provenance to the run manifest. A generated +binding contributes artifact kind, schema identity, media type, canonical +digest, size, and bounded producer provenance. Consumer metadata and checkpoint +fingerprints contain no registry names, aliases, content, paths, or NPC source +ranges. The normalized lane is emitted as `lanes/.json` by the JSON +output module, and warnings and rejection summaries remain in their shared +companion files. diff --git a/docs/integrations/dnd-npc-artifacts.md b/docs/integrations/dnd-npc-artifacts.md index a18e230..3ae545c 100644 --- a/docs/integrations/dnd-npc-artifacts.md +++ b/docs/integrations/dnd-npc-artifacts.md @@ -85,23 +85,43 @@ checks. Relatedness emits bounded warnings when an NPC canonical name or alias is not present near its cited transcript text; opaque campaign references may explain such a warning but do not become evidence. -## Manifest And Sequential Consumption +## Manifest And Artifact Handoff The NPC extractor records prompt and response-schema identities. The durable codec records only `npc_count`; raw names, aliases, descriptions, source references, and payload bytes stay in the lane file rather than manifest -metadata. The normalized lane is independently reusable as a file reference: +metadata. The normalized lane can be consumed by a later ordered step through +the registered canonical codec: -```sh -go run ./cmd/notarius run dnd-spells \ - --config examples/dnd-npc-spell-sequential.config.yml \ - --input examples/seriatim-minimal-transcript.json \ - --reference spells.extract.npcs=./npc-output//lanes/npcs.json +```yaml +steps: + - id: identify-npcs + artifacts: + npcs: + extract: dnd/npcs + normalize: dnd/npcs + - id: grounded-events + references: + npcs: + artifact: + step: identify-npcs + lane: npcs + artifacts: + spells: + extract: dnd/spells + normalize: dnd/spells + combat: + extract: dnd/combat-turns + normalize: dnd/combat-turns ``` -The spell extractor strictly decodes and identity-validates this file, accepts -source references belonging to another session as registry provenance, and -uses only canonical names and aliases for caster grounding. Those NPC source -references are never accepted as spell evidence. The spell run's manifest -keeps raw file provenance under `references` and records only the prepared -registry's semantic digest and count in extractor metadata. +The framework hands only an accepted normalized artifact across the barrier. It +validates the canonical bytes against each consumer slot and clones the +operation-time reference for the spell and combat consumers. Generated +provenance records the artifact kind, schema identity, media type, canonical +digest, size, and producer step/lane/module, but not names, aliases, source +ranges, or payload bytes. External normalized files remain supported as +explicit references and retain their file provenance. + +NPC source references are registry provenance and are never accepted as spell +or combat evidence. Current transcript units remain the only event evidence. diff --git a/docs/integrations/dnd-spell-artifacts.md b/docs/integrations/dnd-spell-artifacts.md index 586295e..8c757ee 100644 --- a/docs/integrations/dnd-spell-artifacts.md +++ b/docs/integrations/dnd-spell-artifacts.md @@ -97,18 +97,20 @@ addressable through `source_refs`. ## Optional NPC Grounding The `dnd/spells` extractor accepts an optional `npcs` reference containing one -normalized NPC artifact as `application/json`, up to 1 MiB. Preparation uses -the approved NPC codec and identity policy to validate the file, re-encodes -canonical durable JSON, and supplies that JSON as a spell-owned prompt input. -It helps the model prefer canonical caster names and recognize aliases; it -does not establish that a spell was cast. +normalized NPC artifact as `application/json`, up to 1 MiB. An external file is +validated during preparation; an ordered generated binding is validated at the +step handoff. Both paths use the approved NPC codec and identity policy, +re-encode canonical durable JSON, and supply that JSON as an operation-time +spell prompt input. It helps the model prefer canonical caster names and +recognize aliases; it does not establish that a spell was cast. NPC source references may identify the run that produced the registry or any other session. They remain registry provenance and are never copied into a spell cast's `source_refs`; every spell evidence range must still identify the -current transcript. When the slot is absent, the prompt receives exactly -`{"npcs":[]}` and the run has no NPC reference provenance or NPC checkpoint -fingerprint. +current transcript. Generated provenance records producer and canonical +artifact identity without payload content or a path. When the slot is absent, +the prompt receives exactly `{"npcs":[]}` and the run has no NPC reference +provenance or NPC checkpoint fingerprint. ## Normalization Behavior @@ -190,15 +192,20 @@ identity fields when that module is selected. Overlay origin, media type, byte size, and raw digest are recorded separately in the manifest's reference provenance; see the [JSON output contract](json-output.md#manifestjson). -The `npc_registry_digest` and `npc_count` fields in the example are present only -when the optional NPC registry is bound. They contain no NPC names, aliases, -source references, paths, or raw bytes. +The `npc_registry_digest` and `npc_count` fields in the example are present for +an external NPC registry when the extractor publishes its prepared module +metadata. They contain no NPC names, aliases, source references, paths, or raw +bytes. A generated registry's identity is instead represented by the framework +handoff provenance and dependency fingerprint, so the consumer module metadata +does not duplicate it. The extractor's prompt hash, private response-schema hash, and effective catalog digest also contribute independently scoped semantic checkpoint fingerprints. Changing any of those prepared contracts intentionally produces a cold checkpoint miss. Fingerprints contain only digests, never prompt, schema, catalog, or reference content. When an NPC registry is bound, its semantic -digest contributes an additional local `npc_registry` fingerprint; the -manifest metadata contains only that digest and `npc_count`. Raw NPC file -provenance remains independently recorded in the manifest's `references` list. +digest contributes an additional local `npc_registry` fingerprint for an +external binding; the manifest metadata contains only that digest and +`npc_count`. Raw NPC file provenance remains independently recorded in the +manifest's `references` list. Generated bindings contribute the canonical +artifact dependency fingerprint and bounded producer provenance instead. diff --git a/docs/internal/modules.md b/docs/internal/modules.md index 31fffc4..9d43961 100644 --- a/docs/internal/modules.md +++ b/docs/internal/modules.md @@ -104,11 +104,12 @@ The NPC identity package owns Unicode comparison keys, deterministic The registry package resolves one optional normalized artifact through the strict codec, validates whole-registry identity, canonicalizes its JSON, and provides immutable records, prompt input, semantic digest, count, and exact -canonical-name/alias lookup. It owns the `npcs` slot and its bounded, -content-safe preparation failures. NPC source references are durable -provenance and are not treated as evidence for a consuming pipeline. The NPC -codec owns the strict durable `dnd/npc-list` JSON boundary and exposes -candidate versus approved encode/decode operations. +canonical-name/alias lookup. External files cross this boundary during +preparation; generated artifacts cross it at the ordered step handoff. It owns +the `npcs` slot and its bounded, content-safe validation failures. NPC source +references are durable provenance and are not treated as evidence for a +consuming pipeline. The NPC codec owns the strict durable `dnd/npc-list` JSON +boundary and exposes candidate versus approved encode/decode operations. The `internal/modules/dnd/codec/combatturns` package owns the durable `dnd/combat-turn-list` schema and candidate versus approved JSON boundary. It @@ -220,9 +221,12 @@ Shared D&D helpers keep prompt input names and source-unit reference conversion consistent with the scene chunker. The extractor also declares the optional `npcs` registry slot and consumes the -prepared immutable registry boundary from `internal/modules/dnd/npcs/registry`. -A bound registry adds only `npc_registry_digest` and `npc_count` to manifest -metadata and an `npc_registry` checkpoint fingerprint. The unbound prompt +immutable registry boundary from `internal/modules/dnd/npcs/registry`. An +external registry is prepared before execution; a generated registry is +validated and supplied at operation time. External bindings may add only +`npc_registry_digest` and `npc_count` to module metadata and an +`npc_registry` checkpoint fingerprint. Generated bindings are represented by +framework handoff provenance and dependency fingerprints. The unbound prompt input is exactly `{"npcs":[]}` and has no registry provenance or fingerprint. The shared NPC grounding fragment is placed immediately after the common campaign reference message and is included in the spell prompt fingerprint. @@ -310,16 +314,18 @@ independently for extraction and normalization. ### `internal/modules/dnd/normalize/combatturns` -The combat normalizer prepares the optional NPC registry once and uses the -immutable prepared view during runtime. It display-normalizes combat fields, +The combat normalizer prepares an external NPC registry before execution or +receives a generated registry at the ordered step handoff, then uses the +immutable view during runtime. It display-normalizes combat fields, rewrites exact canonical-name or alias matches for actors and targets, orders and deduplicates source references, stable-sorts records by source-document position, and collapses only exact duplicate identities with fully valid evidence. It deep-clones output storage and emits bounded warnings scoped to merged input indexes. Its metadata and fingerprints identify the normalization -and NPC identity policies, with registry digest/count only when bound. The -normalizer is included in the production D&D registrar with the default combat -normalization chain. +and NPC identity policies. External bindings may contribute registry +digest/count metadata; generated identity is retained in framework provenance +and dependency fingerprints. The normalizer is included in the production D&D +registrar with the default combat normalization chain. ## Output Encoder diff --git a/docs/internal/overview.md b/docs/internal/overview.md index 77b5370..785166c 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -14,11 +14,13 @@ collaborators, invokes `internal/framework/pipeline`, and places the logical output files returned by the runner. Cache and debug collaborators are supplied at this boundary. -Resolution produces a fixed ordered workflow and a sorted set of artifact -lanes. Preparation constructs the complete module and validator set before the -runner receives source bytes. Source parsing and chunking are serial; extraction -uses a bounded run-wide worker pool, followed by serial per-lane merge and -normalize continuations that may overlap across lanes. +Resolution produces a fixed ordered workflow of steps and globally unique, +sorted artifact lanes. Preparation constructs the complete module and validator +set before the runner receives source bytes. Source parsing and chunking are +serial. Each step then uses a bounded run-wide extraction pool followed by +serial per-lane merge and normalize continuations. A step barrier prevents +later consumers from starting until all earlier lanes are terminal and their +required normalized artifacts have crossed the typed handoff. ## Application Boundary @@ -42,7 +44,7 @@ normalize continuations that may overlap across lanes. | Package | Implemented responsibility | | --- | --- | | `internal/framework/contracts` | Source-stage contracts plus artifact identity, schema, serialized representation, codec, validator, reference, output, and structured-completion interfaces and data types. | -| `internal/framework/pipeline` | Module and artifact-codec registries, option validation, profile resolution, capability checks, reference materialization, complete pipeline preparation, retries, orchestration, warnings, and manifest population. | +| `internal/framework/pipeline` | Module and artifact-codec registries, ordered-step and generated-reference resolution, option validation, profile resolution, capability checks, external reference materialization, complete pipeline preparation, typed handoff, retries, orchestration, warnings, checkpoint decisions, and manifest population. | | `internal/framework/validate` | Shared validator decision and cardinality helpers. | | `internal/framework/llm` | Scriptorium-backed structured completions, prompt/schema registration, scheduling, profile recording, and secret redaction. | | `internal/framework/promptfs` | Builds module prompt filesystems from module-owned and caller-provided shared prompt assets. | @@ -111,12 +113,15 @@ this package. Domain-neutral prompt filesystem composition lives in `internal/framework/promptfs`. The `dnd/npcs/registry` package owns the optional `npcs` registry boundary. -Preparation strictly decodes and identity-validates one normalized JSON -artifact, emits canonical registry JSON to the spell prompt, and records only -its semantic digest and count in prepared metadata. The raw reference remains -independently tracked by pipeline provenance. An absent registry is represented -only by the empty prompt value `{"npcs":[]}`. Spell extraction consumes this -shared registry boundary without changing its public module contract. +External references are strictly decoded and identity-validated during +preparation; generated references are decoded and identity-validated at the +ordered step handoff. Both paths emit canonical registry JSON to operation-time +spell and combat prompt or normalization requests. The framework records +generated identity and bounded producer provenance, while the raw external +reference remains independently tracked by pipeline provenance. An absent +registry is represented only by the empty prompt value `{"npcs":[]}`. Spell +and combat consumers use this shared boundary without changing their public +module contracts. Generic validators under `internal/modules/generic/validate` provide unconditional test decisions, JSON syntax validation, and JSON Schema @@ -144,12 +149,12 @@ Implementation details for all production extensions are in | Surface | Implemented owners | Internal purpose | | --- | --- | --- | | Durable output | Output module, pipeline runner, and CLI writer | Return logical consumer files and place them for a run. | -| Cache checkpoints | `internal/framework/checkpoint` and `internal/cli` | Validate and serialize reusable extract, merge, and normalize outcomes. | +| Cache checkpoints | `internal/framework/checkpoint` and `internal/cli` | Validate and serialize reusable extract, merge, and normalize outcomes, including ordered-step scope and generated-artifact dependency decisions. | | Chunk-plan cache | `internal/framework/chunkplan` and `internal/cli` | Persist and select source-addressed plans before framework materialization. | | Debug bundles | `internal/core/debugbundle`, `internal/framework/debug`, and pipeline instrumentation | Persist redacted summaries and application-owned traces. | Physical layout, cleanup, recovery, and sensitive-data handling are defined -in [Operations](../operations.md). Concrete stage modules receive recorder +in [Operations](../operations.md). Concrete modules receive recorder interfaces and request data, not physical state roots. ## Focused Documentation diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index d7544ec..4804ba5 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -6,13 +6,15 @@ Their fixed workflow and ownership boundaries are defined by defaults, and selectable keys are defined in [Configuration](../config.md#pipelines). -Resolution fixes the selected lanes and all stage bindings; preparation -constructs every selected implementation before the runner begins source work. -After serial input parsing and plan selection or generation, the runner -materializes chunks and dispatches extract work to -one bounded run-wide worker pool in chunk-first, lane-second order. Each lane's -merge and normalize operations remain serial and may overlap other lanes once -all extracts for that lane are terminal. +Resolution fixes the ordered steps, selected lanes, and all stage bindings; +preparation constructs every selected implementation before the runner begins +source work. After serial input parsing and plan selection or generation, the +runner materializes chunks and executes one step at a time. Within a step, +extract work uses one bounded run-wide worker pool in chunk-first, lane-second +order. Each lane's merge and normalize operations remain serial, and lanes in +the same step may overlap once their extracts are terminal. A later step cannot +start across its barrier until every earlier lane is terminal and each required +generated artifact has been accepted and handed off. ## Resolution @@ -22,38 +24,48 @@ calls `pipeline.ResolvePipeline`. `ResolvePipeline`: -1. selects and sorts artifact lanes; -2. completes omitted bindings using the documented configuration defaults; -3. looks up each module and validator spec without constructing it; -4. for a typed extractor, derives its artifact kind, requires the codec, and +1. selects the explicit ordered steps, or creates the implicit `default` step + from the legacy top-level `artifacts` map; +2. selects and sorts artifact lanes within each step while enforcing global lane + identity; +3. completes omitted bindings using the documented configuration defaults; +4. looks up each module and validator spec without constructing it; +5. for a typed extractor, derives its artifact kind, requires the codec, and selects exact-type merger, normalizer, and validator variants; -5. checks required and provided capabilities in workflow order; -6. resolves target-aware reference bindings and validator chains; -7. validates each selected module and validator option set through its registry +6. checks required and provided capabilities in workflow order; +7. resolves external and generated target-aware reference bindings and + validates producer order, consumer slot declarations, and artifact-kind + compatibility; +8. validates each selected module and validator option set through its registry entry; and -8. calculates a digest over the resolved structure, including typed artifact - kind and schema identity and the effective validator policy in its resolved - execution order. +9. calculates a digest over the resolved structure, including step order, step + IDs, lane membership, generated topology, producer and consumer identities, + typed artifact kind and schema identity, and the effective validator policy + in its resolved execution order. -Resolution returns a `ResolvedPipeline` containing ordered lanes, concrete -bindings, validator chains, reference targets, and the digest. It does not read -reference bytes or construct runtime modules. CLI lane and reference selector -syntax is defined in the [CLI reference](../cli.md#run). +Resolution returns a `ResolvedPipeline` containing ordered steps, lanes, +concrete bindings, validator chains, reference targets, and the digest. It does +not read external reference bytes or construct runtime modules. CLI lane and +reference selector syntax is defined in the [CLI reference](../cli.md#run). -The digest includes each resolved validator chain's stage, lane, owning module, -ordered validator bindings, execution classes, targets, and artifact kinds. -Changing a default chain or an explicit override therefore changes pipeline -identity whenever it changes the effective validator policy. +The digest includes each resolved step's ID and lane membership, generated +producer/consumer topology, and each validator chain's stage, lane, owning +module, ordered validator bindings, execution classes, targets, and artifact +kinds. Changing step order, a dependency, a default chain, or an explicit +override therefore changes pipeline identity whenever it changes effective +execution policy. ## Reference Materialization The CLI calls `MaterializeReferences` after resolution and before constructing -the LLM client or running the pipeline. The materializer checks each binding -against its resolved target declaration, reads and validates the file, and -builds both a `contracts.ReferenceSet` and provenance-only metadata on the -corresponding `ResolvedReferenceTarget`. +the LLM client or running the pipeline. For external bindings, the materializer +checks each binding against its resolved target declaration, reads and validates +the file, and builds both a `contracts.ReferenceSet` and provenance-only +metadata on the corresponding `ResolvedReferenceTarget`. A structured +generated binding is declaration-only at this point: its producer bytes do not +exist until the producer lane reaches an accepted normalized result. -Preparation delivers the materialized set for each target through +Preparation delivers the materialized external set for each target through `pipeline.BuildRequest`: chunkers and chunk validators receive the chunk target; extractors and extract validators receive the lane extract target; mergers and merge validators receive the lane merge target; and normalizers and normalize @@ -62,10 +74,20 @@ an empty set because those stages cannot declare references. Every builder gets an isolated deep clone of its target set, so construction-time mutation cannot change another builder, the resolved pipeline, or later runtime requests. +Prepared consumers do not need to be reconstructed when generated content is +available. At the step boundary, the runner encodes the accepted producer value +through its registered canonical codec, validates the generated bytes against +each target slot's kind, schema, media type, and size, and clones one immutable +reference item into the operation request. The item includes canonical digest, +size, and bounded producer provenance but no filesystem URI. A handoff failure +is a framework dependency error and prevents every consumer in that step from +starting. + The runner continues to clone the resulting set into the chunk, extract, merge, or normalize request that owns the target. LLM-backed extensions may convert those items into named prompt inputs. Reference content remains separate from -source evidence and source digests. +source evidence and source digests, whether the item came from a file or a +generated handoff. Binding precedence, path resolution, accepted content, and media-type behavior are configuration contracts; see [Configuration](../config.md#pipelines). @@ -120,10 +142,12 @@ The current production catalog and default chain are listed only in ## Preparation And Runner Boundary `pipeline.Prepare` receives a resolved pipeline, the registries, and shared -module dependencies. It constructs input; chunk and its validators; each lane's -extract, merge, and normalize modules and validator chains in resolved order; -then output. It stops at the first error with pipeline, stage, lane, module, and -validator context as applicable. It never invokes an operation method. +module dependencies. It constructs input; chunk and its validators; every +step's lane extract, merge, and normalize modules and validator chains in +resolved order; then output. It stops at the first error with pipeline, step, +stage, lane, module, and validator context as applicable. It never invokes an +operation method. Generated references are not available during preparation; +the operation request is the handoff boundary. `PreparedPipeline` keeps private constructed executors and exposes cloned resolved input, chunk, lane, and output identities. Prepared components may @@ -136,8 +160,8 @@ stable and must not contain source content, credentials, local paths, timestamps, or other invocation-specific values. `pipeline.RunInput` carries that prepared pipeline, raw source input, run identity and timing, optional -session and profile metadata, a chunk-plan store and mode, and checkpoint/debug -collaborators. The runner +session and profile metadata, a chunk-plan store and mode, a checkpoint +execution policy, and checkpoint/debug collaborators. The runner parses source bytes through the already constructed input adapter. Later stage requests receive the generic source model; extract requests receive chunk-scoped input material, while chunk, merge, and normalize requests retain @@ -145,7 +169,9 @@ access to the original source material. Input, chunk, and output operation requests do not carry raw module options. The chunk request also does not carry an LLM client; an LLM-backed chunker receives the shared client during preparation. Their operation requests retain run-specific source, reference, -profile, session, and metadata context as applicable. +profile, session, metadata, and step-handoff context as applicable. A generated +reference is cloned into each compatible consumer request and is never exposed +as a path. Prepared lanes retain exact-type-checked erased operation closures. The runner uses those closures to keep each value typed through extraction, validation, @@ -173,10 +199,15 @@ The runner: 3. selects a stored plan or executes the configured chunker's `Plan` operation; 4. canonicalizes and materializes the plan, then validates the resulting chunks; -5. dispatches extract jobs in source-chunk then resolved-lane order, starting a - bounded lane continuation when all extracts for that lane are terminal; -6. invokes the prepared output encoder and validates its logical file results; -7. returns the assembled manifest, outcomes, warnings, and files. +5. executes each resolved step in configuration order. For one step, it + dispatches extract jobs in source-chunk then resolved-lane order, starts a + bounded lane continuation when all extracts for that lane are terminal, and + waits for every lane to become terminal; +6. encodes and validates each accepted normalized producer artifact, then + builds the immutable generated reference sets for the next step; +7. invokes the prepared output encoder only after every step succeeds and + validates its logical file results; +8. returns the assembled manifest, outcomes, warnings, and files. Within each artifact lane, it reuses the prepared extractor, merger, normalizer, and validators while performing these transitions: @@ -190,6 +221,13 @@ and validators while performing these transitions: 6. normalize the accepted merge result; 7. validate and append the accepted normalized result. +At a step barrier, a lane with no accepted normalized output is still a regular +rejection unless a later generated binding names that lane as a required +producer. In that case the runner raises a deterministic dependency error and +does not start the consumer step. One accepted typed artifact may fan out to +multiple compatible target slots. Consumers in the same step may run +concurrently after the handoff; no work crosses the barrier early. + Module-provided warnings and payload warnings are promoted only from attempts whose results are accepted and used. @@ -271,10 +309,19 @@ entering the normal handoff path. Typed extract, merge, and normalize checkpoints store codec bytes with artifact kind, schema ID and version, exact schema digest, and media type. Reuse compares that identity with the prepared codec and decodes through the codec; missing identity, mismatches, corrupt -bytes, and decode failures become explicit reuse misses and execute the step +bytes, and decode failures become explicit reuse misses and execute the lane normally. Dependency fingerprints and debug content digests use the same stable codec bytes that cross those boundaries. +Generated references add downstream dependencies containing the producer's +artifact kind, complete schema identity, media type, canonical content digest, +and size. Compatible producer checkpoints may therefore feed a later step +without re-executing the producer. A missing, rejected, corrupt, incompatible, +or changed producer invalidates every transitive dependent lane while leaving +independent work eligible for reuse. The runner records bounded decision +categories: `reused`, `executed`, `forced_recompute`, and +`dependency_invalidated`. + The CLI includes prepared-component fingerprints in the run-wide checkpoint identity alongside resolved configuration, raw input, reference provenance, runtime overrides, and LLM-profile fingerprints. Module metadata is not used @@ -283,6 +330,11 @@ values that can change accepted output. Adding or changing a component fingerprint intentionally produces a cold cache miss. Existing checkpoint schemas and paths remain unchanged. +The CLI's `--recompute-step` policy forces the selected step and all transitive +dependents, but requires reusable checkpoints for all selected producers that +precede it. It changes loader decisions only; it does not alter persistent +checkpoint identity. + Debug instrumentation wraps run, stage, attempt, validator, and structured LLM boundaries. Every executed chunk, extract, merge, and normalize attempt writes one terminal envelope for acceptance, validator rejection, module or validator diff --git a/docs/internal/state.md b/docs/internal/state.md index 86403db..18fa305 100644 --- a/docs/internal/state.md +++ b/docs/internal/state.md @@ -32,6 +32,17 @@ The serialized `workspace_schema_version` identifiers are frozen wire-compatibility fields; they do not describe a current public state surface. +Ordered-step lane checkpoints include the step identity in their storage scope. +When a later lane consumes a generated artifact, its dependency fingerprints +include the producer's artifact kind, complete schema identity, media type, +canonical content digest, and size. A producer checkpoint may be decoded through +the registered codec and handed off without rerunning it. Missing, rejected, +corrupt, incompatible, or changed producer state produces a bounded +`dependency_invalidated` decision for every transitive dependent lane; it does +not permit stale downstream reuse. The CLI's selective recomputation policy +records `forced_recompute` for the selected step and its dependents while +requiring compatible predecessor checkpoints. + `internal/core/fileio` provides confined atomic file writes used by state collaborators. The chunk-plan store retains its stronger entry validation. @@ -48,6 +59,11 @@ boundaries redact sensitive metadata and credential-shaped bytes while allowing application-owned trace material. Debug data is never a checkpoint source or cache input. +Generated reference bytes exist only in cloned operation requests and are not +written as paths into checkpoints, manifests, or debug summaries. Those state +surfaces retain canonical identities and bounded producer provenance so that a +resume decision can be explained without copying generated campaign content. + After allocation, one CLI-owned state value accumulates the known report paths, pipeline outcome counts, and validation status. A single guarded terminalization operation writes the success report, or makes one attempt each to write the diff --git a/docs/operations.md b/docs/operations.md index b1ca307..7339b05 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -40,63 +40,33 @@ names, schemas, and media types inside a run directory. Remove an output run directory only after its consumer data is no longer needed. This is data deletion, not cache cleanup. -## Sequential NPC And Spell Runs +## Ordered D&D Workflow -The maintained [sequential configuration](../examples/dnd-npc-spell-sequential.config.yml) -contains two independent pipelines over the same Seriatim input shape. Run the -NPC pipeline first and retain its normalized payload: +The maintained [NPC-grounded configuration](../examples/dnd-npc-grounded.config.yml) +contains one pipeline with two ordered steps. The first step extracts and +normalizes NPCs. Only after that lane reaches an accepted terminal result does +the second step begin; its generated NPC reference is supplied in memory to +spell extraction, combat extraction, and combat normalization. ```sh -go run ./cmd/notarius run dnd-npcs \ - --config examples/dnd-npc-spell-sequential.config.yml \ +go run ./cmd/notarius run dnd-npc-grounded \ + --config examples/dnd-npc-grounded.config.yml \ --input examples/seriatim-minimal-transcript.json \ - --output-dir ./npc-output + --output-dir ./npc-grounded-output ``` -Then bind that completed run's `lanes/npcs.json` file to the spell extractor: +The NPC artifact grounds canonical names and aliases, not spell or combat +evidence. Current-transcript source ranges remain the only event evidence. The +manifest records generated-reference identity and bounded producer provenance; +it does not record generated payload content, and no generated content is +exposed through a filesystem path. The same producer artifact may fan out to +compatible consumers, while a missing or rejected producer prevents the later +step from starting. -```sh -go run ./cmd/notarius run dnd-spells \ - --config examples/dnd-npc-spell-sequential.config.yml \ - --input examples/seriatim-minimal-transcript.json \ - --reference spells.extract.npcs=./npc-output//lanes/npcs.json -``` - -The NPC file is a reference for canonical caster names and aliases, not spell -evidence. The spell manifest records the bound file's raw reference provenance -and the prepared registry's count and semantic digest separately. The NPC -payload, names, aliases, source references, and file bytes can be sensitive -campaign data; protect both output roots and any checkpoint or debug roots that -retain derived application data. A registry from another session is allowed, -but its source references are never copied into spell output evidence. - -## Sequential NPC And Combat Runs - -The maintained [sequential NPC and combat configuration](../examples/dnd-npc-combat-sequential.config.yml) -also represents two independent runs. Run `dnd-npcs` first, then bind its -normalized `lanes/npcs.json` payload to both combat stages: - -```sh -go run ./cmd/notarius run dnd-npcs \ - --config examples/dnd-npc-combat-sequential.config.yml \ - --input examples/seriatim-minimal-transcript.json \ - --output-dir ./npc-output - -go run ./cmd/notarius run dnd-combat \ - --config examples/dnd-npc-combat-sequential.config.yml \ - --input examples/seriatim-minimal-transcript.json \ - --reference combat.extract.npcs=./npc-output//lanes/npcs.json \ - --reference combat.normalize.npcs=./npc-output//lanes/npcs.json -``` - -Extraction and normalization bindings are stage-local and are intentionally -specified separately. Notarius does not discover the NPC run, copy its source -ranges into combat evidence, or compose the two runs into one workflow. The -combat manifest records both reference bindings and the prepared registry's -semantic digest/count. Changing the referenced NPC payload, prompt or schema, -normalization policy, or registry digest makes affected checkpoint state -incompatible; output, checkpoint, and debug roots remain independent sensitive -state surfaces. +Standalone module configurations continue to support external NPC files when a +workflow intentionally crosses a process or session boundary. Those files are +validated against the consumer slot and must be protected as sensitive +campaign data. They are not part of the maintained ordered handoff workflow. ## Chunk-Plan Cache @@ -177,6 +147,33 @@ cache: Remove an exact checkpoint identity directory or the configured root only when recomputation is acceptable. +### Resume And Selective Recompute + +`--resume` loads compatible accepted work only when checkpoint recording is +enabled. A normal resumed run may reuse source, extract, merge, and normalize +checkpoints independently. Generated references add a dependency fingerprint +for the producer's artifact kind, schema identity, media type, canonical +content digest, and size. If that fingerprint changes or the producer is +missing, dependent checkpoints are invalidated; unrelated work remains eligible +for reuse. + +`--recompute-step ` requires both `--resume` and +`cache.checkpoints.enabled: true`. It forces the named step and all transitive +dependents to execute, while compatible predecessors and unrelated lanes remain +reusable. It cannot be combined with `--only`, and it does not change the +persistent identity of otherwise identical checkpoints. Decisions are bounded +and categorized as `reused`, `executed`, `forced_recompute`, or +`dependency_invalidated`. + +If a required predecessor cannot be reused during selective recomputation, the +run fails before the dependent step starts. The failure manifest retains the +completed upstream outcomes and dependency context but not generated reference +content. For diagnosis, first check the failed step and lane in the manifest, +then inspect checkpoint decision categories and reason codes. A +`dependency_invalidated` decision means the stored producer, codec identity, +schema, or canonical content no longer matches; a missing or rejected producer +requires rerunning it rather than copying an artifact into the checkpoint root. + ## Debug Bundles Only `notarius run --debug` enables debug collection. The selected root contains diff --git a/examples/dnd-npc-combat-sequential.config.yml b/examples/dnd-npc-combat-sequential.config.yml deleted file mode 100644 index dbae917..0000000 --- a/examples/dnd-npc-combat-sequential.config.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: 3 -output: - directory: ./notarius-output -cache: - chunk_plans: - mode: bypass - directory: ./notarius-cache/chunk-plans - checkpoints: - enabled: false - directory: ./notarius-cache/checkpoints -debug: - directory: ./notarius-debug -pipelines: - dnd-npcs: - input: seriatim - chunk: generic - artifacts: - npcs: - extract: - module: dnd/npcs - retries: 2 - normalize: dnd/npcs - dnd-combat: - input: seriatim - chunk: generic - artifacts: - combat: - extract: - module: dnd/combat-turns - retries: 2 - normalize: dnd/combat-turns diff --git a/examples/dnd-npc-spell-sequential.config.yml b/examples/dnd-npc-spell-sequential.config.yml deleted file mode 100644 index 876a9ed..0000000 --- a/examples/dnd-npc-spell-sequential.config.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: 3 -output: - directory: ./notarius-output -cache: - chunk_plans: - mode: bypass - checkpoints: - enabled: false - directory: "" -debug: - directory: ./notarius-debug -pipelines: - dnd-npcs: - input: seriatim - chunk: generic - artifacts: - npcs: - extract: - module: dnd/npcs - retries: 2 - normalize: dnd/npcs - dnd-spells: - input: seriatim - chunk: generic - artifacts: - spells: - extract: dnd/spells - normalize: dnd/spells diff --git a/internal/cli/npc_registry_contract_test.go b/internal/cli/npc_registry_contract_test.go index 9692939..7621eb9 100644 --- a/internal/cli/npc_registry_contract_test.go +++ b/internal/cli/npc_registry_contract_test.go @@ -22,7 +22,7 @@ func TestOversizedNPCRegistryFailsBeforeRuntimeAndCheckpointConstruction(t *test t.Fatal(err) } checkpointRoot := filepath.Join(t.TempDir(), "checkpoints") - content := string(readRepositoryFile(t, "examples", "dnd-npc-spell-sequential.config.yml")) + content := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml")) content = replaceRequiredOnce(t, content, " extract: dnd/spells", " extract:\n module: dnd/spells\n references:\n npcs: "+npcPath) content = replaceRequiredOnce(t, content, " enabled: false\n directory: \"\"", " enabled: true\n directory: "+checkpointRoot) configPath := filepath.Join(t.TempDir(), "config.yml") @@ -46,11 +46,11 @@ func TestOversizedNPCRegistryFailsBeforeRuntimeAndCheckpointConstruction(t *test } var stdout, stderr strings.Builder code := RunWithOptions([]string{ - "run", "dnd-spells", "--config", configPath, + "run", "dnd-session", "--config", configPath, "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--chunk_cache", "bypass", "--output-dir", t.TempDir(), }, &stdout, &stderr, options) - for _, fragment := range []string{`pipeline "dnd-spells"`, `reference slot "npcs"`, "1048577 bytes", "limit 1048576"} { + for _, fragment := range []string{`pipeline "dnd-session"`, `reference slot "npcs"`, "1048577 bytes", "limit 1048576"} { if code == 0 || !strings.Contains(stderr.String(), fragment) { t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, stderr.String(), fragment) } diff --git a/internal/cli/production_contract_test.go b/internal/cli/production_contract_test.go index 29c98c2..cd0c16f 100644 --- a/internal/cli/production_contract_test.go +++ b/internal/cli/production_contract_test.go @@ -504,9 +504,7 @@ func maintainedExampleFiles(t *testing.T) []maintainedExample { {name: "minimal", path: repositoryPath("examples", "dnd-spells.config.yml"), pipelineIDs: []string{"dnd-session"}}, {name: "production", path: repositoryPath("examples", "dnd-spells-production.config.yml"), pipelineIDs: []string{"dnd-session"}}, {name: "npcs", path: repositoryPath("examples", "dnd-npcs.config.yml"), pipelineIDs: []string{"dnd-session"}}, - {name: "sequential", path: repositoryPath("examples", "dnd-npc-spell-sequential.config.yml"), pipelineIDs: []string{"dnd-npcs", "dnd-spells"}}, {name: "combat", path: repositoryPath("examples", "dnd-combat-turns.config.yml"), pipelineIDs: []string{"dnd-combat"}}, - {name: "npc-combat-sequential", path: repositoryPath("examples", "dnd-npc-combat-sequential.config.yml"), pipelineIDs: []string{"dnd-combat", "dnd-npcs"}}, {name: "npc-grounded", path: repositoryPath("examples", "dnd-npc-grounded.config.yml"), pipelineIDs: []string{"dnd-npc-grounded"}}, } } diff --git a/internal/modules/integration/dnd_combat_runner_test.go b/internal/modules/integration/dnd_combat_runner_test.go index a6acc64..36361e1 100644 --- a/internal/modules/integration/dnd_combat_runner_test.go +++ b/internal/modules/integration/dnd_combat_runner_test.go @@ -152,79 +152,6 @@ func TestCombatNormalizerRejectsCampaignReferenceBinding(t *testing.T) { } } -func TestSequentialNPCOutputGroundsCombatAtBothStageLocalReferences(t *testing.T) { - registries := productionNPCRegistries(t) - catalog := moduleCatalog(registries) - configValue := loadSequentialCombatPipelineConfig(t) - npcEffective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npcs", Catalog: catalog}) - if err != nil { - t.Fatalf("Resolve(NPC) error = %v", err) - } - npcClient := &fakeNPCProductionLLMClient{response: npcProductionResponse{NPCs: []npcProductionRecord{ - {Name: "Mira Thorn", Aliases: []string{"The Greencloak", "Mira"}, Description: "A ranger.", Relationships: []npcProductionRelationship{}, SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 2}}}, - {Name: "Hooded Guard", Aliases: []string{}, Description: "A sentry.", Relationships: []npcProductionRelationship{}, SourceRefs: []npcProductionSourceRef{{StartUnitID: 3, EndUnitID: 3}}}, - }}} - npcOutput, err := runPreparedPipeline(t, registries, npcEffective.ResolvedPipeline, npcClient, pipeline.RunInput{RawInput: readNPCFixture(t)}) - if err != nil || len(npcOutput.NormalizeOutputs) != 1 { - t.Fatalf("NPC run error = %v output = %#v, want one normalized NPC lane", err, npcOutput.NormalizeOutputs) - } - npcPayload := npcOutput.NormalizeOutputs[0].Artifact.Content - if _, err := npccodec.New().Decode(npcPayload); err != nil { - t.Fatalf("Decode(NPC output) error = %v", err) - } - npcPath := filepath.Join(t.TempDir(), "npc-run", "lanes", "npcs.json") - if err := os.MkdirAll(filepath.Dir(npcPath), 0o700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(npcPath, npcPayload, 0o600); err != nil { - t.Fatal(err) - } - - combatEffective, err := configValue.Resolve(config.ResolveInput{ - PipelineID: "dnd-combat", - Catalog: catalog, - ReferenceOverrides: []pipeline.ReferenceBinding{ - {Stage: pipeline.StageExtract, LaneID: "combat", SlotName: "npcs", Source: npcPath, BindingSource: contracts.ReferenceBindingSourceCLI}, - {Stage: pipeline.StageNormalize, LaneID: "combat", SlotName: "npcs", Source: npcPath, BindingSource: contracts.ReferenceBindingSourceCLI}, - }, - }) - if err != nil { - t.Fatalf("Resolve(combat) error = %v", err) - } - materialized, warnings, err := pipeline.MaterializeReferences(combatEffective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{}) - if err != nil || len(warnings) != 0 { - t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings) - } - client := &fakeCombatLLMClient{responses: []string{combatTestTurnResponse("The Greencloak", "turn", "watches", "Hooded Guard", 1, 1)}} - output, err := runPreparedPipeline(t, registries, materialized, client, pipeline.RunInput{RawInput: readNPCFixture(t)}) - if err != nil { - t.Fatalf("Run(combat) error = %v", err) - } - value, err := combatcodec.New().Decode(output.NormalizeOutputs[0].Artifact.Content) - if err != nil { - t.Fatalf("Decode(combat output) error = %v", err) - } - if len(value.CombatTurns) != 1 || value.CombatTurns[0].Actor != "Mira Thorn" || value.CombatTurns[0].Actions[0].Targets[0] != "Hooded Guard" { - t.Fatalf("combat value = %#v, want canonical actor and target", value) - } - for _, ref := range value.CombatTurns[0].SourceRefs { - if ref.SourceID != "npc-session" { - t.Fatalf("combat source ref = %#v, want transcript evidence only", ref) - } - } - if len(output.Manifest.References) != 2 { - t.Fatalf("manifest references = %#v, want both stage-local NPC provenance entries", output.Manifest.References) - } - for _, provenance := range output.Manifest.References { - if provenance.SlotName != "npcs" || provenance.LaneID != "combat" || !strings.Contains(provenance.OriginURI, "npcs.json") || provenance.BindingSource != contracts.ReferenceBindingSourceCLI { - t.Fatalf("NPC provenance = %#v, want combat stage-local CLI binding", provenance) - } - } - if len(client.requests) != 1 || string(client.requests[0].Inputs[combatextract.NPCRegistryReferenceSlot].Content) != string(npcPayload) || client.requests[0].Inputs[combatextract.NPCRegistryReferenceSlot].OriginURI != "" { - t.Fatalf("combat NPC prompt input = %#v, want canonical payload without path provenance", client.requests) - } -} - func TestCombatPreparationRejectsMalformedOrOversizedNPCReferencesBeforeExecution(t *testing.T) { registries := productionNPCRegistries(t) catalog := moduleCatalog(registries) @@ -313,19 +240,6 @@ func combatOnlyConfig() config.Config { return cfg } -func loadSequentialCombatPipelineConfig(t *testing.T) config.Config { - t.Helper() - fileConfig, err := config.LoadFileConfig(repositoryPathForIntegration("examples", "dnd-npc-combat-sequential.config.yml")) - if err != nil { - t.Fatalf("LoadFileConfig() error = %v", err) - } - cfg := config.Default() - if err := cfg.ApplyFileConfig(fileConfig); err != nil { - t.Fatalf("ApplyFileConfig() error = %v", err) - } - return cfg -} - func repositoryPathForIntegration(parts ...string) string { _, file, _, _ := runtime.Caller(0) return filepath.Join(append([]string{filepath.Dir(file), "..", "..", ".."}, parts...)...) diff --git a/internal/modules/integration/dnd_npc_spell_sequential_test.go b/internal/modules/integration/dnd_npc_spell_sequential_test.go deleted file mode 100644 index 1afe948..0000000 --- a/internal/modules/integration/dnd_npc_spell_sequential_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package integration_test - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" - npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" -) - -func TestSequentialNPCOutputCanGroundIndependentSpellRun(t *testing.T) { - registries := productionNPCRegistries(t) - catalog := moduleCatalog(registries) - configValue := loadSequentialPipelineConfig(t) - - npcEffective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npcs", Catalog: catalog}) - if err != nil { - t.Fatalf("resolve NPC pipeline: %v", err) - } - npcClient := &fakeNPCProductionLLMClient{response: npcProductionResponse{NPCs: []npcProductionRecord{{ - Name: "Mira Thorn", - Aliases: []string{"The Greencloak"}, - Description: "A guarded ranger who watches the northern road.", - Relationships: []npcProductionRelationship{{ - Target: "Captain Vale", Relationship: "reports to", - }}, - SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 2}}, - }}}} - npcOutput, err := runPreparedPipeline(t, registries, npcEffective.ResolvedPipeline, npcClient, pipeline.RunInput{RawInput: readNPCFixture(t)}) - if err != nil { - t.Fatalf("run NPC pipeline: %v", err) - } - if len(npcOutput.NormalizeOutputs) != 1 || npcOutput.NormalizeOutputs[0].LaneID != "npcs" { - t.Fatalf("NPC normalized outputs = %#v, want one npcs lane", npcOutput.NormalizeOutputs) - } - npcPayload := npcOutput.NormalizeOutputs[0].Artifact.Content - if _, err := npccodec.New().Decode(npcPayload); err != nil { - t.Fatalf("decode normalized NPC payload: %v", err) - } - - npcRunDir := t.TempDir() - npcPath := filepath.Join(npcRunDir, "lanes", "npcs.json") - if err := os.MkdirAll(filepath.Dir(npcPath), 0o700); err != nil { - t.Fatalf("create NPC output directory: %v", err) - } - if err := os.WriteFile(npcPath, npcPayload, 0o600); err != nil { - t.Fatalf("write NPC output payload: %v", err) - } - - spellEffective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-spells", Catalog: catalog}) - if err != nil { - t.Fatalf("resolve spell pipeline: %v", err) - } - spellEffective.ResolvedPipeline.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings = []pipeline.ReferenceBinding{{ - Stage: pipeline.StageExtract, - LaneID: "spells", - SlotName: spells.NPCRegistryReferenceSlot, - Source: npcPath, - BindingSource: contracts.ReferenceBindingSourceCLI, - }} - materialized, warnings, err := pipeline.MaterializeReferences(spellEffective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{WorkingDir: npcRunDir}) - if err != nil { - t.Fatalf("materialize NPC registry reference: %v", err) - } - if len(warnings) != 0 { - t.Fatalf("reference materialization warnings = %#v, want none", warnings) - } - - spellClient := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{{ - Caster: "Mira Thorn", - Spell: "Cure Wounds", - Effect: "Restores the injured ally.", - NarrativeDescription: "Mira Thorn restores the ally after the fight.", - SourceRefs: responseSourceRefs("spell-session", 1, 1), - }}}} - spellOutput, err := runPreparedPipeline(t, registries, materialized, spellClient, pipeline.RunInput{RawInput: readDNDSpellsFixture(t)}) - if err != nil { - t.Fatalf("run spell pipeline: %v", err) - } - if len(spellOutput.NormalizeOutputs) != 1 || spellOutput.NormalizeOutputs[0].LaneID != "spells" { - t.Fatalf("spell normalized outputs = %#v, want one spells lane", spellOutput.NormalizeOutputs) - } - spellValue := decodeRunnerSpellResponse(t, spellOutput.NormalizeOutputs[0].Artifact.Content) - if len(spellValue.SpellCasts) != 1 || spellValue.SpellCasts[0].Caster != "Mira Thorn" { - t.Fatalf("spell output = %#v, want one registry-grounded caster", spellValue) - } - if len(spellValue.SpellCasts[0].SourceRefs) != 1 || spellValue.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" { - t.Fatalf("spell source refs = %#v, want current spell session only", spellValue.SpellCasts[0].SourceRefs) - } - if len(spellClient.requests) != 1 { - t.Fatalf("spell LLM requests = %d, want one", len(spellClient.requests)) - } - registryInput := spellClient.requests[0].Inputs[spells.NPCRegistryReferenceSlot] - if string(registryInput.Content) != string(npcPayload) || registryInput.MediaType != npccodec.MediaType || registryInput.OriginURI != "" { - t.Fatalf("spell NPC prompt input = %#v, want canonical payload without origin", registryInput) - } - if len(spellOutput.Manifest.References) != 1 { - t.Fatalf("spell manifest references = %#v, want one NPC provenance entry", spellOutput.Manifest.References) - } - provenance := spellOutput.Manifest.References[0] - if provenance.Stage != "extract" || provenance.LaneID != "spells" || provenance.SlotName != spells.NPCRegistryReferenceSlot || provenance.BindingSource != contracts.ReferenceBindingSourceCLI || !strings.Contains(provenance.OriginURI, "npcs.json") { - t.Fatalf("spell NPC provenance = %#v, want extract CLI reference provenance", provenance) - } - metadata, ok := spellOutput.Manifest.ArtifactLanes[0].Metadata["extractor"].(map[string]any) - if !ok || metadata["npc_count"] != 1 || metadata["npc_registry_digest"] != registryInput.Digest { - t.Fatalf("spell extractor metadata = %#v, want NPC count and semantic digest", spellOutput.Manifest.ArtifactLanes[0].Metadata) - } -} - -func loadSequentialPipelineConfig(t *testing.T) config.Config { - t.Helper() - data, err := os.ReadFile("testdata/dnd_npc_spell_sequential_pipeline.yml") - if err != nil { - t.Fatalf("read sequential pipeline config: %v", err) - } - fileConfig, err := config.ParseFileConfigYAML(data) - if err != nil { - t.Fatalf("parse sequential pipeline config: %v", err) - } - configValue := config.Default() - if err := configValue.ApplyFileConfig(fileConfig); err != nil { - t.Fatalf("apply sequential pipeline config: %v", err) - } - return configValue -} diff --git a/internal/modules/integration/testdata/dnd_npc_spell_sequential_pipeline.yml b/internal/modules/integration/testdata/dnd_npc_spell_sequential_pipeline.yml deleted file mode 100644 index 2851a39..0000000 --- a/internal/modules/integration/testdata/dnd_npc_spell_sequential_pipeline.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: 3 -output: - directory: ./notarius-output -cache: - chunk_plans: - mode: bypass - checkpoints: {} -debug: - directory: ./notarius-debug -pipelines: - dnd-npcs: - input: seriatim - chunk: generic - artifacts: - npcs: - extract: - module: dnd/npcs - normalize: dnd/npcs - dnd-spells: - input: seriatim - chunk: generic - artifacts: - spells: - extract: dnd/spells - normalize: dnd/spells