Document ordered pipeline operations and retire sequential examples
This commit is contained in:
@@ -30,4 +30,5 @@ Useful references:
|
|||||||
- [Developer guide](docs/development.md)
|
- [Developer guide](docs/development.md)
|
||||||
- [Internal implementation docs](docs/internal/overview.md)
|
- [Internal implementation docs](docs/internal/overview.md)
|
||||||
- [Maintained example config](examples/dnd-spells.config.yml)
|
- [Maintained example config](examples/dnd-spells.config.yml)
|
||||||
|
- [NPC-grounded example config](examples/dnd-npc-grounded.config.yml)
|
||||||
- [Maintained example input](examples/seriatim-minimal-transcript.json)
|
- [Maintained example input](examples/seriatim-minimal-transcript.json)
|
||||||
|
|||||||
62
docs/cli.md
62
docs/cli.md
@@ -9,7 +9,7 @@ For the minimal end-to-end invocation, see the [README](../README.md).
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
notarius help
|
notarius help
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--chunk_cache auto|bypass|refresh] [--output-dir path] [--resume] [--debug [--debug-dir path]] [--llm-profile id] [--session-id id] [--reference selector=path] [--without-reference selector]
|
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--chunk_cache auto|bypass|refresh] [--output-dir path] [--resume] [--recompute-step step-id] [--debug [--debug-dir path]] [--llm-profile id] [--session-id id] [--reference selector=path] [--without-reference selector]
|
||||||
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius pipelines list [--config path/to/config.yml] [--json]
|
notarius pipelines list [--config path/to/config.yml] [--json]
|
||||||
```
|
```
|
||||||
@@ -28,11 +28,17 @@ Flags:
|
|||||||
- `--config path`: config file path. If omitted, Notarius uses the discovery
|
- `--config path`: config file path. If omitted, Notarius uses the discovery
|
||||||
rules in [Configuration](config.md#discovery).
|
rules in [Configuration](config.md#discovery).
|
||||||
- `--only lane-a,lane-b`: run only the named artifact lanes. Values are
|
- `--only lane-a,lane-b`: run only the named artifact lanes. Values are
|
||||||
comma-separated and must be non-empty.
|
comma-separated and must be non-empty. This retains its existing behavior for
|
||||||
|
implicit single-step pipelines; explicit multi-step pipelines reject it
|
||||||
|
rather than inferring dependency closure.
|
||||||
- `--resume`: request checkpoint reuse for this invocation. Checkpoint recording
|
- `--resume`: request checkpoint reuse for this invocation. Checkpoint recording
|
||||||
must be enabled in configuration. See
|
must be enabled in configuration. See
|
||||||
[Operations](operations.md#checkpoint-cache) for prerequisites and reuse
|
[Operations](operations.md#checkpoint-cache) for prerequisites and reuse
|
||||||
behavior.
|
behavior.
|
||||||
|
- `--recompute-step step-id`: with `--resume` and checkpoint recording enabled,
|
||||||
|
force the named ordered step and every transitive dependent lane to execute.
|
||||||
|
Compatible required predecessors and unrelated lanes remain reusable. The
|
||||||
|
value must identify an explicit step; it cannot be combined with `--only`.
|
||||||
- `--chunk_cache auto|bypass|refresh`: select chunk-plan reuse for this
|
- `--chunk_cache auto|bypass|refresh`: select chunk-plan reuse for this
|
||||||
invocation. `auto` reuses a valid plan by canonical source digest, `bypass`
|
invocation. `auto` reuses a valid plan by canonical source digest, `bypass`
|
||||||
performs no plan-cache I/O, and `refresh` regenerates and replaces a valid
|
performs no plan-cache I/O, and `refresh` regenerates and replaces a valid
|
||||||
@@ -58,8 +64,10 @@ rejected output counts, and the output directory. A debug-enabled run also
|
|||||||
prints `debug=<bundle-path>`. If the run completes with warnings, the warning
|
prints `debug=<bundle-path>`. If the run completes with warnings, the warning
|
||||||
count is printed to stderr.
|
count is printed to stderr.
|
||||||
|
|
||||||
Reference flags are resolved against selected chunk, extractor, merger, and
|
Reference flags are external file bindings resolved against selected chunk,
|
||||||
normalizer targets before the run starts. Flat slot names are accepted only
|
extractor, merger, and normalizer targets before the run starts. Generated
|
||||||
|
artifact bindings are configured in ordered steps and cannot be introduced by a
|
||||||
|
CLI path flag. Flat slot names are accepted only
|
||||||
when exactly one selected target declares that slot. For configured reference
|
when exactly one selected target declares that slot. For configured reference
|
||||||
bindings, precedence, path resolution, and validation, see
|
bindings, precedence, path resolution, and validation, see
|
||||||
[Configuration](config.md#pipelines).
|
[Configuration](config.md#pipelines).
|
||||||
@@ -94,30 +102,21 @@ go run ./cmd/notarius run dnd-session \
|
|||||||
--reference spells.extract.glossary=./campaign-glossary.txt
|
--reference spells.extract.glossary=./campaign-glossary.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
For the operator-driven NPC-to-spell workflow, bind the normalized NPC lane
|
For the maintained NPC-grounded workflow, use the explicit ordered pipeline.
|
||||||
payload from the completed NPC run to the spell extractor:
|
The first step produces the normalized NPC artifact; the second step receives
|
||||||
|
it in memory and fans it out to spell extraction, combat extraction, and combat
|
||||||
|
normalization:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go run ./cmd/notarius run dnd-spells \
|
go run ./cmd/notarius run dnd-npc-grounded \
|
||||||
--config examples/dnd-npc-spell-sequential.config.yml \
|
--config examples/dnd-npc-grounded.config.yml \
|
||||||
--input examples/seriatim-minimal-transcript.json \
|
--input examples/seriatim-minimal-transcript.json \
|
||||||
--reference spells.extract.npcs=./npc-run/lanes/npcs.json
|
--output-dir ./npc-grounded-output
|
||||||
```
|
```
|
||||||
|
|
||||||
For the independent NPC-to-combat workflow, bind the same completed NPC lane
|
The generated NPC content remains contextual grounding, not spell or combat
|
||||||
to both combat stages explicitly:
|
evidence. It is represented in manifests and debug summaries by bounded
|
||||||
|
identity and producer provenance, not by payload content or a filesystem path.
|
||||||
```sh
|
|
||||||
go run ./cmd/notarius run dnd-combat \
|
|
||||||
--config examples/dnd-npc-combat-sequential.config.yml \
|
|
||||||
--input examples/seriatim-minimal-transcript.json \
|
|
||||||
--reference combat.extract.npcs=./npc-run/lanes/npcs.json \
|
|
||||||
--reference combat.normalize.npcs=./npc-run/lanes/npcs.json
|
|
||||||
```
|
|
||||||
|
|
||||||
The two selectors are independent stage-local bindings. Binding extraction
|
|
||||||
does not implicitly bind normalization, and Notarius does not discover or
|
|
||||||
schedule the preceding NPC run.
|
|
||||||
|
|
||||||
The same grammar can target chunk, merge, and normalize slots when the configured
|
The same grammar can target chunk, merge, and normalize slots when the configured
|
||||||
modules declare them:
|
modules declare them:
|
||||||
@@ -161,6 +160,23 @@ go run ./cmd/notarius run dnd-session \
|
|||||||
--resume
|
--resume
|
||||||
```
|
```
|
||||||
|
|
||||||
|
To selectively rerun one ordered step and its dependent lanes, use the step ID
|
||||||
|
from the configuration. The selected step and dependents are reported as
|
||||||
|
`forced_recompute`; reusable predecessors are reported as `reused`:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/notarius run dnd-npc-grounded \
|
||||||
|
--config examples/dnd-npc-grounded.config.yml \
|
||||||
|
--input examples/seriatim-minimal-transcript.json \
|
||||||
|
--resume --recompute-step grounded-events
|
||||||
|
```
|
||||||
|
|
||||||
|
Checkpoint decisions use these categories: `reused`, `executed`,
|
||||||
|
`forced_recompute`, and `dependency_invalidated`. The reason code and bounded
|
||||||
|
detail identify the decision without exposing reference content, local paths,
|
||||||
|
or secrets. `--recompute-step` requires checkpoint recording and `--resume`;
|
||||||
|
unknown step IDs, empty values, and combinations with `--only` are rejected.
|
||||||
|
|
||||||
Use `--debug` to retain the redacted summary and trace bundle for one run. The
|
Use `--debug` to retain the redacted summary and trace bundle for one run. The
|
||||||
bundle is allocated before pipeline resolution; once allocated, its path is
|
bundle is allocated before pipeline resolution; once allocated, its path is
|
||||||
also printed to stderr if the command fails. Debug-write failures cause exit
|
also printed to stderr if the command fails. Debug-write failures cause exit
|
||||||
|
|||||||
159
docs/config.md
159
docs/config.md
@@ -22,9 +22,8 @@ The explicit-path option is defined in the [CLI reference](cli.md).
|
|||||||
- [Minimal D&D spell configuration](../examples/dnd-spells.config.yml)
|
- [Minimal D&D spell configuration](../examples/dnd-spells.config.yml)
|
||||||
- [Production-oriented D&D spell configuration](../examples/dnd-spells-production.config.yml)
|
- [Production-oriented D&D spell configuration](../examples/dnd-spells-production.config.yml)
|
||||||
- [D&D NPC configuration](../examples/dnd-npcs.config.yml)
|
- [D&D NPC configuration](../examples/dnd-npcs.config.yml)
|
||||||
- [Sequential D&D NPC and spell configuration](../examples/dnd-npc-spell-sequential.config.yml)
|
|
||||||
- [D&D combat-turn configuration](../examples/dnd-combat-turns.config.yml)
|
- [D&D combat-turn configuration](../examples/dnd-combat-turns.config.yml)
|
||||||
- [Sequential D&D NPC and combat-turn configuration](../examples/dnd-npc-combat-sequential.config.yml)
|
- [D&D NPC-grounded spell and combat configuration](../examples/dnd-npc-grounded.config.yml)
|
||||||
|
|
||||||
All are complete version 3 files. The fragments below illustrate individual
|
All are complete version 3 files. The fragments below illustrate individual
|
||||||
fields and are not alternate complete configurations.
|
fields and are not alternate complete configurations.
|
||||||
@@ -140,13 +139,64 @@ Pipeline fields:
|
|||||||
|
|
||||||
- `input`: required module binding.
|
- `input`: required module binding.
|
||||||
- `chunk`: optional module binding. Default module is `generic`.
|
- `chunk`: optional module binding. Default module is `generic`.
|
||||||
- `artifacts`: required for pipeline resolution. It maps artifact lane IDs to
|
- `artifacts`: the artifact lane map for a single-step pipeline. It is treated
|
||||||
lane definitions.
|
as an implicit step with the stable ID `default`.
|
||||||
|
- `steps`: an ordered, non-empty list of step definitions. A pipeline may use
|
||||||
|
`steps` or `artifacts`, but not both. Step IDs must be unique after trimming.
|
||||||
- `output`: optional module binding. Default module is `json`.
|
- `output`: optional module binding. Default module is `json`.
|
||||||
- `references`: optional map of reference slot names to reference paths. These
|
- `references`: optional map of reference slot names to reference paths. These
|
||||||
bindings are defaults for eligible pipeline targets that declare the matching
|
bindings are defaults for eligible pipeline targets that declare the matching
|
||||||
slot.
|
slot.
|
||||||
|
|
||||||
|
Each explicit step contains an `id`, an optional `references` map, and a
|
||||||
|
non-empty `artifacts` map. Steps share the pipeline input, chunk plan, worker
|
||||||
|
budget, output encoder, manifest, and failure boundary. Lanes within a step
|
||||||
|
retain the fixed extract, validate, merge, validate, normalize, and validate
|
||||||
|
workflow; the next step starts only after the current step is terminal.
|
||||||
|
|
||||||
|
Generated references use the structured `artifact` source form to identify one
|
||||||
|
accepted normalized lane from an earlier step:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
pipelines:
|
||||||
|
dnd-npc-grounded:
|
||||||
|
input: seriatim
|
||||||
|
steps:
|
||||||
|
- id: identify-npcs
|
||||||
|
artifacts:
|
||||||
|
npcs:
|
||||||
|
extract: dnd/npcs
|
||||||
|
normalize: dnd/npcs
|
||||||
|
- id: grounded-events
|
||||||
|
references:
|
||||||
|
npcs:
|
||||||
|
artifact:
|
||||||
|
step: identify-npcs
|
||||||
|
lane: npcs
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract: dnd/spells
|
||||||
|
normalize: dnd/spells
|
||||||
|
combat:
|
||||||
|
extract: dnd/combat-turns
|
||||||
|
normalize: dnd/combat-turns
|
||||||
|
```
|
||||||
|
|
||||||
|
The generated binding is explicit and typed; it is not inferred from module,
|
||||||
|
lane, or slot names. It may be declared at step scope, applying to every
|
||||||
|
selected target in that step that declares the slot, or at one target's
|
||||||
|
`references` map. A producer may fan out to compatible target slots, but a
|
||||||
|
slot accepts only one producer. A producer must be in an earlier step, and a
|
||||||
|
configured generated dependency is required even when the consumer slot is
|
||||||
|
otherwise optional. Aggregating several producer artifacts is unsupported.
|
||||||
|
|
||||||
|
The producer codec supplies the artifact kind, complete schema identity, media
|
||||||
|
type, canonical content digest, and size used for compatibility and checkpoint
|
||||||
|
dependency checks. Only one accepted normalized artifact crosses the boundary;
|
||||||
|
raw extraction results, rejected output, intermediate values, and validator
|
||||||
|
diagnostics do not. Generated content is supplied in memory and is never
|
||||||
|
represented by a filesystem path.
|
||||||
|
|
||||||
Artifact lane fields:
|
Artifact lane fields:
|
||||||
|
|
||||||
- `extract`: required module binding.
|
- `extract`: required module binding.
|
||||||
@@ -164,19 +214,23 @@ See [CLI Reference](cli.md) for command syntax.
|
|||||||
Reference bindings are validated against reference slots declared by eligible
|
Reference bindings are validated against reference slots declared by eligible
|
||||||
chunk, extract, merge, and normalize targets during pipeline resolution. Required slots
|
chunk, extract, merge, and normalize targets during pipeline resolution. Required slots
|
||||||
must be bound after config defaults, target-local references, lane-level
|
must be bound after config defaults, target-local references, lane-level
|
||||||
compatibility bindings, and command-line reference overrides are applied.
|
compatibility bindings, step-local references, and command-line reference
|
||||||
Config-relative paths are resolved relative to the config file; command-line
|
overrides are applied. Config-relative paths are resolved relative to the
|
||||||
reference paths are resolved relative to the current working directory. Bound
|
config file; command-line reference paths are resolved relative to the current
|
||||||
files must be UTF-8 text. Reference media types are inferred from file
|
working directory. Bound files must be UTF-8 text. Reference media types are
|
||||||
extensions and checked when a module restricts accepted types; unknown
|
inferred from file extensions and checked when a module restricts accepted
|
||||||
extensions use `application/octet-stream`. See [CLI Reference](cli.md#run) for
|
types; unknown extensions use `application/octet-stream`. See [CLI Reference](cli.md#run)
|
||||||
command-line selectors and [Operations](operations.md) for recorded provenance
|
for command-line selectors and [Operations](operations.md) for recorded
|
||||||
and sensitive-data handling.
|
provenance and sensitive-data handling.
|
||||||
|
|
||||||
Pipeline-level `references` are defaults. They are valid when at least one
|
Pipeline-level `references` are defaults. They are valid when at least one
|
||||||
eligible target in the full configured pipeline declares the slot, including
|
eligible target in the full configured pipeline declares the slot, including
|
||||||
chunk, extractor, merger, and normalizer targets. During a run, they apply only
|
chunk, extractor, merger, and normalizer targets. During a run, they apply only
|
||||||
to the selected targets that declare the slot:
|
to the selected targets that declare the slot. For external bindings, a
|
||||||
|
step-local binding overrides a pipeline-level default and a target-local
|
||||||
|
binding retains the existing most-specific precedence. Generated and external
|
||||||
|
bindings may not resolve to the same target slot, and a step-scoped generated
|
||||||
|
binding may not be duplicated by a target-local generated binding.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
pipelines:
|
pipelines:
|
||||||
@@ -226,6 +280,18 @@ Target-local reference fields use the same map shape at:
|
|||||||
|
|
||||||
Each binding is valid only when that target module declares the slot.
|
Each binding is valid only when that target module declares the slot.
|
||||||
|
|
||||||
|
Reference source forms are:
|
||||||
|
|
||||||
|
- a scalar string, which is an external file path; or
|
||||||
|
- an object with only `artifact`, containing trimmed `step` and `lane` IDs for
|
||||||
|
an earlier producer lane.
|
||||||
|
|
||||||
|
Pipeline-level references accept only external paths. Generated references are
|
||||||
|
valid at step scope or on a target-local `references` map. Their producer's
|
||||||
|
registered codec is authoritative for schema, media type, and canonical
|
||||||
|
content identity; an external file is not treated as generated merely because
|
||||||
|
its bytes decode as the same artifact.
|
||||||
|
|
||||||
## Module Bindings
|
## Module Bindings
|
||||||
|
|
||||||
Every module binding may use shorthand:
|
Every module binding may use shorthand:
|
||||||
@@ -400,15 +466,17 @@ The extractor uses campaign references only as supporting disambiguation
|
|||||||
material; spell casts still must be present in the source transcript.
|
material; spell casts still must be present in the source transcript.
|
||||||
|
|
||||||
It also declares an optional `npcs` slot for a normalized NPC artifact. The
|
It also declares an optional `npcs` slot for a normalized NPC artifact. The
|
||||||
slot accepts exactly one `application/json` file no larger than 1 MiB. During
|
slot accepts exactly one `application/json` artifact no larger than 1 MiB. An
|
||||||
extractor preparation Notarius strictly decodes and identity-validates the
|
external file is decoded and identity-validated during preparation. A
|
||||||
artifact, then gives the model canonical JSON for caster-name grounding.
|
generated binding is validated at the step handoff and is provided to the
|
||||||
Registry source references may belong to the NPC-producing session and are
|
operation through the same reference contract. In both cases, the model
|
||||||
provenance only; they are not spell evidence. The bound registry contributes a
|
receives canonical JSON for caster-name grounding. Registry source references
|
||||||
semantic digest and NPC count to extractor metadata and checkpoint identity,
|
may belong to the NPC-producing session and are provenance only; they are not
|
||||||
while its names, aliases, content, and path do not appear there. When absent,
|
spell evidence. Generated reference identity and bounded producer provenance
|
||||||
the prompt receives the exact empty value `{"npcs":[]}` and no registry
|
are recorded by the framework; NPC names, aliases, content, and paths are not
|
||||||
provenance or fingerprint is recorded.
|
copied into manifests or checkpoint decisions. When absent, the prompt receives
|
||||||
|
the exact empty value `{"npcs":[]}` and no registry provenance or fingerprint
|
||||||
|
is recorded.
|
||||||
|
|
||||||
The `dnd/spells` normalizer declares the same optional `spell_catalog` slot.
|
The `dnd/spells` normalizer declares the same optional `spell_catalog` slot.
|
||||||
When an overlay is used, bind it independently under
|
When an overlay is used, bind it independently under
|
||||||
@@ -419,29 +487,34 @@ bound.
|
|||||||
|
|
||||||
The `dnd/npcs` extractor declares the same optional campaign slots as the spell
|
The `dnd/npcs` extractor declares the same optional campaign slots as the spell
|
||||||
extractor, but it does not declare the `npcs` registry slot. Its normalizer
|
extractor, but it does not declare the `npcs` registry slot. Its normalizer
|
||||||
accepts no references. To pass an NPC result to a later spell run, bind the
|
accepts no references. The maintained
|
||||||
normalized payload explicitly at runtime; the maintained sequential example
|
[NPC-grounded example](../examples/dnd-npc-grounded.config.yml) binds its
|
||||||
documents that operator workflow.
|
accepted normalized output to the later spell and combat targets through an
|
||||||
|
explicit ordered step.
|
||||||
|
|
||||||
The `dnd/combat-turns` extractor declares the optional campaign slots and the
|
The `dnd/combat-turns` extractor declares the optional campaign slots and the
|
||||||
structured `npcs` slot. Campaign references guide only the LLM extraction
|
structured `npcs` slot. Campaign references guide only the LLM extraction
|
||||||
stage. The deterministic normalizer declares only `npcs`, whose prepared
|
stage. The deterministic normalizer declares only `npcs`, whose operation-time
|
||||||
immutable registry supports the same actor and target canonicalization. Each
|
registry supports the same actor and target canonicalization. Each `npcs` slot
|
||||||
`npcs` slot accepts exactly one UTF-8 `application/json` file no larger than 1
|
accepts exactly one UTF-8 `application/json` artifact no larger than 1 MiB. The
|
||||||
MiB. The registry's source ranges remain provenance for the reference and never
|
registry's source ranges remain provenance for the reference and never become
|
||||||
become combat evidence. Binding `npcs` to extraction and normalization is
|
combat evidence. An ordered step binding fans the same generated NPC artifact
|
||||||
stage-local, so an operator-driven combat run uses two explicit selectors:
|
out to extraction and normalization:
|
||||||
|
|
||||||
```text
|
```yaml
|
||||||
combat.extract.npcs=<npc-run>/lanes/npcs.json
|
references:
|
||||||
combat.normalize.npcs=<npc-run>/lanes/npcs.json
|
npcs:
|
||||||
|
artifact:
|
||||||
|
step: identify-npcs
|
||||||
|
lane: npcs
|
||||||
```
|
```
|
||||||
|
|
||||||
When bound, the combat extractor and normalizer record only the registry's
|
When bound, the combat extractor and normalizer receive the generated registry
|
||||||
semantic digest and count in their metadata and checkpoint fingerprints; names,
|
at operation time. Framework provenance and checkpoint dependencies contain its
|
||||||
aliases, content, and paths are not recorded there. When absent, the combat
|
kind, schema identity, media type, canonical digest, size, and bounded producer
|
||||||
prompt receives the exact empty registry value `{"npcs":[]}` and no registry
|
identity; names, aliases, content, and paths are not recorded there. When
|
||||||
provenance or fingerprint is recorded.
|
absent, the combat prompt receives the exact empty registry value
|
||||||
|
`{"npcs":[]}` and no registry provenance or fingerprint is recorded.
|
||||||
|
|
||||||
## State Surfaces
|
## State Surfaces
|
||||||
|
|
||||||
@@ -572,8 +645,11 @@ Configuration validation checks:
|
|||||||
Pipeline resolution additionally checks:
|
Pipeline resolution additionally checks:
|
||||||
|
|
||||||
- the pipeline ID exists;
|
- the pipeline ID exists;
|
||||||
- at least one artifact lane is declared and selected;
|
- at least one artifact lane is declared and selected in each explicit step;
|
||||||
|
- `artifacts` and `steps` are mutually exclusive, explicit steps are non-empty,
|
||||||
|
and step IDs are unique after trimming;
|
||||||
- lanes selected through the CLI exist in the resolved pipeline;
|
- lanes selected through the CLI exist in the resolved pipeline;
|
||||||
|
- lane IDs are globally unique across ordered steps;
|
||||||
- required module keys are present;
|
- required module keys are present;
|
||||||
- module keys are registered for the expected slot;
|
- module keys are registered for the expected slot;
|
||||||
- module capability requirements are satisfied;
|
- module capability requirements are satisfied;
|
||||||
@@ -583,4 +659,7 @@ Pipeline resolution additionally checks:
|
|||||||
Scriptorium profile IDs;
|
Scriptorium profile IDs;
|
||||||
- bound reference slots are declared by selected chunk, extractor, merger, or
|
- bound reference slots are declared by selected chunk, extractor, merger, or
|
||||||
normalizer targets;
|
normalizer targets;
|
||||||
|
- generated references identify one lane in an earlier step, use a declared
|
||||||
|
compatible artifact kind, and do not conflict with external or target-local
|
||||||
|
generated bindings;
|
||||||
- required reference slots are bound for selected targets.
|
- required reference slots are bound for selected targets.
|
||||||
|
|||||||
@@ -92,6 +92,9 @@ chunk-scoped transcript plus the existing `players`, `party`, and `glossary`
|
|||||||
inputs, and optionally the deprecated `roster` reference through the shared
|
inputs, and optionally the deprecated `roster` reference through the shared
|
||||||
party mapping. The optional `npcs` reference is an approved normalized NPC
|
party mapping. The optional `npcs` reference is an approved normalized NPC
|
||||||
artifact used only for identity grounding; it never supplies combat evidence.
|
artifact used only for identity grounding; it never supplies combat evidence.
|
||||||
|
An external file is validated during preparation. In an ordered pipeline, the
|
||||||
|
same slot may receive the producer's canonical generated artifact at the step
|
||||||
|
handoff.
|
||||||
|
|
||||||
The private response envelope has the same fields and JSON types as the durable
|
The private response envelope has the same fields and JSON types as the durable
|
||||||
turn/action shape except that source references contain only `start_unit_id`
|
turn/action shape except that source references contain only `start_unit_id`
|
||||||
@@ -130,8 +133,10 @@ schema, combat shape, source references, then source relatedness.
|
|||||||
The standalone normalizer uses key `dnd/combat-turns`, requires `merged`,
|
The standalone normalizer uses key `dnd/combat-turns`, requires `merged`,
|
||||||
provides `normalized`, accepts no options, and accepts only the optional
|
provides `normalized`, accepts no options, and accepts only the optional
|
||||||
structured `npcs` reference. Campaign references are LLM extraction context and
|
structured `npcs` reference. Campaign references are LLM extraction context and
|
||||||
are not normalizer inputs. The NPC registry is resolved during preparation;
|
are not normalizer inputs. For an external file, the NPC registry is resolved
|
||||||
runtime normalization uses that immutable prepared view.
|
during preparation; for a generated binding, it is resolved at the operation-
|
||||||
|
time handoff. Runtime normalization uses that immutable prepared or handed-off
|
||||||
|
view.
|
||||||
|
|
||||||
Normalization policy is `dnd.combat_turns.normalize.v1`. It display-normalizes
|
Normalization policy is `dnd.combat_turns.normalize.v1`. It display-normalizes
|
||||||
actor, summary, declarations, targets, and non-null resolutions; canonicalizes
|
actor, summary, declarations, targets, and non-null resolutions; canonicalizes
|
||||||
@@ -143,9 +148,11 @@ merging its actions or prose. Invalid evidence is never eligible for duplicate
|
|||||||
collapse. Every mutation and collapse emits a bounded warning using the merged
|
collapse. Every mutation and collapse emits a bounded warning using the merged
|
||||||
input index in its scope.
|
input index in its scope.
|
||||||
|
|
||||||
The normalizer reports `normalization_policy` and `identity_policy` metadata and
|
The normalizer reports `normalization_policy` and `identity_policy` metadata
|
||||||
fingerprints, plus `npc_registry_digest`, `npc_count`, and `npc_registry` only
|
and fingerprints. An external registry may additionally contribute
|
||||||
when a registry is bound. The normalized-invariants validator is
|
`npc_registry_digest` and `npc_count`; generated registry identity is retained
|
||||||
|
in framework handoff provenance and dependency fingerprints. The
|
||||||
|
normalized-invariants validator is
|
||||||
`normalize/dnd/combat-turns/invariants`; it defers shape and source-reference
|
`normalize/dnd/combat-turns/invariants`; it defers shape and source-reference
|
||||||
failures, then checks display normalization, target identity uniqueness,
|
failures, then checks display normalization, target identity uniqueness,
|
||||||
canonical evidence ordering, chronology, and duplicate identity. It rejects
|
canonical evidence ordering, chronology, and duplicate identity. It rejects
|
||||||
@@ -162,9 +169,10 @@ merge validator chain.
|
|||||||
|
|
||||||
The selectable lane uses extractor and normalizer key `dnd/combat-turns`,
|
The selectable lane uses extractor and normalizer key `dnd/combat-turns`,
|
||||||
`appendorder` for the typed merger, and the durable codec above. A bound `npcs`
|
`appendorder` for the typed merger, and the durable codec above. A bound `npcs`
|
||||||
reference contributes raw-file provenance to the run manifest. Prepared combat
|
reference contributes raw-file provenance to the run manifest. A generated
|
||||||
extractor and normalizer metadata and checkpoint fingerprints contain only the
|
binding contributes artifact kind, schema identity, media type, canonical
|
||||||
NPC registry's semantic digest and count; the registry content, path, and NPC
|
digest, size, and bounded producer provenance. Consumer metadata and checkpoint
|
||||||
source ranges are not copied into combat output. The normalized lane is emitted
|
fingerprints contain no registry names, aliases, content, paths, or NPC source
|
||||||
as `lanes/<lane-id>.json` by the JSON output module, and warnings and rejection
|
ranges. The normalized lane is emitted as `lanes/<lane-id>.json` by the JSON
|
||||||
summaries remain in their shared companion files.
|
output module, and warnings and rejection summaries remain in their shared
|
||||||
|
companion files.
|
||||||
|
|||||||
@@ -85,23 +85,43 @@ checks. Relatedness emits bounded warnings when an NPC canonical name or
|
|||||||
alias is not present near its cited transcript text; opaque campaign
|
alias is not present near its cited transcript text; opaque campaign
|
||||||
references may explain such a warning but do not become evidence.
|
references may explain such a warning but do not become evidence.
|
||||||
|
|
||||||
## Manifest And Sequential Consumption
|
## Manifest And Artifact Handoff
|
||||||
|
|
||||||
The NPC extractor records prompt and response-schema identities. The durable
|
The NPC extractor records prompt and response-schema identities. The durable
|
||||||
codec records only `npc_count`; raw names, aliases, descriptions, source
|
codec records only `npc_count`; raw names, aliases, descriptions, source
|
||||||
references, and payload bytes stay in the lane file rather than manifest
|
references, and payload bytes stay in the lane file rather than manifest
|
||||||
metadata. The normalized lane is independently reusable as a file reference:
|
metadata. The normalized lane can be consumed by a later ordered step through
|
||||||
|
the registered canonical codec:
|
||||||
|
|
||||||
```sh
|
```yaml
|
||||||
go run ./cmd/notarius run dnd-spells \
|
steps:
|
||||||
--config examples/dnd-npc-spell-sequential.config.yml \
|
- id: identify-npcs
|
||||||
--input examples/seriatim-minimal-transcript.json \
|
artifacts:
|
||||||
--reference spells.extract.npcs=./npc-output/<run-id>/lanes/npcs.json
|
npcs:
|
||||||
|
extract: dnd/npcs
|
||||||
|
normalize: dnd/npcs
|
||||||
|
- id: grounded-events
|
||||||
|
references:
|
||||||
|
npcs:
|
||||||
|
artifact:
|
||||||
|
step: identify-npcs
|
||||||
|
lane: npcs
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract: dnd/spells
|
||||||
|
normalize: dnd/spells
|
||||||
|
combat:
|
||||||
|
extract: dnd/combat-turns
|
||||||
|
normalize: dnd/combat-turns
|
||||||
```
|
```
|
||||||
|
|
||||||
The spell extractor strictly decodes and identity-validates this file, accepts
|
The framework hands only an accepted normalized artifact across the barrier. It
|
||||||
source references belonging to another session as registry provenance, and
|
validates the canonical bytes against each consumer slot and clones the
|
||||||
uses only canonical names and aliases for caster grounding. Those NPC source
|
operation-time reference for the spell and combat consumers. Generated
|
||||||
references are never accepted as spell evidence. The spell run's manifest
|
provenance records the artifact kind, schema identity, media type, canonical
|
||||||
keeps raw file provenance under `references` and records only the prepared
|
digest, size, and producer step/lane/module, but not names, aliases, source
|
||||||
registry's semantic digest and count in extractor metadata.
|
ranges, or payload bytes. External normalized files remain supported as
|
||||||
|
explicit references and retain their file provenance.
|
||||||
|
|
||||||
|
NPC source references are registry provenance and are never accepted as spell
|
||||||
|
or combat evidence. Current transcript units remain the only event evidence.
|
||||||
|
|||||||
@@ -97,18 +97,20 @@ addressable through `source_refs`.
|
|||||||
## Optional NPC Grounding
|
## Optional NPC Grounding
|
||||||
|
|
||||||
The `dnd/spells` extractor accepts an optional `npcs` reference containing one
|
The `dnd/spells` extractor accepts an optional `npcs` reference containing one
|
||||||
normalized NPC artifact as `application/json`, up to 1 MiB. Preparation uses
|
normalized NPC artifact as `application/json`, up to 1 MiB. An external file is
|
||||||
the approved NPC codec and identity policy to validate the file, re-encodes
|
validated during preparation; an ordered generated binding is validated at the
|
||||||
canonical durable JSON, and supplies that JSON as a spell-owned prompt input.
|
step handoff. Both paths use the approved NPC codec and identity policy,
|
||||||
It helps the model prefer canonical caster names and recognize aliases; it
|
re-encode canonical durable JSON, and supply that JSON as an operation-time
|
||||||
does not establish that a spell was cast.
|
spell prompt input. It helps the model prefer canonical caster names and
|
||||||
|
recognize aliases; it does not establish that a spell was cast.
|
||||||
|
|
||||||
NPC source references may identify the run that produced the registry or any
|
NPC source references may identify the run that produced the registry or any
|
||||||
other session. They remain registry provenance and are never copied into a
|
other session. They remain registry provenance and are never copied into a
|
||||||
spell cast's `source_refs`; every spell evidence range must still identify the
|
spell cast's `source_refs`; every spell evidence range must still identify the
|
||||||
current transcript. When the slot is absent, the prompt receives exactly
|
current transcript. Generated provenance records producer and canonical
|
||||||
`{"npcs":[]}` and the run has no NPC reference provenance or NPC checkpoint
|
artifact identity without payload content or a path. When the slot is absent,
|
||||||
fingerprint.
|
the prompt receives exactly `{"npcs":[]}` and the run has no NPC reference
|
||||||
|
provenance or NPC checkpoint fingerprint.
|
||||||
|
|
||||||
## Normalization Behavior
|
## Normalization Behavior
|
||||||
|
|
||||||
@@ -190,15 +192,20 @@ identity fields when that module is selected. Overlay origin, media type, byte
|
|||||||
size, and raw digest are recorded separately in the manifest's reference
|
size, and raw digest are recorded separately in the manifest's reference
|
||||||
provenance; see the [JSON output contract](json-output.md#manifestjson).
|
provenance; see the [JSON output contract](json-output.md#manifestjson).
|
||||||
|
|
||||||
The `npc_registry_digest` and `npc_count` fields in the example are present only
|
The `npc_registry_digest` and `npc_count` fields in the example are present for
|
||||||
when the optional NPC registry is bound. They contain no NPC names, aliases,
|
an external NPC registry when the extractor publishes its prepared module
|
||||||
source references, paths, or raw bytes.
|
metadata. They contain no NPC names, aliases, source references, paths, or raw
|
||||||
|
bytes. A generated registry's identity is instead represented by the framework
|
||||||
|
handoff provenance and dependency fingerprint, so the consumer module metadata
|
||||||
|
does not duplicate it.
|
||||||
|
|
||||||
The extractor's prompt hash, private response-schema hash, and effective catalog
|
The extractor's prompt hash, private response-schema hash, and effective catalog
|
||||||
digest also contribute independently scoped semantic checkpoint fingerprints.
|
digest also contribute independently scoped semantic checkpoint fingerprints.
|
||||||
Changing any of those prepared contracts intentionally produces a cold
|
Changing any of those prepared contracts intentionally produces a cold
|
||||||
checkpoint miss. Fingerprints contain only digests, never prompt, schema,
|
checkpoint miss. Fingerprints contain only digests, never prompt, schema,
|
||||||
catalog, or reference content. When an NPC registry is bound, its semantic
|
catalog, or reference content. When an NPC registry is bound, its semantic
|
||||||
digest contributes an additional local `npc_registry` fingerprint; the
|
digest contributes an additional local `npc_registry` fingerprint for an
|
||||||
manifest metadata contains only that digest and `npc_count`. Raw NPC file
|
external binding; the manifest metadata contains only that digest and
|
||||||
provenance remains independently recorded in the manifest's `references` list.
|
`npc_count`. Raw NPC file provenance remains independently recorded in the
|
||||||
|
manifest's `references` list. Generated bindings contribute the canonical
|
||||||
|
artifact dependency fingerprint and bounded producer provenance instead.
|
||||||
|
|||||||
@@ -104,11 +104,12 @@ The NPC identity package owns Unicode comparison keys, deterministic
|
|||||||
The registry package resolves one optional normalized artifact through the
|
The registry package resolves one optional normalized artifact through the
|
||||||
strict codec, validates whole-registry identity, canonicalizes its JSON, and
|
strict codec, validates whole-registry identity, canonicalizes its JSON, and
|
||||||
provides immutable records, prompt input, semantic digest, count, and exact
|
provides immutable records, prompt input, semantic digest, count, and exact
|
||||||
canonical-name/alias lookup. It owns the `npcs` slot and its bounded,
|
canonical-name/alias lookup. External files cross this boundary during
|
||||||
content-safe preparation failures. NPC source references are durable
|
preparation; generated artifacts cross it at the ordered step handoff. It owns
|
||||||
provenance and are not treated as evidence for a consuming pipeline. The NPC
|
the `npcs` slot and its bounded, content-safe validation failures. NPC source
|
||||||
codec owns the strict durable `dnd/npc-list` JSON boundary and exposes
|
references are durable provenance and are not treated as evidence for a
|
||||||
candidate versus approved encode/decode operations.
|
consuming pipeline. The NPC codec owns the strict durable `dnd/npc-list` JSON
|
||||||
|
boundary and exposes candidate versus approved encode/decode operations.
|
||||||
|
|
||||||
The `internal/modules/dnd/codec/combatturns` package owns the durable
|
The `internal/modules/dnd/codec/combatturns` package owns the durable
|
||||||
`dnd/combat-turn-list` schema and candidate versus approved JSON boundary. It
|
`dnd/combat-turn-list` schema and candidate versus approved JSON boundary. It
|
||||||
@@ -220,9 +221,12 @@ Shared D&D helpers keep prompt input
|
|||||||
names and source-unit reference conversion consistent with the scene chunker.
|
names and source-unit reference conversion consistent with the scene chunker.
|
||||||
|
|
||||||
The extractor also declares the optional `npcs` registry slot and consumes the
|
The extractor also declares the optional `npcs` registry slot and consumes the
|
||||||
prepared immutable registry boundary from `internal/modules/dnd/npcs/registry`.
|
immutable registry boundary from `internal/modules/dnd/npcs/registry`. An
|
||||||
A bound registry adds only `npc_registry_digest` and `npc_count` to manifest
|
external registry is prepared before execution; a generated registry is
|
||||||
metadata and an `npc_registry` checkpoint fingerprint. The unbound prompt
|
validated and supplied at operation time. External bindings may add only
|
||||||
|
`npc_registry_digest` and `npc_count` to module metadata and an
|
||||||
|
`npc_registry` checkpoint fingerprint. Generated bindings are represented by
|
||||||
|
framework handoff provenance and dependency fingerprints. The unbound prompt
|
||||||
input is exactly `{"npcs":[]}` and has no registry provenance or fingerprint.
|
input is exactly `{"npcs":[]}` and has no registry provenance or fingerprint.
|
||||||
The shared NPC grounding fragment is placed immediately after the common
|
The shared NPC grounding fragment is placed immediately after the common
|
||||||
campaign reference message and is included in the spell prompt fingerprint.
|
campaign reference message and is included in the spell prompt fingerprint.
|
||||||
@@ -310,16 +314,18 @@ independently for extraction and normalization.
|
|||||||
|
|
||||||
### `internal/modules/dnd/normalize/combatturns`
|
### `internal/modules/dnd/normalize/combatturns`
|
||||||
|
|
||||||
The combat normalizer prepares the optional NPC registry once and uses the
|
The combat normalizer prepares an external NPC registry before execution or
|
||||||
immutable prepared view during runtime. It display-normalizes combat fields,
|
receives a generated registry at the ordered step handoff, then uses the
|
||||||
|
immutable view during runtime. It display-normalizes combat fields,
|
||||||
rewrites exact canonical-name or alias matches for actors and targets, orders
|
rewrites exact canonical-name or alias matches for actors and targets, orders
|
||||||
and deduplicates source references, stable-sorts records by source-document
|
and deduplicates source references, stable-sorts records by source-document
|
||||||
position, and collapses only exact duplicate identities with fully valid
|
position, and collapses only exact duplicate identities with fully valid
|
||||||
evidence. It deep-clones output storage and emits bounded warnings scoped to
|
evidence. It deep-clones output storage and emits bounded warnings scoped to
|
||||||
merged input indexes. Its metadata and fingerprints identify the normalization
|
merged input indexes. Its metadata and fingerprints identify the normalization
|
||||||
and NPC identity policies, with registry digest/count only when bound. The
|
and NPC identity policies. External bindings may contribute registry
|
||||||
normalizer is included in the production D&D registrar with the default combat
|
digest/count metadata; generated identity is retained in framework provenance
|
||||||
normalization chain.
|
and dependency fingerprints. The normalizer is included in the production D&D
|
||||||
|
registrar with the default combat normalization chain.
|
||||||
|
|
||||||
## Output Encoder
|
## Output Encoder
|
||||||
|
|
||||||
|
|||||||
@@ -14,11 +14,13 @@ collaborators, invokes `internal/framework/pipeline`, and places the logical
|
|||||||
output files returned by the runner. Cache and debug collaborators are supplied
|
output files returned by the runner. Cache and debug collaborators are supplied
|
||||||
at this boundary.
|
at this boundary.
|
||||||
|
|
||||||
Resolution produces a fixed ordered workflow and a sorted set of artifact
|
Resolution produces a fixed ordered workflow of steps and globally unique,
|
||||||
lanes. Preparation constructs the complete module and validator set before the
|
sorted artifact lanes. Preparation constructs the complete module and validator
|
||||||
runner receives source bytes. Source parsing and chunking are serial; extraction
|
set before the runner receives source bytes. Source parsing and chunking are
|
||||||
uses a bounded run-wide worker pool, followed by serial per-lane merge and
|
serial. Each step then uses a bounded run-wide extraction pool followed by
|
||||||
normalize continuations that may overlap across lanes.
|
serial per-lane merge and normalize continuations. A step barrier prevents
|
||||||
|
later consumers from starting until all earlier lanes are terminal and their
|
||||||
|
required normalized artifacts have crossed the typed handoff.
|
||||||
|
|
||||||
## Application Boundary
|
## Application Boundary
|
||||||
|
|
||||||
@@ -42,7 +44,7 @@ normalize continuations that may overlap across lanes.
|
|||||||
| Package | Implemented responsibility |
|
| Package | Implemented responsibility |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `internal/framework/contracts` | Source-stage contracts plus artifact identity, schema, serialized representation, codec, validator, reference, output, and structured-completion interfaces and data types. |
|
| `internal/framework/contracts` | Source-stage contracts plus artifact identity, schema, serialized representation, codec, validator, reference, output, and structured-completion interfaces and data types. |
|
||||||
| `internal/framework/pipeline` | Module and artifact-codec registries, option validation, profile resolution, capability checks, reference materialization, complete pipeline preparation, retries, orchestration, warnings, and manifest population. |
|
| `internal/framework/pipeline` | Module and artifact-codec registries, ordered-step and generated-reference resolution, option validation, profile resolution, capability checks, external reference materialization, complete pipeline preparation, typed handoff, retries, orchestration, warnings, checkpoint decisions, and manifest population. |
|
||||||
| `internal/framework/validate` | Shared validator decision and cardinality helpers. |
|
| `internal/framework/validate` | Shared validator decision and cardinality helpers. |
|
||||||
| `internal/framework/llm` | Scriptorium-backed structured completions, prompt/schema registration, scheduling, profile recording, and secret redaction. |
|
| `internal/framework/llm` | Scriptorium-backed structured completions, prompt/schema registration, scheduling, profile recording, and secret redaction. |
|
||||||
| `internal/framework/promptfs` | Builds module prompt filesystems from module-owned and caller-provided shared prompt assets. |
|
| `internal/framework/promptfs` | Builds module prompt filesystems from module-owned and caller-provided shared prompt assets. |
|
||||||
@@ -111,12 +113,15 @@ this package. Domain-neutral prompt filesystem composition lives in
|
|||||||
`internal/framework/promptfs`.
|
`internal/framework/promptfs`.
|
||||||
|
|
||||||
The `dnd/npcs/registry` package owns the optional `npcs` registry boundary.
|
The `dnd/npcs/registry` package owns the optional `npcs` registry boundary.
|
||||||
Preparation strictly decodes and identity-validates one normalized JSON
|
External references are strictly decoded and identity-validated during
|
||||||
artifact, emits canonical registry JSON to the spell prompt, and records only
|
preparation; generated references are decoded and identity-validated at the
|
||||||
its semantic digest and count in prepared metadata. The raw reference remains
|
ordered step handoff. Both paths emit canonical registry JSON to operation-time
|
||||||
independently tracked by pipeline provenance. An absent registry is represented
|
spell and combat prompt or normalization requests. The framework records
|
||||||
only by the empty prompt value `{"npcs":[]}`. Spell extraction consumes this
|
generated identity and bounded producer provenance, while the raw external
|
||||||
shared registry boundary without changing its public module contract.
|
reference remains independently tracked by pipeline provenance. An absent
|
||||||
|
registry is represented only by the empty prompt value `{"npcs":[]}`. Spell
|
||||||
|
and combat consumers use this shared boundary without changing their public
|
||||||
|
module contracts.
|
||||||
|
|
||||||
Generic validators under `internal/modules/generic/validate` provide
|
Generic validators under `internal/modules/generic/validate` provide
|
||||||
unconditional test decisions, JSON syntax validation, and JSON Schema
|
unconditional test decisions, JSON syntax validation, and JSON Schema
|
||||||
@@ -144,12 +149,12 @@ Implementation details for all production extensions are in
|
|||||||
| Surface | Implemented owners | Internal purpose |
|
| Surface | Implemented owners | Internal purpose |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Durable output | Output module, pipeline runner, and CLI writer | Return logical consumer files and place them for a run. |
|
| Durable output | Output module, pipeline runner, and CLI writer | Return logical consumer files and place them for a run. |
|
||||||
| Cache checkpoints | `internal/framework/checkpoint` and `internal/cli` | Validate and serialize reusable extract, merge, and normalize outcomes. |
|
| Cache checkpoints | `internal/framework/checkpoint` and `internal/cli` | Validate and serialize reusable extract, merge, and normalize outcomes, including ordered-step scope and generated-artifact dependency decisions. |
|
||||||
| Chunk-plan cache | `internal/framework/chunkplan` and `internal/cli` | Persist and select source-addressed plans before framework materialization. |
|
| Chunk-plan cache | `internal/framework/chunkplan` and `internal/cli` | Persist and select source-addressed plans before framework materialization. |
|
||||||
| Debug bundles | `internal/core/debugbundle`, `internal/framework/debug`, and pipeline instrumentation | Persist redacted summaries and application-owned traces. |
|
| Debug bundles | `internal/core/debugbundle`, `internal/framework/debug`, and pipeline instrumentation | Persist redacted summaries and application-owned traces. |
|
||||||
|
|
||||||
Physical layout, cleanup, recovery, and sensitive-data handling are defined
|
Physical layout, cleanup, recovery, and sensitive-data handling are defined
|
||||||
in [Operations](../operations.md). Concrete stage modules receive recorder
|
in [Operations](../operations.md). Concrete modules receive recorder
|
||||||
interfaces and request data, not physical state roots.
|
interfaces and request data, not physical state roots.
|
||||||
|
|
||||||
## Focused Documentation
|
## Focused Documentation
|
||||||
|
|||||||
@@ -6,13 +6,15 @@ Their fixed workflow and ownership boundaries are defined by
|
|||||||
defaults, and selectable keys are defined in
|
defaults, and selectable keys are defined in
|
||||||
[Configuration](../config.md#pipelines).
|
[Configuration](../config.md#pipelines).
|
||||||
|
|
||||||
Resolution fixes the selected lanes and all stage bindings; preparation
|
Resolution fixes the ordered steps, selected lanes, and all stage bindings;
|
||||||
constructs every selected implementation before the runner begins source work.
|
preparation constructs every selected implementation before the runner begins
|
||||||
After serial input parsing and plan selection or generation, the runner
|
source work. After serial input parsing and plan selection or generation, the
|
||||||
materializes chunks and dispatches extract work to
|
runner materializes chunks and executes one step at a time. Within a step,
|
||||||
one bounded run-wide worker pool in chunk-first, lane-second order. Each lane's
|
extract work uses one bounded run-wide worker pool in chunk-first, lane-second
|
||||||
merge and normalize operations remain serial and may overlap other lanes once
|
order. Each lane's merge and normalize operations remain serial, and lanes in
|
||||||
all extracts for that lane are terminal.
|
the same step may overlap once their extracts are terminal. A later step cannot
|
||||||
|
start across its barrier until every earlier lane is terminal and each required
|
||||||
|
generated artifact has been accepted and handed off.
|
||||||
|
|
||||||
## Resolution
|
## Resolution
|
||||||
|
|
||||||
@@ -22,38 +24,48 @@ calls `pipeline.ResolvePipeline`.
|
|||||||
|
|
||||||
`ResolvePipeline`:
|
`ResolvePipeline`:
|
||||||
|
|
||||||
1. selects and sorts artifact lanes;
|
1. selects the explicit ordered steps, or creates the implicit `default` step
|
||||||
2. completes omitted bindings using the documented configuration defaults;
|
from the legacy top-level `artifacts` map;
|
||||||
3. looks up each module and validator spec without constructing it;
|
2. selects and sorts artifact lanes within each step while enforcing global lane
|
||||||
4. for a typed extractor, derives its artifact kind, requires the codec, and
|
identity;
|
||||||
|
3. completes omitted bindings using the documented configuration defaults;
|
||||||
|
4. looks up each module and validator spec without constructing it;
|
||||||
|
5. for a typed extractor, derives its artifact kind, requires the codec, and
|
||||||
selects exact-type merger, normalizer, and validator variants;
|
selects exact-type merger, normalizer, and validator variants;
|
||||||
5. checks required and provided capabilities in workflow order;
|
6. checks required and provided capabilities in workflow order;
|
||||||
6. resolves target-aware reference bindings and validator chains;
|
7. resolves external and generated target-aware reference bindings and
|
||||||
7. validates each selected module and validator option set through its registry
|
validates producer order, consumer slot declarations, and artifact-kind
|
||||||
|
compatibility;
|
||||||
|
8. validates each selected module and validator option set through its registry
|
||||||
entry; and
|
entry; and
|
||||||
8. calculates a digest over the resolved structure, including typed artifact
|
9. calculates a digest over the resolved structure, including step order, step
|
||||||
kind and schema identity and the effective validator policy in its resolved
|
IDs, lane membership, generated topology, producer and consumer identities,
|
||||||
execution order.
|
typed artifact kind and schema identity, and the effective validator policy
|
||||||
|
in its resolved execution order.
|
||||||
|
|
||||||
Resolution returns a `ResolvedPipeline` containing ordered lanes, concrete
|
Resolution returns a `ResolvedPipeline` containing ordered steps, lanes,
|
||||||
bindings, validator chains, reference targets, and the digest. It does not read
|
concrete bindings, validator chains, reference targets, and the digest. It does
|
||||||
reference bytes or construct runtime modules. CLI lane and reference selector
|
not read external reference bytes or construct runtime modules. CLI lane and
|
||||||
syntax is defined in the [CLI reference](../cli.md#run).
|
reference selector syntax is defined in the [CLI reference](../cli.md#run).
|
||||||
|
|
||||||
The digest includes each resolved validator chain's stage, lane, owning module,
|
The digest includes each resolved step's ID and lane membership, generated
|
||||||
ordered validator bindings, execution classes, targets, and artifact kinds.
|
producer/consumer topology, and each validator chain's stage, lane, owning
|
||||||
Changing a default chain or an explicit override therefore changes pipeline
|
module, ordered validator bindings, execution classes, targets, and artifact
|
||||||
identity whenever it changes the effective validator policy.
|
kinds. Changing step order, a dependency, a default chain, or an explicit
|
||||||
|
override therefore changes pipeline identity whenever it changes effective
|
||||||
|
execution policy.
|
||||||
|
|
||||||
## Reference Materialization
|
## Reference Materialization
|
||||||
|
|
||||||
The CLI calls `MaterializeReferences` after resolution and before constructing
|
The CLI calls `MaterializeReferences` after resolution and before constructing
|
||||||
the LLM client or running the pipeline. The materializer checks each binding
|
the LLM client or running the pipeline. For external bindings, the materializer
|
||||||
against its resolved target declaration, reads and validates the file, and
|
checks each binding against its resolved target declaration, reads and validates
|
||||||
builds both a `contracts.ReferenceSet` and provenance-only metadata on the
|
the file, and builds both a `contracts.ReferenceSet` and provenance-only
|
||||||
corresponding `ResolvedReferenceTarget`.
|
metadata on the corresponding `ResolvedReferenceTarget`. A structured
|
||||||
|
generated binding is declaration-only at this point: its producer bytes do not
|
||||||
|
exist until the producer lane reaches an accepted normalized result.
|
||||||
|
|
||||||
Preparation delivers the materialized set for each target through
|
Preparation delivers the materialized external set for each target through
|
||||||
`pipeline.BuildRequest`: chunkers and chunk validators receive the chunk target;
|
`pipeline.BuildRequest`: chunkers and chunk validators receive the chunk target;
|
||||||
extractors and extract validators receive the lane extract target; mergers and
|
extractors and extract validators receive the lane extract target; mergers and
|
||||||
merge validators receive the lane merge target; and normalizers and normalize
|
merge validators receive the lane merge target; and normalizers and normalize
|
||||||
@@ -62,10 +74,20 @@ an empty set because those stages cannot declare references. Every builder gets
|
|||||||
an isolated deep clone of its target set, so construction-time mutation cannot
|
an isolated deep clone of its target set, so construction-time mutation cannot
|
||||||
change another builder, the resolved pipeline, or later runtime requests.
|
change another builder, the resolved pipeline, or later runtime requests.
|
||||||
|
|
||||||
|
Prepared consumers do not need to be reconstructed when generated content is
|
||||||
|
available. At the step boundary, the runner encodes the accepted producer value
|
||||||
|
through its registered canonical codec, validates the generated bytes against
|
||||||
|
each target slot's kind, schema, media type, and size, and clones one immutable
|
||||||
|
reference item into the operation request. The item includes canonical digest,
|
||||||
|
size, and bounded producer provenance but no filesystem URI. A handoff failure
|
||||||
|
is a framework dependency error and prevents every consumer in that step from
|
||||||
|
starting.
|
||||||
|
|
||||||
The runner continues to clone the resulting set into the chunk, extract, merge,
|
The runner continues to clone the resulting set into the chunk, extract, merge,
|
||||||
or normalize request that owns the target. LLM-backed extensions may convert
|
or normalize request that owns the target. LLM-backed extensions may convert
|
||||||
those items into named prompt inputs. Reference content remains separate from
|
those items into named prompt inputs. Reference content remains separate from
|
||||||
source evidence and source digests.
|
source evidence and source digests, whether the item came from a file or a
|
||||||
|
generated handoff.
|
||||||
|
|
||||||
Binding precedence, path resolution, accepted content, and media-type behavior
|
Binding precedence, path resolution, accepted content, and media-type behavior
|
||||||
are configuration contracts; see [Configuration](../config.md#pipelines).
|
are configuration contracts; see [Configuration](../config.md#pipelines).
|
||||||
@@ -120,10 +142,12 @@ The current production catalog and default chain are listed only in
|
|||||||
## Preparation And Runner Boundary
|
## Preparation And Runner Boundary
|
||||||
|
|
||||||
`pipeline.Prepare` receives a resolved pipeline, the registries, and shared
|
`pipeline.Prepare` receives a resolved pipeline, the registries, and shared
|
||||||
module dependencies. It constructs input; chunk and its validators; each lane's
|
module dependencies. It constructs input; chunk and its validators; every
|
||||||
extract, merge, and normalize modules and validator chains in resolved order;
|
step's lane extract, merge, and normalize modules and validator chains in
|
||||||
then output. It stops at the first error with pipeline, stage, lane, module, and
|
resolved order; then output. It stops at the first error with pipeline, step,
|
||||||
validator context as applicable. It never invokes an operation method.
|
stage, lane, module, and validator context as applicable. It never invokes an
|
||||||
|
operation method. Generated references are not available during preparation;
|
||||||
|
the operation request is the handoff boundary.
|
||||||
|
|
||||||
`PreparedPipeline` keeps private constructed executors and exposes cloned
|
`PreparedPipeline` keeps private constructed executors and exposes cloned
|
||||||
resolved input, chunk, lane, and output identities. Prepared components may
|
resolved input, chunk, lane, and output identities. Prepared components may
|
||||||
@@ -136,8 +160,8 @@ stable and must not contain source content, credentials, local paths,
|
|||||||
timestamps, or other invocation-specific values.
|
timestamps, or other invocation-specific values.
|
||||||
|
|
||||||
`pipeline.RunInput` carries that prepared pipeline, raw source input, run identity and timing, optional
|
`pipeline.RunInput` carries that prepared pipeline, raw source input, run identity and timing, optional
|
||||||
session and profile metadata, a chunk-plan store and mode, and checkpoint/debug
|
session and profile metadata, a chunk-plan store and mode, a checkpoint
|
||||||
collaborators. The runner
|
execution policy, and checkpoint/debug collaborators. The runner
|
||||||
parses source bytes through the already constructed input adapter. Later stage
|
parses source bytes through the already constructed input adapter. Later stage
|
||||||
requests receive the generic source model; extract requests receive
|
requests receive the generic source model; extract requests receive
|
||||||
chunk-scoped input material, while chunk, merge, and normalize requests retain
|
chunk-scoped input material, while chunk, merge, and normalize requests retain
|
||||||
@@ -145,7 +169,9 @@ access to the original source material. Input, chunk, and output operation
|
|||||||
requests do not carry raw module options. The chunk request also does not carry
|
requests do not carry raw module options. The chunk request also does not carry
|
||||||
an LLM client; an LLM-backed chunker receives the shared client during
|
an LLM client; an LLM-backed chunker receives the shared client during
|
||||||
preparation. Their operation requests retain run-specific source, reference,
|
preparation. Their operation requests retain run-specific source, reference,
|
||||||
profile, session, and metadata context as applicable.
|
profile, session, metadata, and step-handoff context as applicable. A generated
|
||||||
|
reference is cloned into each compatible consumer request and is never exposed
|
||||||
|
as a path.
|
||||||
|
|
||||||
Prepared lanes retain exact-type-checked erased operation closures. The runner
|
Prepared lanes retain exact-type-checked erased operation closures. The runner
|
||||||
uses those closures to keep each value typed through extraction, validation,
|
uses those closures to keep each value typed through extraction, validation,
|
||||||
@@ -173,10 +199,15 @@ The runner:
|
|||||||
3. selects a stored plan or executes the configured chunker's `Plan` operation;
|
3. selects a stored plan or executes the configured chunker's `Plan` operation;
|
||||||
4. canonicalizes and materializes the plan, then validates the resulting
|
4. canonicalizes and materializes the plan, then validates the resulting
|
||||||
chunks;
|
chunks;
|
||||||
5. dispatches extract jobs in source-chunk then resolved-lane order, starting a
|
5. executes each resolved step in configuration order. For one step, it
|
||||||
bounded lane continuation when all extracts for that lane are terminal;
|
dispatches extract jobs in source-chunk then resolved-lane order, starts a
|
||||||
6. invokes the prepared output encoder and validates its logical file results;
|
bounded lane continuation when all extracts for that lane are terminal, and
|
||||||
7. returns the assembled manifest, outcomes, warnings, and files.
|
waits for every lane to become terminal;
|
||||||
|
6. encodes and validates each accepted normalized producer artifact, then
|
||||||
|
builds the immutable generated reference sets for the next step;
|
||||||
|
7. invokes the prepared output encoder only after every step succeeds and
|
||||||
|
validates its logical file results;
|
||||||
|
8. returns the assembled manifest, outcomes, warnings, and files.
|
||||||
|
|
||||||
Within each artifact lane, it reuses the prepared extractor, merger, normalizer,
|
Within each artifact lane, it reuses the prepared extractor, merger, normalizer,
|
||||||
and validators while performing these transitions:
|
and validators while performing these transitions:
|
||||||
@@ -190,6 +221,13 @@ and validators while performing these transitions:
|
|||||||
6. normalize the accepted merge result;
|
6. normalize the accepted merge result;
|
||||||
7. validate and append the accepted normalized result.
|
7. validate and append the accepted normalized result.
|
||||||
|
|
||||||
|
At a step barrier, a lane with no accepted normalized output is still a regular
|
||||||
|
rejection unless a later generated binding names that lane as a required
|
||||||
|
producer. In that case the runner raises a deterministic dependency error and
|
||||||
|
does not start the consumer step. One accepted typed artifact may fan out to
|
||||||
|
multiple compatible target slots. Consumers in the same step may run
|
||||||
|
concurrently after the handoff; no work crosses the barrier early.
|
||||||
|
|
||||||
Module-provided warnings and payload warnings are promoted only from attempts
|
Module-provided warnings and payload warnings are promoted only from attempts
|
||||||
whose results are accepted and used.
|
whose results are accepted and used.
|
||||||
|
|
||||||
@@ -271,10 +309,19 @@ entering the normal handoff path. Typed extract, merge, and normalize
|
|||||||
checkpoints store codec bytes with artifact kind, schema ID and version, exact
|
checkpoints store codec bytes with artifact kind, schema ID and version, exact
|
||||||
schema digest, and media type. Reuse compares that identity with the prepared
|
schema digest, and media type. Reuse compares that identity with the prepared
|
||||||
codec and decodes through the codec; missing identity, mismatches, corrupt
|
codec and decodes through the codec; missing identity, mismatches, corrupt
|
||||||
bytes, and decode failures become explicit reuse misses and execute the step
|
bytes, and decode failures become explicit reuse misses and execute the lane
|
||||||
normally. Dependency fingerprints and debug content digests use the same stable
|
normally. Dependency fingerprints and debug content digests use the same stable
|
||||||
codec bytes that cross those boundaries.
|
codec bytes that cross those boundaries.
|
||||||
|
|
||||||
|
Generated references add downstream dependencies containing the producer's
|
||||||
|
artifact kind, complete schema identity, media type, canonical content digest,
|
||||||
|
and size. Compatible producer checkpoints may therefore feed a later step
|
||||||
|
without re-executing the producer. A missing, rejected, corrupt, incompatible,
|
||||||
|
or changed producer invalidates every transitive dependent lane while leaving
|
||||||
|
independent work eligible for reuse. The runner records bounded decision
|
||||||
|
categories: `reused`, `executed`, `forced_recompute`, and
|
||||||
|
`dependency_invalidated`.
|
||||||
|
|
||||||
The CLI includes prepared-component fingerprints in the run-wide checkpoint
|
The CLI includes prepared-component fingerprints in the run-wide checkpoint
|
||||||
identity alongside resolved configuration, raw input, reference provenance,
|
identity alongside resolved configuration, raw input, reference provenance,
|
||||||
runtime overrides, and LLM-profile fingerprints. Module metadata is not used
|
runtime overrides, and LLM-profile fingerprints. Module metadata is not used
|
||||||
@@ -283,6 +330,11 @@ values that can change accepted output. Adding or changing a component
|
|||||||
fingerprint intentionally produces a cold cache miss. Existing checkpoint
|
fingerprint intentionally produces a cold cache miss. Existing checkpoint
|
||||||
schemas and paths remain unchanged.
|
schemas and paths remain unchanged.
|
||||||
|
|
||||||
|
The CLI's `--recompute-step` policy forces the selected step and all transitive
|
||||||
|
dependents, but requires reusable checkpoints for all selected producers that
|
||||||
|
precede it. It changes loader decisions only; it does not alter persistent
|
||||||
|
checkpoint identity.
|
||||||
|
|
||||||
Debug instrumentation wraps run, stage, attempt, validator, and structured LLM
|
Debug instrumentation wraps run, stage, attempt, validator, and structured LLM
|
||||||
boundaries. Every executed chunk, extract, merge, and normalize attempt writes
|
boundaries. Every executed chunk, extract, merge, and normalize attempt writes
|
||||||
one terminal envelope for acceptance, validator rejection, module or validator
|
one terminal envelope for acceptance, validator rejection, module or validator
|
||||||
|
|||||||
@@ -32,6 +32,17 @@ The serialized
|
|||||||
`workspace_schema_version` identifiers are frozen wire-compatibility fields;
|
`workspace_schema_version` identifiers are frozen wire-compatibility fields;
|
||||||
they do not describe a current public state surface.
|
they do not describe a current public state surface.
|
||||||
|
|
||||||
|
Ordered-step lane checkpoints include the step identity in their storage scope.
|
||||||
|
When a later lane consumes a generated artifact, its dependency fingerprints
|
||||||
|
include the producer's artifact kind, complete schema identity, media type,
|
||||||
|
canonical content digest, and size. A producer checkpoint may be decoded through
|
||||||
|
the registered codec and handed off without rerunning it. Missing, rejected,
|
||||||
|
corrupt, incompatible, or changed producer state produces a bounded
|
||||||
|
`dependency_invalidated` decision for every transitive dependent lane; it does
|
||||||
|
not permit stale downstream reuse. The CLI's selective recomputation policy
|
||||||
|
records `forced_recompute` for the selected step and its dependents while
|
||||||
|
requiring compatible predecessor checkpoints.
|
||||||
|
|
||||||
`internal/core/fileio` provides confined atomic file writes used by state
|
`internal/core/fileio` provides confined atomic file writes used by state
|
||||||
collaborators. The chunk-plan store retains its stronger entry validation.
|
collaborators. The chunk-plan store retains its stronger entry validation.
|
||||||
|
|
||||||
@@ -48,6 +59,11 @@ boundaries redact sensitive metadata and credential-shaped bytes while allowing
|
|||||||
application-owned trace material. Debug data is never a checkpoint source or
|
application-owned trace material. Debug data is never a checkpoint source or
|
||||||
cache input.
|
cache input.
|
||||||
|
|
||||||
|
Generated reference bytes exist only in cloned operation requests and are not
|
||||||
|
written as paths into checkpoints, manifests, or debug summaries. Those state
|
||||||
|
surfaces retain canonical identities and bounded producer provenance so that a
|
||||||
|
resume decision can be explained without copying generated campaign content.
|
||||||
|
|
||||||
After allocation, one CLI-owned state value accumulates the known report paths,
|
After allocation, one CLI-owned state value accumulates the known report paths,
|
||||||
pipeline outcome counts, and validation status. A single guarded terminalization
|
pipeline outcome counts, and validation status. A single guarded terminalization
|
||||||
operation writes the success report, or makes one attempt each to write the
|
operation writes the success report, or makes one attempt each to write the
|
||||||
|
|||||||
@@ -40,63 +40,33 @@ names, schemas, and media types inside a run directory.
|
|||||||
Remove an output run directory only after its consumer data is no longer
|
Remove an output run directory only after its consumer data is no longer
|
||||||
needed. This is data deletion, not cache cleanup.
|
needed. This is data deletion, not cache cleanup.
|
||||||
|
|
||||||
## Sequential NPC And Spell Runs
|
## Ordered D&D Workflow
|
||||||
|
|
||||||
The maintained [sequential configuration](../examples/dnd-npc-spell-sequential.config.yml)
|
The maintained [NPC-grounded configuration](../examples/dnd-npc-grounded.config.yml)
|
||||||
contains two independent pipelines over the same Seriatim input shape. Run the
|
contains one pipeline with two ordered steps. The first step extracts and
|
||||||
NPC pipeline first and retain its normalized payload:
|
normalizes NPCs. Only after that lane reaches an accepted terminal result does
|
||||||
|
the second step begin; its generated NPC reference is supplied in memory to
|
||||||
|
spell extraction, combat extraction, and combat normalization.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go run ./cmd/notarius run dnd-npcs \
|
go run ./cmd/notarius run dnd-npc-grounded \
|
||||||
--config examples/dnd-npc-spell-sequential.config.yml \
|
--config examples/dnd-npc-grounded.config.yml \
|
||||||
--input examples/seriatim-minimal-transcript.json \
|
--input examples/seriatim-minimal-transcript.json \
|
||||||
--output-dir ./npc-output
|
--output-dir ./npc-grounded-output
|
||||||
```
|
```
|
||||||
|
|
||||||
Then bind that completed run's `lanes/npcs.json` file to the spell extractor:
|
The NPC artifact grounds canonical names and aliases, not spell or combat
|
||||||
|
evidence. Current-transcript source ranges remain the only event evidence. The
|
||||||
|
manifest records generated-reference identity and bounded producer provenance;
|
||||||
|
it does not record generated payload content, and no generated content is
|
||||||
|
exposed through a filesystem path. The same producer artifact may fan out to
|
||||||
|
compatible consumers, while a missing or rejected producer prevents the later
|
||||||
|
step from starting.
|
||||||
|
|
||||||
```sh
|
Standalone module configurations continue to support external NPC files when a
|
||||||
go run ./cmd/notarius run dnd-spells \
|
workflow intentionally crosses a process or session boundary. Those files are
|
||||||
--config examples/dnd-npc-spell-sequential.config.yml \
|
validated against the consumer slot and must be protected as sensitive
|
||||||
--input examples/seriatim-minimal-transcript.json \
|
campaign data. They are not part of the maintained ordered handoff workflow.
|
||||||
--reference spells.extract.npcs=./npc-output/<run-id>/lanes/npcs.json
|
|
||||||
```
|
|
||||||
|
|
||||||
The NPC file is a reference for canonical caster names and aliases, not spell
|
|
||||||
evidence. The spell manifest records the bound file's raw reference provenance
|
|
||||||
and the prepared registry's count and semantic digest separately. The NPC
|
|
||||||
payload, names, aliases, source references, and file bytes can be sensitive
|
|
||||||
campaign data; protect both output roots and any checkpoint or debug roots that
|
|
||||||
retain derived application data. A registry from another session is allowed,
|
|
||||||
but its source references are never copied into spell output evidence.
|
|
||||||
|
|
||||||
## Sequential NPC And Combat Runs
|
|
||||||
|
|
||||||
The maintained [sequential NPC and combat configuration](../examples/dnd-npc-combat-sequential.config.yml)
|
|
||||||
also represents two independent runs. Run `dnd-npcs` first, then bind its
|
|
||||||
normalized `lanes/npcs.json` payload to both combat stages:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go run ./cmd/notarius run dnd-npcs \
|
|
||||||
--config examples/dnd-npc-combat-sequential.config.yml \
|
|
||||||
--input examples/seriatim-minimal-transcript.json \
|
|
||||||
--output-dir ./npc-output
|
|
||||||
|
|
||||||
go run ./cmd/notarius run dnd-combat \
|
|
||||||
--config examples/dnd-npc-combat-sequential.config.yml \
|
|
||||||
--input examples/seriatim-minimal-transcript.json \
|
|
||||||
--reference combat.extract.npcs=./npc-output/<run-id>/lanes/npcs.json \
|
|
||||||
--reference combat.normalize.npcs=./npc-output/<run-id>/lanes/npcs.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Extraction and normalization bindings are stage-local and are intentionally
|
|
||||||
specified separately. Notarius does not discover the NPC run, copy its source
|
|
||||||
ranges into combat evidence, or compose the two runs into one workflow. The
|
|
||||||
combat manifest records both reference bindings and the prepared registry's
|
|
||||||
semantic digest/count. Changing the referenced NPC payload, prompt or schema,
|
|
||||||
normalization policy, or registry digest makes affected checkpoint state
|
|
||||||
incompatible; output, checkpoint, and debug roots remain independent sensitive
|
|
||||||
state surfaces.
|
|
||||||
|
|
||||||
## Chunk-Plan Cache
|
## Chunk-Plan Cache
|
||||||
|
|
||||||
@@ -177,6 +147,33 @@ cache:
|
|||||||
Remove an exact checkpoint identity directory or the configured root only when
|
Remove an exact checkpoint identity directory or the configured root only when
|
||||||
recomputation is acceptable.
|
recomputation is acceptable.
|
||||||
|
|
||||||
|
### Resume And Selective Recompute
|
||||||
|
|
||||||
|
`--resume` loads compatible accepted work only when checkpoint recording is
|
||||||
|
enabled. A normal resumed run may reuse source, extract, merge, and normalize
|
||||||
|
checkpoints independently. Generated references add a dependency fingerprint
|
||||||
|
for the producer's artifact kind, schema identity, media type, canonical
|
||||||
|
content digest, and size. If that fingerprint changes or the producer is
|
||||||
|
missing, dependent checkpoints are invalidated; unrelated work remains eligible
|
||||||
|
for reuse.
|
||||||
|
|
||||||
|
`--recompute-step <step-id>` requires both `--resume` and
|
||||||
|
`cache.checkpoints.enabled: true`. It forces the named step and all transitive
|
||||||
|
dependents to execute, while compatible predecessors and unrelated lanes remain
|
||||||
|
reusable. It cannot be combined with `--only`, and it does not change the
|
||||||
|
persistent identity of otherwise identical checkpoints. Decisions are bounded
|
||||||
|
and categorized as `reused`, `executed`, `forced_recompute`, or
|
||||||
|
`dependency_invalidated`.
|
||||||
|
|
||||||
|
If a required predecessor cannot be reused during selective recomputation, the
|
||||||
|
run fails before the dependent step starts. The failure manifest retains the
|
||||||
|
completed upstream outcomes and dependency context but not generated reference
|
||||||
|
content. For diagnosis, first check the failed step and lane in the manifest,
|
||||||
|
then inspect checkpoint decision categories and reason codes. A
|
||||||
|
`dependency_invalidated` decision means the stored producer, codec identity,
|
||||||
|
schema, or canonical content no longer matches; a missing or rejected producer
|
||||||
|
requires rerunning it rather than copying an artifact into the checkpoint root.
|
||||||
|
|
||||||
## Debug Bundles
|
## Debug Bundles
|
||||||
|
|
||||||
Only `notarius run --debug` enables debug collection. The selected root contains
|
Only `notarius run --debug` enables debug collection. The selected root contains
|
||||||
|
|||||||
@@ -1,31 +0,0 @@
|
|||||||
version: 3
|
|
||||||
output:
|
|
||||||
directory: ./notarius-output
|
|
||||||
cache:
|
|
||||||
chunk_plans:
|
|
||||||
mode: bypass
|
|
||||||
directory: ./notarius-cache/chunk-plans
|
|
||||||
checkpoints:
|
|
||||||
enabled: false
|
|
||||||
directory: ./notarius-cache/checkpoints
|
|
||||||
debug:
|
|
||||||
directory: ./notarius-debug
|
|
||||||
pipelines:
|
|
||||||
dnd-npcs:
|
|
||||||
input: seriatim
|
|
||||||
chunk: generic
|
|
||||||
artifacts:
|
|
||||||
npcs:
|
|
||||||
extract:
|
|
||||||
module: dnd/npcs
|
|
||||||
retries: 2
|
|
||||||
normalize: dnd/npcs
|
|
||||||
dnd-combat:
|
|
||||||
input: seriatim
|
|
||||||
chunk: generic
|
|
||||||
artifacts:
|
|
||||||
combat:
|
|
||||||
extract:
|
|
||||||
module: dnd/combat-turns
|
|
||||||
retries: 2
|
|
||||||
normalize: dnd/combat-turns
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
version: 3
|
|
||||||
output:
|
|
||||||
directory: ./notarius-output
|
|
||||||
cache:
|
|
||||||
chunk_plans:
|
|
||||||
mode: bypass
|
|
||||||
checkpoints:
|
|
||||||
enabled: false
|
|
||||||
directory: ""
|
|
||||||
debug:
|
|
||||||
directory: ./notarius-debug
|
|
||||||
pipelines:
|
|
||||||
dnd-npcs:
|
|
||||||
input: seriatim
|
|
||||||
chunk: generic
|
|
||||||
artifacts:
|
|
||||||
npcs:
|
|
||||||
extract:
|
|
||||||
module: dnd/npcs
|
|
||||||
retries: 2
|
|
||||||
normalize: dnd/npcs
|
|
||||||
dnd-spells:
|
|
||||||
input: seriatim
|
|
||||||
chunk: generic
|
|
||||||
artifacts:
|
|
||||||
spells:
|
|
||||||
extract: dnd/spells
|
|
||||||
normalize: dnd/spells
|
|
||||||
@@ -22,7 +22,7 @@ func TestOversizedNPCRegistryFailsBeforeRuntimeAndCheckpointConstruction(t *test
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
|
checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
|
||||||
content := string(readRepositoryFile(t, "examples", "dnd-npc-spell-sequential.config.yml"))
|
content := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml"))
|
||||||
content = replaceRequiredOnce(t, content, " extract: dnd/spells", " extract:\n module: dnd/spells\n references:\n npcs: "+npcPath)
|
content = replaceRequiredOnce(t, content, " extract: dnd/spells", " extract:\n module: dnd/spells\n references:\n npcs: "+npcPath)
|
||||||
content = replaceRequiredOnce(t, content, " enabled: false\n directory: \"\"", " enabled: true\n directory: "+checkpointRoot)
|
content = replaceRequiredOnce(t, content, " enabled: false\n directory: \"\"", " enabled: true\n directory: "+checkpointRoot)
|
||||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||||
@@ -46,11 +46,11 @@ func TestOversizedNPCRegistryFailsBeforeRuntimeAndCheckpointConstruction(t *test
|
|||||||
}
|
}
|
||||||
var stdout, stderr strings.Builder
|
var stdout, stderr strings.Builder
|
||||||
code := RunWithOptions([]string{
|
code := RunWithOptions([]string{
|
||||||
"run", "dnd-spells", "--config", configPath,
|
"run", "dnd-session", "--config", configPath,
|
||||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||||
"--chunk_cache", "bypass", "--output-dir", t.TempDir(),
|
"--chunk_cache", "bypass", "--output-dir", t.TempDir(),
|
||||||
}, &stdout, &stderr, options)
|
}, &stdout, &stderr, options)
|
||||||
for _, fragment := range []string{`pipeline "dnd-spells"`, `reference slot "npcs"`, "1048577 bytes", "limit 1048576"} {
|
for _, fragment := range []string{`pipeline "dnd-session"`, `reference slot "npcs"`, "1048577 bytes", "limit 1048576"} {
|
||||||
if code == 0 || !strings.Contains(stderr.String(), fragment) {
|
if code == 0 || !strings.Contains(stderr.String(), fragment) {
|
||||||
t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, stderr.String(), fragment)
|
t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, stderr.String(), fragment)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -504,9 +504,7 @@ func maintainedExampleFiles(t *testing.T) []maintainedExample {
|
|||||||
{name: "minimal", path: repositoryPath("examples", "dnd-spells.config.yml"), pipelineIDs: []string{"dnd-session"}},
|
{name: "minimal", path: repositoryPath("examples", "dnd-spells.config.yml"), pipelineIDs: []string{"dnd-session"}},
|
||||||
{name: "production", path: repositoryPath("examples", "dnd-spells-production.config.yml"), pipelineIDs: []string{"dnd-session"}},
|
{name: "production", path: repositoryPath("examples", "dnd-spells-production.config.yml"), pipelineIDs: []string{"dnd-session"}},
|
||||||
{name: "npcs", path: repositoryPath("examples", "dnd-npcs.config.yml"), pipelineIDs: []string{"dnd-session"}},
|
{name: "npcs", path: repositoryPath("examples", "dnd-npcs.config.yml"), pipelineIDs: []string{"dnd-session"}},
|
||||||
{name: "sequential", path: repositoryPath("examples", "dnd-npc-spell-sequential.config.yml"), pipelineIDs: []string{"dnd-npcs", "dnd-spells"}},
|
|
||||||
{name: "combat", path: repositoryPath("examples", "dnd-combat-turns.config.yml"), pipelineIDs: []string{"dnd-combat"}},
|
{name: "combat", path: repositoryPath("examples", "dnd-combat-turns.config.yml"), pipelineIDs: []string{"dnd-combat"}},
|
||||||
{name: "npc-combat-sequential", path: repositoryPath("examples", "dnd-npc-combat-sequential.config.yml"), pipelineIDs: []string{"dnd-combat", "dnd-npcs"}},
|
|
||||||
{name: "npc-grounded", path: repositoryPath("examples", "dnd-npc-grounded.config.yml"), pipelineIDs: []string{"dnd-npc-grounded"}},
|
{name: "npc-grounded", path: repositoryPath("examples", "dnd-npc-grounded.config.yml"), pipelineIDs: []string{"dnd-npc-grounded"}},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -152,79 +152,6 @@ func TestCombatNormalizerRejectsCampaignReferenceBinding(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSequentialNPCOutputGroundsCombatAtBothStageLocalReferences(t *testing.T) {
|
|
||||||
registries := productionNPCRegistries(t)
|
|
||||||
catalog := moduleCatalog(registries)
|
|
||||||
configValue := loadSequentialCombatPipelineConfig(t)
|
|
||||||
npcEffective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npcs", Catalog: catalog})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Resolve(NPC) error = %v", err)
|
|
||||||
}
|
|
||||||
npcClient := &fakeNPCProductionLLMClient{response: npcProductionResponse{NPCs: []npcProductionRecord{
|
|
||||||
{Name: "Mira Thorn", Aliases: []string{"The Greencloak", "Mira"}, Description: "A ranger.", Relationships: []npcProductionRelationship{}, SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 2}}},
|
|
||||||
{Name: "Hooded Guard", Aliases: []string{}, Description: "A sentry.", Relationships: []npcProductionRelationship{}, SourceRefs: []npcProductionSourceRef{{StartUnitID: 3, EndUnitID: 3}}},
|
|
||||||
}}}
|
|
||||||
npcOutput, err := runPreparedPipeline(t, registries, npcEffective.ResolvedPipeline, npcClient, pipeline.RunInput{RawInput: readNPCFixture(t)})
|
|
||||||
if err != nil || len(npcOutput.NormalizeOutputs) != 1 {
|
|
||||||
t.Fatalf("NPC run error = %v output = %#v, want one normalized NPC lane", err, npcOutput.NormalizeOutputs)
|
|
||||||
}
|
|
||||||
npcPayload := npcOutput.NormalizeOutputs[0].Artifact.Content
|
|
||||||
if _, err := npccodec.New().Decode(npcPayload); err != nil {
|
|
||||||
t.Fatalf("Decode(NPC output) error = %v", err)
|
|
||||||
}
|
|
||||||
npcPath := filepath.Join(t.TempDir(), "npc-run", "lanes", "npcs.json")
|
|
||||||
if err := os.MkdirAll(filepath.Dir(npcPath), 0o700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(npcPath, npcPayload, 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
combatEffective, err := configValue.Resolve(config.ResolveInput{
|
|
||||||
PipelineID: "dnd-combat",
|
|
||||||
Catalog: catalog,
|
|
||||||
ReferenceOverrides: []pipeline.ReferenceBinding{
|
|
||||||
{Stage: pipeline.StageExtract, LaneID: "combat", SlotName: "npcs", Source: npcPath, BindingSource: contracts.ReferenceBindingSourceCLI},
|
|
||||||
{Stage: pipeline.StageNormalize, LaneID: "combat", SlotName: "npcs", Source: npcPath, BindingSource: contracts.ReferenceBindingSourceCLI},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Resolve(combat) error = %v", err)
|
|
||||||
}
|
|
||||||
materialized, warnings, err := pipeline.MaterializeReferences(combatEffective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
|
|
||||||
if err != nil || len(warnings) != 0 {
|
|
||||||
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
|
|
||||||
}
|
|
||||||
client := &fakeCombatLLMClient{responses: []string{combatTestTurnResponse("The Greencloak", "turn", "watches", "Hooded Guard", 1, 1)}}
|
|
||||||
output, err := runPreparedPipeline(t, registries, materialized, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Run(combat) error = %v", err)
|
|
||||||
}
|
|
||||||
value, err := combatcodec.New().Decode(output.NormalizeOutputs[0].Artifact.Content)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Decode(combat output) error = %v", err)
|
|
||||||
}
|
|
||||||
if len(value.CombatTurns) != 1 || value.CombatTurns[0].Actor != "Mira Thorn" || value.CombatTurns[0].Actions[0].Targets[0] != "Hooded Guard" {
|
|
||||||
t.Fatalf("combat value = %#v, want canonical actor and target", value)
|
|
||||||
}
|
|
||||||
for _, ref := range value.CombatTurns[0].SourceRefs {
|
|
||||||
if ref.SourceID != "npc-session" {
|
|
||||||
t.Fatalf("combat source ref = %#v, want transcript evidence only", ref)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(output.Manifest.References) != 2 {
|
|
||||||
t.Fatalf("manifest references = %#v, want both stage-local NPC provenance entries", output.Manifest.References)
|
|
||||||
}
|
|
||||||
for _, provenance := range output.Manifest.References {
|
|
||||||
if provenance.SlotName != "npcs" || provenance.LaneID != "combat" || !strings.Contains(provenance.OriginURI, "npcs.json") || provenance.BindingSource != contracts.ReferenceBindingSourceCLI {
|
|
||||||
t.Fatalf("NPC provenance = %#v, want combat stage-local CLI binding", provenance)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(client.requests) != 1 || string(client.requests[0].Inputs[combatextract.NPCRegistryReferenceSlot].Content) != string(npcPayload) || client.requests[0].Inputs[combatextract.NPCRegistryReferenceSlot].OriginURI != "" {
|
|
||||||
t.Fatalf("combat NPC prompt input = %#v, want canonical payload without path provenance", client.requests)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCombatPreparationRejectsMalformedOrOversizedNPCReferencesBeforeExecution(t *testing.T) {
|
func TestCombatPreparationRejectsMalformedOrOversizedNPCReferencesBeforeExecution(t *testing.T) {
|
||||||
registries := productionNPCRegistries(t)
|
registries := productionNPCRegistries(t)
|
||||||
catalog := moduleCatalog(registries)
|
catalog := moduleCatalog(registries)
|
||||||
@@ -313,19 +240,6 @@ func combatOnlyConfig() config.Config {
|
|||||||
return cfg
|
return cfg
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadSequentialCombatPipelineConfig(t *testing.T) config.Config {
|
|
||||||
t.Helper()
|
|
||||||
fileConfig, err := config.LoadFileConfig(repositoryPathForIntegration("examples", "dnd-npc-combat-sequential.config.yml"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LoadFileConfig() error = %v", err)
|
|
||||||
}
|
|
||||||
cfg := config.Default()
|
|
||||||
if err := cfg.ApplyFileConfig(fileConfig); err != nil {
|
|
||||||
t.Fatalf("ApplyFileConfig() error = %v", err)
|
|
||||||
}
|
|
||||||
return cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
func repositoryPathForIntegration(parts ...string) string {
|
func repositoryPathForIntegration(parts ...string) string {
|
||||||
_, file, _, _ := runtime.Caller(0)
|
_, file, _, _ := runtime.Caller(0)
|
||||||
return filepath.Join(append([]string{filepath.Dir(file), "..", "..", ".."}, parts...)...)
|
return filepath.Join(append([]string{filepath.Dir(file), "..", "..", ".."}, parts...)...)
|
||||||
|
|||||||
@@ -1,130 +0,0 @@
|
|||||||
package integration_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
||||||
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs"
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestSequentialNPCOutputCanGroundIndependentSpellRun(t *testing.T) {
|
|
||||||
registries := productionNPCRegistries(t)
|
|
||||||
catalog := moduleCatalog(registries)
|
|
||||||
configValue := loadSequentialPipelineConfig(t)
|
|
||||||
|
|
||||||
npcEffective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npcs", Catalog: catalog})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("resolve NPC pipeline: %v", err)
|
|
||||||
}
|
|
||||||
npcClient := &fakeNPCProductionLLMClient{response: npcProductionResponse{NPCs: []npcProductionRecord{{
|
|
||||||
Name: "Mira Thorn",
|
|
||||||
Aliases: []string{"The Greencloak"},
|
|
||||||
Description: "A guarded ranger who watches the northern road.",
|
|
||||||
Relationships: []npcProductionRelationship{{
|
|
||||||
Target: "Captain Vale", Relationship: "reports to",
|
|
||||||
}},
|
|
||||||
SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 2}},
|
|
||||||
}}}}
|
|
||||||
npcOutput, err := runPreparedPipeline(t, registries, npcEffective.ResolvedPipeline, npcClient, pipeline.RunInput{RawInput: readNPCFixture(t)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("run NPC pipeline: %v", err)
|
|
||||||
}
|
|
||||||
if len(npcOutput.NormalizeOutputs) != 1 || npcOutput.NormalizeOutputs[0].LaneID != "npcs" {
|
|
||||||
t.Fatalf("NPC normalized outputs = %#v, want one npcs lane", npcOutput.NormalizeOutputs)
|
|
||||||
}
|
|
||||||
npcPayload := npcOutput.NormalizeOutputs[0].Artifact.Content
|
|
||||||
if _, err := npccodec.New().Decode(npcPayload); err != nil {
|
|
||||||
t.Fatalf("decode normalized NPC payload: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
npcRunDir := t.TempDir()
|
|
||||||
npcPath := filepath.Join(npcRunDir, "lanes", "npcs.json")
|
|
||||||
if err := os.MkdirAll(filepath.Dir(npcPath), 0o700); err != nil {
|
|
||||||
t.Fatalf("create NPC output directory: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(npcPath, npcPayload, 0o600); err != nil {
|
|
||||||
t.Fatalf("write NPC output payload: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
spellEffective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-spells", Catalog: catalog})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("resolve spell pipeline: %v", err)
|
|
||||||
}
|
|
||||||
spellEffective.ResolvedPipeline.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings = []pipeline.ReferenceBinding{{
|
|
||||||
Stage: pipeline.StageExtract,
|
|
||||||
LaneID: "spells",
|
|
||||||
SlotName: spells.NPCRegistryReferenceSlot,
|
|
||||||
Source: npcPath,
|
|
||||||
BindingSource: contracts.ReferenceBindingSourceCLI,
|
|
||||||
}}
|
|
||||||
materialized, warnings, err := pipeline.MaterializeReferences(spellEffective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{WorkingDir: npcRunDir})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("materialize NPC registry reference: %v", err)
|
|
||||||
}
|
|
||||||
if len(warnings) != 0 {
|
|
||||||
t.Fatalf("reference materialization warnings = %#v, want none", warnings)
|
|
||||||
}
|
|
||||||
|
|
||||||
spellClient := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{{
|
|
||||||
Caster: "Mira Thorn",
|
|
||||||
Spell: "Cure Wounds",
|
|
||||||
Effect: "Restores the injured ally.",
|
|
||||||
NarrativeDescription: "Mira Thorn restores the ally after the fight.",
|
|
||||||
SourceRefs: responseSourceRefs("spell-session", 1, 1),
|
|
||||||
}}}}
|
|
||||||
spellOutput, err := runPreparedPipeline(t, registries, materialized, spellClient, pipeline.RunInput{RawInput: readDNDSpellsFixture(t)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("run spell pipeline: %v", err)
|
|
||||||
}
|
|
||||||
if len(spellOutput.NormalizeOutputs) != 1 || spellOutput.NormalizeOutputs[0].LaneID != "spells" {
|
|
||||||
t.Fatalf("spell normalized outputs = %#v, want one spells lane", spellOutput.NormalizeOutputs)
|
|
||||||
}
|
|
||||||
spellValue := decodeRunnerSpellResponse(t, spellOutput.NormalizeOutputs[0].Artifact.Content)
|
|
||||||
if len(spellValue.SpellCasts) != 1 || spellValue.SpellCasts[0].Caster != "Mira Thorn" {
|
|
||||||
t.Fatalf("spell output = %#v, want one registry-grounded caster", spellValue)
|
|
||||||
}
|
|
||||||
if len(spellValue.SpellCasts[0].SourceRefs) != 1 || spellValue.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
|
|
||||||
t.Fatalf("spell source refs = %#v, want current spell session only", spellValue.SpellCasts[0].SourceRefs)
|
|
||||||
}
|
|
||||||
if len(spellClient.requests) != 1 {
|
|
||||||
t.Fatalf("spell LLM requests = %d, want one", len(spellClient.requests))
|
|
||||||
}
|
|
||||||
registryInput := spellClient.requests[0].Inputs[spells.NPCRegistryReferenceSlot]
|
|
||||||
if string(registryInput.Content) != string(npcPayload) || registryInput.MediaType != npccodec.MediaType || registryInput.OriginURI != "" {
|
|
||||||
t.Fatalf("spell NPC prompt input = %#v, want canonical payload without origin", registryInput)
|
|
||||||
}
|
|
||||||
if len(spellOutput.Manifest.References) != 1 {
|
|
||||||
t.Fatalf("spell manifest references = %#v, want one NPC provenance entry", spellOutput.Manifest.References)
|
|
||||||
}
|
|
||||||
provenance := spellOutput.Manifest.References[0]
|
|
||||||
if provenance.Stage != "extract" || provenance.LaneID != "spells" || provenance.SlotName != spells.NPCRegistryReferenceSlot || provenance.BindingSource != contracts.ReferenceBindingSourceCLI || !strings.Contains(provenance.OriginURI, "npcs.json") {
|
|
||||||
t.Fatalf("spell NPC provenance = %#v, want extract CLI reference provenance", provenance)
|
|
||||||
}
|
|
||||||
metadata, ok := spellOutput.Manifest.ArtifactLanes[0].Metadata["extractor"].(map[string]any)
|
|
||||||
if !ok || metadata["npc_count"] != 1 || metadata["npc_registry_digest"] != registryInput.Digest {
|
|
||||||
t.Fatalf("spell extractor metadata = %#v, want NPC count and semantic digest", spellOutput.Manifest.ArtifactLanes[0].Metadata)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadSequentialPipelineConfig(t *testing.T) config.Config {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile("testdata/dnd_npc_spell_sequential_pipeline.yml")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read sequential pipeline config: %v", err)
|
|
||||||
}
|
|
||||||
fileConfig, err := config.ParseFileConfigYAML(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse sequential pipeline config: %v", err)
|
|
||||||
}
|
|
||||||
configValue := config.Default()
|
|
||||||
if err := configValue.ApplyFileConfig(fileConfig); err != nil {
|
|
||||||
t.Fatalf("apply sequential pipeline config: %v", err)
|
|
||||||
}
|
|
||||||
return configValue
|
|
||||||
}
|
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
version: 3
|
|
||||||
output:
|
|
||||||
directory: ./notarius-output
|
|
||||||
cache:
|
|
||||||
chunk_plans:
|
|
||||||
mode: bypass
|
|
||||||
checkpoints: {}
|
|
||||||
debug:
|
|
||||||
directory: ./notarius-debug
|
|
||||||
pipelines:
|
|
||||||
dnd-npcs:
|
|
||||||
input: seriatim
|
|
||||||
chunk: generic
|
|
||||||
artifacts:
|
|
||||||
npcs:
|
|
||||||
extract:
|
|
||||||
module: dnd/npcs
|
|
||||||
normalize: dnd/npcs
|
|
||||||
dnd-spells:
|
|
||||||
input: seriatim
|
|
||||||
chunk: generic
|
|
||||||
artifacts:
|
|
||||||
spells:
|
|
||||||
extract: dnd/spells
|
|
||||||
normalize: dnd/spells
|
|
||||||
Reference in New Issue
Block a user