112 Commits

Author SHA1 Message Date
39388e96d4 Make PromptKit profile handling safer and more consistent 2026-08-03 18:35:40 +00:00
12ac25bd63 Sanitize PromptKit profile fingerprint errors 2026-08-03 17:25:44 +00:00
394278e1f2 Document workload-oriented LLM profile deployment 2026-08-03 17:21:42 +00:00
5cd7f8e737 Expose pipeline LLM profile defaults 2026-08-03 17:15:07 +00:00
bf3fadf9ae Resolve pipeline LLM profile defaults 2026-08-03 17:09:16 +00:00
58815aaf33 Require explicit module execution classes 2026-08-03 17:00:08 +00:00
ce857966f1 Add execution metadata to module specifications 2026-08-03 16:49:51 +00:00
a3bd0c1867 Add D&D extraction fallback profile 2026-08-03 16:39:45 +00:00
b05634ee86 Add fallback PromptKit profile assets 2026-08-03 16:34:48 +00:00
4829f94157 Inspect PromptKit profiles during preflight 2026-08-03 16:26:15 +00:00
67b315099d Execute PromptKit requests from prepared snapshots 2026-08-03 16:17:40 +00:00
b5c86de4d7 Upgrade PromptKit to version 0.5.0 2026-08-03 16:12:29 +00:00
2eeca2ed5a Plan the PromptKit upgrade and profile workflow 2026-08-03 16:07:37 +00:00
b5aaeb1c78 Update future roadmap document with ideas for new feature developments 2026-07-30 16:27:36 +00:00
9171b66a41 Clarify PromptKit configuration and retire the completed plan 2026-07-30 15:38:31 +00:00
b4363b3b73 Document local PromptKit backend support 2026-07-30 05:22:41 +00:00
241e9d2a89 Include local backend target in checkpoint identity 2026-07-30 05:17:39 +00:00
715fff7b72 Register configured local PromptKit backend 2026-07-30 05:14:22 +00:00
d627b91b4f Add local PromptKit backend configuration 2026-07-30 05:09:54 +00:00
a67b3aa76d Upgrade PromptKit to version 0.3.0 2026-07-30 05:06:32 +00:00
a16dcdfa52 Clean up the PromptKit upgrade 2026-07-30 02:58:25 +00:00
46e4466d28 Translate LLM backend capacity failures 2026-07-30 02:23:51 +00:00
71a004bfc8 Publish effective LLM backend provenance 2026-07-30 02:18:14 +00:00
f8333f2c15 Expose run-wide reasoning effort controls 2026-07-30 02:11:35 +00:00
f603f7ac64 Support internal reasoning effort overrides 2026-07-30 02:03:04 +00:00
7a00e7049c Forward sessions through PromptKit requests 2026-07-30 01:58:40 +00:00
2a9db9a957 Upgrade PromptKit to version 0.2.0 2026-07-30 01:54:47 +00:00
de046a8f13 Keep checkpoints aligned with PromptKit profiles 2026-07-28 13:36:01 -05:00
f1a6574013 Document the PromptKit integration 2026-07-28 16:52:49 +00:00
4bca6d3103 Align run provenance with PromptKit 2026-07-28 16:47:07 +00:00
7c569a3d8c Make D&D prompt assets provider neutral 2026-07-28 16:42:41 +00:00
8e04ef9e2b Introduce version 4 PromptKit configuration 2026-07-28 16:38:05 +00:00
53a330587b Replace the Scriptorium adapter with PromptKit 2026-07-28 16:31:36 +00:00
7cfab8ada0 Add a staged implementation plan for migration to the promptkit LLM library 2026-07-28 11:20:31 -05:00
5c82b62856 Retire completed roadmap documents and add ideas for future feature work 2026-07-27 20:20:00 -05:00
de8ed41b34 Document D&D prompt cache ordering 2026-07-27 20:16:11 +00:00
d1eaec4dad Consolidate D&D prompt ordering tests 2026-07-27 20:14:57 +00:00
c0ec068f53 Reorder D&D scene and normalization prompts 2026-07-27 20:08:09 +00:00
5cbd9e56e4 Reorder D&D extraction prompt prefixes 2026-07-27 20:07:09 +00:00
53490cdb59 Add a new feature roadmap and implementation plan for D&D prompt ordering to improve LLM provider caching 2026-07-27 15:04:32 -05:00
7c94b5eeed Clarify evidence context and test lane filtering 2026-07-27 14:03:23 -05:00
4f2864fc96 Stop tracking generated codebase memory files 2026-07-27 13:56:04 -05:00
893b03fccf Document published evidence context 2026-07-27 18:28:02 +00:00
256cc98ddb Publish evidence context in JSON output bundles 2026-07-27 18:23:48 +00:00
e61e522662 Build evidence context for output encoders 2026-07-27 18:17:54 +00:00
a4c7eca87b Add evidence context artifact builder 2026-07-27 18:11:49 +00:00
224a8292c4 Add evidence context policy preparation 2026-07-27 18:04:12 +00:00
64d461fc18 Add a feature roadmap and implementation plan for a context evidence artifact 2026-07-27 12:09:14 -05:00
fb1134e591 Address code review findings from the subprocess enhancements 2026-07-26 18:52:37 -05:00
1da29e6788 Document subprocess run results 2026-07-26 17:22:27 +00:00
0d947549fb Test JSON run result behavior 2026-07-26 17:17:35 +00:00
950fba17ce Add JSON run result mode 2026-07-26 17:14:16 +00:00
678d2c6099 Add run result wire model 2026-07-26 17:12:57 +00:00
db8db5ffc5 Add a plan to implement improved documentation and support for running notarius as a subprocess 2026-07-26 12:08:53 -05:00
94b3eafb1a Finish the documentation refresh 2026-07-26 13:53:38 +00:00
f6981e2264 Harmonize D&D artifact contracts 2026-07-26 13:43:45 +00:00
2f506f4985 Harmonize foundational integration contracts 2026-07-26 13:38:55 +00:00
fdf8c4afd4 Rewrite LLM runtime documentation 2026-07-26 13:33:45 +00:00
74c793e6a1 Document D&D module conventions 2026-07-26 13:30:36 +00:00
b5835fbc37 Rewrite pipeline and component overview documentation 2026-07-26 13:26:20 +00:00
fd3f7b85cc Rewrite operations and state documentation 2026-07-26 13:24:02 +00:00
d86b74f485 Document configuration and CLI internals 2026-07-26 13:20:18 +00:00
59cbf1eb27 Rewrite configuration reference 2026-07-26 13:16:06 +00:00
ee43add75c Rewrite CLI orientation and reference 2026-07-26 13:11:40 +00:00
46761706a2 Add complete D&D example transcript 2026-07-26 13:07:59 +00:00
5a968b64eb Clean up completed roadmap work 2026-07-25 21:50:39 -05:00
7b077c269d Clarify normalize fallback ownership 2026-07-26 02:44:29 +00:00
80ec939383 Validate normalize retry diagnostics 2026-07-26 02:43:28 +00:00
d3c4d6f133 Simplify NPC normalization prompt guidance 2026-07-26 02:40:08 +00:00
5ad661f95f Redact NPC normalization context errors 2026-07-26 02:38:03 +00:00
d63e5c6852 Plan NPC normalization follow-up fixes 2026-07-25 21:34:42 -05:00
fbb8e0d241 Document NPC semantic normalization 2026-07-26 01:46:23 +00:00
8d9a496935 Integrate NPC semantic normalization 2026-07-26 01:41:35 +00:00
d1c48db4bc Add LLM-assisted NPC normalization 2026-07-26 01:36:32 +00:00
6bd781d344 Add NPC normalization prompt context 2026-07-26 01:27:34 +00:00
8a12c56971 Add retryable normalization fallbacks 2026-07-26 01:20:28 +00:00
26bd59a5a2 Plan LLM-assisted NPC normalization 2026-07-25 20:13:12 -05:00
927a7beb88 Clarify and streamline D&D item events 2026-07-25 23:59:56 +00:00
f7059607af Document D&D item event support 2026-07-25 22:16:29 +00:00
63de44c347 Add D&D item event integration coverage 2026-07-25 22:10:53 +00:00
f0ede9dacc Register D&D item event pipeline 2026-07-25 22:05:52 +00:00
5711f8b9e3 Add D&D item event normalization 2026-07-25 22:00:32 +00:00
da83510234 Add D&D item event validators 2026-07-25 21:52:59 +00:00
f51b22bea7 Add D&D item event extractor 2026-07-25 21:49:24 +00:00
f320c2fcee Add D&D item event artifact contract 2026-07-25 21:41:17 +00:00
4ba1e50a89 Plan D&D item event tracking 2026-07-25 21:33:43 +00:00
2a7e025251 Improve scene-aware combat gating 2026-07-25 21:08:24 +00:00
3da20e9d6a Document scene-aware combat extraction 2026-07-25 19:34:33 +00:00
b1c0faa748 Test scene-aware combat handoff behavior 2026-07-25 19:29:15 +00:00
989f2c220b Gate combat extraction on scene descriptions 2026-07-25 19:23:19 +00:00
7e35915b3e Add scene eligibility registry 2026-07-25 19:15:33 +00:00
24238d249e Plan scene-aware combat extraction 2026-07-25 18:59:57 +00:00
9614469b45 Consolidate the D&D configuration examples 2026-07-25 15:48:48 +00:00
29ee68824d Finish the D&D module cleanup 2026-07-25 13:30:23 +00:00
aeaaf44ae0 Remove unused NPC registry and identity aliases 2026-07-25 13:06:40 +00:00
d752c51aec Centralize D&D candidate JSON codecs 2026-07-25 13:05:02 +00:00
8199d95dc1 Reuse document indexes in D&D normalization 2026-07-25 13:01:31 +00:00
97cdb01357 Reuse document indexes in D&D validators 2026-07-25 12:57:31 +00:00
7a66095912 Reuse indexes for D&D citation validation 2026-07-25 12:53:48 +00:00
9d1356a20e Add D&D refactoring plan and isolate merger results 2026-07-25 12:51:36 +00:00
e4471fc300 Fix D&D extraction issues and retire the completed audit 2026-07-25 12:30:42 +00:00
84a2854b5e Close D&D extraction audit documentation 2026-07-24 14:48:13 +00:00
a1b76093ce Document D&D extractor contract 2026-07-24 14:46:19 +00:00
1aa30a73db Protect D&D citation prompt ordering 2026-07-24 14:44:10 +00:00
dc7c0e2f9e Move scene semantics to deterministic validation 2026-07-24 14:40:54 +00:00
e2cb0d901a Centralize D&D extraction request preparation 2026-07-24 14:38:42 +00:00
8e0b029f5f Order combat and NPC interaction extraction by document position 2026-07-24 14:33:34 +00:00
9bbf2535dd Order spell and NPC extraction by document position 2026-07-24 14:29:19 +00:00
83fde83a58 Preserve literal ordering for foreign source references 2026-07-24 14:25:02 +00:00
bef3d1359d Fingerprint D&D extractor mapping policies 2026-07-24 14:23:35 +00:00
1ff449435f Centralize D&D source reference ordering 2026-07-24 14:20:10 +00:00
6e21c83fd8 Add D&D module code audit implementation plan 2026-07-24 14:08:36 +00:00
311 changed files with 20375 additions and 8040 deletions

2
.gitignore vendored
View File

@@ -2,6 +2,7 @@
notarius
notarius-output
workspace/
.codebase-memory/
# ---> Go
# If you prefer the allow list template instead of the deny list, see community template:
@@ -73,4 +74,3 @@ Icon
Network Trash Folder
Temporary Items
.apdisk

View File

@@ -1,35 +1,45 @@
# Notarius
Notarius is a Go CLI for extracting structured artifacts from source material
with explicit, configurable pipeline modules.
Notarius is a Go CLI for turning source material into structured artifacts with
configured extraction pipelines. The implemented D&D workflow reads Seriatim
transcript JSON and can produce scene descriptions, item and currency events,
NPC identities, combat turns, NPC interactions, and spell casts.
The current implementation reads Seriatim transcript JSON, chunks the source
units, extracts D&D spell-cast artifacts with a Scriptorium-backed LLM runtime,
and writes JSON output. Add `--debug` when a per-run inspection bundle is
needed.
## Quickstart
```sh
OPENROUTER_API_KEY=... \
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json
```
Provide an OpenRouter API key through the environment, then run the maintained
minimal example:
This invocation uses the maintained example configuration and input. See the
configuration and operations references for profile selection, credentials, and
run artifacts.
~~~
OPENROUTER_API_KEY=your-api-key \
go run ./cmd/notarius run dnd-session \
--config examples/dnd-minimal.config.yml \
--input examples/seriatim-minimal-transcript.json
~~~
Useful references:
The command publishes a JSON output bundle. Its command syntax and exit
behavior are documented in the [CLI reference](docs/cli.md); configuration,
credentials, and module selection are owned by the
[configuration reference](docs/config.md).
- [CLI reference](docs/cli.md)
- [Configuration reference](docs/config.md)
- [Operations](docs/operations.md)
- [Seriatim input contract](docs/integrations/seriatim.md)
- [JSON output contract](docs/integrations/json-output.md)
- [D&D spell artifact contract](docs/integrations/dnd-spell-artifacts.md)
- [Developer guide](docs/development.md)
- [Internal implementation docs](docs/internal/overview.md)
- [Maintained example config](examples/dnd-spells.config.yml)
- [NPC-grounded example config](examples/dnd-npc-grounded.config.yml)
- [NPC interaction example config](examples/dnd-npc-interactions.config.yml)
- [Maintained example input](examples/seriatim-minimal-transcript.json)
For the complete ordered D&D workflow, use
[the complete configuration](examples/dnd-complete.config.yml) with
[its synthetic transcript](examples/dnd-complete-transcript.json). It
demonstrates all implemented D&D lanes and the supporting campaign references.
## Documentation
- [CLI reference](docs/cli.md) — commands, flags, output streams, and exits.
- [Configuration reference](docs/config.md) — configuration files, profiles,
validation, and module selection.
- [Operations](docs/operations.md) — output, state, recovery, and debug
handling.
- [Integration contracts](docs/integrations/) — Seriatim input and published
artifact formats.
- [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius
from an orchestrator and consume a published result.
- [Internal overview](docs/internal/overview.md) — implemented component map
for maintainers.
- [Developer guide](docs/development.md) — contributor orientation and
validation guidance.
- [Future work](docs/roadmap/future.md) — unimplemented ideas and priorities.

View File

@@ -0,0 +1,49 @@
# ADR-0010: Use workload-oriented LLM profile defaults
**Status:** Accepted
**Date:** 2026-08-03
## Context
LLM-backed D&D operations share an execution-policy choice, but repeating a
provider or model-named profile on every module binding ties pipeline structure
to a deployment decision. Different environments may require different model,
backend, timeout, or reasoning settings while retaining the same workload.
Notarius also needs a usable default for maintained D&D prompts without making
an operator profile mandatory. That default must remain owned by the D&D
family, while generic LLM infrastructure stays unaware of domain-specific
policy.
## Decision
Pipelines may name one workload-oriented default profile, inherited only by
selected LLM-backed bindings and validators. Binding-level profile IDs remain
intentional exceptions, and the run-wide CLI profile override has highest
precedence.
The D&D family owns an embedded fallback profile named `dnd-extraction`.
Operators may provide a complete profile with the same ID through a PromptKit
filesystem source. PromptKit selects the higher-precedence matching definition;
Notarius does not merge profile documents. Production, development, and local
deployments can therefore use different execution policy behind one unchanged
pipeline ID.
## Alternatives considered
- Repeat a model-named profile on every binding. This makes routine deployment
policy changes noisy and obscures the shared workload intent.
- Require every deployment to install a profile file. This adds configuration
friction and leaves maintained D&D prompts without an application-owned
fallback.
- Put D&D profile policy in generic LLM infrastructure. This breaks domain
ownership and makes generic code depend on one workload.
## Consequences
Pipeline configuration expresses workload intent rather than a specific
provider or model. Operators can replace the complete execution policy without
editing bindings, while binding-level and run-wide exceptions remain available.
Profile changes affect resolved pipeline and checkpoint identity, so they may
intentionally cause work to be recomputed. The D&D fallback becomes a
maintained application execution-policy asset.

View File

@@ -1,273 +1,165 @@
# CLI Reference
This is the canonical reference for the implemented Notarius command-line
interface.
interface. For the shortest successful run, see the [README](../README.md).
Configuration fields, discovery rules, and selectable module keys are defined
in [Configuration](config.md); runtime state and recovery procedures are
defined in [Operations](operations.md).
For the minimal end-to-end invocation, see the [README](../README.md).
## Command Summary
## Commands
```text
~~~
notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--chunk_cache auto|bypass|refresh] [--output-dir path] [--resume] [--recompute-step step-id] [--debug [--debug-dir path]] [--llm-profile id] [--session-id id] [--reference selector=path] [--without-reference selector]
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list [--config path/to/config.yml] [--json]
```
~~~
Running `notarius` with no arguments, `notarius help`, `notarius --help`, or
`notarius -h` prints usage and exits successfully.
Running Notarius without arguments, or with **help**, **--help**, or **-h**,
writes the command summary to standard output and exits with status 0.
## `run`
## run
`notarius run <pipeline-id>` executes a configured pipeline against one input
file.
~~~
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
~~~
Flags:
The **run** command executes the named pipeline for one input file. The
pipeline ID and **--input** are required.
- `--input path`: required source input file.
- `--config path`: config file path. If omitted, Notarius uses the discovery
rules in [Configuration](config.md#discovery).
- `--only lane-a,lane-b`: run only the named artifact lanes. Values are
comma-separated and must be non-empty. This retains its existing behavior for
implicit single-step pipelines; explicit multi-step pipelines reject it
rather than inferring dependency closure.
- `--resume`: request checkpoint reuse for this invocation. Checkpoint recording
must be enabled in configuration. See
[Operations](operations.md#checkpoint-cache) for prerequisites and reuse
behavior.
- `--recompute-step step-id`: with `--resume` and checkpoint recording enabled,
force the named ordered step and every transitive dependent lane to execute.
Compatible required predecessors and unrelated lanes remain reusable. The
value may identify an explicit step or the implicit single-step ID `default`;
it cannot be combined with `--only`.
- `--chunk_cache auto|bypass|refresh`: select chunk-plan reuse for this
invocation. `auto` reuses a valid plan by canonical source digest, `bypass`
performs no plan-cache I/O, and `refresh` regenerates and replaces a valid
plan only after chunk validation succeeds. See
[Configuration](config.md#state-surfaces) for the persistent setting, precedence,
and cache-root selection.
- `--output-dir path`: output root. Defaults to `./notarius-output`.
- `--debug`: allocate and retain one debug bundle for this invocation.
- `--debug-dir path`: debug-bundle root override. This flag requires `--debug`.
- `--llm-profile id`: override every effective LLM-capable pipeline module
binding with one Scriptorium profile ID. Validator-specific profiles are not
overridden.
- `--session-id id`: pass a stable prompt session identifier through LLM-backed
module calls.
- `--reference selector=path`: bind a reference path to a chunk, extractor,
merger, or normalizer reference slot. Repeatable.
- `--without-reference selector`: remove a configured optional reference binding.
Repeatable. It accepts the same selector forms as `--reference`, without
`=path`.
| Flag | Meaning |
| --- | --- |
| **--config path** | Use this configuration file. When omitted, configuration discovery applies; see [Configuration](config.md). |
| **--input path** | Source input file to process. Required. |
| **--output-dir path** | Override the configured output root for this run. |
| **--json** | Write the successful run-result receipt as JSON to standard output. |
| **--chunk_cache auto\|bypass\|refresh** | Override chunk-plan cache handling for this run. |
| **--resume** | Reuse compatible recorded checkpoints when checkpoint recording is enabled. |
| **--recompute-step step-id** | With **--resume**, recompute the selected ordered step and its dependent lanes. It cannot be combined with **--only**. |
| **--debug** | Retain a debug bundle for this run. |
| **--debug-dir path** | Override the debug-bundle root. Requires **--debug**. |
| **--only lane-a,lane-b** | Run only the selected comma-separated artifact lanes when that selection is valid for the configured pipeline. |
| **--llm-profile id** | Highest-precedence configured profile for selected LLM-backed bindings and validators; it replaces binding and [pipeline](config.md#pipelines) defaults. |
| **--session-id id** | Supply a non-empty prompt session identifier to LLM-backed module calls. |
| **--reasoning-effort value** | Replace the selected PromptKit profile's reasoning effort for every LLM-backed call in this run. The value must be non-empty and the flag may be specified only once. |
| **--clear-reasoning-effort** | Clear reasoning effort inherited from the selected PromptKit profile for every LLM-backed call in this run. |
| **--reference selector=path** | Add or replace a file reference binding. Repeatable. |
| **--without-reference selector** | Remove a configured optional reference binding. Repeatable. |
On success, the command prints the completed pipeline ID, normalized output and
rejected output counts, and the output directory. A debug-enabled run also
prints `debug=<bundle-path>`. If the run completes with warnings, the warning
count is printed to stderr.
**--chunk_cache** accepts only **auto**, **bypass**, or **refresh**.
**--debug-dir**, **--output-dir**, **--session-id**, and
**--reasoning-effort**, and **--recompute-step** reject explicit empty values.
**--reasoning-effort** and **--clear-reasoning-effort** are mutually exclusive.
When neither is present, reasoning effort comes from the selected PromptKit
profile. These controls apply to the shared run client, including retries and
LLM-backed validators, and do not modify configuration or profile files.
Persistent reasoning settings remain a PromptKit profile concern.
**--recompute-step** requires **--resume**; checkpoint requirements and reuse
behavior are documented in [Operations](operations.md).
Reference flags are external file bindings resolved against selected chunk,
extractor, merger, and normalizer targets before the run starts. Generated
artifact bindings are configured in ordered steps and cannot be introduced by a
CLI path flag. Flat slot names are accepted only
when exactly one selected target declares that slot. For configured reference
bindings, precedence, path resolution, and validation, see
[Configuration](config.md#pipelines).
### Reference selectors
`--reference` binds or replaces one slot for one selected target. Selectors are:
Use **--reference** only for a reference slot declared by the selected
configured target. The accepted selector forms are:
- `slot=path`: valid when exactly one selected target declares `slot`;
- `chunk.slot=path`: target the chunker;
- `merge.slot=path`: valid when exactly one selected merger declares `slot`;
- `lane.slot=path`: valid when exactly one selected extractor, merger, or
normalizer in that lane declares `slot`;
- `lane.extract.slot=path`: target a lane extractor;
- `lane.merge.slot=path`: target a lane merger;
- `lane.normalize.slot=path`: target a lane normalizer.
| Form | Target |
| --- | --- |
| slot=path | The unique selected target that declares slot. |
| chunk.slot=path | The chunker. |
| merge.slot=path | The unique selected merger that declares slot. |
| lane.slot=path | The unique extractor, merger, or normalizer in lane that declares slot. |
| lane.extract.slot=path | The extractor in lane. |
| lane.merge.slot=path | The merger in lane. |
| lane.normalize.slot=path | The normalizer in lane. |
Use `slot=path` when the selected targets declare the slot unambiguously:
**--without-reference** uses the same selector forms without =path. Slot
names, requiredness, and configured bindings are part of the
[configuration contract](config.md).
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference roster=./campaign-roster.txt
```
### Run output
Use an explicit selector when multiple selected targets declare the same slot or
when you want to target a specific target:
Without **--json**, standard output contains the completed pipeline ID, counts
of normalized and rejected outputs, and the output directory. A debug-enabled
run also prints its debug-bundle path to standard output. A successful run with
warnings reports the warning count to standard error. The published JSON bundle
is defined by the [JSON output contract](integrations/json-output.md).
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference spells.extract.glossary=./campaign-glossary.txt
```
With **--json**, successful standard output is exactly one
`notarius.run-result.v1` JSON document followed by a newline, with no
human-oriented status or debug-path line. Its fields and compatibility policy
are defined by the [run-result contract](integrations/run-result.md). A caller
must check for exit status 0 before decoding this output; a failed write can
leave incomplete standard-output bytes that are not a result document.
For the maintained NPC-grounded workflow, use the explicit ordered pipeline.
The first step produces the normalized NPC artifact; the second step receives
it in memory and fans it out to spell extraction, combat extraction, and combat
normalization:
Example:
```sh
go run ./cmd/notarius run dnd-npc-grounded \
--config examples/dnd-npc-grounded.config.yml \
--input examples/seriatim-minimal-transcript.json \
--output-dir ./npc-grounded-output
```
~~~
OPENROUTER_API_KEY=your-api-key \
go run ./cmd/notarius run dnd-session \
--config examples/dnd-minimal.config.yml \
--input examples/seriatim-minimal-transcript.json
~~~
The generated NPC content remains contextual grounding, not spell or combat
evidence. It is represented in manifests and debug summaries by bounded
identity and producer provenance, not by payload content or a filesystem path.
## config validate
The same grammar can target chunk, merge, and normalize slots when the configured
modules declare them:
~~~
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
~~~
```sh
go run ./cmd/notarius run dnd-session \
--config path/to/config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference chunk.scene_guide=./campaign-scenes.txt \
--reference spells.merge.merge_notes=./merge-notes.txt \
--reference spells.normalize.normalization_notes=./normalization-notes.txt
```
This command loads and validates a configuration. With **--pipeline**, it also
resolves that pipeline against the production module catalog. **--only** selects
lanes during that resolution and requires **--pipeline**.
Use `--without-reference` to remove a configured optional binding for a run:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--without-reference glossary
```
Use `--session-id` when an external orchestrator needs all prompt calls from one
run to share an identifier:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--session-id campaign-17-session-04
```
When `cache.checkpoints.enabled` is `true`, runs record checkpoints whether or
not `--resume` is present. Add the resume flag to load and reuse compatible
recorded work; using it while checkpoint recording is disabled is an error:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--resume
```
To selectively rerun one ordered step and its dependent lanes, use the step ID
from the configuration. The selected step and dependents are reported as
`forced_recompute`; reusable predecessors are reported as `reused`:
```sh
go run ./cmd/notarius run dnd-npc-grounded \
--config examples/dnd-npc-grounded.config.yml \
--input examples/seriatim-minimal-transcript.json \
--resume --recompute-step grounded-events
```
Checkpoint decisions use these categories: `reused`, `executed`,
`forced_recompute`, and `dependency_invalidated`. The reason code and bounded
detail identify the decision without exposing reference content, local paths,
or secrets. `--recompute-step` requires checkpoint recording and `--resume`;
unknown step IDs, empty values, and combinations with `--only` are rejected.
The operator meanings of checkpoint reason codes are maintained in
[Operations](operations.md#resume-and-selective-recompute).
Use `--debug` to retain the redacted summary and trace bundle for one run. The
bundle is allocated before pipeline resolution; once allocated, its path is
also printed to stderr if the command fails. Debug-write failures cause exit
code `1`.
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--debug --debug-dir ./notarius-debug
```
Use `refresh` when intentionally replacing the cached plan for the same source:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--chunk_cache refresh
```
Use `bypass` for a one-off run that must not inspect or create plan-cache state:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--chunk_cache bypass
```
`--diagnostics-dir` has been removed. For checkpoint behavior, durable output,
debug-bundle lifecycle, and failure inspection, see [Operations](operations.md).
## `config validate`
`notarius config validate` loads and validates configuration.
Flags:
- `--config path`: config file path. If omitted, Notarius uses the discovery
rules in [Configuration](config.md#discovery).
- `--pipeline pipeline-id`: additionally resolve one configured pipeline against
the production module catalog.
- `--only lane-a,lane-b`: validate resolution for selected artifact lanes. This
flag requires `--pipeline`.
Success is written to standard output as either config "<path>" is valid or
config "<path>" is valid for pipeline "<pipeline-id>".
Examples:
```sh
~~~
go run ./cmd/notarius config validate \
--config examples/dnd-spells.config.yml
--config examples/dnd-minimal.config.yml \
--pipeline dnd-session
go run ./cmd/notarius config validate \
--config examples/dnd-spells.config.yml \
--pipeline dnd-session \
--only spells
```
OPENROUTER_API_KEY=validation-placeholder \
go run ./cmd/notarius config validate \
--config examples/dnd-complete.config.yml \
--pipeline dnd-session
~~~
## `pipelines list`
The placeholder in the second command is sufficient only for offline
validation; it cannot run a provider-backed pipeline.
`notarius pipelines list` prints configured pipeline IDs in sorted order.
## pipelines list
Flags:
~~~
notarius pipelines list [--config path/to/config.yml] [--json]
~~~
- `--config path`: config file path. If omitted, Notarius uses the discovery
rules in [Configuration](config.md#discovery).
- `--json`: print `{"pipelines":[...]}` instead of one ID per line.
This command lists configured pipeline IDs in sorted order. By default, it
writes one ID per line to standard output. **--json** writes an object shaped as
{"pipelines":[...]} instead.
Examples:
```sh
~~~
go run ./cmd/notarius pipelines list \
--config examples/dnd-spells.config.yml
--config examples/dnd-minimal.config.yml
~~~
go run ./cmd/notarius pipelines list \
--config examples/dnd-spells.config.yml \
--json
```
## Output Streams And Exit Statuses
## Exit Codes
Successful commands write their primary result to standard output. Warnings and
errors are written to standard error.
- `0`: command succeeded.
- `1`: command syntax was valid, but loading config, resolving modules, running
the pipeline, calling the provider, writing output, or writing a requested
debug bundle failed.
- `2`: command syntax was invalid, a command was unknown, a required argument
was missing, or a flag value was malformed.
For **run --json**, warnings remain on standard error and standard output is a
machine-readable success result only. Syntax and runtime diagnostics remain on
standard error. Parse the result only after the process exits with status 0.
For YAML structure, defaults, Scriptorium profile sources, environment
overrides, and selectable module and validator keys, see
[Configuration](config.md).
| Status | Meaning |
| --- | --- |
| 0 | The command completed successfully, including root help. |
| 1 | Command syntax was valid but configuration loading or validation, pipeline resolution or execution, provider use, output, or requested debug handling failed. |
| 2 | The command or flag syntax was invalid, including unknown commands, missing required arguments, invalid flag values, or invalid flag combinations. |
The root help spellings are the supported help path. Invoking **--help** on
**run**, **config validate**, or **pipelines list** is handled by the flag
parser as a usage error: it writes an error to standard error and exits with
status 2.

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,74 @@
# Using Notarius As A Subprocess
Use this workflow when an orchestrator runs Notarius and consumes its published
artifacts. The [CLI reference](../cli.md) owns invocation syntax and exit
statuses, while the [run-result receipt](../integrations/run-result.md) and
[Published JSON Output contract](../integrations/json-output.md) own the
durable result formats.
## Run And Check The Process
Optionally preflight a selected configuration and pipeline before work starts:
```sh
notarius config validate --config /path/to/notarius.yml --pipeline pipeline-id
```
Invoke the run with explicit paths and machine-readable output. Capture
standard output and standard error separately; do not combine them before
processing the result.
```sh
notarius run pipeline-id \
--config /path/to/notarius.yml \
--input /path/to/source.json \
--output-dir /path/to/output-root \
--json
```
Use absolute paths for supplied input, configuration, output-root, and
reference files. When a stable prompt session identifier or references are
needed, pass the supported CLI flags. Supply credentials through Notarius's
documented configuration and environment mechanisms, never as command-line
arguments or generated secret-bearing configuration.
Wait for the process before interpreting standard output. Only an exit status
of 0 permits decoding the receipt. On a nonzero exit, retain standard error for
diagnosis and ignore all standard-output bytes: a failed receipt write may have
left a partial document.
## Discover Required Artifacts
Decode the successful receipt and accept the schema versions supported by the
caller. Use its `output_directory` as the bundle root. For the production JSON
output, resolve `index_file` under that root with a confinement check and reject
an absolute path or a result that escapes the root.
Read the resulting `index.json` and locate each artifact by `lane_id`, not by a
guessed filename. Before decoding a selected payload, verify its descriptor's
media type and schema identity against the relevant published artifact
contract. The JSON bundle contract links to the available lane contracts.
If `index.json` has an `evidence_context` descriptor, treat it as a
pipeline-wide artifact rather than a lane entry. Verify its six descriptor
fields before decoding the linked file according to the [Published Evidence
Context contract](../integrations/evidence-context.md). Use each
`evidence_refs` entry as the citation to source material. Its surrounding
context range and included units explain the citation, but do not widen or
replace the cited source reference.
A zero exit status may still report rejected outputs, warnings, or absent
lanes. The caller decides which lane IDs are required for its own work and
which are optional; it should make that decision explicitly rather than infer
failure from the receipt counts alone.
## Preserve Provenance And Handle Data Carefully
Keep the receipt with the published `manifest.json`, and retain
`rejected.json` and `warnings.json` when review or later provenance requires
them. Treat the input, output bundle, cache, debug bundle, and captured process
logs as potentially sensitive data. Apply the caller's access controls and
retention policy, and avoid copying secrets into arguments, logs, or
provenance records. An evidence-context artifact contains source-unit text and
metadata, and selected lanes can cover most of an input; preserve and share it
only when that source content is authorized for the recipient.

View File

@@ -17,11 +17,13 @@ implemented component map.
| Application shape, package boundaries, contracts, dependency direction, runtime guarantees, or safety properties | [Architecture](policy/architecture.md) and relevant [ADRs](adr/) | Architecture defines the intended system and its invariants; ADRs preserve significant decision rationale. |
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
| Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. |
| CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. |
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate caller workflow, durable receipt contract, and CLI implementation behavior. |
| Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. |
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
| Production modules or validators | [Module Internals](internal/modules.md) | It documents implemented module contracts, capabilities, assets, and registration. |
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and Scriptorium integration. |
| Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. |
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
| CLI or user-visible configuration behavior | [CLI Reference](cli.md) and [Configuration](config.md) | These are the canonical user and operator references. |
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |

View File

@@ -1,83 +1,81 @@
# Accepted Chunk Map
This document defines the durable accepted chunk-map artifact that the JSON
output encoder can write as `chunk-map.json`. It describes the exact accepted,
materialized chunks used by a run; it is not a lane artifact and is never an
input to later pipeline steps. Enable it with the JSON output option described
in [Configuration](../config.md#json-output-options).
This document defines the optional durable `chunk-map.json` artifact in a
[published JSON bundle](json-output.md). It describes the accepted,
materialized chunk plan used by one run. It is not a lane payload and is never
an input to a later pipeline step.
## Identity
## Contract Identity
- Artifact kind: `source/chunk-map`
- Logical file: `chunk-map.json`
- Schema ID: `notarius.source.chunk_map`
- Schema name: `notarius_source_chunk_map_v1`
- Schema version: `v1`
- Media type: `application/json`
| Property | Value |
| --- | --- |
| Artifact kind | `source/chunk-map` |
| Logical file | `chunk-map.json` |
| Media type | `application/json` |
| Schema ID | `notarius.source.chunk_map` |
| Schema name | `notarius_source_chunk_map_v1` |
| Schema version | `v1` |
The checked-in [JSON Schema](../../internal/framework/chunkmap/assets/schemas/source_chunk_map.v1.json)
defines the strict wire shape. `chunk-map.json` is listed by the optional
`chunk_map` descriptor in [the JSON output index](json-output.md#indexjson),
not by the lane-oriented `output_files` collection.
The optional `chunk_map` descriptor in `index.json` identifies this artifact.
Export is controlled by the JSON output binding described in
[Configuration](../config.md#module-bindings-and-validators).
## Payload
## Wire Shape
The payload has these required fields:
Every payload has these required fields:
- `source_id`: accepted source-document identity.
- `source_digest`: canonical lower-case `sha256:` digest of that document.
- `plan_digest`: canonical lower-case `sha256:` digest of the accepted logical
plan.
- `requested_chunker`: chunk module selected by the current resolved pipeline.
- `producer`: the original accepted-plan producer, with required
`input_module` and `chunk_module`; `llm_profile` is present only for an
LLM-backed producer.
- `plan_annotations`: accepted plan-level annotation namespace map. It is
`{}` when no namespaces are present.
- `chunks`: non-empty execution-order collection of accepted chunks.
| Field | Meaning |
| --- | --- |
| `source_id` | Accepted source-document identity. |
| `source_digest` | Lower-case `sha256:` digest of that source document. |
| `plan_digest` | Lower-case `sha256:` digest of the logical chunk plan. |
| `requested_chunker` | Chunk module selected by the resolved pipeline. |
| `producer` | Original accepted-plan producer. `input_module` and `chunk_module` are required; `llm_profile` is optional. |
| `plan_annotations` | Plan-level annotation namespace map; `{}` when none are present. |
| `chunks` | Non-empty execution-order chunk collection. |
Each chunk has `id`, zero-based `index`, `source_ref`, positive `unit_count`,
and an explicit `annotations` namespace map. A source reference has the source
ID and inclusive positive `start_unit_id` and `end_unit_id` endpoints.
Annotation values are arbitrary valid JSON under non-empty namespaces. They
are preserved as canonical JSON without interpreting any module-specific
namespace.
Each `chunks` entry contains non-empty `id`, zero-based `index`, `source_ref`,
positive `unit_count`, and an explicit `annotations` map. `source_ref` contains
the same `source_id` as the top-level value plus positive inclusive
`start_unit_id` and `end_unit_id` values. Endpoints identify source units; their
numeric values do not by themselves establish source-document order.
## Invariants
Annotation namespaces are non-empty trimmed strings. Their values are arbitrary
valid JSON and are retained without interpreting a module-specific namespace.
The framework constructs this artifact only after materializing the selected
logical plan and accepting it through the configured chunk validator chain.
Construction proves the source and plan digests, source-document range order,
materialized chunk IDs and indexes, source references, unit membership and
counts, and plan/range annotations agree exactly. Chunk IDs are unique and
indexes are contiguous and agree with array order.
## Ordering And Validation
The codec rejects unknown fixed-object fields, malformed identities or
digests, invalid annotation JSON, trailing JSON content, and any payload whose
reconstructed logical plan does not reproduce `plan_digest`. It makes
defensive copies at serialization and decoding boundaries.
`chunks` are in execution order. Their indexes are contiguous, start at zero,
and equal their array positions; chunk IDs are unique. The emitted map is built
only after the selected plan has been accepted and materialized against the
source document, so its ranges, unit counts, annotations, and digests describe
that exact plan.
## Acceptance And Provenance
The codec rejects malformed JSON, trailing content, unknown fixed-object
fields, invalid identities or digests, invalid annotations, duplicate chunk
IDs, non-contiguous indexes, and a `plan_digest` that does not match the
reconstructed logical plan. The checked-in
[schema](../../internal/framework/chunkmap/assets/schemas/source_chunk_map.v1.json)
defines the strict JSON shape.
The artifact is available only when the chunk plan was accepted. It remains
available when a later extraction, merge, or normalization result is rejected;
it is absent when chunk validation rejects the candidate plan.
## Valid Example
`requested_chunker` describes the current pipeline selection. `producer`
describes who originally produced the accepted plan. On a cache hit these can
differ: the accepted ranges, annotations, digests, and stable materialized IDs
are reused, while the producer remains the stored producer. Cache paths,
actions, references, metadata, warnings, timestamps, and detailed provenance
remain in the run manifest rather than this payload.
The compact
[source chunk-map fixture](../../internal/framework/chunkmap/testdata/source_chunk_map.v1.json)
is decoded by the production codec and demonstrates an accepted map with
annotations, producer identity, and ordered chunks.
## Data Handling
## Publication And Compatibility
The map contains structure, not source content. It excludes transcript bytes,
materialized units, source-unit metadata, chunk content, private model
responses, rejected proposals, debug data, external-reference content, and
filesystem paths.
The map is present only when a chunk plan was accepted and its export is
enabled. It remains publishable if a later lane is rejected, but is absent when
chunk-plan validation rejects the plan. `requested_chunker` identifies the
current pipeline selection, while `producer` identifies the component that
originally produced the accepted plan; they may differ when an accepted plan is
reused.
Annotations can nevertheless be source- or model-derived. Treat an enabled
`chunk-map.json` with the same sensitivity and retention expectations as lane
output. Physical placement, confined atomic writing, and permissions follow
the ordinary [output operation](../operations.md#output).
The map contains structure rather than source content: it excludes transcript
bytes, source-unit metadata, chunk text, private model output, reference
content, debug data, and filesystem paths. Treat the exported map with the
same care as other published output. Publication location and retention are
defined in [Operations](../operations.md#output-bundles).

View File

@@ -1,10 +1,10 @@
# D&D Combat-Turn Artifact Contract
# D&D Combat-Turn Artifact
This document defines the durable artifact, serialization, extraction,
candidate-validation, normalization, and production lane boundaries for D&D
combat turns.
This contract defines the durable combat-action occurrence list produced by
`dnd/combat-turns`. It records source-grounded turns and actions; it is not a
complete initiative tracker, combat summary, or state model.
## Artifact identity
## Identity and compatibility
| Property | Value |
| --- | --- |
@@ -14,150 +14,56 @@ combat turns.
| Schema version | `v1` |
| Media type | `application/json` |
The top-level JSON object contains the required `combat_turns` array, which
may be empty. Every object rejects unknown fields.
`v1` is a strict JSON object with required `combat_turns`; the array may be
empty. Turn and source-reference objects reject unknown fields. An incompatible
shape change requires a new schema version.
## JSON shape
## Wire shape
Each combat turn contains these required fields:
Each combat turn has these required fields:
| Field | Shape |
| Field | Contract |
| --- | --- |
| `actor` | Non-empty string. |
| `turn_kind` | One of `turn`, `reaction`, `legendary_action`, `lair_action`, or `other`. |
| `source_refs` | Required array with at least one source reference. |
| `actor` | Non-empty acting character or creature name. |
| `turn_kind` | `turn`, `reaction`, `legendary_action`, `lair_action`, or `other`. |
| `source_refs` | One or more transcript evidence ranges. |
Source references use the shared source-reference shape:
Each source reference has exactly `source_id`, `start_unit_id`, and
`end_unit_id`. It identifies an inclusive current-transcript range; unit IDs
are positive and the start may not follow the end.
```json
{
"source_id": "session-alpha",
"start_unit_id": 1,
"end_unit_id": 2
"combat_turns": [
{
"actor": "Mira Thorn",
"turn_kind": "turn",
"source_refs": [
{"source_id": "session-7", "start_unit_id": 31, "end_unit_id": 32}
]
}
]
}
```
`source_id` must be non-empty and both unit IDs must be positive integers. The
codec does not resolve references against a source document or enforce source
range ordering; those checks belong to the later source-reference validation
boundary.
## Eligibility, evidence, and normalized form
## Codec behavior
The extractor requires an approved [scene-description artifact](dnd-scene-description-artifacts.md).
It emits combat turns only for a chunk with an exact matching scene classified
`combat`; an exact non-combat scene produces an accepted empty list. The scene
record controls eligibility only: its title, summary, and reference do not
become turn evidence. No exact matching scene also produces an empty list and
the `scene_classification_unavailable` warning.
The codec exposes two representations of the same typed artifact:
An optional normalized [NPC artifact](dnd-npc-artifacts.md) can ground an
actor name. Its registry references are provenance, never combat evidence.
Normalization trims and, where possible, canonicalizes actor names; orders and
deduplicates exact source references; orders valid-evidence turns by source
chronology; and collapses only duplicates with the same actor identity, turn
kind, and complete valid evidence. It does not infer turns, initiative, or
actions from registry or scene data.
- Candidate encode/decode preserves invalid actor and turn-kind values,
collection presence, and source references so later validators can report
them. Candidate decoding still requires valid JSON, one JSON value, known
fields, and compatible JSON types.
- Approved encode/decode enforces the structural rules in this contract.
The codec owns the durable JSON Schema, whose object layers all set
`additionalProperties` to `false`. Codec metadata contains only
`combat_turn_count`.
The maintained compact fixture is
`internal/modules/dnd/codec/combatturns/testdata/dnd_combat_turns.v1.json`.
## Extraction boundary
The standalone extractor uses these identities:
| Property | Value |
| --- | --- |
| Extractor key | `dnd/combat-turns` |
| Capability | `dnd.combat_turns` |
| Prompt ID | `dnd.combat_turns` |
| Prompt version | `v1` |
| Private response-schema key | `dnd_combat_turns_llm` |
| Private response-schema ID | `notarius.dnd.combat_turns.llm` |
| Default profile | `gemini-2-flash` |
It requires `chunks` and `source.transcript`, accepts no options, and makes one
structured completion for each supplied chunk. The prompt receives the
chunk-scoped transcript plus the existing `players`, `party`, and `glossary`
inputs, and optionally the deprecated `roster` reference through the shared
party mapping. The optional `npcs` reference is an approved normalized NPC
artifact used only for identity grounding; it never supplies combat evidence.
An external file is validated during preparation. In an ordered pipeline, the
same slot may receive the producer's canonical generated artifact at the step
handoff.
The private response envelope has the same turn fields and JSON types as the
durable shape except that source references contain only `start_unit_id`
and `end_unit_id`. It enforces required field presence, types, and
unknown-field rejection, while deterministic validators own enum membership,
non-empty values and collections, and positive-number requirements. The
extractor assigns the current source ID, removes exact duplicate ranges, and
stable-sorts turns by the earliest valid source-document position. Numeric unit
IDs are identifiers; source-document slice position determines chronology.
Semantically malformed candidate fields remain in the typed result for the
configured validation and retry boundary.
## Deterministic candidate validation
The standalone validator keys are:
| Validator | Responsibility |
| --- | --- |
| `extract/dnd/combat-turns/shape` | Required list, actor, turn kind, and source references, plus supported turn-kind values. |
| `extract/dnd/combat-turns/source_refs` | Source identity, source-unit existence, and range order through the source document. |
| `extract/dnd/combat-turns/source_relatedness` | At most one advisory warning per turn when the actor is not related to cited transcript text. |
Source-reference and relatedness validators defer malformed shape to the shape
validator. Relatedness also defers when any cited source range is invalid. It
combines overlapping cited ranges once in document order and compares actors
with the shared Unicode-aware NPC identity policy.
The production D&D registrar exposes the extractor and these validators. Its
default extraction chain preserves this order: JSON syntax, combat shape,
source references, private response schema, then source relatedness.
## Normalization boundary
The standalone normalizer uses key `dnd/combat-turns`, requires `merged`,
provides `normalized`, accepts no options, and accepts only the optional
structured `npcs` reference. Campaign references are LLM extraction context and
are not normalizer inputs. For an external file, the NPC registry is resolved
during preparation; for a generated binding, it is resolved at the operation-
time handoff. Runtime normalization uses that immutable prepared or handed-off
view.
Normalization policy is `dnd.combat_turns.normalize.v1`. It display-normalizes
the actor, canonicalizes exact registry actor matches, orders and deduplicates
exact source references, stable-sorts records by earliest valid source-document
position, and collapses only records with the same actor identity, turn kind,
and complete valid evidence set. The first normalized record is retained.
Invalid evidence is never eligible for duplicate collapse. Every mutation and
collapse emits a bounded warning using the merged input index in its scope.
The normalizer reports `normalization_policy` and `identity_policy` metadata
and fingerprints. An external registry may additionally contribute
`npc_registry_digest` and `npc_count`; generated registry identity is retained
in framework handoff provenance and dependency fingerprints. The
normalized-invariants validator is
`normalize/dnd/combat-turns/invariants`; it defers shape and source-reference
failures, then checks actor display normalization, canonical evidence ordering,
chronology, and duplicate identity. It rejects
with `invalid_combat_turn_normalization` under policy
`dnd.combat_turns.validator.normalized.v1`.
The production D&D registrar exposes the normalizer and normalized-invariants
validator. Its default normalization chain is JSON syntax, combat shape,
normalized invariants, source references, durable schema, then source
relatedness. The lane uses the framework's typed append-order merger and has no
merge validator chain.
## Production manifest and references
The selectable lane uses extractor and normalizer key `dnd/combat-turns`,
`appendorder` for the typed merger, and the durable codec above. A bound `npcs`
reference contributes raw-file provenance to the run manifest. A generated
binding contributes artifact kind, schema identity, media type, canonical
digest, size, and bounded producer provenance. Consumer metadata and checkpoint
fingerprints contain no registry names, content, paths, or NPC source ranges.
The component-local registry fingerprint covers only the names projected to the
consumer, while manifest provenance retains the full artifact digest. The
normalized lane is emitted as `lanes/<lane-id>.json` by the JSON
output module, and warnings and rejection summaries remain in their shared
companion files.
The [NPC-interaction artifact](dnd-npc-interaction-artifacts.md) records
broader NPC occurrences. The [JSON output contract](json-output.md) defines
publication, and [D&D module internals](../internal/dnd.md) describes routing
and validation mechanics.

View File

@@ -0,0 +1,78 @@
# D&D Item-Event Artifact
This contract defines the durable item and currency occurrence list produced by
`dnd/item-events`. It records source-grounded discoveries and possession
changes; it does not maintain an inventory, balance, or ledger.
## Identity and compatibility
| Property | Value |
| --- | --- |
| Artifact kind | `dnd/item-event-list` |
| Schema ID | `notarius.dnd.item_events` |
| Schema name | `notarius_dnd_item_events_v1` |
| Schema version | `v1` |
| Media type | `application/json` |
`v1` is a strict JSON object with required `events`; the array may be empty.
Event and source-reference objects reject unknown fields. An incompatible
shape change requires a new schema version.
## Wire shape
Every event has required `name`, `kind`, and `source_refs`. `quantity`, `from`,
and `to` are optional where the event kind permits them.
| Field | Contract |
| --- | --- |
| `name` | Non-empty item or currency display name. |
| `kind` | `discovered`, `acquired`, `lost`, `consumed`, or `transferred`. |
| `quantity` | Optional positive integer; omit it when no count is established. |
| `from` | Optional non-empty losing holder, when allowed by `kind`. |
| `to` | Optional non-empty gaining holder, when allowed by `kind`. |
| `source_refs` | One or more transcript evidence ranges. |
Each source reference has exactly `source_id`, `start_unit_id`, and
`end_unit_id`. It identifies an inclusive current-transcript range; unit IDs
are positive and the start may not follow the end.
```json
{
"events": [
{
"name": "Silver Pieces",
"kind": "acquired",
"quantity": 20,
"to": "party",
"source_refs": [
{"source_id": "session-7", "start_unit_id": 2, "end_unit_id": 2}
]
}
]
}
```
## Holder rules and minimal extraction
`discovered` has neither holder; `acquired` requires `to` and forbids `from`;
`lost` and `consumed` require `from` and forbid `to`; `transferred` requires
both holders. `party` denotes collective possession. A transfer cannot use
`party` for either holder and its two normalized holders must differ.
Only an evidenced discovery or possession change belongs in this artifact.
It does not infer quantities or holders, convert currency denominations,
calculate balances, or merge nearby events. Campaign references may
disambiguate names but are never event evidence. Currency uses the ordinary
`name` field and an explicit `quantity` only when the transcript establishes
one; each denomination remains a separate event.
Normalization trims display whitespace, orders and removes exact duplicate
source references, then orders events by valid source chronology, name identity
and display value, kind, holders, quantity, and reference sequence. It
collapses only entries with the same normalized durable fields and complete
valid evidence.
The [JSON output contract](json-output.md) defines publication. See
[D&D module internals](../internal/dnd.md) for implementation details and the
[NPC-interaction artifact](dnd-npc-interaction-artifacts.md) for a distinct
kind of occurrence.

View File

@@ -1,143 +1,69 @@
# D&D NPC Artifact
This document defines the durable D&D NPC-list artifact, its JSON codec, and
the selectable production NPC pipeline. The normalized JSON payload can be
passed explicitly to the spell extractor as an optional caster-name registry
or to the combat extractor and normalizer as an actor registry. It
remains a reference, not spell or combat evidence.
The NPC interaction extractor and normalizer also consume this registry for
canonical identity; registry source references remain provenance and never
become interaction evidence. Their occurrence contract is defined in the
[D&D NPC interaction artifact](dnd-npc-interaction-artifacts.md).
This contract defines the durable NPC registry produced by `dnd/npcs`. It is a
minimal, source-grounded identity registry for other D&D artifacts, not a
character sheet or a relationship summary.
## Identity
## Identity and compatibility
- Artifact kind: `dnd/npc-list`
- Durable schema ID: `notarius.dnd.npcs`
- Durable schema name: `notarius_dnd_npcs_v1`
- Durable schema version: `v1`
- Media type: `application/json`
- Identity policy: `dnd.npcs.identity.v1`
| Property | Value |
| --- | --- |
| Artifact kind | `dnd/npc-list` |
| Schema ID | `notarius.dnd.npcs` |
| Schema name | `notarius_dnd_npcs_v1` |
| Schema version | `v1` |
| Media type | `application/json` |
| Identity policy | `dnd.npcs.identity.v1` |
The durable JSON Schema is owned by the D&D NPC codec. NPC IDs are derived from
the Unicode-normalized, case-folded canonical name using the identity policy.
The durable codec enforces the artifact shape and ID syntax; registry identity
validation remains a separate deterministic concern.
`v1` accepts one strict JSON object with required `npcs`; the array may be
empty. NPC and source-reference objects reject unknown fields. An incompatible
artifact shape or identity-policy change uses a new version or policy.
The extractor's private LLM response schema is a separate structural transport
contract. It omits framework-assigned NPC and source IDs and admits semantic
candidates for the deterministic shape and source-reference validators; it is
not part of this durable contract.
## Wire shape and identity
## Output Shape
Each NPC has these required fields:
The payload is one object with a required top-level `npcs` array:
| Field | Contract |
| --- | --- |
| `id` | `npc:sha256:` followed by 64 lowercase hexadecimal characters. |
| `name` | Non-empty canonical display name. |
| `source_refs` | One or more transcript evidence ranges for the identity. |
A source reference has exactly `source_id`, `start_unit_id`, and `end_unit_id`.
The source ID identifies the transcript, unit IDs are positive inclusive unit
identifiers, and the start may not follow the end.
```json
{"npcs": []}
{
"npcs": [
{
"id": "npc:sha256:99a16589618a04f535a7d21fdcc71a0b1c05d22f752cd492065b1086d97bc3d7",
"name": "Mira Thorn",
"source_refs": [
{"source_id": "session-7", "start_unit_id": 4, "end_unit_id": 5}
]
}
]
}
```
The array may be empty. Every object and nested object rejects unknown fields.
The ID is deterministic: normalize the name to Unicode NFKC, normalize the
supported apostrophe forms, collapse whitespace, case-fold it, SHA-256 the
result, then prefix the lowercase hexadecimal digest with `npc:sha256:`. Each
canonical identity and ID appears at most once. Normalization collapses records
with the same canonical identity, retains their earliest position, and merges
their canonicalized evidence; it does not add aliases, roles, descriptions, or
relationship fields.
## NPC Fields
## Scope and consumers
Each NPC contains exactly these required fields:
Only individually identifiable NPC names with transcript evidence belong in
this artifact. Groups, generic roles, invented labels, and descriptive
enrichment are excluded. Its source references prove registry provenance; they
do not become evidence for a spell, interaction, or combat occurrence.
- `id`: `npc:sha256:` followed by 64 lowercase hexadecimal characters;
- `name`: the canonical display name;
- `source_refs`: at least one source reference supporting the NPC record.
Each source reference contains required `source_id`, `start_unit_id`, and
`end_unit_id`; unit IDs are positive integers. Source document identity, unit
existence, and range ordering are validated by the source-reference validator
when the artifact is used by a pipeline.
## Codec Boundary
`EncodeCandidate` and `DecodeCandidate` provide strict single-value JSON
serialization while preserving typed values that still need semantic
validation. `Encode` and `Decode` are the approved-artifact boundary and
require all durable structural fields, non-empty required strings, valid source
reference shapes, and the NPC ID pattern.
Codec metadata contains only `npc_count`. Schema bytes and returned metadata
are independent values so callers cannot mutate codec-owned state.
## Production Pipeline
The production identities are:
- extractor: `dnd/npcs`;
- artifact kind: `dnd/npc-list`;
- normalizer: `dnd/npcs`; and
- durable schema: `notarius.dnd.npcs`, version `v1`, media type
`application/json`.
The extractor maps private model records to the current source identity and
assigns deterministic IDs. Extraction validation checks shape, source
references, and source relatedness. The normalizer then consolidates records
only when their normalized canonical names match, preserves the first record's
display and output position, unions exact evidence, and validates the retained
registry's identity. No LLM is used for consolidation.
The extraction prompt asks only for individually identifiable NPC names backed
by source evidence. Groups, generic roles, invented labels, and descriptive or
relationship enrichment are outside the contract.
The default extraction chain is `generic/valid_json`,
`extract/dnd/npcs/shape`, `extract/dnd/npcs/source_refs`,
`generic/valid_json_schema`, and `extract/dnd/npcs/source_relatedness`. The
default normalize chain is `generic/valid_json`, `extract/dnd/npcs/shape`,
`normalize/dnd/npcs/identity`, `extract/dnd/npcs/source_refs`,
`generic/valid_json_schema`, and `extract/dnd/npcs/source_relatedness`.
Relatedness emits bounded warnings when an NPC canonical name is not present
near its cited transcript text; opaque campaign
references may explain such a warning but do not become evidence.
## Manifest And Artifact Handoff
The NPC extractor records prompt and response-schema identities. The durable
codec records only `npc_count`; raw names, source references, and payload bytes
stay in the lane file rather than manifest
metadata. The normalized lane can be consumed by a later ordered step through
the registered canonical codec:
```yaml
steps:
- id: identify-npcs
artifacts:
npcs:
extract: dnd/npcs
normalize: dnd/npcs
- id: grounded-events
references:
npcs:
artifact:
step: identify-npcs
lane: npcs
artifacts:
spells:
extract: dnd/spells
normalize: dnd/spells
combat:
extract: dnd/combat-turns
normalize: dnd/combat-turns
```
The framework hands only an accepted normalized artifact across the barrier. It
validates the canonical bytes against each consumer slot and clones the
operation-time reference for the spell and combat consumers. Generated
provenance records the artifact kind, schema identity, media type, canonical
digest, size, and producer step/lane/module, but not names, source
ranges, or payload bytes. External normalized files remain supported as
explicit references and retain their file provenance.
NPC source references are registry provenance and are never accepted as spell
or combat evidence. Current transcript units remain the only event evidence.
Consumers receive a separate names-only projection in normalized registry
order, for example `{"npcs":[{"name":"Mira Thorn"}]}`. The projection omits
IDs and evidence. Its digest covers the exact projected bytes and is used for
consumer-local checkpoint identity, while the full durable artifact digest
remains the manifest and generated-reference provenance identity. The unbound
projection is exactly `{"npcs":[]}` and also has a projection digest.
This registry can ground actor or caster names in the [spell](dnd-spell-artifacts.md)
and [combat-turn](dnd-combat-turn-artifacts.md) artifacts. It is required to
resolve the canonical `name` in an [NPC interaction](dnd-npc-interaction-artifacts.md).
The [JSON output contract](json-output.md) defines publication, and
[D&D module internals](../internal/dnd.md) owns pipeline mechanics.

View File

@@ -1,20 +1,38 @@
# D&D NPC Interaction Artifact
This document defines the durable D&D NPC-interaction-list artifact and its
two-step production pipeline. It records discrete, source-grounded occurrences
for NPCs already accepted into a normalized NPC registry; it does not expand
the registry or summarize events.
This contract defines the durable occurrence list produced by
`dnd/npc-interactions`. It records discrete, source-grounded interactions with
NPCs already present in a normalized registry; it does not extend that registry
or summarize the session.
## Identity And JSON
## Identity and compatibility
- Artifact kind: `dnd/npc-interaction-list`
- Durable schema ID: `notarius.dnd.npc_interactions`
- Durable schema name: `notarius_dnd_npc_interactions_v1`
- Durable schema version: `v1`
- Media type: `application/json`
| Property | Value |
| --- | --- |
| Artifact kind | `dnd/npc-interaction-list` |
| Schema ID | `notarius.dnd.npc_interactions` |
| Schema name | `notarius_dnd_npc_interactions_v1` |
| Schema version | `v1` |
| Media type | `application/json` |
The payload is one strict JSON object with only an `interactions` array. The
array may be empty. Each item has exactly `name`, `kind`, and `source_refs`:
`v1` is a strict JSON object with required `interactions`; the array may be
empty. Interaction and source-reference objects reject unknown fields. An
incompatible shape change requires a new schema version.
## Wire shape
Each interaction has these required fields:
| Field | Contract |
| --- | --- |
| `name` | Non-empty canonical display name from the required NPC registry. |
| `kind` | One of the interaction categories below. |
| `source_refs` | One or more transcript evidence ranges. |
Each source reference has exactly `source_id`, `start_unit_id`, and
`end_unit_id`. It identifies an inclusive range in the current transcript;
unit IDs are positive and the start may not follow the end. Extraction evidence
for an interaction is confined to its accepted chunk.
```json
{
@@ -30,119 +48,31 @@ array may be empty. Each item has exactly `name`, `kind`, and `source_refs`:
}
```
`name` is the canonical display name from the required NPC registry.
`source_refs` contains one or more current-source ranges with required
`source_id`, `start_unit_id`, and `end_unit_id`; unit IDs are positive integers.
During extraction, every range must be wholly contained in the current accepted
chunk. This prevents a candidate from citing valid units that were not presented
to that extraction call.
Unknown fields are rejected.
## Interaction Categories
`kind` is exactly one of:
## Interaction categories
| Kind | Meaning |
| --- | --- |
| `mentioned` | The NPC is referred to, but is not established as present or communicating in the evidenced passage. |
| `noncombat_presence` | The NPC is present and relevant to the passage but does not meaningfully participate in dialogue or combat. |
| `dialogue` | The NPC speaks, responds, or is directly engaged in a meaningful non-combat exchange. |
| `mentioned` | The NPC is referred to but is not established as present or communicating. |
| `noncombat_presence` | The NPC is present and relevant without meaningful dialogue or combat participation. |
| `dialogue` | The NPC speaks, responds, or meaningfully participates in a non-combat exchange. |
| `combat_ally` | The NPC actively participates in combat on the party's side. |
| `combat_opponent` | The NPC actively participates in combat against the party. |
| `other` | The transcript clearly establishes a direct NPC occurrence that fits none of the preceding kinds. |
| `other` | A clearly evidenced direct occurrence not covered by another category. |
`other` is a residual category for positively evidenced activity, not a fallback
for uncertain classification. When activities overlap, active combat
participation outranks dialogue, presence, and mention; dialogue outranks
non-combat presence and mention; and non-combat presence outranks mention.
Combat alignment is not resolved by precedence: a meaningful change between
ally and opponent creates separate occurrences.
The categories do not represent motives, relationships, state, or events that
the cited transcript does not establish. An `other` entry is not a substitute
for uncertain classification.
These categories do not encode summaries, relationships, state, motives, or
unobserved events.
## Identity, evidence, and order
## Occurrence Boundaries And Ordering
The required normalized [NPC artifact](dnd-npc-artifacts.md) resolves `name`.
Registry references are provenance only and never replace an interaction's own
evidence. Normalization canonicalizes recognized registry names, orders and
deduplicates exact source references, then orders interactions by valid source
chronology, NPC comparison identity, display name, kind, and reference sequence.
Only entries with the same canonical name, kind, and complete valid evidence
sequence are collapsed; distinct categories or evidence remain separate.
One occurrence represents one NPC, one kind, and one locally coherent passage
within one accepted chunk. Repeated evidence belongs to the same occurrence
only while it supports the same uninterrupted activity. A kind change, combat
alignment change, intervening scene or meaningful absence, or transition from
mention to presence starts a new occurrence. Occurrences never span chunks, and
merge or normalization never semantically combines nearby, overlapping, or
cross-chunk records.
Normalization orders records by:
1. earliest valid source-document position;
2. the NPC identity comparison key;
3. the exact canonical NPC display name;
4. interaction kind in lexical order; and
5. the complete canonical source-reference sequence, ordered by source ID and
the source-document positions of each range's start and end.
Only records with identical canonical names, kinds, and complete valid evidence
sequences are duplicates. Different categories, ranges, or separately grounded
occurrences remain separate.
## Evidence, Registry, And Normalization
The registry proves only the canonical NPC identity. Its source references are
registry provenance and are never interaction evidence. Every durable
interaction must cite current transcript units supporting both the name and its
classification.
The extractor receives a names-only registry projection such as
`{"npcs":[{"name":"Mira Thorn"}]}`. The normalizer uses the full immutable
registry for exact canonical-name lookup. It canonicalizes source references
and applies the ordering and exact-duplicate rules above.
## Production Pipeline
The extractor and normalizer key is `dnd/npc-interactions`. Both require the
structured `npcs` slot, so an accepted normalized registry must come from an
earlier step:
```yaml
pipelines:
dnd-npc-interactions:
input: seriatim
steps:
- id: identify-npcs
artifacts:
npcs:
extract: dnd/npcs
normalize: dnd/npcs
- id: extract-interactions
references:
npcs:
artifact:
step: identify-npcs
lane: npcs
artifacts:
interactions:
extract: dnd/npc-interactions
normalize: dnd/npc-interactions
```
The framework passes only the accepted normalized producer. A missing, rejected,
or incompatible NPC artifact prevents the consumer step from executing. It
records generated artifact identity and bounded producer provenance without
copying registry names, source ranges, or payload content into the manifest.
## Validation And Metadata
The default extract chain is `generic/valid_json`, interaction shape, registry,
and source-reference validation, `generic/valid_json_schema`, then warning-only
source relatedness. The normalize chain runs normalized invariants after
registry validation and before source-reference and schema validation, followed
by relatedness. Normalizer and relatedness warnings are bounded and end with an
omission summary when necessary. The codec metadata contains only
`interaction_count`. Extractor metadata identifies its prompt and private
response schema; component-local checkpoint identities include the names-only
registry projection where relevant. Generated registry identity stays in
framework provenance and dependency fingerprints.
See [Configuration](../config.md#implemented-production-modules) for selectable
keys and chains, [the NPC artifact contract](dnd-npc-artifacts.md) for the
registry boundary, and the copyable
[maintained example](../../examples/dnd-npc-interactions.config.yml).
See the [combat-turn artifact](dnd-combat-turn-artifacts.md) for combat-action
occurrences and the [JSON output contract](json-output.md) for publication.
Pipeline mechanics are described in [D&D module internals](../internal/dnd.md).

View File

@@ -1,9 +1,10 @@
# D&D Scene Description Artifact Contract
# D&D Scene-Description Artifact
This document defines the durable `dnd/scene-description-list` artifact
emitted by the D&D scene-description lane.
This contract defines the durable output of `dnd/scene-descriptions`. Each
record classifies one accepted transcript chunk and gives it a minimal
source-grounded title and summary.
## Artifact identity
## Identity and compatibility
| Property | Value |
| --- | --- |
@@ -13,38 +14,32 @@ emitted by the D&D scene-description lane.
| Schema version | `v1` |
| Media type | `application/json` |
The normalized payload is written by the JSON output module to
`lanes/<lane-id>.json`. See [JSON output](json-output.md) for the surrounding
output bundle.
`v1` is a strict JSON object with required non-empty `scenes`. Scene and
source-reference objects reject unknown fields. An incompatible shape change
requires a new schema version.
## JSON shape
## Wire shape
The payload is a JSON object containing exactly one required field, `scenes`.
Each scene object contains exactly these required fields:
Each scene has exactly these required fields:
| Field | Shape and ownership |
| Field | Contract |
| --- | --- |
| `id` | Non-empty accepted chunk ID, assigned by Notarius. |
| `source_ref` | Exact inclusive accepted chunk range, assigned by Notarius. |
| `kind` | One of `combat`, `narrative`, `recap`, or `meta`. |
| `source_ref` | The assigned inclusive source range for that chunk. |
| `kind` | `combat`, `narrative`, `recap`, or `meta`. |
| `title` | Non-empty, trimmed, source-grounded title. |
| `summary` | Non-empty, trimmed, source-grounded summary. |
All object layers reject unknown fields. The `scenes` array must be present and
non-empty. `source_ref` has exactly `source_id`, `start_unit_id`, and
`end_unit_id`; its source ID is non-empty and its unit IDs are positive
integers.
`source_ref` has exactly `source_id`, `start_unit_id`, and `end_unit_id`.
Its source ID identifies the input transcript; its positive unit IDs identify
the chunk's inclusive range, with the start no later than the end.
```json
{
"scenes": [
{
"id": "chunk-000001",
"source_ref": {
"source_id": "session-alpha",
"start_unit_id": 1,
"end_unit_id": 3
},
"source_ref": {"source_id": "session-7", "start_unit_id": 1, "end_unit_id": 3},
"kind": "narrative",
"title": "Arrival at the watchtower",
"summary": "The party reaches the ruined watchtower and begins to investigate it."
@@ -53,68 +48,22 @@ integers.
}
```
`id` and `source_ref` are application-owned identity and evidence. The model
provides only `kind`, `title`, and `summary`; it is not asked for chunk IDs,
source IDs, unit IDs, ranges, participants, or confidence.
## Meaning and normalized form
## Scene kinds
`combat` identifies a chunk where active combat is the central activity.
`narrative` is current in-world play that is not principally combat, recap, or
meta discussion. `recap` is primarily a recounting of an earlier session, and
`meta` is primarily out-of-character discussion. The artifact does not add
participants, confidence, events, or information absent from the chunk.
| Kind | Meaning |
| --- | --- |
| `combat` | Active combat is a substantive central activity. |
| `narrative` | Current-session in-world play that is not principally combat, recap, or meta discussion. |
| `recap` | The table is primarily recounting a previous session. |
| `meta` | Sustained out-of-character discussion is the scene's primary purpose. |
Normalization trims title and summary, orders scenes by source position and
then ID, and removes exact duplicate records. A reused ID with different
durable fields, or the same source range with different kind, title, or
summary, is invalid. It does not merge adjacent ranges, alter prose, or infer
missing scenes.
For a mixed accepted chunk, classification prefers `combat`, then `recap`,
then `meta`, then `narrative`. Brief table talk, rules clarification, or a
short recollection does not replace the main current-session activity.
## Extraction and evidence
The extractor runs once for each accepted chunk and maps one successful model
response to one scene record. It copies the current chunk ID and exact chunk
range, preserves the model kind without repair, and trims only surrounding
whitespace from title and summary. A model response cannot represent an empty
result; extraction failure follows the configured retry and rejection policy.
Optional `players`, `party`, and `glossary` campaign references can help
disambiguate names or setting terms. They never supply scene evidence or add
events absent from the accepted chunk. The lane requires no NPC registry or
other generated artifact reference.
## Merge and normalization
Accepted per-chunk lists are appended in chunk order. Normalization then:
1. validates the current-source range, non-empty ID and prose, and closed kind;
2. trims only title and summary whitespace;
3. sorts records by source-document start position, then ID;
4. removes records only when all five durable fields are identical;
5. rejects a reused ID when any remaining durable field differs; and
6. rejects the same exact range when `kind`, `title`, or `summary` differs.
Two different IDs with the same range and identical model-owned content remain
separate records. Normalization does not join adjacent ranges, rewrite prose,
repair kinds, infer missing scenes, or use chunk annotations.
## Validation and warnings
Extraction validation requires exactly one record with an ID and range exactly
equal to its current chunk. Later validation checks source membership without a
current chunk. Shape, source-range, kind, ID, and normalized-invariant failures
reject the artifact.
Relatedness checks are advisory. They separately warn when a scene title or
summary has no significant lexical token in its cited transcript range. The
check ignores short tokens and common function words, uses transcript text only,
and does not treat campaign references as evidence. Warning diagnostics are
bounded and do not copy transcript or campaign-reference content.
## Configuration
Use `dnd/scenes` to form accepted scene chunks, then select
`dnd/scene-descriptions` for extraction and normalization. The maintained
[scene-description configuration](../../examples/dnd-scene-descriptions.config.yml)
is a copyable minimal lane. Selectable keys and default validator chains are
defined in [Configuration](../config.md).
The [combat-turn artifact](dnd-combat-turn-artifacts.md) uses an exact matching
`combat` scene only as eligibility control; scene title, summary, and source
reference never become combat evidence. Publication is defined by the
[JSON output contract](json-output.md); implementation details live in
[D&D module internals](../internal/dnd.md).

View File

@@ -1,204 +1,73 @@
# D&D Spell Artifact
This document is the durable serialized artifact contract for the production
D&D spell extractor. Selectable extractor keys are cataloged in
[Configuration](../config.md#implemented-production-modules).
This contract defines the durable output of the `dnd/spells` extractor and
normalizer. It records source-grounded spell-casting occurrences; it is not a
spellbook, a rules lookup result, or a record of hypothetical casts.
## Identity
## Identity and compatibility
- Artifact kind: `dnd/spell-list`
- Prompt ID: `dnd.spells`
- Response schema key: `dnd_spells`
- Response schema ID: `notarius.dnd.spells`
- Response schema name: `notarius_dnd_spells_v1`
- Response schema version: `v1`
- Media type: `application/json`
The durable JSON Schema is owned by the D&D spell artifact codec. The
extractor's private LLM response schema is a separate transport contract: its
source-reference objects omit `source_id`, which the extractor assigns while
mapping the response to the canonical artifact. The LLM DTO and transport
schema are not part of this durable contract. The private schema owns required
fields, JSON types, object and array shapes, and unknown-field rejection;
deterministic validators own the durable artifact's semantic constraints.
The output contains canonical spell casts derived from transcript evidence.
Source IDs are assigned from the input identity; source-unit ranges identify
the evidence location.
## Output Shape
The extractor payload is a JSON object with one required top-level array. Its
structure is:
```text
{"spell_casts": [<spell-cast object>, ...]}
```
`spell_casts` must be present. It may be empty when no spell casts are found.
When multiple chunk results are combined, spell casts remain in chunk order.
When the payload is written as durable output, its logical path is derived from
the configured artifact lane ID as defined by the
[JSON output contract](json-output.md#output-payload-files).
## Spell-Cast Fields
Each spell cast contains exactly these required fields:
- `caster`: in-world character or creature casting the spell;
- `spell`: spell name;
- `source_refs`: transcript source references with extractor-assigned source
IDs and evidence unit ranges. It must contain at least one entry.
Both string fields must be non-empty. `caster` is the canonical in-world
caster, not the human player, transcript speaker, or GM when the associated
character or creature can be identified. Player and party references may
disambiguate that identity, but do not independently establish that a cast
occurred. The `spell` value must resolve through the effective SRD-plus-overlay
catalog as either a canonical name or alias. Catalog validation accepts aliases
but does not rewrite them; unknown fields are rejected.
The artifact includes an actual casting event or an unambiguous declared
casting attempt. Spell mentions, hypothetical plans, rules discussion, and
catalog matches without a casting event are excluded. The spell catalog is a
name-recognition policy and never evidence that a cast occurred.
## Source References
Each source reference contains exactly three required fields: `source_id`,
`start_unit_id`, and `end_unit_id`. The source ID must match the input identity.
The unit IDs must be positive integers present in the input, and the start unit
must not appear after the end unit. Unknown fields are rejected.
For each cast, the complete `source_refs` collection identifies the transcript
evidence for the caster, spell name, and occurrence of the cast or declared
attempt. The deterministic validators establish that ranges are structurally
valid and that the spell name is related to cited text. Semantic evidence
sufficiency is an extraction policy and remains subject to evaluation rather
than deterministic proof.
Reference slot keys and accepted file types are defined in
[Configuration](../config.md#implemented-production-modules). References are
supporting disambiguation material, not source evidence, and are not
addressable through `source_refs`.
## Optional NPC Grounding
The `dnd/spells` extractor accepts an optional `npcs` reference containing one
normalized NPC artifact as `application/json`, up to 1 MiB. An external file is
validated during preparation; an ordered generated binding is validated at the
step handoff. Both paths use the approved NPC codec and identity policy,
re-encode canonical durable JSON for registry provenance, and supply only the
registry's ordered names as the operation-time spell prompt input. It helps the
model prefer canonical caster names; it does not establish that a spell was
cast.
NPC source references may identify the run that produced the registry or any
other session. They remain registry provenance and are never copied into a
spell cast's `source_refs`; every spell evidence range must still identify the
current transcript. Generated provenance records producer and canonical
artifact identity without payload content or a path. When the slot is absent,
the prompt receives exactly `{"npcs":[]}` with its projection digest, and the
run has no NPC reference provenance.
## Normalization Behavior
When the `dnd/spells` normalizer is selected, each recognized spell name is
rewritten to the effective catalog's canonical display name. Lookup uses the
catalog's case-insensitive, whitespace-normalizing, apostrophe-normalizing, and
alias rules. Unknown names are preserved exactly for the normalize validators;
the normalizer does not guess or apply fuzzy matching.
Each cast's `source_refs` is copied, sorted by exact `source_id`,
`start_unit_id`, and `end_unit_id`, and stripped of exact structural
duplicates. Adjacent or overlapping ranges are not merged, and the normalizer
does not synthesize references or change their boundaries.
After those per-cast changes, duplicate identity requires the same canonical
spell name, the same caster after case folding and whitespace normalization,
and the same complete, non-empty set of source references valid for the source
document. Only the first occurrence is retained, in stable order. Its caster
and canonical references are preserved. Unknown names, empty or invalid
evidence, and casts with different evidence remain separate.
Mutation and duplicate decisions are returned through the normal warnings
surface. Warning scopes use the merged input index, such as `spell_casts[0]`,
so they remain meaningful even when a later duplicate is removed. The
normalizer uses these reason codes:
| Reason code | Meaning |
| Property | Value |
| --- | --- |
| `spell_name_canonicalized` | A catalog lookup replaced an input name with its canonical display name. |
| `spell_name_unresolved` | A name was not found in the effective catalog and was retained unchanged. |
| `source_references_normalized` | Reference order changed or exact duplicate references were removed. |
| `duplicate_spell_cast_collapsed` | A later cast matched the retained cast's complete duplicate identity. |
| Artifact kind | `dnd/spell-list` |
| Schema ID | `notarius.dnd.spells` |
| Schema name | `notarius_dnd_spells_v1` |
| Schema version | `v1` |
| Media type | `application/json` |
Only warnings from an accepted normalize attempt are promoted to
`warnings.json`. If an unresolved name reaches the default normalize validator
chain, the catalog validator rejects the candidate with `unknown_spell`; the
`spell_name_unresolved` warning remains in the attempt's debug artifact. An
explicit validator override that accepts the candidate promotes the unresolved
warning normally.
`v1` is a single strict JSON object. It requires `spell_casts`; the array may
be empty. Each spell-cast object and source-reference object rejects unknown
fields. An incompatible shape change requires a new schema version.
The default extraction and normalization chains both preserve this registered
order: JSON syntax, spell shape, catalog membership, source references, JSON
Schema, then source relatedness. Extraction validates the private response
schema; normalization validates the durable artifact schema.
## Wire shape
## Manifest Metadata
Each `spell_casts` entry has these required fields:
The extractor adds prompt and response-schema provenance under the artifact lane
manifest metadata:
| Field | Contract |
| --- | --- |
| `caster` | Non-empty in-world character or creature name. |
| `spell` | Non-empty spell name. |
| `source_refs` | One or more transcript evidence ranges. |
Every source reference has exactly `source_id`, `start_unit_id`, and
`end_unit_id`. The source ID identifies the input transcript; the unit IDs are
positive inclusive unit identifiers, and the start may not follow the end in
that source. References are evidence for the cast, not campaign-reference or
NPC-registry provenance.
```json
{
"metadata": {
"extractor": {
"prompt_id": "dnd.spells",
"prompt_version": "v1",
"prompt_sha256": "sha256:...",
"response_schema_key": "dnd_spells",
"response_schema_id": "notarius.dnd.spells",
"response_schema_name": "notarius_dnd_spells_v1",
"response_schema_version": "v1",
"response_schema_sha256": "sha256:...",
"catalog_base_id": "dnd-5e-2014-srd-spells",
"catalog_digest": "sha256:...",
"catalog_overlay_ids": ["campaign.example"],
"npc_registry_digest": "sha256:...",
"npc_count": 3
},
"normalizer": {
"catalog_base_id": "dnd-5e-2014-srd-spells",
"catalog_digest": "sha256:...",
"catalog_overlay_ids": ["campaign.example"]
"spell_casts": [
{
"caster": "Mira Thorn",
"spell": "Fireball",
"source_refs": [
{"source_id": "session-7", "start_unit_id": 12, "end_unit_id": 13}
]
}
}
]
}
```
`catalog_digest` identifies the effective semantic catalog, while
`catalog_overlay_ids` is sorted and empty for a base-only configuration. Raw
prompt, schema, catalog, alias, and local overlay-file content are not
included in manifest metadata. The `normalizer` metadata uses the same catalog
identity fields when that module is selected. Overlay origin, media type, byte
size, and raw digest are recorded separately in the manifest's reference
provenance; see the [JSON output contract](json-output.md#manifestjson).
## Evidence and normalized form
The `npc_registry_digest` and `npc_count` fields in the example are present for
an external NPC registry when the extractor publishes its prepared module
metadata. They contain no NPC names, source references, paths, or raw
bytes. A generated registry's identity is instead represented by the framework
handoff provenance and dependency fingerprint, so the consumer module metadata
does not duplicate it.
An entry represents an actual cast or an unambiguous declared attempt. A spell
mention, rules discussion, plan, or catalog match alone is not an occurrence.
The configured catalog checks the name; it does not establish evidence.
The extractor's prompt hash, private response-schema hash, and effective catalog
digest also contribute independently scoped semantic checkpoint fingerprints.
Changing any of those prepared contracts intentionally produces a cold
checkpoint miss. Fingerprints contain only digests, never prompt, schema,
catalog, or reference content. When an NPC registry is bound, its semantic
digest contributes an additional local `npc_registry` fingerprint for an
external binding; the manifest metadata contains only that digest and
`npc_count`. Raw NPC file provenance remains independently recorded in the
manifest's `references` list. Generated bindings contribute the canonical
artifact dependency fingerprint and bounded producer provenance instead.
When normalization is selected, recognized spell names use the effective
catalog's canonical display name. Source references are put in canonical source
order and exact duplicate references are removed. A later entry is collapsed
only when it has the same canonical spell, the same case- and
whitespace-insensitive caster identity, and the same complete valid reference
sequence. Remaining entries retain their merged order.
The optional normalized [NPC artifact](dnd-npc-artifacts.md) can ground a
caster name. Its own references remain registry provenance and are never copied
into `source_refs`.
## Related contracts
The [spell-catalog overlay contract](dnd-spell-catalog-overlays.md) defines
the configured catalog additions. The [JSON output contract](json-output.md)
defines where this logical artifact is published; [D&D module internals](../internal/dnd.md)
describes extraction and validation mechanics.

View File

@@ -1,19 +1,27 @@
# D&D Spell-Catalog Overlay Contract
# D&D Spell-Catalog Overlays
This document defines the JSON format accepted by the D&D spell catalog
resolver. An overlay supplies campaign-specific spell names and aliases for
recognition. It does not supply spell rules, levels, classes, effects, or
source evidence.
This document defines the optional JSON overlay consumed by the D&D spell
extractor. An overlay contributes campaign spell names and aliases for
recognition. It does not define spell rules, effects, levels, classes, or
transcript evidence. Bind the optional `spell_catalog` reference as described
in [Configuration](../config.md#references-and-ordered-handoffs).
The `dnd/spells` extractor accepts one optional UTF-8 `application/json` overlay
bundle through its `spell_catalog` reference slot. The framework materializes
that file relative to the configuration or command-line binding, enforces the
1 MiB slot limit, and records its origin and raw digest separately from the
effective catalog digest.
## Contract Identity
## Shape
| Property | Value |
| --- | --- |
| Consumer | D&D spell extraction and normalization |
| Reference slot | `spell_catalog` |
| Media type | `application/json` |
| Required schema version | `notarius.dnd.spell-catalog-overlay.v1` |
| Base catalog | Embedded D&D 5e 2014 SRD catalog |
An overlay bundle has this shape:
At most one overlay document may be bound. The maintained example is
[dnd-spell-catalog.json](../../examples/dnd-spell-catalog.json).
## Wire Shape
This is a minimal valid overlay:
```json
{
@@ -22,49 +30,43 @@ An overlay bundle has this shape:
{
"id": "campaign.example",
"ruleset": "dnd-5e-2014",
"source": {
"title": "Example campaign spells",
"version": "1",
"url": "",
"license": ""
},
"spells": [
{
"name": "Aegis of Emberfall",
"aliases": ["Emberfall Aegis"]
}
]
"source": {"title": "Example campaign spells"},
"spells": [{"name": "Aegis of Emberfall"}]
}
]
}
```
The top-level `schema_version` and `catalogs` fields are required. The schema
version must be exactly `notarius.dnd.spell-catalog-overlay.v1`, and at least
one catalog is required. Catalogs require a unique, non-empty, trimmed `id`,
the exact `dnd-5e-2014` `ruleset`, a `source`, and a non-empty `spells` array.
| Field | Required | Meaning and constraints |
| --- | --- | --- |
| `schema_version` | Yes | Exactly `notarius.dnd.spell-catalog-overlay.v1`. |
| `catalogs` | Yes | Non-empty array of catalog objects with unique IDs. |
| `catalogs[].id` | Yes | Non-empty trimmed string. |
| `catalogs[].ruleset` | Yes | Exactly `dnd-5e-2014`. |
| `catalogs[].source.title` | Yes | Non-empty trimmed string. |
| `catalogs[].source.version` | No | String when present. |
| `catalogs[].source.url` | No | String when present. |
| `catalogs[].source.license` | No | String when present. |
| `catalogs[].spells` | Yes | Non-empty array of spell objects. |
| `catalogs[].spells[].name` | Yes | Non-empty trimmed string. |
| `catalogs[].spells[].aliases` | No | Array of non-empty trimmed strings when present. |
`source.title` is required and must be non-empty and trimmed. `source.version`,
`source.url`, and `source.license` are optional strings and may be empty.
Each spell requires a non-empty, trimmed `name`. `aliases` may be omitted or
may be an array of trimmed, non-empty strings; JSON `null` is not an alias
array. Overlay objects contain no other supported spell fields.
Unknown fields are rejected at every object level. The document must contain
one JSON value; `null` is not accepted for optional strings or aliases.
Decoding is strict: unknown fields, malformed JSON, trailing JSON values, and
non-string optional source fields are rejected.
## Composition And Compatibility
## Composition
Notarius starts with the embedded base catalog, then applies overlay catalogs
in ascending catalog-ID order. A new canonical spell name adds a recognition
entry. If an overlay names an existing canonical spell, it augments that spell
with aliases while retaining the established display spelling.
The resolver always starts with the embedded D&D 5e 2014 SRD catalog. Overlay
catalogs are sorted by `id` before composition, so the input order does not
affect the result. A new canonical name adds a recognition entry. A canonical
name matching an existing canonical name augments that spell and keeps the
established canonical display spelling. Repeated aliases for the same spell
are idempotent.
Repeated aliases for the same spell are accepted. A canonical-name, canonical-
to-alias, or alias-to-alias collision between different spells is rejected,
including a collision with the embedded catalog. Matching uses the catalogs
case, whitespace, and apostrophe normalization, so authors should avoid names
or aliases that normalize to another spell.
Canonical-name display conflicts and canonical/alias or alias/alias collisions
between different spells are errors, including collisions with the embedded
catalog. Canonical names and aliases use the catalog's case, whitespace, and
common-apostrophe normalization rules. The effective catalog returns canonical
names in sorted order and produces a semantic SHA-256 digest that is stable
under JSON formatting, object-key, catalog, spell, and alias reordering.
The overlay is a recognition aid only. The durable spell-artifact schema and
source-evidence rules are defined by the
[D&D spell artifact contract](dnd-spell-artifacts.md).

View File

@@ -0,0 +1,116 @@
# Published Evidence Context
This contract defines the optional `source/evidence-context` artifact emitted
by the production JSON output. Its configuration is owned by
[Configuration](../config.md#module-bindings-and-validators); its logical-file
discovery is owned by [Published JSON Output](json-output.md).
## Identity And Discovery
When enabled, the JSON bundle contains `evidence-context.json` and an
`index.json` `evidence_context` descriptor with the same six fields as other
pipeline-wide artifact descriptors.
| Property | Value |
| --- | --- |
| Artifact kind | `source/evidence-context` |
| Media type | `application/json` |
| Schema ID | `notarius.source.evidence_context` |
| Schema name | `notarius_source_evidence_context_v1` |
| Schema version | `v1` |
| Logical file | `evidence-context.json` |
Consumers must discover the file from the descriptor, verify all six descriptor
fields, and decode only a supported schema version. The descriptor is optional:
its absence means evidence publication was not enabled for that bundle.
## Payload
The v1 payload is a JSON object with required `source_id`, `source_digest`,
`window_units`, `selected_lanes`, and `contexts` fields. `selected_lanes` and
`contexts` are always arrays; an enabled configuration with no accepted direct
evidence publishes `contexts: []`.
```json
{
"source_id": "session-alpha",
"source_digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
"window_units": 1,
"selected_lanes": ["npcs", "spells"],
"contexts": [
{
"context_ref": {
"source_id": "session-alpha",
"start_unit_id": 10,
"end_unit_id": 20
},
"evidence_refs": [
{
"lane_id": "spells",
"source_ref": {
"source_id": "session-alpha",
"start_unit_id": 10,
"end_unit_id": 10
}
}
],
"units": [
{
"id": 10,
"kind": "transcript_segment",
"text": "Aria casts Cure Wounds.",
"ref": {
"source_id": "session-alpha",
"start_unit_id": 10,
"end_unit_id": 10
}
},
{
"id": 20,
"kind": "transcript_segment",
"text": "The party regroups.",
"ref": {
"source_id": "session-alpha",
"start_unit_id": 20,
"end_unit_id": 20
}
}
]
}
]
}
```
Each context requires a `context_ref` object and `evidence_refs` and `units`
arrays. `context_ref` identifies the first and last included unit. Each
evidence entry contains a selected `lane_id` and an original `source_ref`. A
unit uses the existing source-unit shape: required `id`, `kind`, `text`, and
self `ref`, plus optional JSON-object `metadata`. Fixed payload objects reject
unknown fields; unit metadata may contain application-defined JSON values.
## Citations And Context
`evidence_refs` are the authoritative citations. They identify the direct
references emitted by accepted normalized artifacts. `context_ref` and the
units collection include those cited units plus nearby source units selected by
the configured window. They are explanatory context, not widened citations.
Only accepted outputs from the configured lane allowlist contribute. Rejected,
failed, absent, and lane-filtered outputs do not contribute. The artifact never
contains raw input bytes, prompts, model responses, auxiliary reference
content, credentials, or filesystem paths.
## Ordering And Compatibility
The selected lane allowlist is lexical. Contexts and units are in source
document position order, not numeric unit-ID order. Direct evidence entries
are deterministically ordered by lane and source reference. Overlapping or
contiguous windows merge, and each source unit appears at most once in the
resulting contexts.
The artifact is additive to the JSON bundle and is not a lane payload,
normalized-output count, checkpoint, or generated reference. Consumers that
do not need it must tolerate the absent optional descriptor. Consumers that do
use it should preserve the artifact and its schema identity with the run
provenance, and should treat its source text and metadata as sensitive durable
content.

View File

@@ -1,201 +1,133 @@
# JSON Output
# Published JSON Output
This document is the durable JSON output file-format contract produced by the
production JSON encoder and written by the CLI. Selectable output-encoder keys
are cataloged in
[Configuration](../config.md#implemented-production-modules).
This document defines the logical JSON bundle emitted by the production JSON
output encoder. The bundles physical destination, atomic publication, and
retention are operational concerns; see [Operations](../operations.md#output-bundles).
Output configuration, including chunk-map and evidence-context publication, belongs in
[Configuration](../config.md#module-bindings-and-validators).
The output module produces the logical bundle described here. The CLI's
physical placement and lifecycle for that bundle are defined in
[Operations](../operations.md#output-directory).
## Bundle Layout
## Files
All paths below are logical, relative, slash-separated bundle paths. The
encoder always emits the first four JSON files below and adds lane or
pipeline-wide artifact files when their corresponding artifacts are available:
The encoder writes:
A subprocess caller first obtains the physical bundle root from the
[run-result receipt](run-result.md), then resolves `index.json` beneath that
root for the logical discovery described here.
- `index.json`
- `manifest.json`
- `lanes/<lane-id>.json`, one file per normalized serialized artifact
- `rejected.json`
- `warnings.json`
- `chunk-map.json`, only when the JSON output binding enables
`include_chunk_map` and the run has an accepted chunk map
| Path | Purpose |
| --- | --- |
| `index.json` | Entry point that names the other published files and lane payloads. |
| `manifest.json` | Run provenance and result summaries. |
| `rejected.json` | Rejected pipeline outputs. |
| `warnings.json` | Accepted-output and run warnings. |
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
| `evidence-context.json` | Optional source-context artifact, when evidence publication is enabled. |
Files are pretty-printed JSON with a trailing newline when the payload is JSON.
Logical file paths are relative, slash-separated, and may not contain `..`.
JSON files are pretty-printed with a trailing newline. Lane payloads are
accepted only when their media type is `application/json`.
## `index.json`
Shape:
`index.json` is the bundles discovery document. An approved run with no
normalized lanes has this valid minimal index:
```json
{
"manifest_file": "manifest.json",
"output_files": [
{
"lane_id": "spells",
"media_type": "application/json",
"file": "lanes/spells.json",
"module_key": "noop",
"schema_id": "notarius.dnd.spells",
"schema_name": "notarius_dnd_spells_v1",
"schema_version": "v1"
}
],
"output_files": [],
"rejected_file": "rejected.json",
"warnings_file": "warnings.json"
}
```
`output_files` is sorted by lane ID. Output file names are produced by
sanitizing the lane ID:
| Field | Required | Meaning |
| --- | --- | --- |
| `manifest_file` | Yes | Always `manifest.json`. |
| `output_files` | Yes | Lane descriptors sorted by `lane_id`. |
| `rejected_file` | Yes | Always `rejected.json`. |
| `warnings_file` | Yes | Always `warnings.json`. |
| `chunk_map` | No | Descriptor for the pipeline-wide `chunk-map.json`; never a lane descriptor. |
| `evidence_context` | No | Descriptor for the pipeline-wide `evidence-context.json`; never a lane descriptor. |
- characters outside `A-Z`, `a-z`, `0-9`, `.`, `_`, and `-` become `_`;
- repeated `..` sequences are replaced;
- leading and trailing `.`, `_`, and `-` are trimmed;
- empty sanitized names are rejected;
- two lanes that sanitize to the same output file are rejected.
Each lane descriptor has required `lane_id` and `file`. It may also include
`media_type`, `module_key`, `schema_id`, `schema_name`, and `schema_version`
when supplied by the normalized artifact. Each pipeline-wide artifact
descriptor (`chunk_map` or `evidence_context`) contains `artifact_kind`,
`file`, `media_type`, `schema_id`, `schema_name`, and `schema_version`. Their
payloads are defined by the [Accepted Chunk Map contract](chunk-map.md) and
[Published Evidence Context](evidence-context.md), respectively.
`manifest_file`, `rejected_file`, and `warnings_file` contain the fixed paths
shown above. Each `output_files` entry requires `lane_id` and `file`. It also
contains the normalized payload `media_type`, normalizer `module_key`, and
response `schema_id`, `schema_name`, and `schema_version` when those values are
available.
The lane path is derived from its lane ID. Characters outside letters, digits,
periods, underscores, and hyphens become underscores; `..` sequences are
neutralized; leading and trailing periods and underscores are removed. A lane
that produces an empty name, or two lanes that produce the same path, makes
output encoding fail.
When present, the top-level optional `chunk_map` descriptor contains exactly
`artifact_kind`, `file`, `media_type`, `schema_id`, `schema_name`, and
`schema_version`. It identifies the pipeline-wide `chunk-map.json`; it is not
a lane output and never appears in `output_files`. The descriptor and file are
both absent when export is disabled or no chunk plan was accepted. Its payload
contract is defined by [Accepted Chunk Map](chunk-map.md).
## Lane Payloads
Each `lanes/<safe-lane-id>.json` file is the codec-owned normalized JSON for
that lane. Consumers should use the index descriptors schema identity rather
than infer a lane schema from its name. The current D&D payload contracts are
[spells](dnd-spell-artifacts.md), [NPCs](dnd-npc-artifacts.md),
[NPC interactions](dnd-npc-interaction-artifacts.md),
[combat turns](dnd-combat-turn-artifacts.md),
[item events](dnd-item-event-artifacts.md), and
[scene descriptions](dnd-scene-description-artifacts.md).
## `manifest.json`
`manifest.json` contains a run manifest. This abridged example shows its core
structure:
`manifest.json` is published provenance, not a copy of lane payloads or a
checkpoint store. Fields without a value may be omitted. Its top-level fields
group into the following externally observable summaries:
```json
{
"run_id": "run-123",
"pipeline_id": "dnd-session",
"artifact_lanes": [
{
"id": "spells",
"extractor": "dnd/spells",
"merger": "appendorder",
"normalizer": "noop"
}
],
"validation_status": "approved",
"started_at": "2026-01-01T00:00:00Z",
"completed_at": "2026-01-01T00:00:01Z"
}
```
| Group | Fields |
| --- | --- |
| Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` |
| Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` |
| Source and references | `source_digests`, `references` |
| Published result summaries | `normalized_outputs`, `rejected_outputs` |
| Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` |
Fields with empty values may be omitted by JSON encoding.
`references` records provenance such as the target, slot, origin, digest,
media type, size, and generated-artifact identity. It does not contain
reference content. `normalized_outputs` and `rejected_outputs` likewise
summarize results without embedding lane payload bytes. A chunk-plan summary is
provenance for the plan used by this run; cache records, debug artifacts, and
other operational state are not published as bundle files.
The manifest fields are:
Each `llm_profiles` entry identifies effective, non-secret LLM execution
provenance:
- `run_id`, `pipeline_id`, and `pipeline_digest`: run and resolved-pipeline
identity;
- `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, and
`output_encoder`: resolved module keys;
- `chunk_plan`: payload-free provenance for the effective chunk plan. `mode`
is the effective cache mode; `action` is `reused`, `generated`,
`refreshed`, or `bypassed` when a plan was materialized. `requested_module`
is the current pipeline chunker, while `producer_input_module`,
`producer_module`, `producer_llm_profile`, `producer_references`,
`producer_metadata`, `source_digest`, `plan_digest`, `plan_schema_version`,
and `created_at` describe the stored or generated producer when available.
A cached plan can therefore identify a producer different from the requested
module. This object never embeds ranges, units, annotations, prompts,
responses, or reference content;
- `module_metadata` and `artifact_lanes`: module and per-lane provenance,
including prompt and response-schema provenance when provided;
- `validator_chains`: resolved validation points and validators;
- `source_digests` and `references`: source and reference provenance;
- `normalized_outputs` and `rejected_outputs`: payload-free result summaries;
- `llm_profiles`: selected profile IDs and provider or model names when
available;
- `metadata`: the effective prompt `session_id`;
- `validation_status`: `approved` or `rejected`;
- `started_at` and `completed_at`: UTC run timestamps.
| Field | Required | Meaning |
| --- | --- | --- |
| `id` | Yes | Selected PromptKit profile identifier. |
| `provider` | No | Notarius adapter provider identifier. |
| `model` | No | Effective provider model identifier. |
| `backend_id` | No | Effective PromptKit backend registration identifier. Endpoint-only profiles omit it. |
| `reasoning_effort` | No | Effective opaque provider reasoning setting. An empty or explicitly cleared setting is omitted. |
`source_digests` contains source document digests only. Bound references are
recorded separately under `references`, which contains provenance only: target
stage, lane ID when present, slot name, origin type and URI, digest, media
type, byte size, and binding source. Reference content is not written to
durable output.
These values describe observed execution; they are not a backend-registration
interface. Entries that differ by backend or effective reasoning remain
distinct even when their profile, provider, and model are otherwise equal.
Reference `stage` is `chunk`, `extract`, `merge`, or `normalize`. `lane_id` is
omitted for chunk references and present for extract, merge, and normalize
references.
## Rejections And Warnings
`validation_status` is `approved` when no outputs were rejected and `rejected`
when one or more outputs were rejected.
`rejected.json` is always an object with a `rejected` array. Each entry has
required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`,
`chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and
`diagnostic_artifact_path` are present only when applicable.
Producer warnings and the current run's chunk-validation warnings remain in
`warnings.json`. The manifest records only provenance and decision summaries;
empty producer-only values are omitted for compatibility with existing readers.
`warnings.json` is always an object with a `warnings` array. Each warning has
`reason_code` and `message`; `scope` is optional. Both arrays are empty when
there is nothing to report.
`validator_chains` records the resolved validator chain for each validation
point. Entries include stage, lane ID when applicable, module key, and validators
with key and execution class. Empty chains are recorded with an empty
`validators` array, including chains resolved from explicit empty config
overrides.
## Compatibility
`normalized_outputs` summarizes each normalized lane output without embedding
payload bytes. Entries include lane ID, normalizer module key, source ID, media
type, and response schema provenance where available.
`rejected_outputs` summarizes rejected module outputs without embedding raw
payload bytes. Entries include stage, lane, module, chunk, validator or reason,
message, attempt count, and optional diagnostic artifact path.
## Output Payload Files
Each normalized serialized artifact is written to
`lanes/<sanitized-lane-id>.json`. The JSON output encoder is domain-neutral and
accepts only artifacts whose codec media type is `application/json`. The file
contains the codec-owned JSON bytes pretty-printed.
The schema of each lane payload is owned by that artifact contract. For the
current D&D lanes, see [D&D Spell Artifact](dnd-spell-artifacts.md),
[D&D NPC Artifact](dnd-npc-artifacts.md), and
[D&D Combat-Turn Artifact](dnd-combat-turn-artifacts.md), and
[D&D Scene Description Artifact](dnd-scene-description-artifacts.md).
## `rejected.json`
Shape:
```json
{
"rejected": []
}
```
When output validation rejects an output, each entry contains `stage` and
`message`. It includes `lane_id`, `module_key`, `chunk_id`, `chunk_index`,
`validator_name`, `reason_code`, `attempt_count`, and
`diagnostic_artifact_path` when applicable.
## `warnings.json`
Shape:
```json
{
"warnings": [
{
"scope": "extract",
"reason_code": "example",
"message": "human-readable warning"
}
]
}
```
`warnings` is an empty array when no warnings are reported.
Each warning requires `reason_code` and `message`; `scope` is omitted when it is
empty.
The index is the authoritative map from a logical lane to its published
payload. Consumers must tolerate omitted optional manifest and descriptor
fields, and should rely on the linked artifact contract for each lanes JSON
shape. This contract describes the published logical bundle only; it does not
promise a filesystem layout or expose internal state formats.

View File

@@ -0,0 +1,106 @@
# PromptKit Integration
Notarius pins
[`gitea.maximumdirect.net/eric/promptkit` v0.5.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0)
as its in-process prompt engine. The upstream
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/consumers/pkg-promptkit.md)
owns the public engine API, and the upstream
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md)
owns prompt, profile, and schema file contracts.
## Supported Boundary
Notarius relies on the root `promptkit` package to:
- construct an `Engine` with filesystem-backed prompt, schema, and optional
operator and application-fallback profile sources;
- prepare one frozen execution from a `RunRequest` with named inline artifacts,
variables, a direct session ID, prompt identity, and profile selection, then
record credential-redacted details and run that exact execution;
- return rendered debug material, validated structured output, selected
profile, backend, effective model metadata, and token usage;
- register the optional conventional `local` backend through `BackendLocal`,
`LocalBackend`, and `WithBackend`;
- distinguish structured-output validation failure from execution failure; and
- identify a missing explicit profile through `ErrProfileNotFound` and backend
admission exhaustion through `ErrCapacityExceeded`.
The pinned
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/backends.go)
owns the registration and backend-capacity contract.
For one completion, the adapter calls `PrepareExecution`, takes a
caller-owned `Details` snapshot, and calls `RunPrepared` for that same opaque
prepared execution. It defers `Discard` for every unexecuted handle. Explicit
profile preflight uses `Engine.InspectProfile`; it does not prepare a synthetic
prompt. PromptKit's prepared handle, inspection result, and capacity-error
types stay inside the Notarius LLM adapter.
When a PromptKit profile and runtime override leave `temperature`, `max_tokens`,
or `top_p` unset, Notarius leaves that control unset as well. Compatible
providers therefore apply their own defaults; an operator that requires a
specific sampling value must select it explicitly in the profile or runtime
override.
Notarius does not use PromptKit's optional `ArtifactReader`. It materializes
source and reference content itself and supplies owned inline artifacts at the
adapter boundary. It also retains responsibility for pipeline retries,
scheduling, debug persistence, redaction, profile provenance, and conversion
from private model responses into durable domain artifacts.
Notarius sends its trimmed run session through PromptKit's direct session
field, which is authoritative for provider session behavior. It also retains
the same value as the `session_id` prompt variable for maintained prompt
compatibility. Session IDs are stable, non-secret correlation identifiers and
may be exposed to providers and provider observability.
Notarius records PromptKit's selected backend ID and effective reasoning
setting as optional run-manifest provenance. Endpoint-only profiles have no
backend ID. Debug prompt material also retains the selected backend ID and
PromptKit's stable lower-case `effective_model_params` JSON, which may include
`backend_id`. Notarius production configuration exposes one optional
conventional `local` registration. It does not expose a general user-defined
PromptKit backend registry. Endpoint-only profiles remain supported unchanged.
Notarius retains its application-wide scheduled client around the PromptKit
adapter. PromptKit may apply a narrower limit for the selected backend;
endpoint-only profiles have no such backend limit. The adapter translates
PromptKit capacity rejection into the provider-neutral Notarius
`ErrLLMCapacityExceeded` contract. It may include the normalized selected
backend ID in safe diagnostic context, without exposing PromptKit's capacity
error type, and leaves retries to the calling pipeline stage.
## Profile Sources And Compatibility
Notarius gives PromptKit the configured operator profile source, registered
application fallback profile assets, and optional backend registration through
the same construction path for inspection and execution. PromptKit owns the
resulting source precedence and strict profile parsing: a matching operator
profile is a complete replacement for a fallback or built-in profile, while an
invalid matching document fails instead of falling through. The operator
configuration and deployment workflow are defined in
[Configuration](../config.md#promptkit-profiles) and
[Operations](../operations.md#promptkit-profile-deployment).
Notarius supports this boundary against PromptKit v0.5.0. Its fallback source,
prepared-execution, inspection, and typed capacity APIs are used as public
upstream contracts; other PromptKit APIs or file-format behavior are not
implicitly supported. A dependency upgrade requires reviewing the adapter,
profile-source construction, and this compatibility statement against the
pinned upstream documentation.
## Notarius Ownership
[LLM Runtime Internals](../internal/llm.md) describes how Notarius mounts
module assets, maps its transport-neutral completion contract, prepares and
executes requests, validates output, records provenance, captures debug
material, redacts errors, and preserves timeout ownership.
[D&D Module Internals](../internal/dnd.md) owns the embedded
`dnd-extraction` fallback profile and the maintained D&D prompt defaults.
[Configuration](../config.md#promptkit-profiles) defines how a Notarius
configuration selects one PromptKit profile source and optionally registers
the conventional local backend.
PromptKit API or format changes outside this boundary are not implicitly
supported. Updating the pinned version requires reviewing the adapter and
profile/configuration contracts against the upstream documentation.

View File

@@ -0,0 +1,68 @@
# Run Result Receipt
`notarius run --json` writes this receipt to standard output when a run
completes successfully. It lets a subprocess caller discover the physical root
of the published output bundle without parsing interactive command output.
Command syntax, streams, and exit statuses are defined in the
[CLI reference](../cli.md); logical files within the bundle are defined in the
[Published JSON Output contract](json-output.md).
## Schema
The current schema version is `notarius.run-result.v1`.
| Field | Required | Meaning |
| --- | --- | --- |
| `schema_version` | Yes | Exactly `notarius.run-result.v1`. |
| `run_id` | Yes | The finalized Notarius run identifier. |
| `pipeline_id` | Yes | The effective pipeline identifier. |
| `output_directory` | Yes | Absolute path to the published, run-specific output bundle. |
| `index_file` | For the production JSON output | Logical path `index.json`; omitted for other output modules. |
| `normalized_output_count` | Yes | Number of final normalized outputs. |
| `rejected_output_count` | Yes | Number of recorded rejected outputs. |
| `warning_count` | Yes | Number of final run warnings. |
| `validation_status` | Yes | The final run manifest validation status. |
| `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. |
For the production `json` output module, `index_file` is present only when the
completed run returned exactly one logical output file named `index.json`.
For another output module, its absence does not indicate a failed run.
```json
{
"schema_version": "notarius.run-result.v1",
"run_id": "run-1770000000000000000-0123456789abcdef0123456789abcdef",
"pipeline_id": "dnd-session",
"output_directory": "/work/results/run-1770000000000000000-0123456789abcdef0123456789abcdef",
"index_file": "index.json",
"normalized_output_count": 6,
"rejected_output_count": 2,
"warning_count": 1,
"validation_status": "rejected"
}
```
## Paths And Bundle Discovery
`output_directory` and `debug_directory`, when present, are lexical absolute
paths. They identify the paths used by Notarius and do not resolve symlinks.
`output_directory` is the run-specific bundle, not the configured output root.
The receipt is a summary and discovery document. It does not contain lane
descriptors, payloads, manifest data, rejections, warnings, or file contents.
For the production JSON output, resolve `index_file` beneath
`output_directory`, reject path escapes, and use the
[Published JSON Output contract](json-output.md) to discover logical files and
lane payloads.
## Delivery And Compatibility
Notarius writes the receipt only after the output bundle has been published and
any requested debug terminal reporting has completed. Standard output is not
transactional: a result-write failure returns a nonzero status and can leave
partial bytes. Consumers must ignore standard output unless the process exits
with status 0.
Future versions may add optional fields to this schema. Consumers must tolerate
unknown fields. An incompatible field or semantic change requires a new
`schema_version` value.

View File

@@ -1,69 +1,73 @@
# Seriatim Transcript JSON
# Seriatim Transcript Input
This document is the external input contract consumed by the production
Seriatim input adapter. Selectable input-adapter keys are cataloged in
[Configuration](../config.md#implemented-production-modules).
This document defines the JSON transcript accepted by the production Seriatim
input adapter. It is a source input, not a durable lane artifact. Configure the
input adapter through [Configuration](../config.md#production-module-keys).
## Adapter
## Contract Identity
- Source format: `application/vnd.seriatim+json`
| Property | Value |
| --- | --- |
| Consumer | Seriatim input adapter |
| Media type | `application/vnd.seriatim+json` |
| Source document kind | `transcript` |
| Source-unit kind | `transcript_segment` |
## Accepted Shape
The input must be one JSON object with top-level `metadata` and `segments`
fields. This covers the maintained minimal fixture and Seriatim intermediate
output that provides the same required segment fields.
The input is one JSON object containing `metadata` and a non-empty `segments`
array. This minimal document is valid:
The maintained example is
[examples/seriatim-minimal-transcript.json](../../examples/seriatim-minimal-transcript.json).
```json
{
"metadata": {"id": "session-alpha"},
"segments": [
{
"id": 1,
"start": 0,
"end": 4,
"speaker": "Aria",
"text": "Aria casts Cure Wounds."
}
]
}
```
Required top-level fields:
The maintained two-segment input is
[seriatim-minimal-transcript.json](../../examples/seriatim-minimal-transcript.json).
- `metadata`: an object. Its entries are accepted as source metadata.
- `segments`: a non-empty array of segment objects.
| Field | Required | Meaning and constraints |
| --- | --- | --- |
| `metadata` | Yes | JSON object. Its entries become source metadata; no particular metadata key is otherwise required. |
| `segments` | Yes | Non-empty array of segment objects, kept in input order. |
| `segments[].id` | Yes | Positive canonical decimal integer, supplied as a JSON number or string. IDs must be unique. |
| `segments[].start` | Yes | Finite, non-negative numeric value, supplied as a JSON number or string. |
| `segments[].end` | Yes | Finite, non-negative numeric value that is not earlier than `start`. |
| `segments[].speaker` | Yes | String that is non-empty after trimming. |
| `segments[].text` | Yes | String that is non-empty after trimming. Its original text is retained. |
Required segment fields:
Additional top-level and segment fields are ignored. A missing required field,
`null` in place of an object or array, malformed JSON, or more than one
top-level JSON value is rejected.
- `id`: a positive integer JSON number or canonical decimal string without
leading zeros or surrounding whitespace;
- `start`: a finite, non-negative JSON number or numeric string;
- `end`: a finite, non-negative JSON number or numeric string that is not less
than `start`;
- `speaker`: a non-empty string;
- `text`: a non-empty string.
## Source Identity And References
Other top-level and segment fields, such as `categories`, are ignored.
The adapter chooses the source ID in this order:
Multiple top-level JSON values are rejected.
1. a non-empty source ID supplied by the calling request;
2. non-empty string `metadata.id`;
3. non-empty string `metadata.source_id`;
4. `seriatim:` followed by the first 16 hexadecimal characters of the raw
inputs SHA-256 digest.
## Validation
Each accepted segment becomes one source unit whose unit ID is `segments[].id`.
Its self-reference uses the derived source ID and the same segment ID for both
range endpoints. Artifact contracts use those segment IDs when they cite
transcript evidence.
The adapter rejects empty input, malformed JSON, multiple top-level JSON values,
non-object segment values, duplicate segment IDs, and any violation of the
shape or field constraints above.
## Compatibility
Segment text is preserved as provided, but it must not be empty after trimming.
## Derived Identity
Notarius identifies the parsed source in this order:
1. `metadata.id`, when it is a non-empty string after trimming;
2. `metadata.source_id`, when it is a non-empty string after trimming;
3. `seriatim:<first-16-hex-chars-of-raw-sha256>`.
The exact raw input SHA-256 remains the basis of the fallback source ID. The
source digest recorded in output provenance is instead the SHA-256 of the
canonical generic source document, excluding the digest field itself. It covers
the derived source identity, document kind and format, ordered units and their
self-references, and accepted metadata. Segment IDs become the unit IDs used by
artifact source references; each produced unit carries a self-reference whose
source ID is the derived document ID and whose start and end IDs both equal the
segment ID.
## Compatibility Limit
This contract covers only Seriatim transcript JSON with the top-level
`metadata` object and `segments` array described here. Broader Seriatim output
schemas are compatible only when they provide these required fields with the
accepted types.
This adapter accepts only the shape described here. A broader Seriatim export
is usable only when it supplies this object, metadata, and segment shape with
the stated types and constraints. Unknown additional fields do not add
Notarius behavior.

163
docs/internal/cli.md Normal file
View File

@@ -0,0 +1,163 @@
# CLI Internals
This document describes **internal/cli**, Notarius's production composition
root. The [CLI reference](../cli.md) owns command syntax and exit statuses;
[Configuration](../config.md) owns configuration values; and
[Operations](../operations.md) owns filesystem layout, recovery, and operator
procedures.
## Inputs, Outputs, And Boundaries
The CLI accepts process arguments, standard streams, and injectable options
used by tests and embedding code. It writes command results to the supplied
streams and returns a process exit status. For a run, it also creates the
production catalog and runtime collaborators, hands a prepared pipeline and
source bytes to the framework, and places the logical files returned by the
runner.
It is the only boundary allowed to compose concrete registries, LLM clients,
cache/checkpoint collaborators, debug recorders, and physical output paths.
Pipeline modules receive interfaces and request data rather than CLI streams or
filesystem roots. The [Architecture](../policy/architecture.md) defines this
composition-root boundary; [Pipeline Internals](pipeline.md) owns resolution,
preparation, and runner mechanics after their inputs are supplied.
## Dispatch And Configuration Handoff
The root dispatcher handles help, configuration validation, pipeline listing,
and a pipeline run. It normalizes injectable options before dispatch so that a
missing production dependency fails as a command error rather than reaching
execution.
Commands that need configuration use one shared loader. The CLI discovers the
file, parses it through **internal/core/config**, starts from defaults, applies
the file and supported environment overrides, and then validates it for the
command. The configured discovery and precedence contract is in
[Configuration](../config.md), while the loading and resolution mechanics are
in [Configuration Internals](configuration.md).
Configuration validation without a selected pipeline checks structural
configuration only. Validation with a selected pipeline also builds the
effective catalog, resolves the pipeline, and verifies every explicit effective
PromptKit profile. Selected LLM-backed input, chunk, lane, output, and validator
profiles are inspected
against the configured PromptKit source and backend registrations without
loading a prompt or performing generation, so an unknown or invalid profile
fails before pipeline preparation. Credential availability remains an
execution-time concern. Pipeline listing validates configuration before
returning normalized, sorted identifiers.
## Production Composition
The production composition helper allocates every framework registry and the
prompt-asset registry, then registers the generic, Seriatim, and D&D module
families in that order. The resulting registries provide both the module
catalog used for resolution and the concrete constructors used for preparation.
Tests may provide a catalog or registries instead; production code must not
silently merge an injected partial catalog with production registrations.
The production LLM factory builds one PromptKit-backed client from the resolved
**promptkit.profile_dir** or **promptkit.profile_file** source, attaches the
profile-provenance recorder, creates one scheduler from the effective global
LLM limit, and wraps the client before it reaches modules. Registration and LLM
construction errors are returned before a pipeline is prepared. Configuration
field definitions remain in [Configuration](../config.md#promptkit-profiles);
the D&D registrar's fallback profile assets and the adapter mechanics remain in
[LLM Runtime](llm.md).
The factory also accepts `LLMRuntimeOverrides`, whose reasoning pointer
preserves inherit, replace, and clear states across the composition boundary.
Run orchestration constructs this value from the mutually exclusive
`--reasoning-effort` and `--clear-reasoning-effort` controls. Absence preserves
a nil pointer, replacement is trimmed, and clear uses a non-nil empty string.
The same override reaches the one shared production client, checkpoint
identity, and debug invocation metadata. Persistent reasoning configuration
remains owned by PromptKit profiles; Notarius configuration has no reasoning
field.
## Run Orchestration
After parsing and validating a run invocation, the CLI performs this ordered
handoff:
1. load and validate configuration, then apply command-level operational
overrides;
2. create and validate a safe run identity, then allocate a debug bundle only
when requested;
3. build the effective catalog, resolve requested reference changes, resolve
the effective pipeline, and inspect its explicit effective PromptKit
profiles;
4. materialize external or generated references and record redacted invocation
and resolution provenance when debug capture is enabled;
5. construct registries, the scheduled LLM client, prepared modules, and the
requested cache/checkpoint collaborators;
6. read the source input and invoke the framework runner; and
7. write the runner's logical output files only after a successful run, then
complete the command report and user-facing result.
Preparation happens before source parsing, so module construction and
dependency failures cannot begin stage execution. The CLI also preserves the
framework's result and warning information when it writes summaries and the
final command result. Detailed state lifecycle, resume handling, and physical
path confinement are maintained in [Run State Internals](state.md) and
[Operations](../operations.md).
For `run --json`, the CLI constructs and encodes its private run-result receipt
after a successful runner result is available, before it publishes logical
output files. It writes the prepared receipt to standard output only after
output publication and requested debug terminalization succeed. A receipt-write
failure exits with runtime status 1 and may leave partial standard-output bytes,
but the already-published output bundle remains complete and requested debug
reporting remains successfully terminalized. The CLI reports a bounded
command-owned error and does not repeat terminal reporting. The receipt remains
a CLI reporting concern rather than a framework or output-module responsibility;
its public contract is the
[run-result receipt](../integrations/run-result.md).
## Failure Mapping And Terminal Reporting
Argument, flag, and invocation-combination failures are reported to standard
error before runtime composition and use the syntax error class. Once an
invocation is syntactically valid, configuration loading and validation,
resolution, registration, profile checks, reference materialization, module
construction, input reads, runner failures, output publication, and requested
debug handling use the runtime failure class. The public status numbers and
stream contract are defined in the [CLI reference](../cli.md#output-streams-and-exit-statuses).
When debug capture has been allocated, one command-state value records the
known run result. Guarded terminalization writes a success report once, or
attempts a failure report and error record once. A persistence failure is
reported in addition to the original failure and never replaces it. If a debug
path exists, failure output includes that path so the retained diagnostic data
is discoverable.
## Invariants To Preserve
- Only the CLI composes production implementations and physical runtime roots.
- Configuration and resolved composition failures occur before module
preparation or source parsing.
- A runner's logical files are published only after a successful run.
- Production registries and a caller-supplied catalog or registries are
alternative composition sources, not an implicit mixture.
- A requested debug bundle has one terminal report attempt; its persistence
errors supplement rather than obscure the primary command error.
- User-facing flags, paths, exit codes, and configuration fields are defined
by their public documentation, not duplicated here.
## Focused Tests
- **internal/cli/command_contract_test.go** covers dispatch, help, syntax and
runtime error classes, discovery, validation, and listing.
- **internal/cli/run_contract_test.go** covers the run handoff, publication,
debug reporting, and command-owned state collaborators.
- **internal/cli/production_contract_test.go** covers registrar composition,
production catalog contents, assets, and representative configuration
validation.
- **internal/cli/reference_contract_test.go** covers CLI reference overrides,
origin separation, and materialization boundaries.
- **internal/cli/state_hardening_test.go** covers safe run identity, state
roots, and failure ordering.
Run **go test ./internal/cli** after changing command composition or command
behavior. Pair it with **go test ./internal/core/config** when the configuration
handoff changes.

View File

@@ -0,0 +1,144 @@
# Configuration Internals
This document describes the maintainer-facing configuration boundary in
**internal/core/config**. The [Configuration](../config.md) reference owns the
file format, fields, defaults, precedence contract, and selectable keys. The
[CLI reference](../cli.md) owns command syntax; this document does not redefine
either interface.
## Boundary
The configuration package turns a selected YAML file and supported environment
values into a validated, independently owned configuration. It then resolves a
requested pipeline against a module catalog before the framework prepares or
runs anything.
| Boundary | Inputs | Outputs | Does not own |
| --- | --- | --- | --- |
| Loading | Selected file path and environment lookup | Parsed file model and a populated **Config** | Choosing the file path or reporting a command result. |
| Validation | **Config** | Structural configuration errors with pipeline, lane, or binding context | Module availability, capabilities, or construction. |
| Resolution | Valid **Config**, selected pipeline and lanes, runtime reference changes, LLM override, and module catalog | **EffectiveConfig** with a **ResolvedPipeline** | Materializing reference bytes, preparing modules, execution, or filesystem state. |
| Summary | **Config** or **EffectiveConfig** | Detached redacted payload suitable for debug summaries | Redacting arbitrary process state or provider traffic. |
The CLI discovers a configuration file, invokes this package, and supplies the
result to the framework. Configuration never reads an input file, constructs a
module, or creates output, cache, or debug paths. Those responsibilities remain
at their respective [CLI](cli.md), [pipeline](pipeline.md), and
[run-state](state.md) boundaries.
## Loading And Validation
The CLI loads configuration in this order:
1. parse the selected YAML file strictly into the file model;
2. start from **Default**;
3. apply the file model; and
4. apply the supported environment overrides.
This establishes the public precedence order without giving environment input a
second file schema. Loading and application reject malformed YAML, unsupported
file versions, unknown fields, invalid values, and identifiers that are empty
or collide after whitespace normalization. The file application also makes the
effective extraction-worker default follow the effective LLM limit. A present
PromptKit local-backend object requires and trims its endpoint, defaults its
omitted concurrency limit to zero, and is copied so the parsed file model
cannot alias the populated **Config**. A pipeline `llm_profile` is
presence-aware: omission remains empty, while a present blank value is
rejected and a non-empty file value is trimmed before it reaches **Config**.
**Config.Validate** checks configuration-only invariants before resolution. It
rejects incompatible profile sources, invalid state-surface values, unsupported
concurrency settings, malformed bindings and references, invalid retries, and
invalid pipeline, step, or lane structure. PromptKit local-backend validation
accepts only an absolute HTTP or HTTPS endpoint with a host and no user
information, query, or fragment, and rejects a negative local concurrency
limit. Its errors retain the closest known pipeline, lane, and binding context.
It deliberately does not require modules to be registered: that requires a
catalog and belongs to resolution.
The exact user-selectable values and validation rules are defined in
[Configuration](../config.md). Keep additions to the file model, an
environment override, its validation, and that reference in the same change.
## Effective Resolution
**Config.Resolve** first recomputes derived concurrency defaults and validates
the configuration. It normalizes the requested pipeline ID, copies the selected
profile, and passes the non-empty command-level LLM profile override, requested
lane selection, and reference changes to the framework resolver.
After module and validator selection, the resolver applies the effective
profile policy to LLM-backed bindings only: command override, binding profile,
pipeline profile, then the prompt default. Deterministic bindings remain
profile-free, and no second inheritance decision occurs during execution. The
public field definitions and precedence are owned by
[Configuration](../config.md#pipelines).
The framework resolver supplies defaults, selects lanes, resolves validator
chains, checks registered module and artifact compatibility, validates module
options, and returns the fixed ordered pipeline shape. The resulting
**EffectiveConfig** retains the selected ID, requested selection and reference
changes, a clone of the input configuration, and the resolved pipeline.
Callers may therefore retain or modify their input slices and maps without
changing the resolved result, and later consumers cannot mutate the original
configuration through the effective value. This ownership includes the nested
PromptKit local-backend value.
Resolution failures stop before module construction and source parsing. They
include an error path for an unconfigured pipeline, missing module, missing
capability, incompatible artifact variant, invalid option, invalid reference,
or invalid lane selection. CLI code maps these valid-invocation failures to the
runtime error class described in the [CLI reference](../cli.md#output-streams-and-exit-statuses).
## Resolved Identity And Redaction
The framework assigns the resolved pipeline a deterministic SHA-256 digest
after defaults, lane selection, module bindings, reference bindings, validator
chains, effective LLM profiles, and artifact schema identity have been
resolved. The digest excludes
its own stored value. It identifies resolved composition rather than raw YAML
bytes, a debug payload, or all runtime state. The CLI records it as invocation
provenance before execution; cache and checkpoint identity have additional
owners in [Run State Internals](state.md).
Configuration summaries must use **Redacted**, **RedactedSummaryPayload**, or
**RedactedResolvedPipelinePayload**, never a direct configuration marshal.
Those methods copy every binding and nested option container, replace values
whose key is credential-shaped with **[REDACTED]**, and omit materialized
reference content while retaining safe binding and reference provenance. The
payload must not alias the source configuration or resolved pipeline.
PromptKit's local endpoint and concurrency limit are preserved as non-secret
configuration metadata in the independently owned summary; the object contains
no credential value. This redaction is deliberately narrow: it protects
configuration summaries and does not authorize recording arbitrary environment
values or provider requests.
## Invariants To Preserve
- Defaults, YAML values, and environment values are applied in one direction;
later sources may override only their supported operational settings.
- A configuration is structurally valid before it is resolved, and a resolved
pipeline is compatible with the supplied catalog before preparation begins.
- Whitespace-normalized identifiers are unique wherever they identify a
pipeline, step, lane, worker, or reference slot.
- Resolution and summary generation return detached data. Redaction must cover
every configured and resolved binding, including nested validator bindings.
- The resolved digest changes when resolved composition changes and never
includes itself.
## Focused Tests
- **internal/core/config/file_config_contract_test.go** covers strict file
parsing, normalization, file application, and structural rejection.
- **internal/core/config/env_contract_test.go** covers supported operational
overrides and their precedence.
- **internal/core/config/validation_contract_test.go** covers configuration
invariants and contextual failures.
- **internal/core/config/effective_config_contract_test.go** covers defaults,
selections, overrides, resolution context, digest changes, and ownership.
- **internal/core/config/redaction_test.go** covers recursive credential
redaction, reference-content exclusion, and non-aliasing payloads.
Run **go test ./internal/core/config** after changing this boundary. Changes to
the handoff or resolved-composition semantics also need the focused framework
pipeline tests.

155
docs/internal/dnd.md Normal file
View File

@@ -0,0 +1,155 @@
# D&D Module Internals
This guide records the conventions shared by the production D&D module family.
It complements [Module Internals](modules.md), which owns generic registration
and extension mechanics, and [Configuration](../config.md), which owns the
selectable keys, bindings, reference syntax, and default validator chains.
## Durable Artifact Contracts
The six lanes have separate durable wire contracts. This guide deliberately
does not repeat their JSON shapes or schemas.
| Lane | Durable contract |
| --- | --- |
| Spells | [spell artifacts](../integrations/dnd-spell-artifacts.md) |
| NPCs | [NPC artifacts](../integrations/dnd-npc-artifacts.md) |
| Combat turns | [combat-turn artifacts](../integrations/dnd-combat-turn-artifacts.md) |
| Item events | [item-event artifacts](../integrations/dnd-item-event-artifacts.md) |
| NPC interactions | [NPC-interaction artifacts](../integrations/dnd-npc-interaction-artifacts.md) |
| Scene descriptions | [scene-description artifacts](../integrations/dnd-scene-description-artifacts.md) |
## Family Composition
The D&D registrar registers the familys artifact codecs, extractors, typed
append-order mergers, normalizers, validators, prompt assets, fallback LLM
profile asset, and default validator chains. Each extractor and normalizer has
a stable module spec, explicit execution class, strict option decoding, and a
typed builder. Scene chunking, every extractor, and NPC normalization are
registered as `llm_backed`; the remaining current D&D mergers and normalizers
are `deterministic`. The metadata is available to catalog inspection and
resolved-pipeline debug data and determines which selected bindings inherit the
pipeline profile. Configuration remains the canonical owner of the exact keys,
profile precedence, and validator order.
Private structured-LLM response schemas are deliberately minimal. They reject
invalid JSON structure, missing required fields, incompatible types, and
unknown fields, while preserving semantic candidates for deterministic
validation. Do not promote a private response envelope into a durable schema;
the contracts above define durable data.
## Prompt Construction
D&D extractors assemble prompts from an ordered manifest of shared and
module-owned assets. Reuse the shared D&D system, evidence, identity,
reference, and transcript assets instead of copying their text into individual
modules. A manifests declared sequence, including cache-control placement, is
part of the prompt behavior.
Every maintained D&D LLM prompt selects `dnd-extraction` as its default
profile. The D&D registrar embeds that fallback profile with the maintained
OpenRouter model, timeout, and service-tier policy. An operator may provide a
complete profile with the same ID through the configured PromptKit source; that
definition replaces the fallback rather than merging with it. The fallback
leaves reasoning and optional sampling controls unspecified. Deployment profile
selection and the maintained operator example are documented in
[Configuration](../config.md#promptkit-profiles).
All extraction prompts share this four-message rendered prefix: the system
message without cache control, the identity message without cache control, the
campaign-reference message with ephemeral cache control, and the chunk
transcript message with ephemeral cache control. This gives equivalent
extraction requests the same reusable prefix through their source material.
Extraction-evidence policy, generated NPC registries, spell catalogs, module
tasks, and instructions follow the transcript because they are not universal
across all extraction lanes. The final instructions message carries ephemeral
cache control; evidence, registry, catalog, and task messages do not. Preserve
this division when changing an extractor or its assets so prompt-cache behavior
remains stable.
The other D&D LLM prompts intentionally follow different patterns. Scene
chunking has no sibling extraction lane with which to share its full transcript,
so it renders campaign references before its task and instructions, then places
the cacheable full transcript last. NPC normalization keeps its task and
cacheable instructions before the candidate collection, followed by the
cacheable transcript windows: candidates must be available before their
supporting evidence is evaluated, and those windows are not a cross-lane
prefix. Mounted assets and their declared message order determine the prompt
fingerprint, so intentional prompt edits continue to invalidate stale
checkpoints.
All extractors use the shared prompt-input preparation rules. The current chunk
is copied into transcript material; player, party, glossary, and compatible
campaign references are context for disambiguation, not source evidence.
Reference prompt material is canonically ordered before it is rendered, which
keeps equivalent inputs stable across runs.
## Evidence, Candidates, And Normalization
The current transcript is the only durable evidence source. Extractors assign
the current source identity, preserve candidate evidence ranges for validators,
and canonically order or remove exact duplicate ranges without asking the
model to repair semantic errors. Campaign context and generated artifacts may
ground names or control routing, but they never establish evidence for a D&D
result.
Default chains keep responsibilities separate: structural validators assess the
candidate, source-reference validators resolve cited ranges against the current
source, durable-schema validation checks an approved representation, and
relatedness validators report advisory evidence concerns. The configured order
is documented in
[Configuration](../config.md#production-validator-keys-and-default-chains).
Normalizers are deterministic for spells, combat turns, item events, NPC
interactions, and scene descriptions. They canonicalize display values and
evidence, use source-document order for stable output, and issue bounded
warnings for changes or collapsed duplicates. The NPC normalizer is the
intentional exception: it first produces a deterministic candidate set, then
uses a bounded structured-LLM proposal to reconcile identity groups. Invalid
or unusable proposals retain the deterministic result and surface retry or
fallback diagnostics; the model does not directly replace durable records.
## Generated References And Grounding
Normalized D&D artifacts can be handed to a later step through a generated
reference binding. The framework verifies artifact compatibility and retains
producer provenance; consumers resolve the handed-off artifact into an
immutable, validated projection for each operation. External files are checked
during preparation, while generated artifacts are resolved at the handoff.
NPC registries are names-only grounding projections: they may canonicalize
actors for spells and combat turns and are required for NPC interactions, but
they do not supply evidence. Scene-description registries are eligibility-only
projections: they retain the current chunks classification data, not scene
prose or evidence, and exist to route combat extraction.
## Lane-Specific Rules
The following differences are intentional and should remain explicit when a
shared helper changes.
| Lane | Intentional behavior |
| --- | --- |
| Spells | May use a spell-catalog overlay and optional NPC grounding; the catalog validator supplies domain-specific semantic checks. |
| NPCs | Does not consume an NPC registry. Its normalizer is the LLM-assisted reconciliation exception described above. |
| Combat turns | Requires a scene-description artifact. It calls the LLM only for an exact `combat` classification; exact non-combat classifications return an accepted empty result, while missing or mismatched classifications return an empty result with a bounded warning. Optional NPC grounding never becomes evidence. |
| Item events | Uses campaign context for disambiguation but has no NPC-registry or scene-description dependency. |
| NPC interactions | Requires the normalized NPC registry at extraction and normalization, using it for canonical actor grounding only. |
| Scene descriptions | Produces the classifications consumed by combat routing; it does not consume an NPC registry or provide evidence for combat artifacts. |
The combat and scene-description contracts describe their exact handoff and
empty-result behavior in more detail:
[combat turns](../integrations/dnd-combat-turn-artifacts.md) and
[scene descriptions](../integrations/dnd-scene-description-artifacts.md).
## Focused Verification
When changing D&D behavior, test the affected codec, extractor, normalizer,
validator, prompt-asset manifest, and registry projection. Also test generated
handoffs at the integration boundary and run the full D&D module suite:
~~~sh
go test ./internal/modules/dnd/...
go test ./internal/modules/integration/...
~~~

View File

@@ -1,235 +1,251 @@
# LLM Runtime Internals
`internal/framework/llm` implements Notarius's transport boundary for structured
completion. It contains the Scriptorium adapter, concurrency scheduler,
prompt/schema registries, selected-profile recording, and provider-error
redaction.
`internal/framework/llm` is Notariuss provider-independent structured
completion boundary. It adapts framework requests to PromptKit, bounds
provider calls, assembles registered prompt and schema assets, records selected
profiles, and redacts provider errors. The architectural boundary is defined in
[Architecture](../policy/architecture.md#llm-boundary); profile sources,
credentials, and concurrency settings belong in
[Configuration](../config.md#promptkit-profiles) and
[Configuration](../config.md#concurrency-output-cache-and-debug).
Provider-neutral ownership rules are defined in
[Architecture](../policy/architecture.md#llm-boundary). Profile sources,
credentials, and concurrency settings are defined in
[Configuration](../config.md).
## Structured Completion Boundary
## Structured Contract
Modules and LLM-backed validators depend only on
`contracts.StructuredLLMClient`. A completion request supplies a prompt ID and
version, optional profile and session IDs, named input material, variables, and
a caller-owned decode target. The successful response returns the validated raw
structured bytes together with non-secret provider, model, profile, and token
metadata.
Modules and LLM-backed validators depend on
`contracts.StructuredLLMClient.CompleteStructured`. A request identifies a
prompt and optional profile/session, supplies named input materials and
variables, and provides a caller-owned decoding target. A successful response
contains the validated raw structured bytes plus non-secret provider, model,
profile, and token metadata.
The caller owns the domain behavior: it chooses the prompt, prepares inputs,
selects the private response schema, and interprets the decoded result. The
adapter does not own source evidence, artifact conversion, normalization, or
durable schemas. Those responsibilities remain with the module and its
[integration contract](../integrations/).
The caller owns prompt selection, response-schema selection, and interpretation
of the decoded result. `LLMInputMaterial` keeps source and reference bytes with
their origin metadata so the adapter can pass named artifacts to Scriptorium
without exposing Scriptorium types through stage contracts.
`PromptKitClient` validates the request target and prompt identity, maps each
named material to a PromptKit inline artifact while preserving its origin URI,
maps the trimmed request session to PromptKit's direct per-run session field,
retains the same value as the `session_id` prompt variable for maintained
prompt compatibility, and forwards profile selection. It then creates one
frozen prepared execution, captures its caller-owned credential-redacted
details for debug material, and executes that exact snapshot through
PromptKit's prepared-execution boundary. The direct field
is authoritative for provider session behavior. A session ID is a stable,
non-secret correlation identifier and may be exposed to providers and provider
observability. The adapter returns PromptKits validated raw bytes rather than
re-encoding the decoded target. An empty optional material is represented as
one space so its named input is retained by PromptKit.
## Production Construction
Client construction may also receive a run-wide reasoning-effort override from
the CLI factory boundary. The adapter copies the caller-owned pointer and
creates a fresh PromptKit execution override for each request: a nil pointer
inherits the selected profile, a non-empty value replaces it, and an empty
value clears inherited reasoning. The CLI's mutually exclusive
`--reasoning-effort` and `--clear-reasoning-effort` controls select those
states. With neither flag, profile behavior remains unchanged. Because
production constructs one shared client, the selected state applies uniformly
to module calls, retries, and LLM-backed validators for the whole run.
`internal/cli` constructs the production runtime by:
An empty request profile lets the prompt select its configured default. Before a
run begins, the CLI asks the adapter to inspect every explicit profile on the
resolved selected LLM-backed bindings and validators, including inherited
pipeline profiles. Inspection resolves the profile and its selected backend and
target without loading a prompt, reading credentials, admitting capacity, or
contacting a provider, so a missing or invalid explicit profile fails before
stage execution while a valid `api_key_env` may remain unset. Calls record the
profile actually selected by PromptKit. The recorder trims and deduplicates
non-secret profile identity, provider, model, selected backend ID, and
effective reasoning values for manifest use. Entries that differ in backend or
reasoning remain distinct and deterministically ordered. Endpoint-only profiles
retain an empty backend ID, which the published JSON omits. Successful
completion responses and recorded profile manifests identify the adapter
provider as `promptkit`.
1. allocating the asset registry populated by the generic, Seriatim, and D&D
package-family registrars;
2. creating a `ScriptoriumClient` from the effective profile source;
3. attaching an `LLMProfileRecorder`;
4. creating a scheduler from the effective concurrency limit;
5. returning a `ScheduledClient` wrapper;
6. decorating that shared client before preparation when debug recording is
enabled; and
7. injecting that one shared client into complete pipeline preparation before
the source file is read or the runner is invoked.
The CLI's profile-inspection engine and the production adapter use the same
profile-source construction to apply the configured profile directory or file,
the optional registered fallback profile assets, and the optional conventional
`local` backend. Preflight therefore resolves the same profile sources and
backend membership as runtime without performing generation. Fallback assets
are mounted only when at least one source is registered. The production D&D
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
prompts select that logical ID by default. PromptKit owns source precedence and
profile parsing: an operator-provided matching profile takes precedence over a
fallback profile without Notarius merging either document.
When the registration is absent, a profile selecting `backend: local` fails
inspection instead of falling back to a built-in or endpoint-only target.
The D&D scene chunker and spell, NPC, combat-turn, NPC-interaction, and
scene-description extractors retain this
injected client and use it for every structured completion. Operation requests
do not carry an LLM client.
Before execution, the adapter also contributes a non-secret checkpoint
fingerprint for the effective PromptKit profile source. It combines the
identity of PromptKit's compiled-in profile catalog with a deterministic digest
of every YAML profile in the configured profile directory, or of the configured
profile file, and a deterministic digest of the flattened fallback profile
assets. The fingerprint contains neither profile content nor source paths. It
covers inherited pipeline profiles, explicit binding profiles, and
prompt-selected defaults, so changing a model or other profile setting cannot
reuse checkpoints created under the
prior profile source. This cache identity is independent of durable
profile provenance: run manifests continue to list only profiles actually
observed during LLM calls. When the local backend is registered, a second
fingerprint hashes its trimmed endpoint behind a stable marker. Changing that
semantic execution target invalidates checkpoint reuse. The raw endpoint is not
stored in checkpoint identity, and the local concurrency limit is excluded
because it changes scheduling rather than execution semantics.
The CLI separately gathers explicit profile IDs from resolved LLM-capable stage
and validator bindings. It prepares a small internal check prompt for each ID so
missing or invalid profiles fail before pipeline execution. The runtime profile
override syntax and scope are defined in the
[CLI reference](../cli.md#run); binding rules are defined in
[Configuration](../config.md#module-bindings).
## Shared Provider-Call Limit
## Scriptorium Adapter
Production construction creates one PromptKit client and wraps it in one
scheduled client. The scheduler has a fixed, positive permit limit, serves
queued calls in FIFO order, and removes a queued call when its context is
cancelled. A granted permit is released exactly once on every completion path.
`ScriptoriumClient` converts a Notarius request into a Scriptorium `RunRequest`.
It validates the decoding target and prompt identity, maps named input materials
to inline artifacts, forwards explicit profile and session context, delegates
rendering/provider execution/structured validation, and unmarshals successful
JSON into the caller target.
The scheduled wrapper surrounds every `CompleteStructured` call, so concurrent
lanes, pipeline retries, and LLM-backed validators share the same provider-call
ceiling. This ceiling is independent of pipeline worker concurrency; changing
worker counts cannot exceed the configured LLM limit. The configuration field
and its effective default are owned by
[Configuration](../config.md#concurrency-output-cache-and-debug).
Empty optional input material is represented by a single space so Scriptorium
retains the named input. The client returns Scriptorium's validated structured
bytes rather than re-encoding the caller target, allowing modules to preserve
the runtime result exactly.
Selected profile, provider, model, and token metadata are mapped into the
Notarius response. The recorder deduplicates profiles by identity and supplies
manifest-safe profile summaries after actual calls; manifest population does
not guess the selected prompt default in advance.
Generated-output validation failures and provider failures are wrapped with
prompt context. Error strings pass through bearer-token redaction before they
cross the runtime boundary.
## Scheduling
`Scheduler` uses a bounded permit count and a FIFO waiter queue. Immediate
acquisition increments the in-flight count; queued acquisition waits for a
permit or context cancellation. Cancellation removes a queued waiter, while a
cancelled waiter that has already received a permit releases it.
`ScheduledClient` acquires a permit around each structured completion and
defers release on every result path. The effective limit and default are
configuration facts in [Configuration](../config.md#defaults).
This provider-call ceiling is independent of the pipeline's extract worker
limit. Concurrent lanes, retries, and validators all use the same scheduled
client, so increasing framework workers cannot exceed `total_llm`. Pipeline
dispatch and cancellation mechanics are documented in
[Pipeline Internals](pipeline.md#execution-flow).
PromptKit applies a second, independent admission limit when the selected
profile names a limited backend. It sits beneath the Notarius scheduled client,
so it may narrow but cannot expand the application-wide limit. Built-in
OpenRouter profiles select PromptKit's reserved backend and its upstream
capacity policy. A positive configured local-backend limit bounds active local
generations inside PromptKit; zero leaves that backend unlimited there.
Endpoint-only profiles do not select a PromptKit backend and remain limited
only by the Notarius scheduler.
## Prompt And Schema Assets
`AssetRegistry` combines caller-owned prompt filesystems under stable prefixes
and rejects invalid or conflicting registrations. Production module packages
register their own prompt and schema assets; generic framework code contains no
D&D prompt content. `internal/framework/promptfs` provides the domain-neutral
filesystem composition helper used to combine module-owned files with shared
domain prompt fragments.
An `AssetRegistry` collects prompt, schema, and optional fallback-profile
filesystems from production module families. It flattens registered roots into
the corresponding PromptKit filesystems and rejects invalid roots, unreadable
assets, duplicate paths, and missing prompt or schema files during preparation.
Fallback assets receive a safe content digest for checkpoint identity; raw
paths and bytes are never included. The frameworks `promptfs` helper combines
module-owned prompt files with reusable domain fragments without making the
framework depend on D&D content.
The D&D scene chunker and spell, NPC, combat-turn, NPC-interaction, and
scene-description extractors each declare an
ordered prompt asset manifest. The manifest lists the package-owned YAML and
Markdown files, then the exact shared fragments rendered by that prompt; the
same ordered list drives both filesystem mounting and the prompt fingerprint.
Unused shared assets are neither mounted nor fingerprinted. Universal
extraction-evidence and output policy lives only in the shared extraction
assets; package-owned prompt files retain artifact-specific rules. The scene
prompt keeps its separate output rule because it does not render the
extraction-evidence asset.
Each LLM-backed module owns its prompt declaration, package-specific assets,
and private response schema. Shared D&D wording is owned by the D&D shared
asset package; the detailed D&D conventions are in
[D&D Module Internals](dnd.md). The mounted prompt assets used by a module also
determine its prompt fingerprint. Schema loaders validate JSON, attach identity
and digest metadata, make defensive copies, and expose diagnostics without raw
schema bytes.
### D&D Extraction Prompt Ordering And Cache Boundaries
Private response schemas validate a model transport envelope. They are not the
durable artifact schema and should not be documented as an external wire
contract. Durable formats and compatibility rules remain in the
[integration contracts](../integrations/).
D&D extraction prompts order messages from the most reusable content to the
most variable content. New extraction lanes use these tiers in order:
## Prompt Maintenance And Backend Caching
1. universal shared content, including the system, extraction-evidence, and
in-world identity messages;
2. stable campaign or run context shared across lanes, including campaign
references;
3. stable subset- and lane-specific context and instructions, including an NPC
registry, catalog, task, or extraction instructions when applicable;
4. the chunk transcript as the final user message.
Prompt message order and shared asset bytes are runtime behavior. Backend cache
reuse depends on identical preceding roles, rendered bytes, and cache-control
metadata—not merely equivalent meaning. Keep reusable shared assets
byte-identical and preserve each prompts declared ordering and cache controls
when editing it.
This ordering lets requests reuse the longest identical prefix before the
per-chunk transcript changes. Cache reuse requires the preceding message
sequence and content to be exactly identical; semantic similarity is not
sufficient. Cache boundaries belong at the ends of reusable stable tiers,
subject to the provider's cache-boundary limit. The shared identity and
campaign-reference messages form the first two extraction boundaries. Spell,
combat, and interaction prompts add a boundary at the shared NPC registry. Each extraction
prompt places its final boundary on its lane-specific instructions, immediately
before the transcript. The transcript does not carry cache control because no
reusable content follows it.
For sibling prompts that can reuse the same source material, order universal
shared context first, request source material next, and module-specific
suffixes last. Put a cache boundary at a reusable prefix that is useful to the
backend. Redundant intermediate cache boundaries do not extend that reusable
prefix and add no value.
Accordingly, the common prefix of the spell, NPC, combat, and interaction
extraction prompts is system,
extraction evidence, identity, and campaign references. The NPC prompt then
renders task, instructions, and transcript. Spell renders the NPC registry,
catalog, task, instructions, and transcript. Combat renders the NPC registry,
task, instructions, and transcript. NPC interaction renders the names-only NPC
registry, task, instructions, and transcript. The
scene chunker is not an extraction lane: it retains its separate system,
transcript, campaign-reference, task, and instruction order and marks its
transcript and campaign-reference messages ephemeral.
Prompt-family owners may choose a different sequence when their inputs and
reuse pattern differ. The D&D familys extraction, scene-chunking, and NPC
normalization policies are maintained in [D&D Module Internals](dnd.md#prompt-construction).
Do not add tests that enforce prompt prose; prompt tests should verify the
meaningful input placement and cache controls of the prompt being changed.
The scene-description extractor deliberately omits the citation-oriented
`common-dnd-extraction-evidence.md` asset because Notarius attaches the whole
accepted chunk range itself. Its manifest is system, shared identity, shared
campaign references, lane task, lane instructions, then the transcript. The
identity, campaign-reference, and instruction messages are ephemeral cache
boundaries; the transcript is last and has no cache control. Compatible shared
messages remain canonical shared assets rather than copied package text.
## Validation, Repair, And Retries
Shared wording belongs in the canonical assets under
`internal/modules/dnd/shared`; extraction packages reference those assets in
their manifests instead of copying similar text into package-local files.
Package-local assets contain only lane-specific content. An extraction lane may
depart from the tier order only when prompt-quality evidence or a provider
constraint makes the exception necessary; document the exception and rationale
here when it becomes implemented behavior.
PromptKit performs prompt rendering, provider execution, and the prompts
structured-output validation. The adapter reports an empty result, validation
failure, empty structured body, or decode failure as
`ErrInvalidStructuredOutput`, while retaining the returned raw bytes and debug
material when they exist. Provider failures remain operational errors rather
than output-validation failures.
Schema helpers load embedded JSON Schema with identity and digest metadata,
return defensive copies, and expose a diagnostics map that omits schema bytes.
The small framework registry contains only generic test schemas; production
schemas remain package-owned.
When PromptKit rejects backend admission before generation, the adapter maps
`promptkit.ErrCapacityExceeded` to
`contracts.ErrLLMCapacityExceeded`, retaining prompt context and a redacted
upstream diagnostic without exposing the PromptKit sentinel or capacity-error
type as a framework contract. When supplied, the normalized selected backend
ID appears only in that safe application-owned diagnostic context. A canceled
caller context takes precedence. The adapter does not retry capacity failures;
the pipeline's existing binding attempt policy sees the operational error and
decides whether to rerun the complete operation.
The spell, NPC, combat, NPC-interaction, and scene-description extractors'
package-owned prompts declare their
structured JSON inputs and private response schemas. Each private response
schema remains separate from its durable artifact codec schema; this work does
not use shared schema fragments or schema generation. Those private schemas own
the transport envelope—required fields, JSON types, nullability, and
unknown-field rejection—while deterministic validators own semantic constraints
such as enum membership, non-empty values and collections, and positive
numbers. The spell extractor's prompt declares a required
`application/json` `spell_catalog` input and an optional `application/json`
`npcs` input. The extractor generates
the catalog input from its prepared
effective catalog as `{"spell_names":[...]}` using sorted canonical names only.
The shared D&D prompt assets include a generic NPC grounding fragment directly
after the campaign reference message for spell, combat, and interaction prompts. When an NPC
registry is bound, the
domain registry boundary strictly decodes and identity-validates one durable
artifact, re-encodes canonical JSON for provenance, and separately generates a
names-only prompt projection. The unbound projection is exactly `{"npcs":[]}`.
Prompt input and component-local checkpoint digests cover the projected bytes;
manifests retain the optional full registry digest/count rather than names,
overlay bytes, registry paths, or source metadata. Combat and interaction prompt,
response-schema, mapping, normalization, identity, and registry-projection
fingerprints remain separate semantic inputs to checkpoint identity.
Prompt-declared repair is executed within PromptKits structured-output flow.
The current production D&D prompt manifests set repair attempts to zero. That
setting does not replace pipeline retry behavior: a bindings configured retry
count reruns its stage attempt after an error or rejection, and an exhausted
rejection is a recorded output rather than a provider error. The pipeline owns
attempt lifecycle, validation chains, and retry diagnostics; see
[Pipeline Internals](pipeline.md#validation-retries-and-output) and the
[binding reference](../config.md#module-bindings-and-validators).
## Debug And Redaction Boundaries
## Timeout Ownership
The pipeline may wrap the client with a debug recorder that captures prepared
prompt/response material for an explicitly requested debug run. Debug summaries
and manifests receive identities, hashes, usage, and selected profile summaries
rather than prompt, source, reference, schema, or response content.
The caller context remains the outer cancellation authority. PromptKit applies
a positive effective generation timeout as an inner request deadline; an
explicit zero disables only that generation deadline. The HTTP client timeout
is a separate transport-wide cap. Notarius forwards the caller context and
does not install another timeout wrapper around PromptKit.
The Scriptorium error wrapper removes bearer credential values from surfaced
provider errors; `RedactSecrets` and `ErrorWithSecretsRedacted` support known
secret values elsewhere in the runtime. Config summaries use a separate
clone-and-redact path in `internal/core/config`. These mechanisms implement the
security invariant in
[Architecture](../policy/architecture.md#state-output-and-safety); operator
handling of debug data is defined in [Operations](../operations.md#debug).
The selected PromptKit profile owns generation settings. Notarius binding
retries remain outside the adapter and repeat the complete module operation
and validation chain. PromptKit does not add a provider retry loop.
Operator-facing behavior is summarized in
[Operations](../operations.md#operational-limits), and the pinned upstream
contract is identified in
[PromptKit Integration](../integrations/pkg-promptkit.md).
## Failure Behavior
## Observability And Redaction
- Invalid targets, missing prompt IDs, malformed structured output, and
Scriptorium failures return contextual errors to the calling module.
- Scheduler construction rejects non-positive limits; acquisition respects
context cancellation.
- Asset registration rejects invalid roots, missing content, and path conflicts.
- Schema loading distinguishes missing assets, invalid JSON, and invalid
metadata.
- Profile validation errors occur during CLI preparation when an explicit
selected ID cannot be prepared.
When debug recording is enabled, the pipeline decorates the shared client. The
wrapper records prepared prompt and response material, timing, selected profile
and backend, effective model parameters, and call identifiers in the runs
debug bundle, including material available from a failed structured completion.
Effective parameters use PromptKit's stable lower-case JSON field names and may
include `backend_id`. For a successful completion, a debug-write failure is
surfaced; when the completion already failed, its call error remains the
result. Debug-bundle location, retention, and handling are operational concerns
documented in [Operations](../operations.md#debug-bundles).
## Tests To Inspect
Run manifests receive selected profile summaries, including optional effective
backend and reasoning provenance, and component identities—not prompt, schema,
source, reference, or response content. The published field semantics belong
to the [JSON output contract](../integrations/json-output.md#manifestjson).
Provider error text is wrapped with prompt context and bearer credentials are
redacted before it crosses the runtime boundary. Known-secret redaction is
available to other runtime collaborators; it does not make prompt or response
contents safe for general logging.
- `internal/framework/llm/scriptorium_client_test.go`: adapter mapping and local
HTTP integration.
- `internal/framework/llm/scheduler_test.go` and
`scheduled_client_test.go`: permits, FIFO behavior, cancellation, and wrapper
release.
- `internal/framework/llm/asset_registry_test.go` and
`schema_registry_test.go`: asset composition, validation, and defensive
copies.
- `internal/framework/llm/secrets_test.go`: provider-error redaction.
- `internal/cli/run_contract_test.go`: profile validation, production client
wiring, manifest recording, and debug integration.
- Module-local `scriptorium_assets_test.go` files: prompt inputs and package
asset registration.
## Failure Boundaries
- Construction fails for missing asset registries, mutually exclusive profile
sources, invalid asset registration, or a non-positive scheduler limit.
- Preparation failures, unavailable explicit profiles, provider failures, and
context cancellation propagate to the calling stage with context.
- Backend admission exhaustion is a provider-neutral operational error and is
not classified as invalid structured output or validator rejection.
- Malformed or schema-invalid provider output is classified separately as
invalid structured output so the module or pipeline can apply its own retry
and rejection policy.
- Domain semantic checks, evidence decisions, and deterministic normalization
run outside the provider adapter.
## Focused Verification
Read the LLM adapter, scheduler, asset registry, schema loader, and redaction
tests when changing this boundary. Prompt changes also require the owning
modules asset tests, and retry or debug changes require focused pipeline or
CLI coverage. The focused runtime and D&D checks are:
~~~sh
go test ./internal/framework/llm/... ./internal/modules/dnd/...
~~~

View File

@@ -1,591 +1,113 @@
# Module And Validator Internals
Production module and validator implementations live under their domain-first
trees in `internal/modules`.
The selectable keys, configuration options, reference slots, and default
validator chain are canonical in the
[module](../config.md#implemented-production-modules) and
[validator](../config.md#implemented-production-validators) catalogs in
Configuration.
## Extension Pattern
A stage module package provides a stable key, constructor, contract
implementation, `ModuleSpec`, `Register`, and focused behavior and registration
tests. A validator package follows the same pattern with `ValidatorSpec` and the
validator registry. Package-family registrars compose those leaf registrations
into the production catalog and own family-level policy such as default
validator chains and prompt asset collection.
Production input, chunk, output, and D&D spell-, NPC-, combat-, interaction-, and scene-description-extract packages
register strict option decoders and run-local builders. Preparation decodes their options into
implementation-owned values and injects dependencies plus the materialized
reference set for the selected target. Each builder receives an isolated clone
of that set; input and output builders receive no references. The spell, NPC,
combat, interaction, and scene-description extractors are typed over the canonical D&D model. D&D validators, merge,
and normalize use typed variants; JSON representation validators use serialized
requests; and unconditional validators expose separate chunk and typed
variants. The D&D production registrar registers the canonical typed spell,
NPC, combat, interaction, and scene-description implementations, including their kind-specific merge and
normalize behavior.
For D&D artifact defaults, generic JSON syntax validation runs first. Rejecting
domain validators then own semantic diagnostics before generic JSON Schema
validation provides the final rejecting representation backstop; warning-only
relatedness validators run last. This default composition does not reorder an
explicitly configured validator chain.
Prepared extractors, extract validators, and codecs may be reused concurrently
by the run-wide extract pool. Production implementations are immutable after
construction: they retain only typed options, immutable assets, or the shared
concurrency-safe LLM client. Implementations that introduce mutable state must
synchronize that state without creating a separate provider scheduler.
Specs expose capability and execution metadata without constructing an
implementation. Registry entries separately expose option validation and
run-local construction. Chunk, extract, merge, and normalize modules that accept
auxiliary material declare identical reference slots from both
`ReferenceSlots()` and `ModuleSpec().ReferenceSlots`; registration tests enforce
that agreement. Runtime delivery uses the corresponding stage request's
`References` field.
LLM-backed extensions own their prompt definitions and response schemas under
package-local embedded assets. Shared filesystem composition belongs in
`internal/framework/promptfs`; reusable D&D prompt fragments, reference
declarations, prompt-input assembly, and source-unit/citation helpers belong in
`internal/modules/dnd/shared`, which also owns bounded D&D diagnostics. The
D&D scene chunker and spell, NPC, combat-turn, NPC-interaction, and scene-description extractors use ordered
package-local prompt manifests for both rendering and prompt fingerprinting, so
only the shared fragments each prompt actually renders participate in either
operation. Extraction prompts place stable shared and lane-specific context
before the variable transcript and use shared assets for wording common across
lanes. The canonical ordering and cache-boundary policy is documented in
[LLM Runtime](llm.md#dd-extraction-prompt-ordering-and-cache-boundaries). Stage
contracts expose only Notarius structured-completion types, not Scriptorium
public types.
The shared `ChunkPromptMaterial` helper owns common transcript material
preparation for the spell, NPC, combat-turn, and NPC-interaction extractors. It clones supplied
source metadata, falls back to the materialized chunk when content is absent,
checks that content remains chunk-identical, and fills only the common default
fields. Extractors retain their request validation and wrap helper errors with
their module context.
Reference material may inform a module or prompt but must not become source
evidence. The resolver and materializer behavior is described in
[Pipeline Internals](pipeline.md#reference-materialization).
## Domain Reference Data
### `internal/modules/dnd/spells/catalog`
The spell catalog package owns the embedded, versioned D&D 5e 2014 SRD spell
reference data. Its strict JSON asset contains one canonical record per spell,
including spell level and all applicable class memberships. `LoadSRD5E2014`
validates catalog identity, provenance metadata, ordering, uniqueness, levels,
classes, aliases, and lookup-key collisions before exposing immutable copies.
Lookup is case-insensitive and normalizes whitespace and common apostrophe
variants while preserving source punctuation in canonical display names. The
catalog contains 319 unique spells and 779 class memberships. Source and
license details live beside the asset in `SOURCES.md`. This domain-owned data is
separate from `internal/modules/dnd/shared`, which is reserved for reusable
prompt and source-reference machinery.
`ResolveEffectiveCatalog` builds the immutable recognition view used by the
spell extractor and catalog validator. It starts with the embedded SRD catalog
and optionally applies one strict JSON overlay from the `spell_catalog` item in
a materialized reference set. Overlay catalogs are ordered by ID, may add names
and aliases, and may augment an existing canonical spell without replacing its
display name. Cross-spell lookup collisions are errors. The effective view
exposes sorted canonical names, normalized lookup, overlay identities, and a
semantic digest; overlay content remains contextual reference material rather
than source evidence. Its external JSON contract is defined in the
[spell-catalog overlay contract](../integrations/dnd-spell-catalog-overlays.md).
### `internal/modules/dnd/npcs/identity`, `internal/modules/dnd/npcs/registry`, and `internal/modules/dnd/codec/npcs`
The NPC identity package owns Unicode comparison keys, deterministic
`npc:sha256:` IDs, display normalization, and whole-registry collision issues.
The registry package resolves one optional normalized artifact through the
strict codec, validates whole-registry identity, canonicalizes its JSON, and
provides immutable records, a names-only prompt projection, distinct durable
and projection digests, count, and exact canonical-name lookup. External files cross this boundary during
preparation; generated artifacts cross it at the ordered step handoff. It owns
the `npcs` slot and its bounded, content-safe validation failures. NPC source
references are durable provenance and are not treated as evidence for a
consuming pipeline. The NPC codec owns the strict durable `dnd/npc-list` JSON
boundary and exposes candidate versus approved encode/decode operations.
The `internal/modules/dnd/codec/combatturns` package owns the durable
`dnd/combat-turn-list` schema and candidate versus approved JSON boundary. It
is registered by the production D&D family registrar for the selectable combat
lane.
The `internal/modules/dnd/codec/npcinteractions` package owns the durable
`dnd/npc-interaction-list` schema and candidate versus approved JSON boundary.
It is registered by the production D&D family registrar for the selectable
interaction lane. Its external contract is documented in the
[D&D NPC interaction artifact contract](../integrations/dnd-npc-interaction-artifacts.md).
The `internal/modules/dnd/codec/scenedescriptions` package owns the durable
`dnd/scene-description-list` schema and candidate versus approved JSON boundary.
It is registered by the production D&D family registrar. Its external contract
is documented in the
[D&D scene-description artifact contract](../integrations/dnd-scene-description-artifacts.md).
## Input Adapter
### `internal/modules/seriatim/input/transcript`
The adapter decodes the supported transcript JSON, selects the source identity,
computes canonical source provenance, validates segments, and maps each segment
into a generic source unit with a self-reference plus speaker and timestamp
metadata. It accepts no module options. Its spec advertises the transcript
capabilities consumed by D&D modules.
Parsing is strict about required values and duplicate unit IDs but deliberately
ignores unrelated Seriatim fields. The external format and derived-identity
rules are defined in the
[Seriatim contract](../integrations/seriatim.md).
## Chunkers
Chunkers implement `contracts.Chunker.Plan`. A plan identifies ordered source
unit ranges and may carry optional namespaced JSON annotations; it does not
contain materialized chunk content. The framework canonicalizes annotations,
validates ranges against the current source, and materializes chunk IDs,
indexes, references, content, units, and generic metadata. Materialized source
unit metadata is independently owned. Annotation
namespaces remain optional data: generic framework code and downstream modules
must not require D&D scene annotations or import `dnd/scenes`.
### `internal/modules/generic/chunk/units`
The generic chunker validates the source document and returns ranges over units
in configured windows. Overlap changes the next window start but never reorders
units. Framework materialization derives the resulting chunk identity and
generic metadata from those ranges.
The accepted options and defaults are defined in
[Configuration](../config.md#implemented-production-modules). Generic
framework validation canonicalizes the returned unit slices before extraction.
The chunker decodes its options during construction and retains only the typed
window settings used by `Plan`.
### `internal/modules/dnd/chunk/scenes`
The scene chunker prepares a structured Scriptorium request from the full
transcript, session, and optional D&D reference inputs. It validates the model's
inclusive source-unit endpoints against document position and converts them
into deterministic plan ranges. Preparation injects the shared structured LLM
client into the chunker; `Plan` supplies only the run-specific profile, session,
source, references, and metadata.
Scene validation requires sequential, contiguous, non-overlapping coverage from
the first source unit through the last. Its private response contains only the
boundary endpoints; the accepted plan has no D&D-specific annotations and
produces no boundary warnings. Malformed structured output is returned as an
error; there is no fallback chunker.
The package embeds its prompt and response schema and reports their non-secret
identity and hashes through singleton module metadata. Shared D&D assets supply
reference declarations and prompt inputs; their user-facing keys and accepted
file types remain canonical in [Configuration](../config.md).
## Extractor
### `internal/modules/dnd/extract/spells`
The spell extractor prepares a structured request from one chunk, the
chunk-scoped source input, the session, and optional D&D reference inputs. It
decodes the model response, assigns the generic source identity to every source
reference, canonicalizes duplicate references, orders spell casts by their
earliest cited unit, and returns `dnd.SpellList`.
Its private response schema admits only the structural transport envelope:
required fields, JSON types, array and object shapes, and unknown-field
rejection. It maps integer source-unit candidates directly without repairing
semantic values, so the deterministic shape, catalog, and source-reference
validators own blank values, empty evidence, and invalid or unresolved ranges.
The extractor owns its private model-response DTO, embedded prompt, LLM response
schema, strict option decoder, injected shared LLM client, and prompt/schema
manifest metadata. During preparation it resolves the optional `spell_catalog`
reference into an immutable effective catalog and adds a generated
canonical-name-only JSON input to every structured completion request. Overlay
failures therefore stop construction before source parsing or an LLM call;
campaign references remain separate disambiguation inputs and never become
source evidence.
The prompt includes only actual casting events and unambiguous declared casting
attempts. Spell mentions, plans, rules discussion, and catalog matches without
a casting event are excluded. Shared extraction-evidence and identity rules
require transcript-supported caster and spell facts, while the catalog,
campaign references, and NPC names only disambiguate source text. Structural
source validation remains deterministic; semantic evidence sufficiency is
enforced through extraction policy and evaluation.
Both the extractor and deterministic catalog validator expose
the effective base-plus-overlay semantic digest as scoped prepared-component
checkpoint identity. Raw overlay provenance independently covers file-byte
changes, while the semantic digest also invalidates reuse when the embedded
catalog or catalog composition changes. The extractor additionally fingerprints
its complete prompt assets and private response schema, so either semantic
contract changing invalidates previously recorded extraction checkpoints. The
separate `internal/modules/dnd/codec/spells` package
owns the durable schema and stable JSON representation for artifact kind
`dnd/spell-list`. The runner keeps the result typed through validators and later
stages, using the codec only for checkpoint, debug, and output boundaries.
Shared D&D helpers keep prompt input names and source-unit reference conversion
consistent with the scene chunker.
The extractor also declares the optional `npcs` registry slot and consumes the
immutable registry boundary from `internal/modules/dnd/npcs/registry`. An
external registry is prepared before execution; a generated registry is
validated and supplied at operation time. Bound external registries add only
the full `npc_registry_digest` and `npc_count` to module metadata. The local
`npc_registry` checkpoint fingerprint always covers the names-only projection,
including its exact unbound value. Generated bindings are represented by
framework handoff provenance and dependency fingerprints. The unbound prompt
input is exactly `{"npcs":[]}` and has no registry provenance.
The shared NPC grounding fragment is placed immediately after the common
campaign reference message and is included in the spell prompt fingerprint.
The durable payload and manifest metadata shapes are defined in the
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
### `internal/modules/dnd/extract/npcs`
The NPC extractor maps private model output to the canonical `dnd.NPCList`,
assigns source identity and deterministic NPC IDs, and preserves source
references for deterministic validation. It uses the shared campaign
references only for disambiguation and does not consume the optional NPC
registry slot. Its prompt and private response schema are package-owned. The
private response contains only a name and model-facing evidence ranges for each
record; anonymous groups, generic roles, invented labels, descriptions,
aliases, and relationships are outside its contract. The
prompt follows the shared D&D extraction ordering and cache policy documented
in [LLM Runtime](llm.md#dd-extraction-prompt-ordering-and-cache-boundaries).
The private response schema owns only structural transport validation and maps
integer source-unit candidates unchanged. Required semantic content, non-empty
evidence, and valid source ranges are rejected by the deterministic shape and
source-reference validators.
### `internal/modules/dnd/extract/scenedescriptions`
The scene-description extractor makes one structured completion for each
accepted chunk and maps its private `kind`, `title`, and `summary` response to
one `dnd.SceneDescription`. It assigns the current chunk ID and exact range,
preserves kind without repair, and trims only title and summary whitespace.
Optional players, party, and glossary references can disambiguate prompt terms
but do not supply evidence. The package owns its private schema, prompt assets,
and mapping fingerprint; deterministic validators own the durable semantic
checks. The durable contract is defined in the
[D&D scene-description artifact contract](../integrations/dnd-scene-description-artifacts.md).
### `internal/modules/dnd/extract/combatturns`
The combat extractor prepares one structured request per supplied chunk using
the shared extraction-evidence, identity, campaign-reference, NPC-grounding,
and transcript prompt inputs. It
maps the private response to `dnd.CombatTurnList`, assigns the current source
identity, removes exact duplicate source ranges, and orders turns by valid
source-document position while preserving malformed candidate fields for
deterministic validators. Its package-owned private response schema enforces
only the structural JSON envelope; semantic artifact constraints remain with
the validator chain. Its prepared metadata and checkpoint fingerprints contain
only prompt/schema/mapping identities plus an optional NPC registry digest.
The prompt follows the shared D&D extraction ordering and cache policy
documented in
[LLM Runtime](llm.md#dd-extraction-prompt-ordering-and-cache-boundaries). The
package exposes typed registration and is included in the production D&D
registrar with the default combat extraction chain.
The combat normalizer accepts only the optional structured NPC registry.
Campaign references remain extractor-only LLM context and are not materialized
for deterministic normalization.
### `internal/modules/dnd/extract/npcinteractions`
The NPC interaction extractor requires the structured `npcs` registry slot. It
uses the registry's names-only prompt projection with shared extraction
evidence, identity, and transcript material, then maps private model records to
`dnd.NPCInteractionList` with the current source identity. Registry source
references are never reused as interaction evidence. The private response
schema carries only name, bounded interaction kind, and source-unit ranges;
deterministic validators own registry membership, source validity, and
relatedness. Extract-stage source validation additionally requires every cited
range to be wholly contained in the current materialized chunk. Prompt, schema,
mapping, and the names-only registry projection
participate in checkpoint identity, while generated producer identity remains
framework provenance.
The domain-owned `internal/modules/dnd/npcinteractions` package defines
canonical source-reference and occurrence ordering, valid-evidence eligibility,
and collision-safe exact identity. The interaction normalizer and normalized
invariants validator both consume those rules, so their production and checking
paths cannot drift. Normalizer and relatedness warning lists use the shared D&D
diagnostic cap and emit a final omission-summary warning when truncated.
### `internal/modules/dnd/normalize/npcs`
The NPC normalizer performs deterministic identity-aware consolidation in
merged input order. It consolidates only equal canonical-name comparison keys,
retains the first display record, and unions exact source references. It exposes
the identity policy as its local checkpoint fingerprint and emits bounded
normalization warnings.
## Merger And Normalizer
### `internal/modules/generic/merge/appendorder`
The merger passes typed values to an injected combine function in framework
source-chunk order. The D&D registrar specializes it with a spell-list append
function.
### `internal/modules/generic/normalize/noop`
The normalizer returns the merged domain value unchanged and is reusable for
any registered artifact type.
### `internal/modules/dnd/normalize/spells`
The typed spell normalizer resolves the optional `spell_catalog` reference into
the same immutable SRD-plus-overlay effective catalog used by spell extraction
and catalog validation. It performs no LLM calls. For each spell cast it
canonicalizes recognized names using the catalog's case, whitespace,
apostrophe, and alias rules; sorts source references by source identity and
unit boundaries; removes only exact reference duplicates; and emits bounded,
scoped warnings for each mutation or unresolved name.
After those per-cast changes, it collapses only casts with the same canonical
spell, case-folded and whitespace-normalized caster, and complete non-empty
valid source-reference set. It retains the first occurrence and its caster,
source references, and stable order. Unknown names, empty or invalid evidence,
and adjacent or overlapping but different ranges remain unchanged for
validation.
The normalizer exposes the effective catalog digest as its independently scoped
`effective_catalog` checkpoint fingerprint and reports catalog base ID, digest,
and overlay IDs as manifest metadata. Catalog contents, reference paths, and
raw overlay bytes are not included in either surface. The normalize-stage
reference is stage-local, so an overlay-capable pipeline binds the catalog
independently for extraction and normalization.
### `internal/modules/dnd/normalize/combatturns`
The combat normalizer prepares an external NPC registry before execution or
receives a generated registry at the ordered step handoff, then uses the
immutable view during runtime. It display-normalizes actors,
rewrites canonical-name matches for actors, orders and deduplicates source
references, stable-sorts records by source-document position, and collapses
only exact duplicate identities with fully valid evidence. It deep-clones
output storage and emits bounded warnings scoped to merged input indexes. Its
metadata and fingerprints identify the normalization and NPC identity policies.
External bindings may contribute registry
digest/count metadata; generated identity is retained in framework provenance
and dependency fingerprints. The normalizer is included in the production D&D
registrar with the default combat normalization chain.
### `internal/modules/dnd/normalize/npcinteractions`
The interaction normalizer requires the same immutable NPC registry. It
canonicalizes exact registry-name matches, orders and de-duplicates source
references, stable-sorts occurrences by source-document position, and collapses
only exact interaction identities with valid evidence. It does not infer,
merge, or summarize distinct occurrences. Its metadata and fingerprints expose
the normalization and NPC identity policies; generated registry identity stays
in framework provenance and checkpoint dependencies.
### `internal/modules/dnd/normalize/scenedescriptions`
The scene-description normalizer has no options or references. It validates
each source range against the source document, trims title and summary
whitespace, orders records by source position then ID, removes only exactly
identical records, and rejects conflicting reused IDs or ranges. Its policy
fingerprint identifies this deterministic behavior; the matching invariant
validator checks the normalized result in the production chain.
## Output Encoder
### `internal/modules/generic/output/json`
The JSON encoder sorts normalized results by lane, derives collision-checked
safe logical names, pretty-prints JSON payloads, and assembles the logical index,
manifest, rejected-result, warning, and lane files. Invalid JSON, unsupported
media types, unsafe names, and sanitized-name collisions are errors.
Its strict `include_chunk_map` option is disabled by default. When enabled, it
validates the framework-supplied accepted chunk map through its codec and adds
the pipeline-wide `chunk-map.json` plus its index descriptor; it does not treat
the map as a lane payload. The external shape is owned by the
[Accepted Chunk Map contract](../integrations/chunk-map.md).
The encoder returns logical files only. The CLI places them on disk, and the
[JSON output contract](../integrations/json-output.md) defines their external
paths and schemas.
## Generic Validators
The generic validator implementations live under
`internal/modules/generic/validate`.
The unconditional accept and reject validators provide explicit chunk and
typed-artifact variants used primarily for controlled composition and tests.
The serialized JSON syntax validator uses `encoding/json` to reject malformed
representation bytes. The serialized JSON Schema validator requires schema
bytes, parses the instance and schema with `jsonschema`, and distinguishes
payload rejection from schema loading or compilation errors. The framework
serialized-validation request carries either canonical chunk bytes or artifact
codec bytes according to its target context. Neither validator calls the LLM.
## D&D Spell Validators
All four validators receive `dnd.SpellList` directly. The shape validator
rejects a missing list, blank caster or spell names, and empty reference lists.
The catalog validator defers when shape is invalid, then checks every non-empty
spell name against the immutable effective SRD and overlay catalog. It accepts
normalized canonical names and aliases without rewriting the artifact; unknown
names reject the complete result with bounded, stable index/name diagnostics. The
source-reference validator defers malformed shapes, validates every cited
range, and reports all range defects through a bounded aggregate while
preserving `invalid_source_refs`. The relatedness validator resolves all cited
ranges through the shared document-order traversal, then warns when a normalized
consecutive spell-name token sequence is absent from the cited source text.
Invalid shape
or cited ranges produce no relatedness warnings; the shape and source-reference
validators own those defects.
These validators are deterministic. Shape, source-reference, and relatedness
each expose a local semantic `policy` checkpoint fingerprint. The catalog
validator instead exposes its effective catalog digest as its semantic
checkpoint identity and does not add a separate policy fingerprint. Their
selectable keys and production order are defined in
[Configuration](../config.md#implemented-production-validators); their durable
payload rules are defined in the
[artifact contract](../integrations/dnd-spell-artifacts.md).
## D&D NPC Validators
NPC shape validation checks the required ID and name strings, list presence, and source-reference
shape. The source-reference validator defers malformed shapes, checks
current-document identity, unit existence, and range ordering, and reports all
defects through bounded aggregates. Source relatedness uses the shared
document-order traversal and normalized consecutive-token matching, emitting at
most one bounded warning per record when the canonical name does not occur near
its cited text. Invalid shape or cited ranges produce no relatedness warnings.
Normalize identity validation checks deterministic IDs, canonical names, and
duplicate canonical-name or ID ownership.
All are deterministic and expose the policy fingerprints used by the
production chains.
## D&D Combat Validators
Combat shape validation owns the required list, actor, supported turn kind, and
non-empty source-reference collection. Combat source-reference validation defers invalid
shape, checks source identity, unit existence, and range order, and reports all
defects through bounded aggregates. Combat source-relatedness defers invalid
shape or ranges, uses the shared traversal to combine overlapping cited units
in document order, and emits at most one bounded advisory warning per turn for
an unrelated actor. Actors use normalized consecutive-token matching. The
normalized-invariants validator owns actor display normalization, canonical
source-reference order, chronology, and exact duplicate identity; it defers
shape and source-reference failures. All four validators are deterministic and
expose local policy fingerprints. In the registered defaults, JSON syntax runs
first; combat shape, normalized invariants when applicable, and source-reference
validation precede JSON Schema validation; warning-only relatedness runs last.
## D&D NPC Interaction Validators
Interaction shape validation owns the required list, registry name, supported
kind, and non-empty source-reference collection. Registry validation checks
exact membership in the required immutable NPC registry. Source-reference and
relatedness validation use the current transcript only; malformed candidates
are deferred by later validators and produce no relatedness warning. The
normalized-invariants validator owns canonical registry names, source-reference
order, chronology, and exact duplicate identity. The production chains run
shape, registry, and source-reference checks before JSON Schema validation;
relatedness remains warning-only and last.
## D&D Scene Description Validators
Scene-description shape validation owns the non-empty list, trimmed ID and
prose, closed kind, and basic source-reference shape. Extract-stage source
validation additionally requires the one record to attach exactly to the
current accepted chunk; later source validation checks source membership.
Relatedness checks the title and summary independently against only their cited
transcript range and emits bounded advisory warnings. The normalized-invariants
validator owns ordering, exact duplicate elimination, and conflicting ID or
range detection. The production chains run shape and source-reference checks
before JSON Schema validation; the warning-only relatedness check is last.
## Production Registration
Production composition occurs through family registrars. The CLI allocates one
complete framework registry set and one LLM asset registry. It invokes
`internal/modules/generic/register`,
`internal/modules/seriatim/register`, and `internal/modules/dnd/register` in
that order, then exposes the matching catalog for resolution. The generic and
Seriatim registrars own their production leaf registrations. The D&D registrar
owns D&D leaf registrations, typed spell, NPC, combat, interaction, and scene-description default-validator
chains, typed append-order specializations, and D&D prompt/schema asset
collection. Its registration helpers group module, validator, prompt-asset, and
chain composition while retaining artifact-specific merge and clone behavior in
the registrar.
Concrete implementation packages do not import generic implementation
packages directly. A concrete family's `register` package is its composition
point for specializing reusable generic implementations, while the generic
registrar composes only generic children.
Core and framework production packages do not import production extensions.
CLI production code is the sole application composition root for extensions
and imports only exact family registrar packages. Other production packages,
including commands and newly introduced package trees, do not import module
packages directly. Compatibility tests in the CLI, core, and framework trees
may import roots and implementation leaves directly. Other non-module tests do
not receive that exemption. White-box tests within module families retain the
production family boundaries. `internal/modules/integration` is test
infrastructure: its black-box tests may compose multiple families, but it is
not a production module family or production dependency target.
## Adding An Extension
When adding a production module or validator:
1. implement the stage or validator contract and package-local key;
2. expose and test its spec, constructor, and registration function;
3. keep format or domain parsing inside the concrete package;
4. add package-owned prompt/schema assets when the extension is LLM-backed;
new LLM-backed D&D extraction modules must follow the stable-to-variable
prompt ordering, shared-asset ownership, and cache-boundary policy in
[LLM Runtime](llm.md#dd-extraction-prompt-ordering-and-cache-boundaries), or
document the implemented exception and its evidence there;
5. register it through its package-family registrar and add a default chain
there only when production policy requires one;
6. add resolution and composition coverage for capabilities, options,
references, and validation behavior;
7. update the selectable-key catalog in [Configuration](../config.md), the
relevant external contract, this inventory, and maintained examples when
user-visible behavior changes.
Do not add the extension to `docs/development.md`; that file routes by task and
does not inventory implementations.
## Tests To Inspect
- Package-local `*_test.go` files under the module or validator being changed.
- `internal/framework/pipeline/typed_resolution_test.go`: typed registry, spec,
and heterogeneous artifact composition.
- `internal/framework/pipeline/profile_test.go`: framework binding defaults and
profile resolution.
- `internal/cli/production_contract_test.go`: production catalog, config
resolution, and composition smoke coverage.
- `internal/cli/example_contract_test.go`: maintained example ownership.
- `internal/framework/promptfs/*_test.go` and
`internal/modules/dnd/shared/*_test.go`: shared prompt and reference assembly.
- `internal/modules/integration/*_test.go`: black-box composition across
production extension domains.
# Module Internals
This guide owns the mechanics for implementing and registering production
modules. [Configuration](../config.md) owns selectable keys, binding syntax,
reference configuration, and default validator chains. Durable input and output
shapes belong in [integration contracts](../integrations/).
The D&D family has additional shared conventions and domain-specific
exceptions. See [D&D Module Internals](dnd.md) rather than adding them here.
## Module Boundary
A module is a typed implementation registered for one pipeline stage. Its
`ModuleSpec` is the public-to-the-framework declaration of its stable key,
stage, execution class, required and provided capabilities, artifact kind, and
accepted reference slots. The execution class states whether a module is
`deterministic` or `llm_backed`; registries retain it for catalog inspection and
resolved-pipeline debug data without constructing the module. The framework
uses the declaration to resolve a configured binding before it builds the
implementation. After selection, the resolver applies profile inheritance only
to bindings whose declared execution class is `llm_backed` and rejects a
binding-specific profile on a deterministic module. The user-facing precedence
contract belongs in [Configuration](../config.md#pipelines).
Implementations that accept options must provide both an option validator and
a builder. The validator is used while resolving configuration; the builder
decodes the same options and constructs the implementation from the prepared
`BuildRequest`. Reject unknown options in both paths. A builder receives only
the dependencies and materialized references that the framework prepared for
that operation, so it must not re-read configuration or files.
Registry helpers register the typed builder for a stage-specific registry.
They are preferable to hand-written untyped registration because they retain
the artifact type at the framework boundary. Registrars validate the registries
they need, register each leaf implementation, and add any family-owned assets
or default validator chains. They return contextual errors so production
composition fails at startup rather than at the first run.
An artifact family can register an optional typed evidence projector alongside
its codec. The projector returns defensive copies of the artifact's direct
generic source references and must use the codec's exact Go type. It does not
interpret surrounding context or publish files; the pipeline validates the
capability during preparation and the output boundary owns publication. See
the [Published Evidence Context contract](../integrations/evidence-context.md)
for the durable result.
## Production Composition
Production composition is intentionally split by family:
- The generic registrar provides the unit chunker, generic JSON validators,
and JSON output encoder.
- The Seriatim registrar provides the transcript input adapter. Its external
input behavior is defined by the [Seriatim contract](../integrations/seriatim.md).
- The D&D registrar provides its codecs, extractors, mergers, normalizers,
validators, prompt assets, fallback profile asset, and default chains. Its behavioral conventions
are documented in [D&D Module Internals](dnd.md).
The CLI owns the composition that invokes these registrars. A module package
may register its own family but must not assemble the CLI or make framework
packages depend on production extensions.
## Adding Or Changing A Module
1. Choose the pipeline stage and the typed artifact boundary. Put external
input or durable artifact formats in the relevant integration contract,
not in this guide or in a private LLM response type.
2. Define a stable `ModuleSpec` with an explicit execution class, the exact
capabilities, and reference slots needed for the operation. Model a
producer/consumer handoff as an artifact-compatible slot; configuration
then chooses an external file or a generated binding.
3. Implement strict option decoding, construction, and the typed stage
interface. Preserve caller ownership: do not retain mutable request data
and return defensive copies where an implementation exposes stored data.
4. Register the module through its typed registry helper and add it to the
owning family registrar. Add a default validator chain only when that
family owns the behavior; otherwise require an explicit compatible chain.
5. Update the selectable-key and chain reference in
[Configuration](../config.md#production-module-keys), the applicable
integration contract, and focused tests. Keep the configuration document
as the sole list of production keys and validator order.
## Validation And References
Validators operate on the value produced at their configured stage. A default
chain is ordered behavior, not a set: JSON parsing, structural checks,
domain-specific checks, durable-schema checks, and advisory checks may have
different responsibilities and failure handling. The active default chains and
override rules are maintained in
[Configuration](../config.md#production-validator-keys-and-default-chains).
Reference slots are part of the module specification. They describe the
accepted artifact kind, media type, size, and whether a binding is required;
the framework validates those constraints before construction. An external
reference is materialized during preparation. A generated reference is a
compatible normalized artifact handed from an earlier pipeline step at
operation time. The configuration reference rules, including precedence and
ordered-handoff requirements, are maintained in
[Configuration](../config.md#references-and-ordered-handoffs).
## Focused Verification
Exercise the leaf implementation and its registration path when changing a
module. Registry and registrar tests cover duplicate keys, required registries,
and typed construction; pipeline resolution tests cover capabilities, options,
and reference compatibility. Domain packages should additionally test their
codecs, validators, normalizers, and any integration handoffs they own.
Run the affected package tests while iterating. The complete module suite is:
~~~sh
go test ./internal/modules/...
~~~

View File

@@ -1,185 +1,58 @@
# Internal Overview
This document inventories the implemented Notarius components. Normative
This document is the implemented component map for Notarius. Normative
boundaries and dependency direction belong in
[Architecture](../policy/architecture.md); external behavior belongs in the
[CLI](../cli.md), [Configuration](../config.md),
[Architecture](../policy/architecture.md). User and operator contracts belong
in the [CLI](../cli.md), [Configuration](../config.md),
[Operations](../operations.md), and [integration contracts](../integrations/).
## Execution Path
`cmd/notarius` delegates to `internal/cli`, the production composition root.
The CLI loads configuration, builds the production catalogs and runtime
collaborators, invokes `internal/framework/pipeline`, and places the logical
output files returned by the runner. Cache and debug collaborators are supplied
at this boundary.
~~~
cmd/notarius -> internal/cli -> configuration and production composition
-> internal/framework/pipeline -> logical output files
-> internal/cli -> durable output and optional state/debug data
~~~
Resolution produces a fixed ordered workflow of steps and globally unique,
sorted artifact lanes. Preparation constructs the complete module and validator
set before the runner receives source bytes. Source parsing and chunking are
serial. Each step then uses a bounded run-wide extraction pool followed by
serial per-lane merge and normalize continuations. A step barrier prevents
later consumers from starting until all earlier lanes are terminal and their
required normalized artifacts have crossed the typed handoff.
The CLI is the application boundary: it discovers configuration, composes
production registries and runtime collaborators, invokes the framework, and
places returned files. The framework resolves and prepares a fixed extraction
pipeline, then returns logical results without owning process behavior or
physical state roots.
## Application Boundary
## Components
| Package | Implemented responsibility |
| --- | --- |
| `cmd/notarius` | Executable entry point and process exit delegation. |
| `internal/cli` | Command parsing, config discovery, package-family registrar invocation, LLM client construction, reference materialization, state collaborator setup, durable writes, and user-facing results. |
## Core Packages
| Package | Implemented responsibility |
| --- | --- |
| `internal/core/artifacts` | Run-manifest and provenance models. |
| `internal/core/config` | Defaults, YAML parsing, environment overrides, validation, redaction, and effective pipeline resolution. |
| `internal/core/debugbundle` | Explicit per-run debug-bundle allocation and redacted summary writing. |
| `internal/core/fileio` | Generic confined atomic file and JSON writes with caller-selected permissions. |
| `internal/core/source` | Generic source documents, units, chunks, canonical references, validation, deterministic source digests, and independent metadata materialization. |
## Framework Packages
| Package | Implemented responsibility |
| --- | --- |
| `internal/framework/contracts` | Source-stage contracts plus artifact identity, schema, serialized representation, codec, validator, reference, output, and structured-completion interfaces and data types. |
| `internal/framework/pipeline` | Module and artifact-codec registries, ordered-step and generated-reference resolution, option validation, profile resolution, capability checks, external reference materialization, complete pipeline preparation, typed handoff, retries, orchestration, warnings, checkpoint decisions, and manifest population. |
| `internal/framework/validate` | Shared validator decision and cardinality helpers. |
| `internal/framework/llm` | Scriptorium-backed structured completions, prompt/schema registration, scheduling, profile recording, and secret redaction. |
| `internal/framework/promptfs` | Builds module prompt filesystems from module-owned and caller-provided shared prompt assets. |
| `internal/framework/checkpoint` | Root-based checkpoint loading, recording, identity, and payload serialization. |
| `internal/framework/chunkplan` | Source-addressed chunk-plan filesystem storage, envelope validation, and atomic publication. |
| `internal/framework/chunkmap` | Strict durable accepted chunk-map construction, schema, validation, cloning, and serialization. |
| `internal/framework/debug` | Root-based framework and LLM debug recording. |
Framework contracts provide typed artifact, provenance-wrapper, chunk-validator,
serialized-validator, and
typed-validator interfaces. The runner owns handoff provenance, validation
sequencing, rejection handling, checkpoint and debug boundaries, and final
manifest assembly.
Artifact registries support heterogeneous typed extraction entries and
kind-specific merger, normalizer, and validator variants. Resolution derives a
lane's kind from its extractor, requires the matching codec, verifies exact Go
type equality across the lane, and records schema identity in the resolved lane
and pipeline digest. Registry entries carry separate option-validation and
run-local construction closures. Preparation injects shared dependencies and
constructs input, chunk, validators, ordered lanes, and output before source
parsing. Production modules use strict construction-time option decoding, and
LLM-backed modules retain the injected shared client. The D&D family registers
the canonical `dnd/spell-list`, `dnd/npc-list`, `dnd/combat-turn-list`,
`dnd/npc-interaction-list`, and `dnd/scene-description-list` codecs, typed
spell, NPC, combat, interaction, and scene-description extractors and
normalizers, validators,
plus kind-specific generic merge strategies; generic JSON validators use the
serialized-validation contract. The runner executes lanes through
private exact-type-checked closures, coordinates extract results independently
of completion timing, and serializes artifacts only through their codec at
checkpoint, debug, and output boundaries.
## Production Extensions
The canonical catalogs of user-selectable
[module](../config.md#implemented-production-modules) and
[validator](../config.md#implemented-production-validators) keys are in
Configuration. The implemented module packages are:
| Package | Implemented responsibility |
| --- | --- |
| `internal/modules/seriatim/input/transcript` | Parses the supported Seriatim transcript format into the generic source model. |
| `internal/modules/generic/chunk/units` | Splits ordered source units by unit count and overlap. |
| `internal/modules/dnd/chunk/scenes` | Produces contiguous D&D scene chunks from structured model output. |
| `internal/modules/dnd` | Owns the canonical D&D spell-list, spell-cast, NPC-list, NPC, combat-turn-list, combat-turn, NPC-interaction-list, and scene-description-list artifact types. |
| `internal/modules/dnd/codec/spells` | Strictly decodes and stably encodes the durable D&D spell-list representation. |
| `internal/modules/dnd/codec/npcs` | Strictly decodes and stably encodes the durable D&D NPC-list representation. |
| `internal/modules/dnd/codec/combatturns` | Strictly decodes and stably encodes the durable D&D combat-turn-list representation. |
| `internal/modules/dnd/codec/npcinteractions` | Strictly decodes and stably encodes the durable D&D NPC-interaction-list representation. |
| `internal/modules/dnd/codec/scenedescriptions` | Strictly decodes and stably encodes the durable D&D scene-description-list representation. |
| `internal/modules/dnd/extract/spells` | Maps private structured model output to canonical source-grounded D&D spell lists. |
| `internal/modules/dnd/extract/npcs` | Maps private structured model output to canonical source-grounded D&D NPC lists. |
| `internal/modules/dnd/extract/combatturns` | Maps private structured model output to source-grounded D&D combat-turn candidates and preserves chronology and invalid candidate values for validators. |
| `internal/modules/dnd/extract/npcinteractions` | Maps private structured model output to current-source NPC interaction candidates grounded by a required registry. |
| `internal/modules/dnd/extract/scenedescriptions` | Maps one private scene description to the current accepted chunk's ID and exact range. |
| `internal/modules/dnd/npcinteractions` | Owns canonical source-reference ordering, occurrence ordering, valid-evidence checks, and exact interaction identity shared by normalization and invariant validation. |
| `internal/modules/dnd/normalize/combatturns` | Canonicalizes and orders merged combat turns, applies exact NPC identity matches, and collapses only exact valid-evidence duplicates. |
| `internal/modules/dnd/normalize/npcinteractions` | Canonicalizes required-registry names, orders interaction occurrences, and collapses only exact valid-evidence duplicates. |
| `internal/modules/dnd/normalize/scenedescriptions` | Trims, source-orders, and removes only exactly identical scene descriptions while rejecting ID and range conflicts. |
| `internal/modules/dnd/validate/combatturns` | Provides deterministic shape, source-reference, source-relatedness, and normalized-invariant validation for the production combat chains. |
| `internal/modules/dnd/validate/npcinteractions` | Provides deterministic shape, registry, source-reference, source-relatedness, and normalized-invariant validation for interaction chains. |
| `internal/modules/dnd/validate/scenedescriptions` | Provides deterministic shape, exact extraction attachment, source-relatedness, and normalized-invariant validation for scene-description chains. |
| `internal/modules/dnd/npcs/registry` | Resolves validated normalized NPC references into immutable grounding data and exact identity lookup. |
| `internal/modules/dnd/npcs/identity` | Owns Unicode-aware NPC identity, ID derivation, and registry collision validation. |
| `internal/modules/dnd/spells/catalog` | Embeds and validates the versioned D&D 5e 2014 SRD catalog, composes optional overlays, and provides immutable effective lookup. |
| `internal/modules/generic/merge/appendorder` | Combines accepted extraction results in chunk order. |
| `internal/modules/generic/normalize/noop` | Preserves accepted merged output. |
| `internal/modules/dnd/normalize/spells` | Canonicalizes catalog-backed spell names and exact source references, conservatively collapses duplicate casts, and reports deterministic warnings and independently scoped catalog checkpoint identity. |
| `internal/modules/dnd/normalize/npcs` | Consolidates NPC records deterministically by canonical name, unions exact evidence, and reports bounded warnings. |
| `internal/modules/generic/output/json` | Encodes manifests, lane payloads, warnings, rejections, and an explicitly enabled accepted chunk map as logical JSON files. |
`internal/modules/dnd/shared` owns reusable D&D prompt fragments,
reference declarations, prompt input assembly, source-unit reference helpers,
and bounded diagnostics under `internal/modules/dnd/shared/diagnostics`.
The shared NPC grounding fragment is mounted for D&D prompts and is owned by
this package. Domain-neutral prompt filesystem composition lives in
`internal/framework/promptfs`.
The `dnd/npcs/registry` package owns the optional `npcs` registry boundary.
External references are strictly decoded and identity-validated during
preparation; generated references are decoded and identity-validated at the
ordered step handoff. Both paths retain canonical registry JSON for provenance
and emit a names-only projection to operation-time spell, combat, and
interaction prompts. Combat and interaction normalization use the canonical
registry for exact name lookup. The
framework records generated identity and bounded producer provenance, while
the raw external reference remains independently tracked by pipeline
provenance. An absent registry is represented only by the empty prompt value
`{"npcs":[]}`. Spell
and combat consumers use this shared boundary without changing their public
module contracts. Interaction consumers require it and retain only current
transcript references as durable evidence.
Generic validators under `internal/modules/generic/validate` provide
unconditional test decisions, JSON syntax validation, and JSON Schema
validation. D&D spell validators under `internal/modules/dnd/validate/spells`
consume the canonical spell-list type directly to provide shape,
effective-catalog, source-reference, and source-relatedness decisions.
Production composition is grouped behind package-family registrars, and every
implemented production extension uses its domain-first tree:
| Package | Implemented responsibility |
| --- | --- |
| `internal/modules/generic/register` | Registers domain-neutral chunk, merge, normalize, output, and validator implementations. |
| `internal/modules/seriatim/register` | Registers the Seriatim input adapter. |
| `internal/modules/dnd/register` | Registers D&D modules, validators, default validator policy, and prompt/schema assets. |
The CLI allocates the framework registries and asset registry, then invokes
these registrars in generic, Seriatim, and D&D order.
Implementation details for all production extensions are in
[Module Internals](modules.md).
## Run-State Components
| Surface | Implemented owners | Internal purpose |
| Area | Implemented owners | Responsibility |
| --- | --- | --- |
| Durable output | Output module, pipeline runner, and CLI writer | Return logical consumer files and place them for a run. |
| Cache checkpoints | `internal/framework/checkpoint` and `internal/cli` | Validate and serialize reusable extract, merge, and normalize outcomes, including ordered-step scope and generated-artifact dependency decisions. |
| Chunk-plan cache | `internal/framework/chunkplan` and `internal/cli` | Persist and select source-addressed plans before framework materialization. |
| Debug bundles | `internal/core/debugbundle`, `internal/framework/debug`, and pipeline instrumentation | Persist redacted summaries and application-owned traces. |
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. |
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |
| LLM and prompt runtime | **internal/framework/llm**, **internal/framework/promptfs** | Provider-neutral structured completions, scheduling, profile recording, prompt assets, schema registration, and credential-shaped-value redaction. |
| Runtime state | **internal/core/fileio**, **internal/core/debugbundle**, **internal/framework/checkpoint**, **internal/framework/chunkplan**, **internal/framework/chunkmap**, **internal/framework/debug** | Confined atomic files, debug bundles, checkpoint and chunk-plan state, accepted chunk maps, and pipeline-facing debug recording. |
| Production extensions | **internal/modules/generic**, **internal/modules/seriatim**, **internal/modules/dnd** | Domain-neutral extensions, Seriatim input support, and D&D extraction families registered into the production catalog. |
Physical layout, cleanup, recovery, and sensitive-data handling are defined
in [Operations](../operations.md). Concrete modules receive recorder
interfaces and request data, not physical state roots.
Generic core and framework packages do not depend on production extensions.
Concrete extensions depend inward on their contracts and are registered only at
the CLI composition boundary.
## Focused Documentation
- [Pipeline Internals](pipeline.md): resolution, execution, validation, retries,
checkpoint/debug hooks, and result assembly.
- [Module Internals](modules.md): production modules, validators, assets,
registration, and the contributor recipe for adding an extension.
- [LLM Runtime](llm.md): structured completion contracts, Scriptorium adapter,
assets, scheduling, profile recording, and redaction.
- [Configuration Internals](configuration.md): loading, validation, effective
resolution, redaction, and resolved-composition identity.
- [CLI Internals](cli.md): command dispatch, production composition, run
orchestration, and terminal reporting.
- [Pipeline Internals](pipeline.md): resolution, preparation, execution,
validation, typed handoff, and framework state hooks.
- [Run State Internals](state.md): output, cache, debug collaborator
composition, and path safety.
- [LLM Runtime](llm.md): structured completion, scheduling, prompt assets,
profiles, and secret handling.
- [Module Internals](modules.md): generic extension registration, module
construction, validation, and reference mechanics.
- [D&D Module Internals](dnd.md): shared D&D extractor conventions, generated
reference projections, and lane-specific exceptions. Durable D&D and
Seriatim data shapes remain in the [integration contracts](../integrations/).
Use this map to find an owner, then read the focused document and its tests
before changing behavior.

View File

@@ -1,462 +1,185 @@
# Pipeline Internals
The implemented resolver and runner live in `internal/framework/pipeline`.
Their fixed workflow and ownership boundaries are defined by
[Architecture](../policy/architecture.md#system-shape). Configuration fields,
defaults, and selectable keys are defined in
[Configuration](../config.md#pipelines).
This document describes the framework-owned pipeline mechanics in
**internal/framework/pipeline**. [Configuration](../config.md) owns selectable
profiles, bindings, and retry settings; [Operations](../operations.md) owns
state lifecycle and recovery; and the [integration contracts](../integrations/)
own durable output shapes. Concrete production extensions are covered by
[Module Internals](modules.md).
Resolution fixes the ordered steps, selected lanes, and all stage bindings;
preparation constructs every selected implementation before the runner begins
source work. After serial input parsing and plan selection or generation, the
runner materializes chunks and executes one step at a time. Within a step,
extract work uses one bounded run-wide worker pool in chunk-first, lane-second
order. Each lane's merge and normalize operations remain serial, and lanes in
the same step may overlap once their extracts are terminal. A later step cannot
start across its barrier until every earlier lane is terminal and each required
generated artifact has been accepted and handed off.
## Boundary
## Resolution
The pipeline framework accepts a resolved composition, registries, shared
dependencies, input bytes, and state/debug collaborators. It returns logical
output files, normalized artifacts, recorded rejections and warnings, manifest
provenance, and checkpoint decisions. The CLI owns process arguments,
configuration discovery, physical roots, and placement of returned output
files.
`internal/core/config.Config.Resolve` validates the loaded configuration,
selects the named profile, applies the runtime inputs supplied by the CLI, and
calls `pipeline.ResolvePipeline`.
The framework has one fixed shape:
`ResolvePipeline`:
~~~
input -> chunk -> extract -> merge -> normalize -> output
~~~
1. selects the explicit ordered steps, or creates the implicit `default` step
from the legacy top-level `artifacts` map;
2. selects and sorts artifact lanes within each step while enforcing global lane
identity;
3. completes omitted bindings using the documented configuration defaults;
4. looks up each module and validator spec without constructing it;
5. for a typed extractor, derives its artifact kind, requires the codec, and
selects exact-type merger, normalizer, and validator variants;
6. checks required and provided capabilities in workflow order;
7. resolves external and generated target-aware reference bindings and
validates producer order, consumer slot declarations, and artifact-kind
compatibility;
8. validates each selected module and validator option set through its registry
entry; and
9. calculates a digest over the resolved structure, including step order, step
IDs, lane membership, generated topology, producer and consumer identities,
typed artifact kind and schema identity, and the effective validator policy
in its resolved execution order.
Input and chunking are pipeline-wide. A selected artifact lane owns extract,
merge, and normalize; output aggregates the terminal lane outcomes. A pipeline
is an ordered list of steps, not an arbitrary workflow graph.
Resolution returns a `ResolvedPipeline` containing ordered steps, lanes,
concrete bindings, validator chains, reference targets, and the digest. It does
not read external reference bytes or construct runtime modules. CLI lane and
reference selector syntax is defined in the [CLI reference](../cli.md#run).
## Resolve, Materialize, Prepare
The digest includes each resolved step's ID and lane membership, generated
producer/consumer topology, and each validator chain's stage, lane, owning
module, ordered validator bindings, execution classes, targets, and artifact
kinds. Changing step order, a dependency, a default chain, or an explicit
override therefore changes pipeline identity whenever it changes effective
execution policy.
Resolution turns a configured pipeline profile into a **ResolvedPipeline**.
It normalizes the pipeline and lane identities, applies stage defaults, selects
requested lanes where that is supported, resolves validator chains, checks
module capabilities and typed artifact compatibility, validates options, and
assigns a deterministic resolved-composition digest. The resolved pipeline
contains bindings and declared reference targets, not external reference bytes.
After selection, the resolver applies command, binding, and pipeline profile
precedence to LLM-backed bindings and validators only; prompt defaults remain
an empty resolved binding profile. Deterministic bindings remain profile-free.
These effective values are part of the digest, so execution and checkpoint
consumers do not repeat profile inheritance.
Configuration resolution supplies the selected profile and catalog; see
[Configuration Internals](configuration.md).
## Reference Materialization
External reference materialization happens before preparation. The materializer
checks that each slot is declared by the selected module, resolves a file path
relative to the correct configuration or working-directory origin, reads
UTF-8 text, verifies media type and size limits, and retains bounded
provenance. A generated-artifact selector remains declared but has no bytes
until its producing step completes.
The CLI calls `MaterializeReferences` after resolution and before constructing
the LLM client or running the pipeline. For external bindings, the materializer
checks each binding against its resolved target declaration, reads and validates
the file, and builds both a `contracts.ReferenceSet` and provenance-only
metadata on the corresponding `ResolvedReferenceTarget`. A structured
generated binding is declaration-only at this point: its producer bytes do not
exist until the producer lane reaches an accepted normalized result.
Preparation is the construction boundary. It validates the resolved shape and
registry set, clones the resolved data, then constructs the input adapter,
chunker, stage-local validators, every typed lane, and output encoder with
cloned options, references, and shared dependencies. It also collects stable
checkpoint fingerprints. Missing registrations, incompatible typed entries,
nil implementations, and constructor failures are reported before source
parsing or any stage operation begins.
Preparation delivers the materialized external set for each target through
`pipeline.BuildRequest`: chunkers and chunk validators receive the chunk target;
extractors and extract validators receive the lane extract target; mergers and
merge validators receive the lane merge target; and normalizers and normalize
validators receive the lane normalize target. Input and output builders receive
an empty set because those stages cannot declare references. Every builder gets
an isolated deep clone of its target set, so construction-time mutation cannot
change another builder, the resolved pipeline, or later runtime requests.
An output encoder can opt into source-evidence publication through its output
policy. Preparation keeps the configured lane allowlist and active lanes
separate, then verifies an exact typed evidence projector and registered codec
for each active lane. The resulting private plan is immutable; lanes excluded
by invocation filtering remain configured but do not acquire a projector for
that run.
Prepared consumers do not need to be reconstructed when generated content is
available. At the step boundary, the runner encodes the accepted producer value
through its registered canonical codec, validates the generated bytes against
each target slot's kind, schema, media type, and size, and clones one immutable
reference item into the operation request. The item includes canonical digest,
size, and bounded producer provenance but no filesystem URI. A handoff failure
is a framework dependency error and prevents every consumer in that step from
starting.
## Typed Lanes And References
The runner continues to clone the resulting set into the chunk, extract, merge,
or normalize request that owns the target. LLM-backed extensions may convert
those items into named prompt inputs. Reference content remains separate from
source evidence and source digests, whether the item came from a file or a
generated handoff.
Each resolved lane has one artifact kind, codec, and exact Go type. The
framework uses private type erasure only around those typed operations; every
handoff checks exact type and codec identity and reports incompatibility as an
error rather than panicking. Encoding through the registered codec is the
boundary for output, checkpoints, debug records, and generated references.
Binding precedence, path resolution, accepted content, and media-type behavior
are configuration contracts; see [Configuration](../config.md#pipelines).
Durable provenance is defined in the
[JSON output contract](../integrations/json-output.md#manifestjson), while
runtime sensitive-data handling belongs in [Operations](../operations.md).
Reference targets are stage- and lane-specific. External reference bytes are
cloned into the operation request. Generated references are built at the next
step boundary from exactly one accepted normalized producer output. The
framework decodes and re-encodes that output with the registered producer
codec, checks its complete schema and media identity, and records a content
digest plus bounded producer provenance. A missing, ambiguous, invalid, or
incompatible producer prevents the consumer step from starting.
## Registries And Specs
## Execution And Ordering
`pipeline.Registries` holds option validators and run-local builders used during
resolution and preparation.
`pipeline.ModuleCatalog` exposes their specs during configuration validation and
resolution. Separate registries exist for every stage and for validators;
`ValidatorChainRegistry` stores production default-chain mappings. Both
containers also carry an `ArtifactCodecRegistry`. Generic registration records
one codec per stable artifact kind, validates its schema metadata and JSON
Schema, retains the exact schema digest and Go type, and safely encodes or
decodes framework-erased values with typed errors on incompatibility.
The runner validates its input, installs no-op state collaborators when none
were supplied, and serially performs source parsing and chunk-plan selection.
An accepted plan is materialized into source-addressed chunks and passes the
configured chunk validators before any lane runs. A chunk rejection is a
recorded pipeline outcome: lanes do not start, but the output stage can encode
the terminal result.
Typed extractor entries are keyed by module key and declare one artifact kind.
Merger, normalizer, and typed-validator variants are keyed by module or
validator key plus artifact kind. Chunk and serialized validators occupy
separate target namespaces; serialized registrations declare whether they
support chunks, artifacts, or both. Duplicate variants and exact Go-type
mismatches are rejected deterministically.
For each ordered step, the runner first builds generated reference sets from
the accepted normalized outputs of earlier steps. It then executes the step's
lanes. Later steps do not begin until the current step is terminal and its
generated handoffs have succeeded.
Lane-sensitive merger and normalizer spec discovery always supplies the
extractor's artifact kind, so variants under one reusable key may declare
different capabilities and reference slots. Kind-neutral registry inspection
selects the first registered artifact kind in sorted order.
Within a step, the lane engine dispatches extraction jobs in deterministic
chunk-first, lane-second order to a bounded worker group. When all extraction
jobs for one lane are terminal, a bounded continuation group can run that
lane's merge and normalize work while extraction for other lanes continues.
The framework does not create an unbounded goroutine per chunk or lane.
Production composition registers the D&D spell-list, NPC-list, combat-turn-list,
NPC-interaction-list, and scene-description-list codecs and typed lane
variants, plus serialized JSON validators. Every artifact lane resolves through
the typed registries and a matching codec.
Completion timing does not determine public results. The coordinator restores
lane and chunk order before merging results, and selects a framework error by
stable stage, lane, and chunk position. A validator rejection records a lane
outcome without cancelling unrelated work. A framework error or parent
cancellation cancels derived work, prevents queued work from starting, waits
for started workers, and prevents output encoding.
A `ModuleSpec` declares its stage plus required and provided capabilities.
Chunk, extract, merge, and normalize specs may also declare reference slots.
Registry implementations defensively copy spec metadata, reject duplicate keys,
and verify that a constructed implementation reports the registered key.
Builder registrations accept `ModuleDependencies` and cloned configuration
options through one `BuildRequest`. Builders decode those options and retain
typed values or injected dependencies in the constructed implementation.
Extractors declare their artifact kind, and merger, normalizer, and validator
resolution selects the matching typed variant.
## Validation, Retries, And Output
A `ValidatorSpec` declares a validator key and execution class. Resolution uses
the execution class to reject incompatible profile bindings before execution.
The current production catalog and default chain are listed only in
[Configuration](../config.md#implemented-production-validators).
Every chunk, extract, merge, and normalize candidate passes its resolved
validator chain. Validators receive immutable canonical input appropriate to
their target: chunks, typed values, or serialized codec bytes. They may
approve, approve with warnings, reject, or fail. A rejection is an ordinary
pipeline result; a validator error is a framework error.
## Preparation And Runner Boundary
The runner applies the binding's retry policy around a stage operation and its
complete validation chain. It preserves warnings only from the final accepted
or rejected attempt. Cancellation stops retries. Normalizer-specific retry
directives consume this same budget and validate any final safe fallback through
the normalizer chain.
`pipeline.Prepare` receives a resolved pipeline, the registries, and shared
module dependencies. It constructs input; chunk and its validators; every
step's lane extract, merge, and normalize modules and validator chains in
resolved order; then output. It stops at the first error with pipeline, step,
stage, lane, module, and validator context as applicable. It never invokes an
operation method. Generated references are not available during preparation;
the operation request is the handoff boundary.
`PreparedPipeline` keeps private constructed executors and exposes cloned
resolved input, chunk, lane, and output identities. Prepared components may
implement `pipeline.CheckpointFingerprintProvider` to contribute explicit
semantic identities to checkpoint reuse. Preparation trims and validates each
non-secret name and value, prefixes it with the component's stage, lane,
module, and validator scope, rejects duplicates, and retains the resulting
sorted collection behind a defensive-copy accessor. Fingerprints must be
stable and must not contain source content, credentials, local paths,
timestamps, or other invocation-specific values.
`pipeline.RunInput` carries that prepared pipeline, raw source input, run identity and timing, optional
session and profile metadata, a chunk-plan store and mode, a checkpoint
execution policy, and checkpoint/debug collaborators. The runner
parses source bytes through the already constructed input adapter. Later stage
requests receive the generic source model; extract requests receive
chunk-scoped input material, while chunk, merge, and normalize requests retain
access to the original source material. Input, chunk, and output operation
requests do not carry raw module options. The chunk request also does not carry
an LLM client; an LLM-backed chunker receives the shared client during
preparation. Their operation requests retain run-specific source, reference,
profile, session, metadata, and step-handoff context as applicable. A generated
reference is cloned into each compatible consumer request and is never exposed
as a path.
Prepared lanes retain exact-type-checked erased operation closures. The runner
uses those closures to keep each value typed through extraction, validation,
merge, and normalization.
Source validation requires every unit to carry a canonical self-reference to
its containing document and its own unit ID. Explicit clone, checkpoint, and
debug boundaries retain that reference, and the canonical source digest covers
it deterministically. Chunks use the same source model and carry one canonical
reference spanning the first selected unit through the last.
`pipeline.RunOutput` carries the run manifest, accepted normalized serialized
artifacts with lane and normalizer provenance,
rejected results, warnings, checkpoint events, and logical files returned by the
output encoder. The CLI owns debug-summary and durable filesystem writes after
the runner returns.
## Execution Flow
The pipeline-wide coordinator owns the ordered step loop, generated-reference
sets at each barrier, and deterministic merging of step outcomes. For one step,
one run-local lane engine owns worker lifecycle, cancellation, dispatch,
continuation queues, and result collection. It initializes checkpoint state in
lane order, dispatches bounded extract work, advances terminal lanes through
serial merge and normalize work, selects failures by stable pipeline scope, and
merges lane-local outcomes back in resolved order. Completion timing never
becomes public ordering.
The runner:
1. validates its prepared input;
2. parses the raw input with the prepared adapter and validates the generic
source document;
3. selects a stored plan or executes the configured chunker's `Plan` operation;
4. canonicalizes and materializes the plan, then validates the resulting
chunks;
5. builds the framework-owned accepted chunk map from the accepted source,
logical plan, and exact materialized chunks, then supplies it to the output
request independently of output-module options;
6. executes each resolved step in configuration order. For one step, it
dispatches extract jobs in source-chunk then resolved-lane order, starts a
bounded lane continuation when all extracts for that lane are terminal, and
waits for every lane to become terminal;
7. encodes and validates each accepted normalized producer artifact, then
builds the immutable generated reference sets for the next step;
8. invokes the prepared output encoder only after every step succeeds and
validates its logical file results;
9. returns the assembled manifest, outcomes, warnings, and files.
Within each artifact lane, it reuses the prepared extractor, merger, normalizer,
and validators while performing these transitions:
1. extract once per accepted chunk and add runner-owned lane, source, and chunk
provenance;
2. validate each extract result and omit rejected results from merge input;
3. skip the rest of the lane when no extract result is accepted;
4. merge accepted extract results in their existing order;
5. validate the merge result and skip normalization on rejection;
6. normalize the accepted merge result;
7. validate and append the accepted normalized result.
At a step barrier, a lane with no accepted normalized output is still a regular
rejection unless a later generated binding names that lane as a required
producer. In that case the runner raises a deterministic dependency error and
does not start the consumer step. One accepted typed artifact may fan out to
multiple compatible target slots. Consumers in the same step may run
concurrently after the handoff; no work crosses the barrier early.
Module-provided warnings and payload warnings are promoted only from attempts
whose results are accepted and used.
## Chunk Plans And Reuse
`Chunker.Plan` returns a `source.ChunkPlan`: the canonical source digest,
ordered unit-ID ranges, and optional plan or range annotations. The framework
owns plan canonicalization and materialization. It creates the generic chunks
and therefore owns their IDs, indexes, source references, JSON content, units,
media type, and generic metadata. Plan and range annotations are independently
owned raw JSON and become `Chunk.PlanAnnotations` and `Chunk.Annotations`.
In `auto`, the runner looks up the source digest before invoking the chunker. A
valid hit is materialized and sent through the current run's configured chunk
validators; it does not invoke the chunk module, consume its retry budget, or
make a chunk-stage LLM call. A missing, invalid, or unmaterializable record
generates a candidate. `refresh` generates without lookup; `bypass` generates
without cache access. Generated plans are published only after the full chunk
validator chain approves them. A validator rejection is a regular rejected
pipeline outcome and never replaces a cached plan.
The store is source-addressed, not pipeline-addressed. Changes to pipeline
configuration, requested chunker, options, references, lanes, validators, or
LLM profile do not prevent a source-digest hit. The manifest records both the
currently requested chunker and the effective plan producer. Cache state and
paths are configured and operated outside the runner; see
[Configuration](../config.md#state-surfaces) and [Operations](../operations.md).
For an accepted plan, the runner also constructs the strict framework-owned
[Accepted Chunk Map](../integrations/chunk-map.md) before lane execution. It
uses the current resolved chunker as `requested_chunker` and the stored or
generated record as `producer`, preserving that distinction on reuse. Chunk
rejection supplies no map; later lane rejection does not discard it. Output
encoders receive a defensively owned serialized value and may explicitly
ignore it.
The extract job channel has the same capacity as the effective extract worker
count, so dispatch applies backpressure. A fixed continuation executor prevents
ready or checkpoint-reused lanes from creating one goroutine each. Workers and
continuations publish lane-local results; the coordinator is the only writer of
aggregate output and merges those results in resolved lane and source-chunk
order.
## Plan Canonicalization And Chunk Materialization
Plan canonicalization requires canonical JSON annotations, a matching source
digest, at least one range, existing ordered boundaries, and increasing range
starts. Ranges may overlap or leave gaps; a chunker may impose stricter policy.
Materialization deterministically reconstructs each range from the current
source document, deep-clones JSON-shaped source-unit metadata, and copies
annotations without interpreting their namespaces. Materialized chunks and
separate materializations do not share mutable unit metadata; unsupported or
cyclic metadata fails materialization with context.
Before lane execution, generic chunk validation checks the materialized chunks'
identities, order, source references, content, media type, units, and metadata.
No chunk checkpoint participates in plan selection: plan storage is the only
chunk-reuse mechanism. Extract, merge, and normalize checkpoints continue to
use materialized chunk digests as their dependencies.
## Validation And Retries
Chunk, extract, merge, and normalize results pass through the resolved validator
chain for their stage and module. Chunk validators receive canonical chunks;
typed validators receive the domain value; and serialized validators receive
canonical chunk JSON or artifact codec bytes. Validators execute in resolved
order and stop at the first error or rejection. An empty chain approves the
result.
Production D&D artifact chains keep generic JSON syntax validation first, then
run every rejecting domain validator before generic JSON Schema validation. The
domain validator therefore owns expected semantic diagnostics; the generic
schema validator remains the final rejecting representation backstop, before
warning-only relatedness validation. Explicitly configured validator chains
retain their configured order.
`runWithRetry` applies the effective retry policy around module execution and
its complete validation chain. A module or validator error becomes a framework
error when attempts are exhausted. A rejection becomes a recorded
`RejectedOutput` when attempts are exhausted. Cancellation stops retry
processing immediately.
Rejected output is a non-fatal pipeline outcome and does not advance. Warnings
from discarded attempts are not promoted. Configuration owns retry counts and
validator overrides; see [Module Bindings](../config.md#module-bindings).
After terminal lane work, the runner assembles manifest provenance, normalized
artifacts, rejections, warnings, and an optional accepted chunk map. When an
output policy selected evidence lanes, it decodes accepted serialized normalize
outputs through their registered codecs and invokes the prepared typed
projectors. Rejected or absent lanes contribute nothing. This reconstruction is
also used after normalized-checkpoint reuse, so no second typed output channel
is retained. The runner passes the resulting owned artifact to the output
encoder, which returns logical files and does not choose a physical directory.
The CLI publishes those files only after the runner returns without a framework
error. Logical file names and schemas are defined by the [output integration
contracts](../integrations/).
## Checkpoint And Debug Hooks
The runner depends on recorder and loader interfaces, using no-op
implementations when collaborators are absent. Each checkpointed workflow
boundary records a running, succeeded, or failed transition. Reuse decisions
are consulted in workflow order and accepted payloads are cloned before
entering the normal handoff path. Typed extract, merge, and normalize
checkpoints store codec bytes with artifact kind, schema ID, name, version and
exact digest, and media type. Reuse compares that identity with the prepared
codec and decodes through the codec; missing identity, mismatches, corrupt
bytes, and decode failures become explicit reuse misses and execute the lane
normally. Dependency fingerprints and debug content digests use the same stable
codec bytes that cross those boundaries.
The runner receives checkpoint and debug interfaces rather than roots. It
records workflow transitions and reuse decisions through the supplied
collaborators, and clones reusable artifacts before they re-enter normal typed
handoff. Generated-reference dependencies participate in checkpoint decisions.
Selective recomputation can require a canonical accepted normalized predecessor
before a dependent lane starts.
That progressive extract, merge, and normalize reuse is the ordinary resume
path. A lane marked as a required predecessor for selective recomputation takes
a separate accepted-output path before extract scheduling. The loader reads the
existing successful normalize manifest and payload by step, lane, and
normalizer, without consulting extract or merge dependencies. It requires the
current non-empty checkpoint identity to match, so the invocation identity
still binds the input, resolved topology and configuration, references, runtime
overrides, profiles, and component fingerprints.
Debug recording is attempt-scoped and application-owned. A failure to persist
required debug data is a framework error. State roots, persistence, reason-code
meanings, resume, and cleanup are intentionally owned by
[Run State Internals](state.md) and [Operations](../operations.md).
The runner decodes and canonically re-encodes each reusable artifact once with
the prepared codec, requiring exact kind, schema identity and digest, media
type, canonical bytes, content digest, and producer provenance. A valid accepted
producer becomes a runner-owned cloned normalized output, restores only
normalize-checkpoint warnings, and records one `accepted_artifact_reused`
normalize decision. It does not invoke or record extract, merge, normalize, or
their validators. Invalid or unavailable accepted state records its decision
and fails the producer step; the dependent step never starts and the producer
is not implicitly rerun. If a later required lane fails during initialization,
already hydrated terminal lanes remain in the failed output in resolved order.
## Invariants To Preserve
Generated references add downstream dependencies containing the producer's
artifact kind, complete schema identity, media type, canonical content digest,
and size. Compatible accepted producer outputs may therefore feed a later step
without re-executing the producer. Forced lanes bypass accepted-output
hydration and execute normally. A missing, rejected, corrupt, incompatible, or
changed producer blocks its dependent while leaving independent work eligible
for reuse. The runner records bounded decision
categories: `reused`, `executed`, `forced_recompute`, and
`dependency_invalidated`. Operator meanings for the stable reason codes belong
to [Operations](../operations.md#resume-and-selective-recompute).
- The six fixed stages remain explicit; a pipeline is not a general DAG.
- Resolution and preparation reject statically discoverable incompatibility
before parsing or execution.
- Every typed lane uses one compatible artifact kind, codec, and exact Go type.
- Generated references come only from one earlier accepted normalized producer
and carry canonical identity rather than an unverified value.
- Rejections are recorded outcomes; framework errors cancel derived work and
prevent output encoding.
- Public ordering and selected errors are independent of goroutine completion
order.
- Pipeline modules receive collaborators and data, never CLI streams or
physical output, cache, or debug roots.
The CLI includes prepared-component fingerprints in the run-wide checkpoint
identity alongside resolved configuration, raw input, reference provenance,
runtime overrides, and LLM-profile fingerprints. Module metadata is not used
implicitly for cache identity: components opt in only with stable semantic
values that can change accepted output. Adding or changing a component
fingerprint intentionally produces a cold cache miss. Existing checkpoint
schemas and paths remain unchanged.
## Focused Tests
The CLI's `--recompute-step` policy forces the selected step and all transitive
dependents, but requires accepted normalized artifacts for every unselected
producer on which that closure depends. It changes execution policy only; it
does not alter persistent checkpoint identity.
- **internal/framework/pipeline/profile_test.go** and
**typed_resolution_test.go** cover resolution, defaults, ordered steps,
compatibility, validators, references, and resolved identity.
- **internal/framework/pipeline/preparation_test.go** covers complete
construction before execution and contextual construction failures.
- **internal/framework/pipeline/references_test.go** and **handoff_test.go**
cover external materialization, generated references, provenance, and typed
producer checks.
- **internal/framework/pipeline/runner_concurrency_test.go** covers bounded
execution, ordered steps, stable error selection, rejections, and
cancellation.
- **internal/framework/pipeline/runner_chunk_plan_test.go**,
**runner_typed_checkpoint_test.go**, and
**runner_accepted_checkpoint_test.go** cover state hooks and reuse behavior.
- **internal/framework/pipeline/runner_attempt_debug_test.go** and
**runner_terminal_debug_test.go** cover attempt and terminal debug behavior.
Debug instrumentation wraps run, stage, attempt, validator, and structured LLM
boundaries. Every executed chunk, extract, merge, and normalize attempt writes
one terminal envelope for acceptance, validator rejection, module or validator
error, or applicable candidate or final serialization error. The envelope
contains its attempt-local warnings, any available candidate and rejection,
and terminal error text; failures before a candidate exists omit that payload.
Only LLM calls made by the module operation belong to the module attempt.
Validator calls retain independent scopes under `validate/` and are not
duplicated into the module envelope. A failed terminal-envelope write is a
non-retryable framework error and is joined with any primary attempt error.
Debug data is never used as a checkpoint source. Typed artifact debug envelopes
are domain-neutral, redact sensitive metadata and bytes through the common
debug policy, and record codec identity plus schema and content digests.
Merge and normalize attempts serialize their in-memory candidate with the
codec's required candidate encoder before typed validation. Serialized
validators and attempt debug use that candidate representation, which carries
the codec media type and schema identity but is never checkpointed or passed
downstream. Only a validator-approved value is encoded through the strict final
codec and made eligible for a checkpoint or stage output.
Checkpoint identity, physical layout, reuse behavior, and debug artifact
handling are operator contracts in [Operations](../operations.md). Serialization
and recorder implementation are inventoried in
[Internal Overview](overview.md#run-state-components).
## Results And Failures
The runner owns manifest assembly and handoff summaries but not the durable JSON
schema. It records resolved module and lane provenance, validator chains,
source/reference identities, selected LLM profiles, normalized and rejected
summaries, status, and timing. Serialized artifact content remains outside the manifest.
Module metadata providers may add non-secret singleton or lane-scoped metadata.
Execution errors include stage, module, lane, or validator context. Once a
manifest exists, a failing run returns it with failed status and completion
time. Successful status reflects whether any result was rejected. The
durable manifest and logical file schemas are defined in the
[JSON output contract](../integrations/json-output.md).
On a framework failure, the runner cancels its derived context, stops submitting
new extract work, drains started tasks, and skips the output encoder. Parent
cancellation takes precedence. Otherwise context-cancellation fallout is
discarded when a substantive error exists, and the primary error is selected by
stage, resolved lane, and source chunk rather than completion time.
## Tests To Inspect
- `internal/core/config/effective_config_test.go`: config-to-resolution boundary.
- `internal/framework/pipeline/profile_test.go`: selection, defaults,
capabilities, validator chains, and digest behavior.
- `internal/framework/pipeline/artifact_codec_registry_test.go`: typed codec
metadata, registration, erasure safety, strict decoding, and cloning.
- `internal/framework/pipeline/typed_resolution_test.go`: heterogeneous typed
lane resolution and preparation, target-specific validators,
incompatibilities, ordering, and schema-sensitive pipeline identity.
- `internal/framework/pipeline/runner_concurrency_test.go`: bounded dispatch and
continuations, reverse completion, stable errors, rejection, cancellation,
retries, and independent provider-call limits.
- `internal/framework/pipeline/preparation_test.go`: option validation,
construction order, dependency failures, and the before-source-work boundary.
- `internal/framework/pipeline/references_test.go`: target resolution and
materialization.
- `internal/cli/run_contract_test.go`: production run transitions, retries,
rejections, warnings, CLI recomputation controls, debug hooks, and manifests.
- `internal/cli/recompute_execution_contract_test.go`: filesystem-backed
selective recomputation and accepted-producer recovery.
- `internal/cli/production_contract_test.go`: production composition and
configuration-resolution smoke coverage.
- `internal/cli/example_contract_test.go`: maintained example resolution and
execution ownership.
- `internal/modules/integration/*_test.go` and
`internal/modules/seriatim/input/transcript/runner_test.go`: typed runner
composition across concrete module families.
- `internal/framework/checkpoint/*_test.go`: checkpoint serialization and reuse
collaborators.
Run **go test ./internal/framework/pipeline ./internal/cli** after changing a
pipeline boundary. Use the more focused tests above while iterating.

View File

@@ -2,7 +2,8 @@
This document describes the implementation collaborators behind output, cache,
and debug state. User-visible fields belong in [Configuration](../config.md),
and layouts and lifecycle belong in [Operations](../operations.md).
and physical layout, retention, recovery, reason codes, and cleanup belong in
[Operations](../operations.md).
## Composition
@@ -12,6 +13,12 @@ constructs cache collaborators, writes logical output files, and reports paths.
Pipeline modules receive interfaces and request data, never output, cache, or
debug roots.
The CLI creates no chunk-plan store in bypass mode. It creates a checkpoint
recorder only when recording is enabled and a checkpoint loader only for a
resume invocation. It allocates debug state only after a safe run identity has
been generated and only when debug capture was requested. These choices keep
the three state families independently composable.
## Output And Cache
The pipeline runner returns logical output files. After validating every
@@ -32,7 +39,9 @@ codecs, loader, and recorder. The CLI constructs a recorder whenever checkpoint
recording is enabled and constructs a loader only for a `--resume` invocation.
Identity incorporates explicit stable semantic fingerprints collected from
prepared modules and validators in addition to configuration, input,
references, runtime overrides, and LLM profiles.
references, runtime overrides, observed LLM profiles, and the LLM runtime's
non-secret effective profile-source identity. A profile source change therefore
causes a cold miss even when the configured profile ID remains unchanged.
The serialized
`workspace_schema_version` identifiers are frozen wire-compatibility fields;
they do not describe a current public state surface.
@@ -68,12 +77,19 @@ names only the step, lane, and stable reason code. Decision detail is selected
from code-owned descriptions by reason code and then UTF-8 normalized and
bounded; callers cannot supply arbitrary diagnostic prose. Typed categories and
codes remain intact through pipeline events and become strings only in manifest
and debug-summary JSON. [Operations](../operations.md#resume-and-selective-recompute)
is the canonical operator-facing reason-code reference.
and debug-summary JSON.
[Operations](../operations.md#checkpoint-recording-resume-and-recompute) is the
canonical operator-facing reason-code reference.
`internal/core/fileio` provides confined atomic file writes used by state
collaborators. The chunk-plan store retains its stronger entry validation.
The CLI constructs selective-recomputation policy from resolved generated
artifact dependencies. It forces the selected step and transitive consumers,
while marking unforced producers as required reusable inputs. The runner owns
the actual hydration and rejection decisions; the [Operations guide](../operations.md#checkpoint-recording-resume-and-recompute)
owns the operator workflow and stable reason-code meanings.
## Debug Bundles
`internal/core/debugbundle` allocates an explicitly requested per-run bundle
@@ -98,12 +114,27 @@ operation writes the success report, or makes one attempt each to write the
failure report and error log. Terminal persistence failures are reported
separately and never replace the command's primary error.
## Invariants To Preserve
- Modules receive state collaborators and request data, never physical roots.
- Output logical paths are validated before a run directory is allocated, and
files are atomically written within that directory.
- Chunk-plan publication occurs only for accepted plans; bypass does not
construct or touch a plan store.
- Checkpoint recording and checkpoint loading remain separate collaborators.
- Debug state is opt-in, is not cache input, and terminal reporting does not
obscure the command's primary failure.
## Tests To Inspect
- `internal/cli/run_contract_test.go`: command-owned state allocation,
terminalization, and output/report boundaries.
- `internal/cli/cache_contract_test.go`: cache-mode precedence, root selection,
and resume collaborator construction.
- `internal/cli/state_hardening_test.go`: independent roots, reuse, failures,
permissions, cleanup, and redaction.
- `internal/cli/recompute_policy_test.go`: forced dependents and required
reusable predecessors for selective recomputation.
- `internal/cli/recompute_execution_contract_test.go`: selective recomputation,
filesystem recovery, deterministic decisions, and failed predecessor state.
- `internal/cli/production_contract_test.go`: production composition and

View File

@@ -1,294 +1,291 @@
# Operations
This is the canonical guide to operating Notarius filesystem state. Command
syntax is in the [CLI reference](cli.md); field definitions and precedence are
in [Configuration](config.md).
This is the canonical guide for operating Notarius runtime state. The
[CLI reference](cli.md) owns command syntax and exit statuses, while
[Configuration](config.md) owns fields, defaults, and precedence. Maintainers
who need implementation mechanics should read [Run State Internals](internal/state.md).
## State Model
## State Surfaces
Notarius uses three independent filesystem surfaces:
Each run can use independent roots with different retention and access-control
needs.
- output is durable user data;
- cache is reconstructible chunk-plan and checkpoint state; and
- debug is explicitly requested inspection data.
| Surface | Purpose | Created when | Retention |
| --- | --- | --- | --- |
| Output | Durable user-facing result bundle | A pipeline completes and returns logical output files | Keep until consumers no longer need it. |
| Chunk-plan cache | Reconstructible source-addressed plan | The configured cache mode permits cache I/O | Keep while reuse is useful. |
| Checkpoint cache | Reconstructible execution and recovery state | Checkpoint recording is enabled | Keep only while recovery or reuse is useful. |
| Debug bundle | Explicit diagnostic record | A run requests debug collection | Keep only under an intentional sensitive-data retention policy. |
Choose separate roots and access controls for each surface. A normal run writes
durable output, may use the chunk-plan cache, and records checkpoints when
`cache.checkpoints.enabled` is true. It does not create debug state unless its
invocation includes `--debug`.
Output, cache, and debug roots are never merged or cleaned automatically. Use
separate locations and permissions for operators or services that must not
share application data.
## Output
## Roots And Permissions
Durable logical files are written under:
The configured output and debug directories are exact roots. An empty cache
directory selects a per-user root:
```text
~~~
<os.UserCacheDir>/notarius/chunk-plans
<os.UserCacheDir>/notarius/checkpoints
~~~
The field definitions and configuration examples are in [Configuration](config.md).
On supported Unix systems, output directories and files are created with
requested modes **0755** and **0644**. Chunk-plan, checkpoint, and debug
directories and files use **0700** and **0600**. The operating system's umask
may impose stricter output modes. Cache and debug roots may contain sensitive
source-derived data, so provision them for one trusted account or service. An
output bundle can also contain source content when its JSON output enables
evidence publication. Apply an appropriate umask and output-root access policy
before enabling that option; the requested output modes alone may not be
suitable for transcript-bearing bundles.
## PromptKit Profile Deployment
Profile deployment has four distinct layers:
| Layer | Owner | Operational role |
| --- | --- | --- |
| Prompts and schemas | Notarius module families | Embedded request and structured-output definitions. They are not deployment profile files. |
| Fallback profiles | Notarius module families | Embedded application defaults, including D&D's `dnd-extraction` profile. |
| Built-in profiles | PromptKit | Upstream catalog entries available when no higher-precedence source defines an ID. |
| Operator profiles | Deployment filesystem | Complete environment-specific definitions selected by `promptkit.profile_file` or `promptkit.profile_dir`. |
The maintained D&D pipeline uses the workload ID `dnd-extraction`. The
embedded fallback makes that ID usable without an operator file. Production,
development, and local deployments can each install a different complete
definition for the same ID, retaining the pipeline while choosing their own
model, backend, timeout, or reasoning policy. An operator definition wins over
the fallback; it is not merged with it. The configuration field and full
precedence rules are owned by [Configuration](config.md#promptkit-profiles).
Use a profile source owned by the service account, keep it readable only by
the intended operator, and supply provider credentials through the service
environment—not in the Notarius configuration or profile YAML. The maintained
[operator profile](../examples/profiles/dnd-extraction.yml) is secret-free and
can be copied as a format starting point. Validate a deployment without a
provider call or credentials:
~~~sh
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
~~~
Profile paths are currently resolved from the process working directory, not
from the configuration file. The complete example's
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
invocation only. Use absolute paths such as
`/etc/notarius/profiles/dnd-extraction.yml` for services and containers.
## Run Lifecycle
Use the [run command](cli.md#run) to start a pipeline. A valid invocation loads
and resolves configuration before module preparation and source parsing. It
then performs any permitted cache lookup, executes the pipeline, and publishes
logical output files only after a successful runner result.
On success, the command reports the output bundle path. A warning-bearing run
still succeeds and reports its warning count on standard error. Errors and
their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses).
## Output Bundles
Each successful run receives a generated safe run identifier and writes beneath:
~~~
<output-root>/<run-id>/
```
~~~
The CLI generates one run ID in the form
`run-<started-at-unix-nanoseconds>-<32-lowercase-hex-characters>` and uses it
for output, manifests, and any requested debug bundle. It validates every
logical output name before exclusively creating the run directory. If that
directory already exists, the invocation fails without changing it.
The [JSON output contract](integrations/json-output.md) owns the logical files
and their schemas. Before creating the run directory, Notarius validates every
logical output path. It refuses an existing run directory without changing it.
Files are written atomically; if a later write fails, the newly created partial
run directory remains for inspection and is never removed automatically.
Each output file is written atomically. A later file-write failure leaves the
newly allocated partial run directory in place for inspection; Notarius never
automatically removes output. The
[JSON output contract](integrations/json-output.md) owns the logical file
names, schemas, and media types inside a run directory.
An enabled JSON `include_chunk_map` option adds an accepted chunk map to durable
output. Its annotations may contain source- or model-derived data, so retain
and protect it like lane output. The map is opt-in and does not alter existing
bundles; its payload exclusions are defined in the
[Accepted Chunk Map contract](integrations/chunk-map.md).
Remove an output run directory only after its consumer data is no longer
needed. This is data deletion, not cache cleanup.
## Ordered D&D Workflow
The maintained [NPC-grounded configuration](../examples/dnd-npc-grounded.config.yml)
contains one pipeline with two ordered steps. The first step extracts and
normalizes NPCs. Only after that lane reaches an accepted terminal result does
the second step begin; its generated NPC reference is supplied in memory to
spell extraction, combat extraction, and combat normalization.
```sh
go run ./cmd/notarius run dnd-npc-grounded \
--config examples/dnd-npc-grounded.config.yml \
--input examples/seriatim-minimal-transcript.json \
--output-dir ./npc-grounded-output
```
The NPC artifact grounds canonical names through a names-only prompt projection, not spell or combat
evidence. Current-transcript source ranges remain the only event evidence. The
manifest records generated-reference identity and bounded producer provenance;
it does not record generated payload content, and no generated content is
exposed through a filesystem path. The same producer artifact may fan out to
compatible consumers, while a missing or rejected producer prevents the later
step from starting.
Standalone module configurations continue to support external NPC files when a
workflow intentionally crosses a process or session boundary. Those files are
validated against the consumer slot and must be protected as sensitive
campaign data. They are not part of the maintained ordered handoff workflow.
Treat an output bundle as durable user data. Do not use cache-cleanup policy to
remove it. An optional accepted chunk map is also durable output and can carry
source- or model-derived annotations; its content and compatibility contract
are defined in [Accepted Chunk Map](integrations/chunk-map.md). An optional
[evidence context](integrations/evidence-context.md) contains source-unit text
and metadata. It is not a cache or debug artifact: retain it with the output
bundle only for as long as consumers need it, and apply source-content access
controls to the entire bundle. Selected lanes may collectively cite most of a
transcript, so a broad allowlist can make the evidence artifact nearly as
sensitive and large as the source itself.
## Chunk-Plan Cache
Chunk plans are stored at:
Chunk plans live beneath the selected chunk-plan root:
```text
~~~
<chunk-plan-root>/<source-sha256-hex>/plan.json
```
~~~
`auto` reuses a complete valid plan or regenerates missing or invalid state.
`refresh` regenerates and atomically replaces a plan after chunk validation.
`bypass` performs no plan-cache I/O and does not resolve or create the root.
Plan selection is source-addressed and independent of checkpoint and debug
roots.
One validated canonical plan is active for each source digest. The plan stores
boundaries and provenance, not a second copy of the entire source. This
source-addressed policy is recorded in [ADR-0005](adr/0005-cache-canonical-chunk-plans-by-source.md).
When its directory is empty in configuration, the root is
`<os.UserCacheDir>/notarius/chunk-plans`. A configured directory is the exact
root; no suffix is appended. Directories and files created by the store use
`0700` and `0600` permissions on supported Unix systems. The configured root
is a trust boundary: do not share it among mutually untrusted users.
The configured cache mode controls one invocation:
Remove an exact digest directory or the configured root only when accepting the
cost of recomputing plans and any chunk-stage work. Cache publication is atomic;
there is no history, locking, garbage collection, or rollback facility.
- **auto** looks for a valid active plan. Missing or invalid state causes a new
plan to be generated; an accepted new plan is atomically published.
- **refresh** skips lookup, generates a plan with the configured chunker, and
atomically replaces the active plan after it is accepted.
- **bypass** performs no chunk-plan cache I/O. It does not resolve or create a
chunk-plan root.
For a Linux service account, provision a dedicated restrictive root such as:
A reused plan is still materialized and validated against the current source.
If a prior plan no longer gives acceptable results, use a refresh run rather
than editing cache files. Deleting a plan is recoverable but can repeat costly
chunking work.
```yaml
cache:
chunk_plans:
directory: /var/cache/notarius/chunk-plans
```
## Checkpoint Recording, Resume, And Recompute
## Checkpoint Cache
Checkpoint recording is an explicit configuration choice and is disabled by
default. When enabled, each run records stage transitions and the state needed
for compatible recovery. A run records checkpoints even when it does not ask
to reuse them. Checkpoint payloads can contain source-derived and intermediate
application data, so treat the entire root as sensitive.
Checkpoint recording is controlled by `cache.checkpoints.enabled`, which
defaults to `false`. When enabled, every run records running, succeeded, and
failed transitions and reusable validator-approved results. Successful,
rejected, and failed runs may therefore all leave checkpoint state. The
`--resume` flag additionally loads compatible completed work before executing
missing or incompatible stages. Without `--resume`, a recording-enabled run
never loads checkpoints. Using `--resume` while recording is disabled is an
error.
Checkpoint loading is separate: [**--resume**](cli.md#run) asks a run to reuse
compatible recorded work. A resume request fails when checkpoint recording is
disabled. Without **--resume**, a recording-enabled run executes normally and
does not load checkpoint state. Compatibility includes the resolved pipeline,
input, selected lanes, runtime overrides, reference provenance, LLM-profile
provenance, the effective PromptKit profile-source fingerprint, and
prepared-component fingerprints. When a local PromptKit backend is configured,
compatibility also includes a non-secret fingerprint of its endpoint. Changing
profile content or the local endpoint causes a cold miss; changing only the
local concurrency limit does not. A changed identity produces a cold miss;
Notarius does not migrate, rewrite, or delete older checkpoint directories.
Reasoning-effort inheritance, replacement, and explicit clearing are distinct
runtime identities, so checkpoints created under one state are not reused by
either of the others.
Checkpoints use the selected root and the existing identity hierarchy:
Checkpoint state is confined below an identity-specific path:
```text
<checkpoint-root>/<pipeline-id>/<input-key>-<source-or-input-digest>/<pipeline-digest>/<identity-digest>/...
```
~~~
<checkpoint-root>/<pipeline-id>/<input-key>-<source-or-input-digest-prefix>/<pipeline-digest-prefix>/<identity-digest-prefix>/
~~~
The final identity digest includes stable semantic fingerprints explicitly
contributed by prepared modules and validators. Adding or changing one of
these fingerprints intentionally causes a cold cache miss; old checkpoint
directories are left in place and are never migrated or deleted automatically.
### Selective Recompute
An empty configured directory selects
`<os.UserCacheDir>/notarius/checkpoints`. The root is exact when configured.
Created directories and files use `0700` and `0600` permissions on supported
Unix systems.
[**--recompute-step**](cli.md#run) requires both **--resume** and enabled
checkpoint recording. It forces the selected ordered step and every lane that
depends on it through generated artifact references. Unrelated lanes remain
eligible for reuse.
Checkpoint payloads can contain source text, intermediate artifacts, metadata,
warnings, and content digests. Treat them as sensitive derived application
data. Compatible files from a former checkpoint root remain reusable when
`cache.checkpoints.directory` names that exact existing root. They are not
moved, migrated, or deleted automatically. The frozen serialized identifier
`workspace_schema_version` remains part of checkpoint compatibility; it is not
a configuration setting.
For an earlier producer required by a forced consumer, Notarius requires a
compatible accepted normalized artifact. It validates that artifact before
hydrating it and does not silently rerun the producer. If that state is
missing, rejected, corrupt, non-canonical, or incompatible, the run stops
before its dependent starts. Rerun the required producer deliberately instead
of copying or editing checkpoint files.
For a Linux service account, independently provision:
## Checkpoint Decisions And Recovery
```yaml
cache:
checkpoints:
enabled: true
directory: /var/cache/notarius/checkpoints
```
Checkpoint events classify work as **executed**, **reused**,
**forced_recompute**, or **dependency_invalidated**. Their stable reason codes
are written to run diagnostics and provenance. Use the code, not a copied
error message, to decide what to repair.
Remove an exact checkpoint identity directory or the configured root only when
recomputation is acceptable.
### Resume And Selective Recompute
`--resume` loads compatible accepted work only when checkpoint recording is
enabled. A normal resumed run may reuse source, extract, merge, and normalize
checkpoints independently and may recompute a stage after a cache miss.
Generated references add a dependency fingerprint
for the producer's artifact kind, schema identity, media type, canonical
content digest, and size. If that fingerprint changes or the producer is
missing, dependent checkpoints are invalidated; unrelated work remains eligible
for reuse.
`--recompute-step <step-id>` requires both `--resume` and
`cache.checkpoints.enabled: true`. It forces the named step and all transitive
dependents to execute, while compatible predecessors and unrelated lanes remain
reusable. The ID may be an explicit configured step or `default` for an
implicit single-step pipeline. It cannot be combined with `--only`, and it does
not change the persistent identity of otherwise identical checkpoints.
Decisions are bounded and categorized as `reused`, `executed`,
`forced_recompute`, or `dependency_invalidated`.
For an unselected producer required by a recomputed step, Notarius loads the
accepted normalized artifact directly. Valid normalize state is sufficient even
when that producer's extract or merge checkpoint is missing or corrupt. The
normalize manifest must be successful and match workspace schema v3, the exact
current invocation identity, step, lane, and normalizer; its payload digest and
canonical codec representation must also validate. A forced producer bypasses
this lookup and executes.
If a required predecessor's accepted normalized artifact is missing, rejected,
corrupt, non-canonical, or incompatible, the run fails before the dependent
step starts. It does not fall back to rerunning that predecessor. The failure
manifest retains completed upstream outcomes and dependency context but not
generated reference content. For diagnosis, first check the producer step and
lane in the manifest, then inspect checkpoint decision categories and reason
codes. Rerun the producer explicitly rather than copying an artifact into the
checkpoint root.
The decision that caused a required-predecessor failure is retained before the
run returns, and the CLI error identifies its step, lane, and reason code.
Checkpoint reason codes are stable diagnostic identifiers:
| Reason code | Operator meaning |
| Reason code | Recovery meaning |
| --- | --- |
| `loading_disabled` | This invocation did not enable checkpoint loading. |
| `checkpoint_missing` | The requested checkpoint file does not exist. |
| `checkpoint_path_invalid` | The requested checkpoint location failed confinement validation. |
| `checkpoint_read_failed` | An existing checkpoint could not be read. |
| `checkpoint_decode_failed` | Checkpoint JSON could not be decoded. |
| `workspace_schema_incompatible` | The stored workspace schema is not supported by this build. |
| `identity_mismatch` | The stored invocation identity differs from the current invocation. |
| `stage_mismatch`, `step_mismatch`, `lane_mismatch`, `module_mismatch` | Stored scope does not match the requested pipeline scope. |
| `status_not_reusable` | The stored operation did not finish in a reusable status. |
| `dependency_mismatch` | Stored dependencies differ; the category is `dependency_invalidated`. |
| `artifact_payload_invalid` | Stored artifact payload structure or encoding is invalid. |
| `artifact_digest_mismatch` | Stored artifact bytes do not match their recorded digest. |
| `artifact_codec_incompatible` | Stored artifact identity is incomplete or incompatible with the codec contract. |
| `artifact_not_canonical` | The codec can decode the artifact, but its bytes are not canonical. |
| `checkpoint_reused` | The stored checkpoint passed validation and was reused. |
| `accepted_artifact_reused` | A required producer's accepted normalized artifact was canonically validated and hydrated. |
| `recompute_step` | Selective recomputation forced execution of this lane. |
| **loading_disabled** | This invocation did not permit checkpoint loading. |
| **checkpoint_missing**, **checkpoint_path_invalid**, **checkpoint_read_failed**, **checkpoint_decode_failed** | The stored checkpoint could not be located or read safely; normal resume work can execute again. |
| **workspace_schema_incompatible**, **identity_mismatch**, **stage_mismatch**, **step_mismatch**, **lane_mismatch**, **module_mismatch** | Stored state belongs to a different compatible scope or identity; allow a fresh run to create new state. |
| **status_not_reusable** | The recorded operation did not end in reusable state. |
| **dependency_mismatch** | A dependency changed; dependent work is invalidated rather than reused. |
| **artifact_payload_invalid**, **artifact_digest_mismatch**, **artifact_codec_incompatible**, **artifact_not_canonical** | A stored artifact cannot safely be hydrated; rerun the producer instead of modifying the cache. |
| **checkpoint_reused** | A normal checkpoint passed compatibility checks. |
| **accepted_artifact_reused** | A required predecessor's accepted normalized artifact was safely hydrated. |
| **recompute_step** | Selective recomputation deliberately forced this work. |
Decision detail is bounded explanatory text derived from the stable reason code,
not caller-supplied prose or a data-recovery channel. It never contains
checkpoint paths, artifact or reference content, source content, credentials,
or environment values.
Reason detail is bounded code-owned text. It is diagnostic information, not a
path-discovery or data-recovery mechanism, and does not contain checkpoint,
source, reference, credential, or environment content.
## Debug Bundles
Only `notarius run --debug` enables debug collection. The selected root contains
one retained bundle per invocation:
Only a [debug-enabled run](cli.md#run) creates a bundle:
```text
~~~
<debug-root>/<run-id>/
summary/
trace/
```
~~~
`summary/` contains redacted invocation, effective-configuration, resolved
pipeline and reference provenance, checkpoint and chunk-plan decisions, run
manifest, warnings, report, and any available error text. It excludes raw
source, references, annotations, prompts, model responses, credentials, and
malformed cache bytes.
The summary contains redacted invocation and resolution information plus run,
warning, checkpoint, chunk-plan, and terminal reporting artifacts. The trace
contains allowlisted application diagnostic records and can include source or
derived application data. Neither surface is a cache input. Do not treat a
debug bundle as safe to share merely because its configuration summary is
redacted. Invocation metadata omits reasoning effort when it is inherited,
records the replacement value when one is supplied, and records an empty value
when inherited reasoning was explicitly cleared.
`trace/` contains application-owned execution detail, including source and
stage material, plans, chunks, validator attempts, prompts, model responses,
timing, and serialized artifacts. It may retain application data omitted from
output. Credentials, credential-shaped values, sensitive metadata, unrelated
environment values, and unrelated filesystem content are not captured.
Bundles inherit the sensitivity of the application data they capture. Their
additional risk comes from copying and aggregating that data, so restrict
access, avoid shared roots between untrusted users, and define retention outside
Notarius. Created bundle directories use `0700` and files use `0600` on
supported Unix systems.
Notarius never automatically deletes a requested bundle. If allocation
succeeds, its path is reported on success and failure. A requested summary or
trace write failure makes the command fail, preserving whatever bundle data was
already written for inspection. Every allocated bundle makes one best-effort
attempt to record a terminal `run-report.json`.
## Failures And Warnings
Failures before debug allocation are reported on stderr without a bundle.
Failures after allocation report the bundle path on stderr and make independent
attempts to write a failure `run-report.json` and `error.log`. The report retains
the paths and pipeline outcome fields known at the failure point. If either
terminal write fails, the original command error remains first on stderr,
followed by the persistence error and bundle path. An output-write failure
leaves the allocated bundle in place. A successful run with warnings exits `0`,
reports a warning count on stderr, and records warnings in durable output and
any requested debug summary.
Notarius never creates debug state without an explicit request and never
automatically deletes a requested bundle. If allocation succeeds, the command
reports its path on both success and later failure. A summary, trace, or
terminal-report persistence failure fails the command while preserving any
already-written diagnostic data for inspection.
## Cleanup
Use exact paths for manual cleanup. Examples:
Cleanup is manual and destructive. First inspect the exact leaf directory,
then remove only that leaf; do not use a glob or a parent root as the target.
```sh
rm -rf ./notarius-output/run-1721300000000000000-0123456789abcdef0123456789abcdef
rm -rf /var/cache/notarius/chunk-plans/0123abcd
rm -rf /var/cache/notarius/checkpoints/pipeline/input-0123/pipeline-4567/identity-89ab
rm -rf ./notarius-debug/run-1721300000000000000-0123456789abcdef0123456789abcdef
```
~~~
rm -rf -- /srv/notarius/output/run-1721300000000000000-0123456789abcdef0123456789abcdef
rm -rf -- /srv/notarius/chunk-plans/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
rm -rf -- /srv/notarius/checkpoints/example/seriatim-0123456789abcdef/0123456789abcdef/0123456789abcdef
rm -rf -- /srv/notarius/debug/run-1721300000000000000-0123456789abcdef0123456789abcdef
~~~
Avoid broad recursive cleanup against a parent root unless it is an explicit
operator policy. Output deletion is permanent user-data loss. Cache deletion is
recoverable but can repeat expensive work. Debug deletion removes troubleshooting
evidence and any retained application-data copy.
Deleting output permanently removes user data. Deleting chunk plans or
checkpoints is recoverable but may repeat expensive provider or pipeline work.
Deleting a debug bundle removes troubleshooting evidence and a retained copy of
application data. Notarius has no cache garbage collector, rollback operation,
or automatic cleanup command.
## Operational Limits
Provider retries and timeouts are handled by Scriptorium according to the
selected execution profile. Pipeline module retry settings are defined in
[Configuration](config.md#module-bindings). Extract worker concurrency and
actual provider-call concurrency are separate limits; their fields and
validation are defined in [Configuration](config.md#concurrency). Notarius
writes local files only; remote storage and archive management are outside the
implemented CLI.
Provider execution settings and the generation timeout come from the selected
PromptKit profile. The invocation-only **--reasoning-effort** and
**--clear-reasoning-effort** controls may replace or clear that profile setting
for all LLM-backed calls in one run without changing the profile. PromptKit
v0.5.0 does not add a provider retry loop. Notarius binding retries rerun the
complete module operation and validation chain as defined by
[module bindings](config.md#module-bindings-and-validators).
Timeouts are layered. Caller cancellation is the outer authority. A positive
effective generation timeout adds an inner request deadline, while zero
disables only that generation deadline. The HTTP client timeout remains a
transport-wide cap. Notarius does not add another timeout around PromptKit.
The pinned upstream boundary and profile-format links are in
[PromptKit Integration](integrations/pkg-promptkit.md).
Concurrency has two independent layers. Notarius **total_llm** is the
application-wide provider-call limit shared by all backends, modules, retries,
and validators. PromptKit may impose a narrower admission limit for the
selected backend. The effective active-generation bound is the intersection of
both limits and can therefore be lower than **total_llm**. Built-in OpenRouter
profiles use PromptKit's upstream backend limit; endpoint-only profiles have no
PromptKit backend limit and remain bounded by Notarius. For the configured
local backend, a zero **concurrency_limit** leaves only the Notarius scheduler
as a call limit. A positive value makes the effective active local-generation
bound the smaller of **total_llm** and that local limit.
For a positive local limit, PromptKit owns its default waiting capacity and
admission behavior. When a PromptKit backend has admitted all active and queued
work, a new call fails as capacity exhaustion before generation. The adapter
maps that failure to Notarius's existing provider-neutral capacity error and
does not retry it. The calling stage's configured retry policy applies
normally, and the run fails if those attempts are exhausted. Caller
cancellation remains authoritative. Configuration contracts are documented
under [PromptKit profiles](config.md#promptkit-profiles) and
[concurrency](config.md#concurrency-output-cache-and-debug). Extract-worker
limits and actual provider-call limits are independent. Notarius writes local
filesystem state only; remote storage, archival, and retention automation are
outside the implemented CLI.

View File

@@ -79,6 +79,14 @@ Pipeline resolution requires a compatible codec and matching kind-specific
variants before a typed lane can be accepted. Framework-owned erasure remains
private and must report type incompatibility as an error rather than a panic.
An artifact kind may additionally provide a typed evidence projection that
copies its direct generic source references. Preparation proves that projection
matches the artifact codec's exact Go type before retaining it for an output
policy. The runner reconstructs evidence only from accepted serialized
normalized artifacts, and the output boundary owns any resulting publication.
Generic framework code never infers evidence by inspecting domain JSON or
depends on domain artifact types.
Auxiliary references provide context or disambiguation. They are not source
evidence and must not be converted into source references.
@@ -169,7 +177,8 @@ individual modules.
The application-wide LLM scheduler bounds actual provider calls independently
of framework worker limits. Every LLM-backed module, retry, and validator uses
the single injected scheduled client, including work performed by overlapping
lanes.
lanes. Provider runtime adapters may enforce a narrower backend-specific limit
beneath this mandatory application-wide scheduler.
## Configuration And Provenance

View File

@@ -1,766 +0,0 @@
# D&D Extraction Module Refactoring Audit Strategy
Status: Strategy complete; final audit results recorded below
## Purpose
Define a disciplined audit of the five production D&D extraction modules:
- `dnd/spells`;
- `dnd/npcs`;
- `dnd/combat-turns`;
- `dnd/npc-interactions`; and
- `dnd/scene-descriptions`.
The audit will determine whether these modules follow a coherent set of
conventions, whether repeated implementation can be replaced by appropriately
scoped shared code or assets, and whether the accumulated design suggests other
maintainability improvements.
This document defines how to perform the audit. It does not contain audit
findings and does not authorize production changes.
## Audit Principles
The audit must distinguish consistency from uniformity. The five modules should
use the same conventions where they perform the same responsibility, but a
module should remain different when its artifact semantics, evidence model,
reference requirements, or normalization policy require it.
Recommendations must follow these principles:
- Prefer evidence from current code, tests, prompts, schemas, configuration,
and documentation over naming or visual similarity.
- Treat a divergence as a finding only when it is unexplained, increases
maintenance or correctness risk, or violates an intended convention.
- Do not recommend an abstraction solely to reduce line count. A shared helper
must own one coherent invariant and make future correct changes easier.
- Preserve package ownership of artifact semantics, private response DTOs,
private structured-output schemas, and lane-specific prompts.
- Keep D&D-specific behavior in D&D packages. Move behavior into a generic
framework package only when the contract is demonstrably domain-neutral and
has a non-D&D consumer or a clear framework-owned responsibility.
- Evaluate prompt sharing by byte identity and semantic ownership. Prompt
caching benefits only when repeated message content and ordering are exactly
identical.
- Apply the testing policy to proposed refactors. Prefer behavioral protection
at stable boundaries and do not add change-detector tests for helper usage,
prompt length, exact hashes, or private file layout.
## Scope
### Primary scope
Inspect the complete package-owned implementation beneath:
- `internal/modules/dnd/extract/spells`;
- `internal/modules/dnd/extract/npcs`;
- `internal/modules/dnd/extract/combatturns`;
- `internal/modules/dnd/extract/npcinteractions`; and
- `internal/modules/dnd/extract/scenedescriptions`.
For each package, include:
- module identity, capabilities, construction, options, registration, and
execution class;
- reference-slot declarations and construction-time or operation-time
reference handling;
- checkpoint fingerprints and manifest metadata;
- request validation and structured LLM request preparation;
- private response DTOs, response-schema loading, and response mapping;
- source-reference resolution, evidence attachment, canonicalization,
ordering, and exact deduplication;
- embedded prompt manifests, shared and local assets, message order, cache
boundaries, and schema assets;
- errors, warnings, diagnostics, cloning, and mutation safety; and
- package-local tests and test support.
### Contextual scope
Inspect a neighboring component only when needed to determine ownership,
duplication, or compatibility:
- `internal/modules/dnd/shared` and focused D&D subpackages used by more than
one extractor;
- the five artifact model and codec contracts;
- corresponding merge, normalize, and validate variants;
- production registration and default validator composition;
- pipeline reference, fingerprint, and LLM contracts;
- canonical current-behavior documentation and integration contracts; and
- representative production and integration tests.
Contextual inspection is not a request to redesign every lane stage. Findings
outside extraction should be reported only when they directly explain an
extractor inconsistency or reveal a misplaced responsibility.
### Exclusions
Do not use this audit to:
- change durable artifact schemas or extraction policy;
- redesign the fixed pipeline shape or ordered-step model;
- combine distinct artifacts into a larger D&D result;
- evaluate live-model output quality;
- introduce schema generation, a dependency-injection framework, or a general
module superclass;
- move domain rules into the generic framework;
- rewrite tests merely to make their file layout look alike; or
- implement any recommended refactor.
If the audit exposes a product-contract concern, record it separately from
refactoring recommendations and identify the additional decision required.
## Comparison Method
### 1. Establish a module inventory
Create one row per module in a working comparison matrix. Record exact current
facts rather than inferred conventions:
| Dimension | Facts to record |
| --- | --- |
| Identity | Module key, artifact kind, capabilities, execution class |
| Files | Production files, embedded assets, focused test files |
| Construction | Dependencies, options, reference decoding, immutable prepared state |
| Registration | `ModuleSpec`, builder, option validation, declared slots |
| Provenance | Manifest metadata and checkpoint fingerprint keys and values |
| Prompt | Prompt ID/version, manifest messages, inputs, shared assets, cache boundaries |
| Schema | Private schema identity, strictness, loader, diagnostics behavior |
| Execution | Request validation, LLM call, response mapping, errors and warnings |
| Evidence | Source identity, range resolution, canonicalization, ordering, deduplication |
| Tests | Contract owner, malformed cases, integration coverage, test-only helpers |
Use the matrix to identify exact agreement, intentional variation, and
unexplained variation. Do not infer a preferred convention from whichever
module was implemented first. Determine the preferred shape from architecture,
current documentation, shared contracts, and the clearest implementation.
### 2. Classify every divergence
Assign each observed difference one classification:
- **Required specialization:** the artifact or reference contract requires the
difference. No harmonization is recommended.
- **Permitted variation:** implementations differ without meaningful
maintenance or correctness cost.
- **Convention drift:** equivalent responsibilities use different names,
layouts, error behavior, metadata, validation, or tests without a reason.
- **Architectural divergence:** responsibility is placed in the wrong layer or
bypasses a shared contract.
- **Undetermined:** more evidence or a policy decision is needed.
For required specialization, document the reason briefly so a future audit
does not repeatedly flag it. For drift or architectural divergence, identify
the preferred convention and why it is preferable.
### 3. Build a duplication inventory
Search for three kinds of repetition:
1. **Exact duplication:** identical Go logic, prompt text, schema fragments, or
test support.
2. **Structural duplication:** the same algorithm or lifecycle expressed with
renamed domain types.
3. **Policy duplication:** the same invariant is independently encoded in
several production or test layers.
Trace callers and consumers before recommending extraction. Record:
- the repeated responsibility;
- participating modules;
- meaningful semantic differences;
- change history or likely change cadence when discoverable;
- defect risk if copies drift;
- proposed owner and API shape; and
- code or assets that would remain module-owned.
Similar code is not sufficient evidence. Prefer a shared abstraction when at
least one of the following is true:
- three or more modules independently implement the same nontrivial invariant;
- two modules share correctness-sensitive behavior that must evolve together;
- existing duplicated code has already drifted or caused a defect; or
- an existing shared contract is being reimplemented locally.
Avoid extraction when the shared API would need artifact-specific callbacks,
large configuration objects, type erasure, or branching on module identity.
Those are signs that visual similarity is masking separate responsibilities.
## Convention Review
Evaluate the following conventions across all five modules.
### Package organization
- Comparable responsibilities use predictable filenames and package-local
ownership.
- Optional specialized files, such as catalog or registry wiring, are present
only where the module has that responsibility.
- Exported identifiers are limited to framework and registration contracts.
- Test helpers remain local unless sharing them improves test clarity without
coupling independent suites.
### Construction and registration
- Required dependencies fail during construction.
- Options are decoded and unknown options rejected consistently.
- `ModuleSpec`, reference slots, artifact kind, execution class, and builder
behavior agree with runtime behavior.
- Construction resolves stable reference-derived state where possible, while
operation requests own genuinely run- or chunk-specific inputs.
- Returned specs, metadata, fingerprints, and byte slices have consistent
defensive-copy behavior.
### Prompt and schema boundary
- Shared prompt messages come from canonical shared assets; package assets
contain only lane-specific wording.
- Shared content is exactly identical across manifests and appears in the
documented cache-friendly order.
- Stable messages and cache boundaries precede the variable transcript.
- Declared prompt inputs match reference slots and generated projections.
- Private response schemas are strict structural envelopes and reject unknown
fields.
- Semantic validation remains in deterministic code at the intended boundary.
- Schema identity, version, digest, and diagnostics are exposed consistently
without leaking schema or prompt content.
### Extraction and evidence
- Request validation and contextual error wrapping follow one recognizable
pattern.
- Reference material aids disambiguation but never becomes transcript evidence.
- Source unit IDs are resolved against document order rather than numeric
assumptions.
- Mapping performs only the deterministic transformations owned by extraction.
- Canonical source references, stable ordering, and exact deduplication use
consistent policies where artifact semantics agree.
- Whole-chunk evidence in scene descriptions is treated as an intentional
specialization rather than forced through citation-oriented helpers.
- Results do not share mutable state with model responses, references, or
requests.
### Provenance and diagnostics
- Prompt and response-schema identities and hashes appear consistently in
manifest metadata.
- Checkpoint fingerprints cover every stable semantic input that could change
accepted output, without including credentials, paths, timestamps, or source
content.
- Registry and catalog projections use canonical bytes and retain bounded
provenance.
- Errors and diagnostics are contextual, bounded, and free of raw reference or
secret content.
### Tests
- Each important contract has one clear test owner.
- Equivalent risks receive comparable coverage without requiring identical test
file layouts.
- Schema tests cover required structure and unknown-field rejection without
duplicating every semantic validator case.
- Extractor tests cover request validation, response mapping, evidence,
ordering, references, and provider failures at the narrowest stable boundary.
- Prompt tests prepare real embedded assets and protect input placement,
ordering, cache boundaries, and content-safety properties without freezing
exact prompt text.
- Registration or production tests prove assembly once and do not repeat
package-local behavior unnecessarily.
- Obsolete, redundant, representation-specific, or impossible fixtures are
identified for deletion or simplification.
## Shared-Code Decision Framework
Recommend the narrowest owner that matches the repeated responsibility:
1. Keep artifact semantics in the module that owns the artifact.
2. Use `internal/modules/dnd/shared` for D&D-wide mechanics with identical
semantics, such as prompt inputs or source-unit reference handling.
3. Use a focused D&D subpackage for behavior shared by a subset of lanes, such
as NPC registry projection, when it has a coherent domain contract.
4. Use a framework package only for transport-neutral or domain-neutral
behavior owned by the framework.
Evaluate these candidate categories without assuming they should be extracted:
- extractor request precondition validation;
- private response-schema loading and metadata assembly;
- prompt asset registration and hashing;
- source-reference conversion, canonical ordering, and exact deduplication;
- manifest metadata and checkpoint fingerprint assembly;
- NPC registry resolution and names-only projection;
- strict option decoding and module registration;
- diagnostic redaction and bounded error context; and
- repeated test fixtures or schema-validation utilities.
For prompts and schemas:
- Factor prompt wording into a shared asset only when every consumer needs the
exact same text and should receive future changes atomically.
- Prefer an existing shared asset over a new near-duplicate.
- Do not create shared prompt fragments solely because prose is similar.
- Keep private response schemas package-owned unless a genuine shared wire
contract exists.
- Do not introduce shared JSON Schema fragments or generation unless the audit
demonstrates a maintenance problem that outweighs tooling and indirection.
## Additional Quality Review
Beyond consistency and duplication, inspect:
- functions with high cognitive complexity or responsibilities that can be
separated without obscuring the extraction flow;
- repeated linear scans inside loops, avoidable serialization, unnecessary
allocations, or per-chunk reconstruction of stable state;
- hidden mutation, aliasing, or inconsistent clone boundaries;
- fingerprint omissions that could permit stale checkpoint reuse;
- prompt inputs or references repeated unnecessarily across messages;
- unreachable defensive checks or validation performed redundantly at several
layers;
- errors that lose module, field, source, or stage context;
- stale documentation, fixtures, names, compatibility aliases, or comments;
- public or package abstractions that have only one artificial consumer; and
- opportunities to delete code after a shared helper replaces it.
Performance recommendations must identify a plausible workload and complexity
impact. Do not recommend micro-optimization without evidence.
## Evidence Collection
Perform the audit in this order:
1. Read the architecture, testing, documentation, module, LLM, pipeline, and
relevant integration contracts.
2. Build the five-module comparison matrix from definitions and assets.
3. Use graph similarity only to identify candidates; read the complete
functions and trace their callers before classifying them.
4. Compare prompt manifests and shared assets byte-for-byte, then compare local
prompt semantics.
5. Compare private schemas structurally and map each field to its DTO, mapper,
durable artifact, and validator owner.
6. Trace reference and fingerprint data from construction through the LLM
request and checkpoint identity.
7. Review focused tests alongside the behavior they own.
8. Run the existing focused and repository-wide validation commands to
distinguish current failures from maintainability observations.
Do not modify production code, tests, prompts, schemas, examples, or
current-behavior documentation during the audit.
## Finding Standard
Every reported finding must contain:
- severity: high, medium, or low;
- category: convention drift, duplication, architecture, correctness,
performance, testing, or documentation;
- affected modules and exact file or symbol references;
- observed behavior and the convention or invariant it is compared against;
- concrete maintenance, correctness, cost, or security impact;
- recommended target state and ownership;
- why the recommendation is preferable to leaving the code separate; and
- validation or migration considerations.
Order findings by severity and impact, not by module. Separate confirmed
findings from optional improvements. State explicitly when no issue is found in
a comparison area.
For each apparent duplication, the audit must choose one outcome:
- extract now;
- harmonize without sharing;
- retain intentionally separate; or
- defer pending a named missing requirement.
Do not report speculative abstractions as findings. Record them, if useful, as
rejected or deferred candidates with the reason.
## Audit Deliverable
The completed audit should provide:
1. an executive conclusion addressing convention consistency, shared-code
opportunities, and overall code quality;
2. the completed five-module comparison matrix;
3. prioritized findings with evidence and recommendations;
4. intentional differences that should be preserved;
5. rejected or deferred sharing candidates and rationale;
6. a proposed refactoring sequence grouped into independently safe changes;
and
7. validation commands and any residual risks.
The audit should be actionable enough to support a later decision-complete
implementation plan, but it must not implement or silently commit any
recommendation.
## Completion Criteria
The strategy has been followed when:
- all five primary packages and their prompts, schemas, registration,
provenance, reference handling, mapping, and tests have been compared;
- every divergence has a classification;
- every repeated candidate has an ownership and keep/share decision;
- architecture and testing-policy constraints are applied explicitly;
- findings cite exact evidence and explain impact;
- intentional specialization is documented alongside drift;
- repository validation results are recorded; and
- no production changes were made as part of the audit.
## Audit Results
Status: Complete
### Executive Conclusion
The five D&D extraction modules follow a coherent overall convention: all are
typed, production-registered extractors with strict private response schemas,
package-owned artifact mapping, bounded provenance, shared canonical prompt
assets, defensive result ownership, and deterministic downstream validation.
Their differences in catalog/registry state, reference slots, local prompt
assets, whole-chunk versus cited evidence, and scene prose cleanup are explained
by artifact semantics and should remain.
The consistency question is therefore **mostly yes, with two material
exceptions**. Spell artifact ordering and all four citation extractors'
reference ordering assume numeric unit IDs instead of source-document order,
and spell/NPC checkpoint identities omit stable mapping policies. Both can
change durable results or reuse results produced under different semantics.
Smaller drift exists in common request preflight, scene semantic-validation
ownership, prompt cache-boundary coverage, and package surface conventions.
The shared-code question is **yes, but only for two narrow D&D-wide
responsibilities**: common chunk-extraction preflight and document-aware
source-reference ordering/canonicalization. They belong in
`internal/modules/dnd/shared`, not in the framework. Private response DTOs,
artifact mapping, structured LLM calls, prompt prose, response schemas,
metadata assembly, error prefixes, and typed test fakes should remain
package-owned; sharing them would require callbacks, type erasure, module
branching, or broad configuration.
Overall code quality is **good**. Construction, registration, cloning,
diagnostics, schema loading, prompt composition, and validator composition are
clear and consistently tested. The repository passes all current validation.
The recommended work is targeted correction and consolidation, not a redesign
of the extractor family.
### Final Module Comparison Matrix
| Module | Common contract | Required specialization | Prompt/schema boundary | Evidence and ordering | Provenance and tests |
| --- | --- | --- | --- | --- | --- |
| Spells | `dnd/spells` -> `dnd/spell-list`; typed LLM extractor; strict private DTO/schema; append merge and typed normalize/validate ([spec](../../internal/modules/dnd/extract/spells/extractor.go#L207)) | Prepared spell catalog and optional NPC registry projection; catalog prompt input ([constructor](../../internal/modules/dnd/extract/spells/extractor.go#L73)) | Shared evidence/input messages plus local task, instructions, catalog, and transcript; semantic catalog validation remains deterministic ([manifest](../../internal/modules/dnd/extract/spells/assets/prompts/dnd.spells.yaml#L23)) | Attaches current transcript ID and preserves invalid candidates, but sorts artifacts/references by numeric unit ID rather than document position ([canonicalization](../../internal/modules/dnd/extract/spells/canonicalize.go#L10)) | Prompt/schema/catalog/NPC projection fingerprints, but no mapping-policy fingerprint; broad extractor tests, incomplete full prompt cache-order assertion ([fingerprints](../../internal/modules/dnd/extract/spells/extractor.go#L144)) |
| NPCs | `dnd/npcs` -> `dnd/npc-list`; same typed lifecycle ([spec](../../internal/modules/dnd/extract/npcs/extractor.go#L159)) | Deterministic NPC identity derivation; campaign references remain request context ([mapping](../../internal/modules/dnd/extract/npcs/canonicalize.go#L94)) | Shared evidence/input messages plus NPC-local task/instructions/transcript; private schema owns transport only ([manifest](../../internal/modules/dnd/extract/npcs/assets/prompts/dnd.npcs.yaml#L17)) | Artifact order resolves document positions; reference list still sorts numeric IDs before exact deduplication ([canonicalization](../../internal/modules/dnd/extract/npcs/canonicalize.go#L11)) | Prompt/schema/identity fingerprints, but no mapping-policy fingerprint; weakest focused preflight matrix and partial prompt ordering coverage ([fingerprints](../../internal/modules/dnd/extract/npcs/extractor.go#L91)) |
| Combat turns | `dnd/combat-turns` -> `dnd/combat-turn-list`; same typed lifecycle ([spec](../../internal/modules/dnd/extract/combatturns/extractor.go#L201)) | Optional prepared NPC registry; all combat kinds remain raw validator candidates ([constructor](../../internal/modules/dnd/extract/combatturns/extractor.go#L68)) | Shared evidence/NPC/input messages plus combat-local task/instructions/transcript ([manifest](../../internal/modules/dnd/extract/combatturns/assets/prompts/dnd.combat_turns.yaml#L20)) | Artifact order uses valid source positions; extractor, normalizer, and invariant validator duplicate differing reference-order mechanics ([extractor](../../internal/modules/dnd/extract/combatturns/canonicalize.go#L10), [normalizer](../../internal/modules/dnd/normalize/combatturns/normalizer.go#L228)) | Mapping policy is fingerprinted; complete request preconditions, mapping, registry, and provider tests, but incomplete full prompt cache-order assertion ([fingerprints](../../internal/modules/dnd/extract/combatturns/extractor.go#L126)) |
| NPC interactions | `dnd/npc-interactions` -> `dnd/npc-interaction-list`; same typed lifecycle ([spec](../../internal/modules/dnd/extract/npcinteractions/extractor.go#L203)) | Required NPC registry and names-only projection; exact identity remains in focused domain package ([constructor](../../internal/modules/dnd/extract/npcinteractions/extractor.go#L68)) | Shared evidence/NPC/input messages plus interaction-local task/instructions/transcript ([manifest](../../internal/modules/dnd/extract/npcinteractions/assets/prompts/dnd.npc_interactions.yaml#L20)) | Extractor references sort numeric IDs, while the interaction model already owns correct document-aware canonicalization at too-narrow a layer ([extractor](../../internal/modules/dnd/extract/npcinteractions/canonicalize.go#L10), [model helper](../../internal/modules/dnd/npcinteractions/canonical.go#L17)) | Mapping, identity, prompt, schema, and registry projection are fingerprinted; focused suite omits nil-receiver and full prompt cache-order cases ([fingerprints](../../internal/modules/dnd/extract/npcinteractions/extractor.go#L126)) |
| Scene descriptions | `dnd/scene-descriptions` -> `dnd/scene-description-list`; same typed lifecycle ([spec](../../internal/modules/dnd/extract/scenedescriptions/extractor.go#L174)) | One summary per chunk, whole-chunk evidence, and fingerprinted prose trimming; no catalog/NPC projection ([mapping](../../internal/modules/dnd/extract/scenedescriptions/extractor.go#L150)) | Local task/instructions plus shared input/transcript; schema currently duplicates semantic enum/non-empty policy owned by shape validation ([schema](../../internal/modules/dnd/extract/scenedescriptions/assets/schemas/dnd_scene_descriptions_llm.v1.json#L8), [validator](../../internal/modules/dnd/validate/scenedescriptions/shape/validator.go#L49)) | Whole materialized chunk becomes one source range; citation ordering is not applicable | Mapping policy is fingerprinted; strongest full prompt order/cache test, but preflight omits nil receiver/context cases ([fingerprints](../../internal/modules/dnd/extract/scenedescriptions/extractor.go#L98), [prompt test](../../internal/modules/dnd/extract/scenedescriptions/scriptorium_assets_test.go#L15)) |
All five reject unknown options, register through the production D&D registrar,
return independently owned results, and use package-local structured response
types. No inconsistent secret handling, raw prompt/schema diagnostic exposure,
hidden result aliasing, or unregistered audited extractor was found.
### Prioritized Findings
#### High
1. **Source references and spell artifacts can be durably ordered contrary to
transcript order.**
- **Category:** correctness and architecture.
- **Affected modules:** spell, NPC, combat-turn, and NPC-interaction
extractors; related spell/combat/NPC normalizers and combat invariants.
- **Evidence:** spell selects the smallest positive numeric unit ID
([`earliestSourceUnit`](../../internal/modules/dnd/extract/spells/canonicalize.go#L62));
all four citation extractors sort reference endpoints numerically
([spell](../../internal/modules/dnd/extract/spells/canonicalize.go#L30),
[NPC](../../internal/modules/dnd/extract/npcs/canonicalize.go#L31),
[combat](../../internal/modules/dnd/extract/combatturns/canonicalize.go#L30),
[interaction](../../internal/modules/dnd/extract/npcinteractions/canonicalize.go#L30)).
Valid source documents require unique positive IDs, not monotonically
increasing IDs
([`ValidateDocument`](../../internal/core/source/validation.go#L8)).
The interaction model demonstrates the correct document-aware comparison
([`SourceRefLess`](../../internal/modules/dnd/npcinteractions/canonical.go#L51)).
- **Impact:** valid evidence can be reordered away from transcript order;
spell casts with invalid or later evidence can precede earlier valid
casts. This changes durable list order, evidence presentation, merge input,
and checkpointed results.
- **Target state and owner:** move a document-backed `SourceRefOrder` to
`internal/modules/dnd/shared`, with `EarliestValid` and `Canonicalize`
operations. Preserve invalid candidates, exact deduplication, stable ties,
nil/empty distinction, cloning, and deterministic invalid fallback.
Artifact comparison, DTO conversion, source-ID attachment, and repair
accounting remain local.
- **Why shared:** the rule is a D&D-wide evidence invariant already
implemented by extract, normalize, and validate consumers; continued
copies have already diverged. A shared position index also reduces repeated
`UnitIndex` scans from approximately `O(A log A * R * U)` ordering work to
`O(U + A*R + A log A)` for `A` artifacts, `R` references, and `U` units.
- **Migration/validation:** add non-monotonic, invalid, duplicate, nil/empty,
aliasing, stable-tie, and repair-count tests; migrate the interaction model
and combat normalize/invariant pair before extractors; add/bump mapping
policy fingerprints so old checkpoints miss intentionally.
2. **Spell and NPC checkpoint identities omit stable mapping policies.**
- **Category:** correctness and data integrity.
- **Affected modules:** spell and NPC extractors.
- **Evidence:** both extractors perform deterministic ordering,
canonicalization, source attachment, and mapping
([spell mapping](../../internal/modules/dnd/extract/spells/canonicalize.go#L10),
[NPC mapping](../../internal/modules/dnd/extract/npcs/canonicalize.go#L11)),
but their fingerprint providers name catalog/projection or identity
policies without a mapping policy
([spell fingerprints](../../internal/modules/dnd/extract/spells/extractor.go#L144),
[NPC fingerprints](../../internal/modules/dnd/extract/npcs/extractor.go#L91)).
Prepared fingerprints are lane-scoped
([collector](../../internal/framework/pipeline/prepared_fingerprints.go#L23))
and restore requires exact normalized equality
([comparison](../../internal/framework/checkpoint/loader.go#L301)).
- **Impact:** a mapping-policy code change can reuse a checkpoint produced
under older artifact/evidence semantics when prompt and schema bytes are
unchanged.
- **Target state and owner:** each extractor owns an explicit stable
mapping/canonicalization policy fingerprint. Metadata assembly stays
package-local.
- **Why preferable:** a local named semantic fingerprint directly closes
the reuse gap; a generic metadata builder would only hide module-specific
omissions behind configuration.
- **Migration/validation:** add provider and prepared-checkpoint restore
tests. Adding a fingerprint safely invalidates existing identities by full
list mismatch; no artifact payload migration is required.
#### Medium
3. **Common extraction preflight is copied across five production callers and
has already drifted in test protection.**
- **Category:** duplication and testing.
- **Affected modules:** all five extractors.
- **Evidence:** every `Extract` validates the same context/source/chunk/unit
prerequisites before calling
[`ChunkPromptMaterial`](../../internal/modules/dnd/shared/extraction_inputs.go#L12),
but focused coverage ranges from a complete spell/combat matrix to only
cancellation and source mismatch for NPCs
([spell tests](../../internal/modules/dnd/extract/spells/extractor_test.go#L235),
[NPC tests](../../internal/modules/dnd/extract/npcs/extractor_test.go#L139)).
- **Impact:** validation order, error behavior, or a newly required common
precondition can diverge silently among modules.
- **Target state and owner:** add
`shared.PrepareChunkExtraction(ctx, req) (contracts.LLMInputMaterial,
error)` for cancellation, non-nil source/chunk, non-empty units, and
matching cloned material. Receiver/client checks, specialized references,
provider calls, typed results, and contextual wrapping remain local.
- **Why shared:** this extends the existing common material boundary with
one coherent invariant and serves five real callers without callbacks or
module configuration.
- **Migration/validation:** shared table tests own common inputs; package
tests retain nil receiver/dependency, specialized reference, wrapped
error, and provider cases. Preserve current validation order and useful
package context.
4. **Scene semantic validity has two production owners.**
- **Category:** architecture.
- **Affected module:** scene descriptions.
- **Evidence:** the private response schema enforces the scene-kind enum and
non-empty title/summary
([schema](../../internal/modules/dnd/extract/scenedescriptions/assets/schemas/dnd_scene_descriptions_llm.v1.json#L8));
deterministic shape validation independently enforces the same policy
([validator](../../internal/modules/dnd/validate/scenedescriptions/shape/validator.go#L49)).
The documented schema boundary assigns semantic enum/non-empty rules to
deterministic validators
([LLM internals](../internal/llm.md#L169)).
- **Impact:** the two policies can drift and produce provider-dependent
rejection before typed validation, while non-LLM artifacts see only the
validator.
- **Target state and owner:** retain JSON type, required/nullability, and
unknown-field constraints in the package-private schema; make the scene
shape validator the sole semantic owner.
- **Why harmonize without sharing:** only scene has this duplicated policy;
moving it to the existing validator removes an owner without inventing an
abstraction.
- **Migration/validation:** update schema structural tests and validator
semantic tests, then expect the schema digest/checkpoint identity to
change. Verify provider-decoded invalid candidates reach deterministic
validation.
#### Low
5. **Four prompt suites do not fully protect the documented message-order and
cache-boundary contract.**
- **Category:** testing.
- **Affected modules:** spells, NPCs, combat turns, and NPC interactions.
- **Evidence:** scene descriptions asserts the complete prepared role order
and ephemeral/no-cache placement
([test](../../internal/modules/dnd/extract/scenedescriptions/scriptorium_assets_test.go#L15));
the four citation suites cover registration and selected inputs but not
the full documented sequence
([spell tests](../../internal/modules/dnd/extract/spells/scriptorium_assets_test.go),
[NPC tests](../../internal/modules/dnd/extract/npcs/scriptorium_assets_test.go),
[combat tests](../../internal/modules/dnd/extract/combatturns/scriptorium_assets_test.go),
[interaction tests](../../internal/modules/dnd/extract/npcinteractions/scriptorium_assets_test.go)).
- **Impact:** manifest edits can move variable content into a cacheable
prefix or reorder stable grounding without a focused failure, increasing
request cost or reducing prompt quality.
- **Target state and owner:** each package test should assert its complete
documented prepared sequence and cache flags using the real registry.
- **Why local:** lane inputs differ and the behavior belongs to each prompt
manifest; shared test setup would obscure the boundary.
- **Migration/validation:** assert roles/input identities/cache flags, not
exact prompt text, byte counts, or hashes.
6. **Small exported-surface and defensive-behavior drift remains.**
- **Category:** convention drift.
- **Affected modules:** spells, combat turns, and D&D shared.
- **Evidence:** spell and combat export unused singular `ArtifactType`
constants while production uses typed `ArtifactKind`
([spell](../../internal/modules/dnd/extract/spells/extractor.go#L17),
[combat](../../internal/modules/dnd/extract/combatturns/extractor.go#L17));
spell `ManifestMetadata` lacks the nil guard used by the other four
([spell](../../internal/modules/dnd/extract/spells/extractor.go#L122),
[NPC example](../../internal/modules/dnd/extract/npcs/extractor.go#L74));
[`SourceRefCandidate`](../../internal/modules/dnd/shared/unit_refs.go#L90)
has only a test caller, ignores its document parameter, and is unsafe for
extractor provenance because it trusts model-supplied source identity.
- **Impact:** the package surface presents competing artifact vocabulary,
zero-value behavior is inconsistent, and an artificial shared API invites
incorrect reuse. Immediate runtime impact is limited.
- **Target state and owner:** delete the unused constants; align spell nil
metadata behavior locally; delete `SourceRefCandidate` when the correct
shared reference API lands.
- **Why preferable:** deletion and local harmonization clarify existing
contracts without adding a helper.
- **Migration/validation:** graph search found no production consumers;
compile all internal packages and add one spell zero-value metadata test.
#### Optional improvement
The current module documentation lists only four production consumers of
`ChunkPromptMaterial`, omitting scene descriptions
([documentation](../internal/modules.md#L67)). Correct that list when the shared
preflight boundary is documented. This is localized documentation maintenance,
not a separate production design finding.
No actionable issue was found in option strictness, production registration,
typed codec selection, result cloning, prompt/schema content redaction, local
prompt semantic ownership, catalog/NPC source-evidence separation, provider
error wrapping, or secret handling.
### Intentional Differences To Preserve
- **Prepared specialized state:** spells retains a catalog and optional NPC
resolver; combat retains an optional NPC resolver; interactions requires one;
NPC and scene extraction need only request material. Construction-time
identity belongs only where the artifact uses it.
- **Reference slots:** catalog and NPC registry slots follow grounding needs.
The interaction registry is required; combat/spell registry use is optional.
Campaign references remain prompt context and never become evidence.
- **Evidence model:** scene descriptions cites the whole materialized chunk;
forcing it through citation DTO/canonicalization machinery would weaken its
one-summary-per-chunk contract.
- **Prompt assets:** only byte-identical canonical messages are shared. Local
task, instruction, catalog, identity, and evidence wording changes with the
artifact and must not be coupled.
- **Private schemas and DTOs:** similar `source_refs` fragments do not form a
separately versioned wire contract. Package ownership keeps response changes
aligned with mapping and diagnostics.
- **Prose handling:** scene title/summary trimming is a deliberate fingerprinted
mapping policy; citation-lane names and enum candidates remain raw for their
normalizers and validators.
- **Test organization:** file layout, typed fakes, schema compilation helpers,
and real-registry prompt setup remain package-local. Equivalent risks need
comparable coverage, not identical fixtures or filenames.
### Shared-Code Decisions
| Decision | Owner and scope | Why this is the narrow correct boundary |
| --- | --- | --- |
| Extract common preflight | `internal/modules/dnd/shared`; validate common typed extraction request state and return cloned matching `LLMInputMaterial` | Five current callers repeat one prerequisite to the existing `ChunkPromptMaterial` boundary. Receiver/dependency/specialized checks remain readable and local. |
| Extract source-reference order | `internal/modules/dnd/shared`; document-backed `SourceRefOrder.EarliestValid` and `.Canonicalize` over durable `[]source.SourceRef` | Multiple extract, normalize, model, and validate consumers must evolve together; the rule is D&D-wide but has no demonstrated non-D&D/framework consumer. |
| Harmonize mapping fingerprints | Spell and NPC extractor packages | The missing values are module-semantic; local named fingerprints are safer than a configurable metadata builder. |
| Harmonize scene validation ownership | Private scene schema plus `validate/scenedescriptions/shape` | Remove semantic keywords from the transport schema and keep the already registered durable validator as sole owner. No sharing is needed. |
| Harmonize prompt contract tests | Four citation extractor test suites | Each manifest owns its role/input/cache sequence; consistent assertions should stay beside distinct assets. |
| Harmonize preflight coverage | All five package suites after shared preflight | Shared tests own common branches; local tests retain package-visible context and specializations. |
| Harmonize package hygiene | Spell/combat extractors and D&D shared | Delete unused exports/artificial API and align spell metadata nil behavior without introducing a new abstraction. |
### Rejected Or Deferred Candidates
| Candidate | Decision and rationale |
| --- | --- |
| Deprecated `roster` slot asymmetry | **Deferred pending a product compatibility decision.** Configuration documents `roster` as a deprecated `party` alias ([config](../config.md#L526)), while scene removes it. Choose a removal release or uniform alias lifetime before changing slots and migration guidance. |
| Spell response-schema key/ID naming | **Deferred pending a provenance migration decision.** The spell names differ from the `_llm`/`.llm` convention, but schema identity is persisted metadata. Renaming needs an alias or an explicit manifest/checkpoint compatibility break. |
| Prompt registration/hash wrapper | **Rejected.** `shared.PromptAssetManifest` already owns coherent composition/hashing. Hiding filesystems, registries, `sync.Once`, and diagnostic nouns would require broad configuration. |
| Manifest/fingerprint builder | **Rejected.** Stable inputs are module-specific; a generic builder would accept the same keys/policies as arguments and could conceal omissions such as the current spell/NPC gap. |
| Generic artifact mapper | **Rejected.** Private DTO fields, identity, enums, and artifact types require callbacks, type erasure, or module branching. Only reference ordering is genuinely common. |
| Generic structured-LLM call wrapper | **Rejected.** Generic output and request configuration would hide an already clear framework client boundary without removing semantic work. |
| Shared error-prefix helper | **Rejected.** It would add a module-name parameter to replace three transparent lines and weaken local diagnostic ownership. |
| Shared response schemas or prompt prose | **Retain intentionally separate.** Similar structure/text is not an atomic shared contract; private schema and lane prompt changes should not propagate together. |
| Shared schema-test or provider-fake utilities | **Retain intentionally separate.** Typed DTOs and package assets are the behavior under test; central fixtures would couple suites and obscure failures. |
| Framework-level preflight/reference API | **Rejected.** No non-D&D consumer or framework-owned invariant was found. The D&D shared layer preserves dependency direction and domain ownership. |
### Recommended Refactoring Sequence
1. **Land checkpoint identity protection independently.** Add spell and NPC
mapping-policy fingerprints with provider and checkpoint-restore tests.
This is a small package-local change and protects all later semantic
migrations from stale reuse.
2. **Consolidate source-reference mechanics.** Add and exhaustively test the
D&D shared document-position index/API. Migrate the existing interaction
model helper first, then combat normalize plus invariants, then NPC/spell
normalizers, and finally the four citation extractors. Keep package mapping
and repair accounting local; remove numeric comparators and the unused
`SourceRefCandidate` only after all consumers move.
3. **Consolidate common extraction preflight.** Extend the existing shared
chunk-material boundary, migrate one extractor to establish error/validation
compatibility, then migrate the remaining four. Move common branch tables
to shared tests and retain package-context smoke tests.
4. **Resolve scene validation ownership independently.** Adjust schema tests,
remove semantic enum/non-empty constraints from the private schema, retain
validator cases, and verify the expected schema fingerprint/checkpoint miss.
5. **Close prompt contract coverage.** Add full role/input/cache-boundary
assertions to each citation prompt suite without shared fixtures or exact
text/hash assertions.
6. **Apply low-risk package/documentation cleanup.** Remove unused
`ArtifactType` constants, add the spell metadata nil guard/test, and update
module documentation for the shared helper consumers and boundaries.
Each scope can be reviewed and reverted independently. The sequence is not a
decision-complete implementation plan; implementation should still pin exact
fingerprint values, exported names, error compatibility, and per-package test
cases.
### Validation And Residual Risks
Final validation on 2026-07-24:
```text
go test -count=1 ./... PASS
go vet ./... PASS
go build ./cmd/notarius PASS
gofmt -l . PASS (no files listed)
git diff --check PASS
git diff --no-index --check /dev/null docs/roadmap/audit.md
PASS (no whitespace errors; exit 1 denotes differences)
```
The audit also checked that cited relative paths exist in the current working
tree. The audit changed no production code, tests, prompts, schemas, examples,
or current-behavior documentation.
Residual risks and limits:
- This was a static and deterministic-test audit; it did not evaluate live
model extraction quality, prompt effectiveness, provider cache-hit rates, or
token cost. Prompt changes still need representative human/model evaluation.
- Existing tests use mostly monotonic unit IDs, so the ordering defect is not a
current failing test. The proposed non-monotonic fixtures are required before
changing behavior.
- The exact durable order expected for invalid references must remain
deterministic and diagnostics-friendly during API design; invalid references
must not be discarded merely because default validators usually reject them
later.
- Mapping and schema fingerprint additions intentionally invalidate prior
checkpoint identities. Operators should be told to expect recomputation; no
serialized artifact migration is otherwise indicated.
- The `roster` alias lifetime and spell schema-identity migration remain human
product/compatibility decisions.
- Passing repository checks establishes current deterministic correctness, not
absence of model-quality regressions or correctness under unrepresented
source-document shapes.

View File

@@ -1,200 +0,0 @@
# D&D NPC Interactions
Status: Implemented
The current durable and configuration contract is documented in the
[D&D NPC interaction artifact](../integrations/dnd-npc-interaction-artifacts.md).
## Purpose
The normalized NPC registry intentionally answers only who was identified in a
session. It does not answer whether an NPC was merely mentioned, participated
in dialogue, or fought alongside or against the party. Add a separate,
ordered NPC-interaction artifact for that occurrence-level information rather
than expanding the identity registry.
This feature preserves the minimal-extractor policy: the model identifies one
bounded kind of interaction and its supporting transcript evidence. It does
not summarize the interaction, infer relationships, or maintain NPC state.
## Desired End State
A D&D pipeline can run NPC extraction first and supply its accepted normalized
registry to a later `dnd/npc-interactions` lane. The later lane emits an ordered
list of evidenced interaction occurrences involving registry NPCs.
The production identities should be:
- extractor and normalizer key: `dnd/npc-interactions`;
- artifact kind: `dnd/npc-interaction-list`;
- durable schema ID: `notarius.dnd.npc_interactions`;
- durable schema name: `notarius_dnd_npc_interactions_v1`;
- durable schema version: `v1`; and
- media type: `application/json`.
The lane must use the existing D&D module organization, shared prompt assets,
typed artifact pipeline, codec boundary, registration pattern, and default
validator composition.
## Artifact Contract
The durable payload is an object containing an `interactions` array. The array
may be empty. Each interaction contains exactly:
- `name`: the canonical NPC name from the supplied registry;
- `kind`: one value from the bounded interaction vocabulary; and
- `source_refs`: one or more current-transcript ranges supporting both the NPC
identity and the classified interaction.
Every object rejects unknown fields. The model-facing response should contain
only the corresponding name, kind, and source-unit range candidates. Notarius
attaches the current source identity deterministically; the model must not
reproduce it.
Do not add a separate interaction ID in the durable contract. Stable ordering,
the canonical NPC name, the bounded kind, and exact evidence ranges are enough
to identify and audit an occurrence for the present use cases. Revisit durable
cross-artifact identity only with a concrete consumer requirement.
## Interaction Vocabulary
Use this closed vocabulary:
| Kind | Meaning |
| --- | --- |
| `mentioned` | The NPC is referred to, but is not established as present or communicating in the evidenced passage. |
| `noncombat_presence` | The NPC is present and relevant to the passage but does not meaningfully participate in dialogue or combat. |
| `dialogue` | The NPC speaks, responds, or is directly engaged in a meaningful non-combat exchange. |
| `combat_ally` | The NPC actively participates in combat on the party's side. |
| `combat_opponent` | The NPC actively participates in combat against the party. |
| `other` | The transcript clearly establishes a direct NPC occurrence that fits none of the preceding kinds. |
`other` is a residual category for positively evidenced activity, not an
escape hatch for uncertain classification. Omit a candidate when the
transcript does not support one category.
When activities overlap within one occurrence, apply this precedence:
1. active combat participation outranks dialogue, presence, and mention;
2. dialogue outranks non-combat presence and mention;
3. non-combat presence outranks mention; and
4. `other` applies only when none of the defined categories describes the
evidenced activity.
Combat alignment is not resolved by precedence. An NPC cannot be both a combat
ally and combat opponent in one occurrence; split the record when its alignment
meaningfully changes.
## Occurrence Boundaries And Ordering
An occurrence represents one NPC, one interaction kind, and one locally
coherent transcript passage. Combine repeated evidence only while it supports
the same uninterrupted activity. Create separate occurrences when:
- the interaction kind changes;
- combat alignment changes;
- a scene or meaningful absence separates repeated activity; or
- the NPC is first mentioned and later becomes present.
Extraction remains chunk-scoped, so an occurrence must not span accepted chunk
boundaries. Merge and normalization must not semantically combine occurrences
from different chunks.
Order the durable list by the earliest valid source position. Use canonical
name, interaction kind, and canonical source ranges as deterministic
tie-breakers. Remove only exact duplicate records; do not use model judgment to
collapse nearby occurrences.
## NPC Grounding And Evidence
The extractor declares the existing `npcs` reference slot and requires a bound,
accepted NPC registry. The normal same-run configuration binds the normalized
output of an earlier NPC lane through an ordered generated reference. Existing
framework support for a compatible external registry may remain available, but
must not weaken the artifact contract.
Present the registry's names-only projection to the model. An emitted name must
match one canonical registry name exactly after the registry's established
lookup rules are applied, and the durable artifact retains that canonical
display name. Do not copy NPC IDs or registry source references into an
interaction.
The registry establishes available identity, not occurrence. Every interaction
must cite current transcript ranges that independently support both the NPC and
the selected kind. Campaign references and generated artifacts may disambiguate
a name, but they never become interaction evidence.
If no registry NPC has an evidenced interaction, the correct artifact is an
empty list. A missing, rejected, or incompatible required registry handoff is a
pipeline dependency failure rather than a request to extract ungrounded names.
## Prompt And Model Boundary
Follow the established D&D prompt ordering and cache-boundary policy. Stable
shared instructions, lane instructions, campaign references, and the NPC
names-only projection precede the variable transcript. Factor wording shared
with the spell and combat lanes into the existing shared asset pattern rather
than creating nearly identical package-local messages.
The prompt must:
- define the closed vocabulary and precedence rules;
- ask for occurrences involving only supplied registry NPCs;
- distinguish mention from presence and dialogue;
- require current-transcript evidence for identity and kind;
- forbid summaries, relationship inference, sentiment, aliases, and invented
names; and
- permit an empty result.
Keep the private JSON Schema structural. Deterministic code owns canonical-name
resolution, enum enforcement at the durable boundary, source-range validation,
ordering, exact deduplication, and evidence invariants.
## Validation And Quality
Provide production validators and default chains at extraction and
normalization boundaries consistent with the existing D&D artifacts. The
append-only merge does not require a separate default validator chain.
Deterministic validation must reject:
- missing or extra fields;
- empty or unrecognized NPC names;
- names absent from the supplied registry;
- unknown interaction kinds;
- empty, malformed, out-of-source, or reversed evidence ranges; and
- records whose evidence comes from a reference rather than the current
transcript.
Any relatedness validator should remain warning-only unless evaluation
demonstrates a reliable deterministic rejection rule. Diagnostics must be
bounded and must not leak reference contents.
Evaluate the lane on a small human-reviewed transcript set that includes every
category, transitions between categories, multiple occurrences for one NPC,
mentions followed by appearances, alignment changes, repeated evidence across
chunks, and empty output. Review category agreement, evidence sufficiency,
ordering, duplicate behavior, and reliability on the smaller models the
application is intended to support. Treat model-output evaluation as a human
development aid, not a brittle deterministic test oracle.
## Documentation Outcomes
When implemented, document the durable artifact in `docs/integrations/`, add
the selectable module and validator contracts to configuration documentation,
update the current module and pipeline internals, and provide a maintained
ordered-pipeline example showing NPC extraction followed by interaction
extraction. Future behavior must remain in this roadmap until it exists.
## Non-Goals
This scope does not:
- add occurrence fields to the normalized NPC registry;
- summarize dialogue, combat, or NPC behavior;
- infer disposition, relationships, factions, motives, or persistent state;
- identify player characters or anonymous groups as NPCs;
- add scene participants or duplicate scene-description responsibilities;
- reconcile NPC aliases or perform LLM-assisted semantic deduplication;
- derive interaction records from registry evidence; or
- introduce a DAG, concurrent cross-lane reconciliation, or a new reference
mechanism.

View File

@@ -7,6 +7,37 @@ not as committed release dates.
## Near-Term D&D Pipeline
### Combat Enemy Ledger
- Add a D&D artifact that identifies enemies faced during combat and supports
an end-of-session encounter ledger.
- Track each enemy's observed state using a small controlled vocabulary such as
`active`, `killed`, `fled`, `captured`, or `incapacitated`, while preserving
an explicit unresolved state when the transcript does not establish an
outcome.
- Preserve the evidence for enemy participation and state changes rather than
inferring a terminal outcome from combat ending or an enemy disappearing
from the conversation.
- Define how repeated mentions, groups of unnamed enemies, summoned or allied
creatures, and the same enemy appearing in multiple combats affect identity
and ledger entries.
- Evaluate whether the ledger should be extracted directly, derived from
combat-turn artifacts, or use a sequential pipeline that consumes combat
turns and the normalized NPC registry as grounding references.
### Location Extraction
- Add a D&D artifact for locations visited by the party or otherwise mentioned
in the transcript.
- Distinguish observed visits from references, plans, recalled places, and
uncertain or inferred locations so a mention alone is not reported as a
visit.
- Preserve transcript evidence for each visit or mention and reconcile aliases,
nested places, and repeated appearances without collapsing distinct
locations that share a generic name.
- Define how the location artifact should ground later narrative reports and
whether future event artifacts should retain canonical location identities.
### Evaluate Spell Extraction And Normalization
- Evaluate ordinary extraction retries and the completed normalization path
@@ -16,19 +47,77 @@ not as committed release dates.
validator, and normalizer development. Treat model-quality review as an
iterative human evaluation aid, not a deterministic correctness gate.
### Use D&D Scene Chunking
### Evaluate The Shared D&D Scene Plan
- Allow the combat extractor to no-op for chunks classified as non-combat only
after the scene-description artifact can be supplied through an explicit
ordered dependency. Do not make generic chunk materialization depend on a D&D
classification.
- Use ordered pipeline steps whenever a later artifact needs an accepted
earlier artifact as context. Keep independent lanes in the same step and do
not introduce a general DAG or concurrent cross-lane reconciliation model.
- Reassess whether one shared scene plan provides enough context for NPC,
spell, combat, interaction, and scene-description lanes after real-world use.
Add more complex chunking only in response to demonstrated failures.
## Cross-Cutting LLM Runtime
### Deterministic Prompt Session Identity
- Replace the source-document-ID default for prompt sessions with one
predictable, procedurally generated session ID for the complete
source-processing workload.
- Preserve an explicit non-empty `--session-id` as the highest-precedence
override. Otherwise, derive the default only from the effective input module
identity and the exact raw input bytes.
- Use a versioned, bounded representation such as
`notarius:v1:<sha256(input-module + NUL + raw-input)>`. The exact encoding
must fit PromptKit's session length contract and must not embed source
content.
- Keep the derived session stable across runs, pipelines, selected lanes,
ordered steps, retries, resume, recomputation, LLM profiles, reasoning
overrides, and output, debug, or cache settings.
- Do not include file-backed references, generated references, reference
contents, or the composition of a reference bundle in session derivation.
References may change between prompt calls within one pipeline without
changing routing affinity.
- Resolve the authoritative session before checkpoint construction and use the
same value for checkpoint runtime identity, every prompt-facing module,
PromptKit's direct session field, the compatibility `session_id` prompt
variable, run-manifest metadata, and debug metadata.
- Keep routing identity separate from cache and checkpoint content identity.
Exact prompt prefixes, reference contents, model settings, and other
generation-affecting inputs must continue to participate in their existing
hashes and checkpoint fingerprints even though they do not change the
session.
- Treat the generated value as a provider-visible, stable pseudonymous
correlation identifier. Do not introduce an installation-specific HMAC or
secret unless a concrete multi-tenant or privacy requirement justifies
sacrificing deterministic identity across installations.
### Raise The Default Application-Wide LLM Limit
- Raise the default `concurrency.total_llm` value from 1 to 16 so ordinary
single-backend runs can use PromptKit's expected OpenRouter capacity and
lower-capacity local backends without an unnecessarily narrower Notarius
limit.
- Keep the Notarius application-wide scheduler mandatory and require
`total_llm` to remain a positive integer. Do not make the default unlimited:
endpoint-only profiles, an unrestricted local backend, injected clients, and
aggregate work across several backends may have no narrower PromptKit limit.
- Continue defaulting `concurrency.stage_workers.extract` to the effective
`total_llm`, making its default 16 as part of the same change. Preserve an
explicit lower extract-worker setting when an operator wants less queued or
concurrent extraction work.
- Define effective provider concurrency as the intersection of the Notarius
application-wide limit, the selected PromptKit backend limit when present,
and the work made available by stage execution. A Notarius limit of 16 does
not narrow a backend already limited to 16, while a local backend limited to
4 remains bounded at 4.
- Treat the default as an application-wide safety ceiling across profiles,
backends, modules, retries, and validators. A run that intentionally needs
the combined capacity of several backends may configure a higher
`total_llm` and an appropriate extract-worker count explicitly.
- Retain the existing configuration and environment override surfaces. Update
canonical configuration, operations, and internal documentation together
when the default changes.
- Reconsider decoupling the extract-worker default from `total_llm` only after
mixed-backend workloads demonstrate a need for a high global emergency
ceiling with a lower default work-production rate.
## Shared Normalization And Quality Work
### Generic LLM-Assisted Deduplication

View File

@@ -1,425 +1,586 @@
# D&D Extraction Module Audit Execution Plan
# PromptKit v0.5 Implementation Plan
Status: Ready for audit execution
## Objective
This document divides the audit defined in
[D&D Extraction Module Refactoring Audit Strategy](audit.md) into five prompts.
Execute the stages in order. Each stage is a read-only code audit: it may update
`docs/roadmap/audit.md`, but it must not change production code, tests, prompts,
schemas, examples, current-behavior documentation, or configuration.
Implement the target state in
[PromptKit v0.5 Integration And LLM Profile Policy](promptkit.md). Each numbered
stage is intended to be one implementation prompt for a GPT-5.6-Terra coding
agent. Complete stages in order and leave the repository buildable, tested, and
internally coherent after every stage.
The word "implementation" in this filename refers to implementing the audit
strategy, not implementing refactors.
Follow [Architecture](../policy/architecture.md),
[Testing Policy](../policy/testing.md), and
[Documentation Policy](../policy/documentation.md) throughout. Preserve
unrelated user changes. Use `apply_patch` for source and documentation edits,
run `gofmt` on changed Go files, and add only tests that protect the behaviors
and risks assigned to that stage.
## Global Instructions
Do not implement the separate deterministic session-ID or default-concurrency
roadmap items as part of this plan. Do not perform paid or credentialed LLM
calls.
Every stage must:
## Background Summary
1. Read `docs/roadmap/audit.md` in full before beginning.
2. Follow `AGENTS.md`, `docs/development.md`, and all policies under
`docs/policy/`.
3. Prefer the codebase knowledge graph for code discovery and call tracing.
Use direct file and text searches for prompt assets, JSON Schemas,
configuration, documentation, and exact string comparison.
4. Inspect all five extractors where the stage calls for comparison:
`spells`, `npcs`, `combatturns`, `npcinteractions`, and
`scenedescriptions`.
5. Cite exact files and symbols for every observation written to `audit.md`.
6. Distinguish confirmed facts, provisional interpretations, intentional
specialization, and actionable findings.
7. Avoid treating visual similarity, file count, or line count as sufficient
evidence for sharing.
8. Preserve findings from earlier stages unless new evidence disproves them.
When revising an earlier conclusion, edit it in place and record the reason;
do not append contradictory conclusions.
9. Keep implementation out of scope. Do not modify code to test whether a
proposed refactor is convenient.
10. Leave the repository otherwise unchanged and report any pre-existing dirty
worktree state before proceeding.
Notarius currently pins PromptKit v0.3.0, calls `Prepare` and then `Run` for one
completion, validates profiles through a synthetic prompt, has no application
fallback profile source, and accepts LLM profiles only at individual bindings
or through the run-wide CLI override. PromptKit v0.5.0 is source-compatible
with the current tree; a temporary v0.5.0 module override has already passed
`go test ./...`.
## Working Results Structure
The implementation must nevertheless treat the upstream optional-parameter
change as intentional: unset `temperature`, `max_tokens`, and `top_p` remain
unset and are omitted from compatible provider requests. Do not restore the old
implicit `top_p: 1` default.
Stages 1 through 4 maintain one working area at the end of `audit.md`:
## Stage 1: Upgrade The PromptKit Dependency
```markdown
## Audit Results
### Goal
Status: In progress
Establish a clean PromptKit v0.5.0 baseline before adopting its new APIs.
### Baseline And Module Matrix
### Divergence Register
### Prompt, Schema, And LLM Review
### Extraction And Canonicalization Review
### Duplication Register
### Contextual Architecture And Ownership Review
### Candidate Decisions
```
### Work
Create headings when their owning stage begins. Later stages may refine earlier
sections but must not duplicate them. Use compact tables where they improve
five-way comparison. Keep detailed evidence in the relevant review section and
keep the divergence and duplication registers concise.
- Update `go.mod` and `go.sum` from PromptKit v0.3.0 to v0.5.0 and run
`go mod tidy`.
- Change the PromptKit built-in profile-catalog marker in
`internal/framework/llm/promptkit_profile_fingerprint.go` to identify
v0.5.0. This deliberately invalidates LLM checkpoints tied to the prior
catalog identity.
- Review PromptKit-facing compile errors or test failures against the v0.4.0
and v0.5.0 release guides. Do not adopt prepared execution, inspection, or
fallback profiles in this stage.
- Replace the existing test assertion for one exact built-in fingerprint hash
with durable assertions that the fingerprint is deterministic, non-empty,
non-secret, and changes when a semantic profile source changes. Do not add a
new version-constant or exact-hash change detector.
- Update `docs/integrations/pkg-promptkit.md` to pin and link v0.5.0 and state
the implemented dependency-level behavior: unset optional sampling controls
are provider defaults. Do not document later stages as implemented.
- Update any other canonical text that explicitly claims the dependency is
v0.3.0, but defer descriptions of unimplemented v0.5 APIs.
Each divergence-register entry must have:
### Tests And Validation
- a stable identifier such as `D-01`;
- affected modules;
- observed difference;
- provisional classification from the audit strategy;
- evidence links;
- impact or reason it may be harmless; and
- status: open, confirmed, intentional, rejected, or superseded.
- `go test ./internal/framework/llm ./internal/cli`
- `go test ./...`
- `go vet ./...`
- `go build ./cmd/notarius`
- `rg -n 'promptkit v0\.3\.0|promptkit@v0\.3\.0|PromptKit v0\.3\.0' .`
- `git diff --check`
Each duplication-register entry must have:
### Completion Criteria
- a stable identifier such as `R-01`;
- participating modules;
- repeated responsibility;
- exact, structural, or policy-duplication classification;
- meaningful differences;
- candidate owner, if any;
- evidence links; and
- provisional outcome: extract, harmonize without sharing, retain separately,
or defer.
- The repository directly pins v0.5.0 and all default offline checks pass.
- The profile-source fingerprint identifies the new upstream catalog without a
brittle literal-hash test.
- Current documentation no longer identifies v0.3.0 as the supported version.
Do not assign final severity until Stage 5 has traced ownership and evaluated
impact.
## Stage 2: Execute One Frozen Prepared Snapshot
## Stage 1: Inventory And Convention Matrix
### Goal
### Objective
Make Notarius debug details and generation use one exact PromptKit preparation.
Establish the complete, factual five-module baseline and identify convention
differences without yet recommending shared abstractions.
### Work
### Required review
- Refactor `PromptKitClient.CompleteStructured` to call
`PrepareExecution`, immediately defer `Discard`, obtain a caller-owned
`Details` value, and execute with `RunPrepared`.
- Preserve the existing Notarius request mapping, cancellation precedence,
validation classification, raw structured bytes, response decoding,
profile recording, usage reporting, and credential redaction.
- Ensure every preparation, execution, validation, empty-result, and decode
error retains useful Notarius prompt context without exposing prepared handle
state or secrets.
- Use `errors.As` to obtain `*promptkit.CapacityError` on admission rejection.
Preserve `contracts.ErrLLMCapacityExceeded` as the stable classification and
add a nonblank backend ID only to safe application-owned diagnostic context.
Do not expose `promptkit.CapacityError` outside the LLM adapter.
- Update `docs/internal/llm.md` and the implemented-mechanics portion of
`docs/integrations/pkg-promptkit.md` to describe the single frozen execution
snapshot and structured capacity adaptation.
For every extractor, inspect:
### Tests And Validation
- production and test file inventory;
- module key, artifact kind, capabilities, and execution class;
- constructor inputs and retained prepared state;
- option decoding and unknown-option behavior;
- `ModuleSpec`, registration builder, and reference slots;
- manifest metadata and checkpoint fingerprint providers;
- exported versus package-private surface;
- embedded asset registration;
- principal extractor entry point and result type; and
- package-local test organization.
- Adapt existing PromptKit client tests to the prepared-execution path.
- Retain or add one behavioral test proving that the debug prompt details match
the request actually passed to generation when a backing prompt source could
otherwise change between independent preparations. Test the resulting
snapshot consistency, not a private helper call count.
- Retain capacity tests proving `errors.Is` reaches
`contracts.ErrLLMCapacityExceeded`, the selected backend can appear in safe
diagnostic context, and provider calls are not made after rejected
admission.
- Run `go test ./internal/framework/llm` and
`go test -race ./internal/framework/llm`.
- Run `go test ./...` and `git diff --check`.
Trace registration into production composition far enough to confirm that the
declared contract is the one actually selected. Inspect neighboring packages
only as required to verify an identity or ownership fact.
### Completion Criteria
### Write to `audit.md`
- `CompleteStructured` no longer calls independent `Prepare` and `Run`
operations for one request.
- Debug prompt material and generation result originate from the same frozen
PromptKit snapshot.
- Capacity remains a provider-neutral Notarius error classification.
Create `## Audit Results`, mark it `Status: In progress`, and add:
## Stage 3: Replace Synthetic Profile Validation With Inspection
1. `### Baseline And Module Matrix`
- one row per module;
- columns for every comparison dimension in the audit strategy;
- concise facts with links rather than judgments; and
- an explicit note where a dimension is not applicable.
2. `### Divergence Register`
- record every observed organizational, naming, construction, registration,
provenance, or test-layout difference;
- classify only as required specialization, permitted variation, convention
drift, architectural divergence, or undetermined; and
- do not propose helper extraction in this stage.
### Goal
Also add a short baseline-validation note recording:
Validate profiles through PromptKit's exact profile-inspection boundary and
centralize engine profile-source construction.
### Work
- Introduce a small provider-adapter-owned profile inspection or validation
function in `internal/framework/llm`. Its public internal signature must use
Notarius-owned configuration and result/error types rather than returning
PromptKit types to the CLI.
- Share the code that applies `profile_dir`, `profile_file`, and registered
backend options between the production PromptKit engine and the inspection
engine. Preserve the mutual-exclusion and local-backend rules.
- Change CLI explicit-profile preflight to use `Engine.InspectProfile` through
that LLM boundary.
- Remove `profileCheckPromptID`, `profileCheckPromptFS`, the `testing/fstest`
production dependency, and the synthetic `Prepare` request.
- Preserve distinct, useful errors for an absent profile, invalid profile,
unknown backend registration, cancellation, and invalid profile source.
- Do not require `api_key_env` to be populated during configuration validation.
Inspection may report credential requirements internally, but actual
preparation remains responsible for credential availability before a model
call.
- Update current-behavior sections in `docs/internal/cli.md` and
`docs/internal/llm.md`. Keep field definitions in `docs/config.md`.
### Tests And Validation
- Replace synthetic-prompt tests with profile inspection tests covering:
configured local backend success; missing local backend failure; absent
profile; malformed profile; and an otherwise valid profile whose credential
environment variable is intentionally unset.
- Prove validation performs no provider HTTP call and remains offline.
- Run `go test ./internal/framework/llm ./internal/cli` and `go test ./...`.
- Run `git diff --check`.
### Completion Criteria
- No production synthetic profile-check prompt remains.
- Profile validation uses the same ordinary profile source and backend
registrations as execution.
- Configuration validation succeeds for structurally valid profiles without
reading credential values.
## Stage 4: Add Application Fallback Profile Asset Plumbing
### Goal
Allow module families to register application-owned fallback profile YAML
without placing domain policy in generic LLM code.
### Work
- Extend `internal/framework/llm.AssetRegistry` with a separate fallback
profile source collection, registration method, flattened filesystem, and
safe content digest.
- Reuse the existing asset-source path validation and flattening behavior where
appropriate. Reject invalid roots, unreadable assets, and duplicate flattened
paths. Do not parse PromptKit profile YAML in Notarius.
- Add `promptkit.WithFallbackProfileFS` to production engine options only when
at least one fallback profile source is registered.
- Supply the identical assembled fallback source to the profile-inspection
engine. Adjust CLI composition so pipeline-aware profile validation can use
the production LLM asset registry without exposing PromptKit types.
- Extend profile-source checkpoint identity to include the exact fallback
profile asset digest in addition to the PromptKit catalog marker and operator
source. Keep the resulting fingerprint hash-only and path/content/credential
free.
- Keep operator source precedence owned by PromptKit. Do not implement profile
merging or duplicate PromptKit source resolution in Notarius.
- Update `docs/internal/llm.md` only for the new implemented generic asset and
fingerprint mechanics. No domain fallback exists until Stage 5.
### Tests And Validation
- Add focused AssetRegistry tests for successful flattening, invalid roots,
duplicate paths, and hash changes when fallback bytes change.
- Add adapter-level tests showing that the fallback filesystem reaches both
execution construction and inspection construction.
- Extend checkpoint tests to prove fallback content changes profile-source
identity without exposing raw YAML or paths. Use relational comparisons, not
a fixed hash literal.
- Run `go test ./internal/framework/llm ./internal/cli` and `go test ./...`.
- Run `git diff --check`.
### Completion Criteria
- Generic plumbing can carry application fallback profiles while remaining
unaware of D&D IDs or model settings.
- Inspection, execution, and checkpoint identity use the same fallback asset
source.
## Stage 5: Adopt The D&D `dnd-extraction` Fallback
### Goal
Give the D&D module family one stable embedded workload profile that operators
can replace.
### Work
- Add a D&D-owned embedded PromptKit profile asset with ID `dnd-extraction`
under `internal/modules/dnd`. Use the exact baseline defined in
`promptkit.md`: OpenRouter, `openai/gpt-5.6-luna`, no explicit reasoning
effort, a 240-second timeout, flex service tier, and no selected temperature,
token limit, or `top_p`. The omitted reasoning value intentionally allows
OpenAI's backend to apply its `medium` default.
- Register the profile filesystem from the D&D registrar through the generic
fallback profile asset boundary. Keep D&D policy out of
`internal/framework/llm` and the CLI composition root.
- Change every maintained D&D LLM prompt definition—including scene chunking,
all D&D extractors, and NPC normalization—from the model-named default to
`default_profile: dnd-extraction`.
- Add an integration-level profile-resolution test proving that:
- the fallback resolves when no operator source defines the ID;
- a valid operator profile with the same ID wins completely; and
- an invalid matching operator profile fails rather than falling through.
- Test through Notarius's assembled production assets and PromptKit boundary;
do not duplicate every upstream source-precedence case.
- Update the implemented profile ownership and prompt-default behavior in
`docs/internal/dnd.md`, `docs/internal/llm.md`, and
`docs/integrations/pkg-promptkit.md`. Defer the complete operator walkthrough
and examples to Stage 10.
### Tests And Validation
- Run focused D&D prompt preparation tests and the production composition
tests.
- Run `go test ./internal/modules/dnd/... ./internal/framework/llm
./internal/cli`.
- Run `go test ./...`.
- Verify `rg -n 'default_profile: gemini-2-flash' internal/modules/dnd`
returns no matches.
- Run `git diff --check`.
### Completion Criteria
- All maintained D&D prompts use the application-owned logical profile ID.
- The fallback works without an operator profile and remains authoritatively
overridable by a matching valid operator definition.
## Stage 6: Introduce Module Execution-Class Metadata
### Goal
Make each production module's ability to use an LLM statically discoverable
without yet changing profile inheritance.
### Work
- Add `ExecutionClass contracts.ExecutionClass` to `pipeline.ModuleSpec` and
preserve it through normalization, cloning, catalogs, registries, JSON/debug
views, and lookup helpers.
- In this transitional stage only, allow an omitted execution class to
normalize to deterministic so existing test-only fixtures can be migrated in
Stage 7 without breaking the repository midway.
- Explicitly classify every production module:
- D&D scene chunking, every D&D extractor, and D&D NPC normalization as
`llm_backed`;
- all other current production input, chunk, merge, normalize, and output
modules as `deterministic`.
- Update production module specification tests and production catalog tests to
assert the semantic class alongside stage, artifact kind, and capabilities.
- Add catalog lookup support needed by later resolution to retrieve a selected
module's execution class by stage and key without constructing it.
- Do not implement pipeline-level profile inheritance or reject deterministic
profiles yet.
- Update `docs/internal/modules.md` and `docs/internal/dnd.md` to identify
execution class as registered module metadata, while noting only implemented
uses.
### Tests And Validation
- Run module registration/spec tests across generic, Seriatim, and D&D
families.
- Run `go test ./internal/framework/pipeline ./internal/modules/...`.
- Run `go test ./...` and `git diff --check`.
### Completion Criteria
- Every production module has an explicit correct execution class.
- Catalog consumers can retrieve that class without a concrete module
instance.
- Test-only omitted classes remain the only temporary compatibility behavior.
## Stage 7: Enforce Execution Metadata And Remove Runtime Probing
### Goal
Finish the execution-class contract so missing metadata cannot cause future
profile drift.
### Work
- Update every framework, CLI, and integration test module specification to
declare an explicit execution class appropriate to the fake behavior.
- Change module-spec validation so an empty or unsupported execution class is a
registration error. Remove the transitional deterministic default from
Stage 6.
- Replace the chunk runner's special `ChunkExecutionClassProvider` probe with
specification-derived behavior. Remove the now-redundant provider interface,
implementation methods, and tests when they have no remaining consumer.
- Ensure chunk producer provenance remains unchanged: it records a non-empty
effective binding profile for an LLM-backed chunker, while a deterministic
chunker records no profile. A profile selected only through the prompt
default remains represented by PromptKit's actual-profile manifest rather
than being invented as an explicit chunk binding.
- Review helper constructors and fixtures for opportunities to set execution
class once without obscuring the class under test. Do not introduce an
elaborate test-spec framework.
- Update internal documentation if the removal changes any described runtime
mechanics.
### Tests And Validation
- Add or retain focused registration tests for missing and invalid execution
classes.
- Retain chunk-plan provenance tests for LLM-backed and deterministic
chunkers.
- Run `go test ./internal/framework/pipeline ./internal/modules/...`.
- Run `go test ./...`, `go vet ./...`, and `git diff --check`.
### Completion Criteria
- No registered module specification relies on an implicit execution class.
- Pipeline metadata, not a concrete runtime type assertion, owns module
execution classification.
## Stage 8: Resolve Programmatic Pipeline Profile Defaults
### Goal
Implement profile inheritance and precedence inside the pipeline resolver
before exposing the field through YAML configuration.
### Work
- Add an optional trimmed `LLMProfile` field to
`pipeline.PipelineProfile`. Add a non-empty runtime override field to
`pipeline.ResolveOptions` so all precedence decisions occur in the resolver
rather than through pre-resolution mutation.
- After module selection, `--only` filtering, default validator-chain
selection, and validator compatibility resolution, apply effective profiles
to every selected input, chunk, extract, merge, normalize, output, and
validator binding according to the precedence in `promptkit.md`.
- Apply profiles only when the selected module or validator execution class is
`llm_backed`.
- Reject a binding-specific `llm_profile` on any deterministic module or
validator. Do not reject or inspect an unused pipeline default when no
selected LLM-backed binding consumes it.
- Leave an LLM-backed binding empty when no CLI, binding, or pipeline profile is
selected so PromptKit can use the prompt's `default_profile`.
- Store the effective values on resolved bindings before digest construction.
Do not add a second inheritance decision to execution.
- Ensure semantically equivalent repeated binding profiles and one inherited
default produce the same resolved pipeline digest. Ensure any changed
effective profile changes the digest.
- Do not modify file configuration or CLI parsing in this stage.
### Tests And Validation
- Add pipeline package tests for the complete precedence matrix:
runtime override; binding-specific exception; pipeline default; prompt
fallback; and deterministic bindings.
- Cover default and explicitly configured validator chains, all relevant stage
categories, `--only` lane selection, unused defaults, deterministic-profile
rejection, and semantic digest equivalence.
- Prefer table-driven package-level tests over assertions on private traversal
helpers.
- Run `go test ./internal/framework/pipeline` and `go test ./...`.
- Run `git diff --check`.
### Completion Criteria
- Programmatic pipelines resolve one canonical effective profile policy.
- Only LLM-backed resolved bindings can contain a profile.
- Runtime override, binding, pipeline, and prompt precedence is unambiguous and
digest-stable.
## Stage 9: Expose Pipeline Defaults Through Configuration And CLI
### Goal
Make the profile-default workflow available to operators while preserving
validation and override behavior.
### Work
- Add optional `pipelines.<id>.llm_profile` support to the version 4 file
configuration model. Use presence-aware decoding so an explicitly set blank
value is rejected, while omission remains valid.
- Preserve the field through file application, configuration cloning,
effective configuration, and programmatic profile copies without aliasing or
trimming drift.
- Remove `applyLLMProfileOverride`. Pass the CLI override through the resolver's
runtime-override input so deterministic bindings are never populated.
- Update effective profile-ID collection to cover every selected LLM-backed
module stage and LLM-backed validator, including future LLM-backed input and
output modules. Do not inspect deterministic or unselected profiles.
- Ensure `run`, `config validate --pipeline`, resume/checkpoint identity, and
relevant dry preflight paths all use the same resolved effective profiles.
- Preserve `--llm-profile` as the highest-precedence non-empty run-wide
override and preserve binding-specific profiles as exceptions when no CLI
override is present.
- Do not increment the configuration version.
- Update current configuration and CLI contracts in `docs/config.md` and
`docs/cli.md` in the same stage. Link to operations for the deployment
workflow rather than duplicating it prematurely.
### Tests And Validation
- Add file-config tests for omission, trimming, explicit blank rejection,
unknown-key behavior, cloning, and round-trip application.
- Add effective-config and CLI contract tests for precedence, LLM-only
application, inherited-profile inspection failure before factory execution,
`--only`, and digest changes.
- Retain offline operation and do not require credentials for
`config validate --pipeline`.
- Run `go test ./internal/core/config ./internal/framework/pipeline
./internal/cli`.
- Run `go test ./...`, `go vet ./...`, and `git diff --check`.
### Completion Criteria
- Operators can select `dnd-extraction` once per pipeline.
- Configuration and CLI paths share the resolver's precedence policy.
- Unknown effective profiles fail preflight, while deterministic and unused
profiles do not cause spurious inspection.
## Stage 10: Complete Operator Documentation, Examples, And Decision Record
### Goal
Make the implemented workflow understandable, copyable, and maintainable
without duplicating canonical facts.
### Work
- Create an ADR using the next sequential number for the durable decision to
use workload-oriented pipeline defaults with operator-overridable application
fallback profiles. Record context, decision, alternatives, and consequences;
do not turn the ADR into a field reference or implementation log.
- Complete `docs/config.md` as the canonical owner of profile-source fields,
`pipelines.<id>.llm_profile`, validation, and precedence.
- Complete `docs/operations.md` with an operator workflow that distinguishes
Notarius embedded prompts, Notarius fallback profiles, PromptKit built-ins,
and deployment filesystem profiles. Include production/development/local use
of the same `dnd-extraction` ID, credential handling, absolute-path guidance,
and the fact that current relative profile paths use the process working
directory rather than the configuration file's directory.
- Complete `docs/integrations/pkg-promptkit.md` with the v0.5.0 boundary,
prepared execution, inspection, fallback and ordinary source precedence,
optional provider controls, capacity adaptation, and compatibility policy.
- Update `docs/internal/configuration.md`, `docs/internal/pipeline.md`,
`docs/internal/cli.md`, `docs/internal/llm.md`, `docs/internal/modules.md`, and
`docs/internal/dnd.md` only for their owned implementation details. Link to
canonical configuration, operations, and upstream format contracts rather
than restating them.
- Keep exactly the existing two D&D configuration examples. Add
`llm_profile: dnd-extraction` to the minimal and complete pipelines and remove
the now-redundant model-named binding override from the complete example.
- Add one secret-free maintained operator profile at
`examples/profiles/dnd-extraction.yml`. It should be a complete valid profile
for the same logical ID and may mirror the embedded baseline; its purpose is
to demonstrate file ownership and format, not claim automatic environment
detection. Link it from the configuration and operations documentation.
- If the complete example selects the external profile file, use a path that
is valid for the documented repository-root invocation and explicitly note
the working-directory rule. Keep the minimal example dependent only on the
embedded fallback.
- Add or extend maintained-example validation so both configuration examples
and the profile YAML are checked without generation or credentials.
- Remove the now-implemented `Pipeline-Level LLM Profile Defaults` section from
`docs/roadmap/future.md`. Preserve the unrelated deterministic session and
concurrency items.
- Do not delete `promptkit.md` or this implementation plan during the feature
implementation; retire them only after post-implementation review.
### Tests And Validation
- Run maintained example/configuration tests and relevant CLI help/parser
tests.
- Run `go test ./...`.
- Run `rg -n 'gemini-2-flash' examples docs` and review every remaining match
for intentional model-policy or historical context.
- Run `rg -n 'v0\.3\.0|profileCheckPrompt|applyLLMProfileOverride' .` and resolve
stale production or current-documentation matches.
- Verify all new links and `git diff --check`.
### Completion Criteria
- Every current fact has one canonical documentation owner.
- Operators can distinguish and deploy all profile layers without reading Go
source.
- Both maintained configurations and the maintained external profile are valid,
secret-free, and tested offline.
- Implemented profile work no longer remains in `future.md`.
## Stage 11: Final Verification And Quality Review
### Goal
Verify the complete migration as one integrated change and correct only defects
or omissions found during that review.
### Work
- Review the final diff against every acceptance criterion in `promptkit.md`.
- Confirm provider-specific PromptKit types remain inside the LLM integration
boundary and D&D policy remains inside the D&D module family.
- Confirm execution and inspection receive identical ordinary, fallback, and
backend configuration.
- Confirm no paths, profile YAML, endpoints, credentials, or prepared handle
state leak into fingerprints or ordinary diagnostics.
- Confirm all production module specs have explicit correct execution classes
and every resolved deterministic binding is profile-free.
- Confirm prompt default, pipeline default, binding override, and CLI override
behavior through representative assembled configurations.
- Review tests for redundancy and remove obsolete synthetic-prompt,
runtime-probe, exact-hash, or duplicated upstream-behavior tests superseded by
stronger contract tests.
- Perform an optional manual D&D quality comparison if credentials and an
evaluation transcript are deliberately supplied. Record no private input or
credential material, and do not make this comparison a completion gate.
### Validation Commands
```sh
go test -count=1 ./...
gofmt -w <changed-go-files>
go test ./...
go test -race ./internal/framework/llm ./internal/core/config ./internal/framework/pipeline ./internal/cli
go vet ./...
go build ./cmd/notarius
gofmt -l .
git diff --check
```
If a command fails, record the exact command, affected package, and concise
failure classification. Do not fix it during the audit.
Also run focused stale-contract searches:
### Completion gate
Stage 1 is complete when every matrix cell is populated or marked not
applicable, every observed baseline divergence has a stable register entry, and
the repository's starting validation state is recorded.
## Stage 2: Prompts, Schemas, And LLM Boundaries
### Objective
Determine whether the five modules consistently present stable context,
references, lane instructions, transcripts, and structured-output contracts to
the LLM, and identify exact or near-duplicate assets without changing them.
### Required review
For every extractor:
- read the complete prompt manifest and every referenced local and shared
asset;
- record the exact ordered message sequence, role, input, and cache-control
boundary;
- compare shared message files by identity and bytes, not by paraphrased
meaning;
- verify stable-to-variable ordering against `docs/internal/llm.md`;
- map declared prompt inputs to module reference slots and generated inputs;
- trace optional, required, empty, and generated reference projections;
- inspect prompt and schema registration, hashing, and diagnostic redaction;
- inspect the complete private response schema for identity, required fields,
nullability, strict objects, and semantic constraints;
- map schema fields to private DTOs and response mapping; and
- compare prompt and schema tests at their behavioral boundaries.
Treat the scene-description whole-chunk evidence model as a specialization to
explain, not a presumption of drift. Likewise, treat catalog and NPC registry
inputs as subset-specific responsibilities unless evidence shows inconsistent
handling of the same contract.
### Write to `audit.md`
Add `### Prompt, Schema, And LLM Review` containing:
- a five-way prompt-order and cache-boundary table;
- a prompt-input and reference-projection table;
- a private-schema and DTO ownership table;
- exact shared-asset usage;
- local assets with identical or near-identical content;
- content-safety and diagnostic observations; and
- evidence-backed deviations from documented LLM conventions.
Update the divergence register for confirmed or newly discovered prompt,
schema, reference, provenance, and testing differences.
Create `### Duplication Register` and add prompt-, schema-, metadata-, and
asset-related candidates. For each prompt candidate, state whether the text is
byte-identical, merely similar, or semantically different. Do not recommend a
shared prompt asset unless all intended consumers should receive future edits
atomically.
### Completion gate
Stage 2 is complete when every prompt message and input is accounted for, every
private schema field has an owner, cache-prefix claims are based on exact
message identity, and every asset-sharing candidate has a provisional keep or
share outcome.
## Stage 3: Extraction, Evidence, And Canonicalization
### Objective
Compare runtime extraction flow and identify repeated algorithms or policy
without erasing artifact-specific semantics.
### Required review
For every extractor, inspect and trace:
- request, context, source, chunk, and dependency validation;
- construction-time state versus operation-time overrides;
- structured request assembly and provider-error wrapping;
- response-to-artifact mapping;
- source identity attachment and unit-ID resolution;
- evidence range validation assumptions;
- canonical source-reference ordering and exact deduplication;
- artifact ordering and deterministic tie-breakers;
- enum or canonical-name handling;
- warnings and diagnostics;
- cloning and aliasing boundaries; and
- focused extractor, model, canonicalization, and malformed-output tests.
Compare complete algorithms rather than function names alone. For similar
canonicalization helpers, identify which parts are:
- identical source-reference mechanics;
- artifact-specific mapping;
- artifact-specific ordering policy; or
- validation that belongs to a later validator rather than extraction.
Use call traces and complexity data to inspect repeated scans, allocations,
serialization, or high-cognitive-complexity paths. Report performance only when
a plausible workload and complexity impact exist.
### Write to `audit.md`
Add `### Extraction And Canonicalization Review` containing:
- a five-way extraction-flow table;
- an evidence and ordering-policy table;
- clone, mutation, error, and warning observations;
- test-ownership comparisons; and
- code-quality or performance candidates with concrete impact.
Update the divergence register, revising earlier provisional classifications
where runtime evidence explains or contradicts them.
Expand the duplication register with request-validation, reference,
canonicalization, ordering, deduplication, mapping, error, and test-support
candidates. Give each candidate a provisional owner and explicitly identify the
artifact-specific code that must remain local.
### Completion gate
Stage 3 is complete when the full extraction path of all five modules is
accounted for, every similar canonicalization path has been decomposed into
shared mechanics versus domain policy, and every runtime duplication candidate
has a provisional ownership decision.
## Stage 4: Contextual Architecture And Ownership Review
### Objective
Validate candidate findings against the surrounding D&D and framework
architecture, and decide which apparent similarities should actually be shared.
### Required review
For every open divergence and duplication candidate:
- trace callers and consumers;
- inspect existing facilities in `internal/modules/dnd/shared`;
- inspect focused D&D registry, identity, catalog, codec, normalize, and
validate packages as relevant;
- verify the typed artifact and reference contracts;
- inspect production registration, default validator composition, and
checkpoint fingerprint assembly;
- verify documentation ownership and current durable contracts; and
- check whether a proposed generic helper has a genuine domain-neutral owner.
Apply the shared-code hierarchy from `audit.md`:
1. module-owned artifact semantics;
2. D&D-wide shared mechanics;
3. focused subset-specific D&D packages; and
4. framework-owned domain-neutral behavior.
Reject or defer candidates whose API would require artifact-specific callbacks,
type erasure, module-key branching, a broad configuration object, or speculative
future consumers.
### Write to `audit.md`
Add `### Contextual Architecture And Ownership Review` containing:
- traced ownership evidence for every open candidate;
- dependency-direction and layer-boundary conclusions;
- checkpoint, provenance, and reference compatibility conclusions;
- documentation or test ownership implications; and
- any product-contract questions that cannot be decided as refactors.
Add `### Candidate Decisions`, with one row per divergence and duplication
identifier. Choose exactly one outcome:
- extract now;
- harmonize without sharing;
- retain intentionally separate;
- reject as harmful abstraction; or
- defer pending a named missing requirement or product decision.
For extract or harmonize outcomes, specify:
- target owner;
- minimal responsibility and proposed API shape;
- participating modules;
- behavior that remains package-owned;
- migration order;
- relevant tests; and
- principal risks.
Update every register entry to confirmed, intentional, rejected, superseded, or
explicitly deferred. No entry may remain merely open at the end of this stage.
### Completion gate
Stage 4 is complete when all candidates have traced ownership and a final
keep/share/harmonize/defer decision, no proposed helper violates dependency
direction, and all required product decisions are separated from executable
refactoring recommendations.
## Stage 5: Synthesis And Final Audit
### Objective
Turn the working evidence into one concise, internally consistent audit that
can support roadmap decisions and a later implementation plan.
### Required work
1. Re-read the complete strategy and all working audit results.
2. Recheck every cited file and symbol against the current working tree.
3. Reconcile duplicate, overlapping, or contradictory observations.
4. Assign severity only to confirmed actionable findings:
- **high:** correctness, security, data integrity, or architectural failure
with substantial impact;
- **medium:** meaningful drift, duplication, or design weakness likely to
cause defects or costly divergence;
- **low:** localized maintainability, clarity, test-quality, or
documentation issue with limited immediate impact.
5. Keep optional improvements separate from findings.
6. Confirm intentional specializations and rejected sharing candidates are
documented so future work does not repeatedly reopen them without evidence.
7. Run the repository-wide validation commands from Stage 1 again and record
the final result. Do not modify code in response to failures.
### Rewrite `audit.md`
Preserve the strategy sections above `## Audit Results`, but replace the
provisional working area with this final structure:
```markdown
## Audit Results
Status: Complete
### Executive Conclusion
### Final Module Comparison Matrix
### Prioritized Findings
### Intentional Differences To Preserve
### Shared-Code Decisions
### Rejected Or Deferred Candidates
### Recommended Refactoring Sequence
### Validation And Residual Risks
```sh
rg -n 'gitea.maximumdirect.net/eric/promptkit v0\.3\.0|PromptKit v0\.3\.0' .
rg -n 'default_profile: gemini-2-flash|profileCheckPrompt|applyLLMProfileOverride' internal docs examples
```
Requirements for the final sections:
Review any matches rather than deleting intentional historical references
blindly.
- `Executive Conclusion` directly answers the user's three audit questions.
- `Final Module Comparison Matrix` remains factual and compact.
- `Prioritized Findings` follows the finding standard in `audit.md`, is ordered
by severity and impact, and cites exact evidence.
- `Intentional Differences To Preserve` explains why harmonization would be
incorrect.
- `Shared-Code Decisions` records extract and harmonize recommendations,
proposed ownership, and why sharing is better than continued separation.
- `Rejected Or Deferred Candidates` records superficially attractive
abstractions and why they should not be pursued now.
- `Recommended Refactoring Sequence` groups accepted recommendations into
independently safe, dependency-ordered scopes. It is a sequence, not a
decision-complete implementation plan.
- `Validation And Residual Risks` records commands, results, limits of the
static audit, and any human or model-quality evaluation still needed.
### Completion Criteria
Remove the working divergence and duplication registers after their evidence
has been incorporated into the final sections. Do not leave raw stage notes,
provisional severities, or superseded conclusions in the completed audit.
### Completion gate
Stage 5 is complete when:
- the final audit satisfies every deliverable and completion criterion in
`audit.md`;
- the three user questions receive explicit answers;
- every recommendation has evidence, ownership, scope, and rationale;
- no finding relies only on similarity or stylistic preference;
- validation results are current;
- no code or current-behavior files changed; and
- `git status --short` shows only the intended `audit.md` audit-result changes
plus any pre-existing user changes.
- All automated checks pass offline and without real credentials.
- The implemented behavior matches `promptkit.md` with no known architecture,
provenance, checkpoint, profile-precedence, or documentation gap.
- Any optional live evaluation is clearly separate from correctness testing.
## Open Questions
None. The audit stages, working-document structure, decision rules, and final
deliverable are fully specified.
None. The roadmap decisions are sufficient to implement every stage without an
additional product or architecture choice.

318
docs/roadmap/promptkit.md Normal file
View File

@@ -0,0 +1,318 @@
# PromptKit v0.5 Integration And LLM Profile Policy
## Purpose
This roadmap defines the target state for upgrading Notarius from PromptKit
v0.3.0 to v0.5.0 and adopting the upstream runtime and profile facilities that
directly improve Notarius. It also defines the application policy for stable,
domain-oriented LLM profile names, operator overrides, pipeline inheritance,
profile validation, provider defaults, checkpoint identity, and documentation.
The ordered work needed to reach this state belongs in
[the implementation plan](implementation.md). Current behavior remains defined
by the canonical documentation outside `docs/roadmap/` until the corresponding
work is implemented.
## Background
Notarius currently pins PromptKit v0.3.0. Its adapter prepares a request once
for debug material and then independently runs the original request, causing
PromptKit to prepare the same logical call a second time. The CLI validates an
explicit profile by preparing a synthetic prompt. PromptKit profile selection
can be repeated on individual module bindings or replaced for one invocation
with `--llm-profile`, but a configured pipeline cannot yet declare one inherited
profile policy.
PromptKit v0.4.0 and v0.5.0 add the upstream boundaries needed to improve these
areas:
- [v0.4.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/releases/v0.4.0.md)
adds opaque prepared executions, exact profile and prompt inspection, and a
typed backend-capacity error;
- [v0.5.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/releases/v0.5.0.md)
adds application fallback profile filesystems and stops sending unset
optional sampling controls as framework-selected provider values; and
- the [v0.5.0 format contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md)
defines the resulting profile-source and execution-setting precedence.
A source-compatibility test of the current Notarius repository against
PromptKit v0.5.0 completed successfully. The work is therefore primarily an
intentional runtime and configuration migration rather than a repair for a
breaking Go API change.
## Goals
- Pin and document PromptKit v0.5.0 as Notarius's supported upstream contract.
- Execute the exact prepared request snapshot whose safe details are recorded
in Notarius debug material.
- Validate configured PromptKit profiles through the upstream inspection API
without synthetic prompts, provider calls, or credential-value access.
- Give Notarius an application-owned, operator-overridable
`dnd-extraction` profile fallback.
- Let a pipeline choose one default LLM profile without repeating that ID on
every LLM-backed binding.
- Apply profile inheritance and run-wide overrides only where the resolved
module or validator can use an LLM.
- Preserve accurate checkpoint invalidation, effective profile provenance,
redaction, cancellation, concurrency, and provider-neutral module contracts.
- Provide operators with one clear deployment pattern for production,
development, and local profile definitions.
## Target End State
### PromptKit Runtime Boundary
Notarius depends on PromptKit v0.5.0 and uses its public APIs rather than
reimplementing source or execution resolution.
For each structured completion, the adapter:
1. builds one PromptKit run request from the provider-neutral Notarius request;
2. calls `PrepareExecution` once;
3. immediately arranges an idempotent `Discard` for every unexecuted handle;
4. obtains credential-redacted `Details` for debug and response metadata; and
5. calls `RunPrepared` so generation uses that exact frozen snapshot.
The debug prompt and successful result therefore describe the same selected
profile, rendered messages, input bytes, session, output contract, and effective
settings even when a filesystem-backed source changes concurrently. PromptKit
handle types remain private to `internal/framework/llm`.
PromptKit admission failures continue to match Notarius's provider-neutral
`ErrLLMCapacityExceeded` contract. When PromptKit supplies a `CapacityError`,
the adapter obtains the normalized backend ID through `errors.As` and may add it
to safe application-owned diagnostics without parsing upstream error wording.
The backend ID does not become a provider-specific module contract.
### Optional Provider Controls
Notarius accepts PromptKit v0.5.0's new behavior for `temperature`,
`max_tokens`, and `top_p`: an unset setting is omitted from compatible provider
requests and the provider chooses its own default. Notarius does not restore
PromptKit's former implicit `top_p: 1` value globally.
An operator who requires a particular value specifies it in the selected
PromptKit profile. The application fallback described below intentionally
leaves these controls unset. A human-reviewed D&D extraction comparison should
be performed after the upgrade, but paid or nondeterministic model output is
not part of the default automated test suite.
### Profile Inspection
Pipeline-aware configuration validation uses `Engine.InspectProfile` for every
effective explicit profile ID. It verifies that the profile exists, parses and
validates, resolves its backend and target, and is compatible with the engine's
registered backends. It does not create a synthetic prompt, load prompt inputs,
contact a provider, or require credential values to exist in the validation
process environment.
Credential availability is execution-time state. PromptKit preparation still
enforces the selected profile's credential contract before generation. This
keeps `notarius config validate` useful in build and deployment validation
environments where secrets are deliberately absent.
PromptKit construction for inspection and execution uses one shared internal
profile-source and backend-option path. The CLI does not expose PromptKit public
types across the Notarius LLM boundary merely to perform inspection.
`InspectPrompt` is not adopted merely because it exists. It remains available
for a later, separately defined module-to-prompt interface preflight if a
concrete validation requirement justifies that additional contract.
### Application And Operator Profile Sources
Notarius embeds one ordinary PromptKit YAML profile with the stable ID
`dnd-extraction`. It is an application fallback registered through
`WithFallbackProfileFS`, is owned by the D&D module family, and initially
preserves the current effective D&D baseline:
- backend: PromptKit's built-in `openrouter` backend;
- model: `openai/gpt-5.6-luna`;
- reasoning effort: unset, allowing OpenAI's backend to apply its default of
`medium`;
- generation timeout: 240 seconds;
- service tier: `flex`; and
- no application-selected `temperature`, `max_tokens`, or `top_p`.
All maintained D&D LLM prompt definitions use `dnd-extraction` as their
`default_profile`. The ID communicates workload intent rather than a provider,
model, or environment. Changing the embedded fallback is an intentional
Notarius execution-policy change and participates in checkpoint identity.
Effective profile definitions resolve in PromptKit's order:
1. programmatic in-memory profiles used by tests or explicit consumers;
2. the operator source configured by `promptkit.profile_file` or
`promptkit.profile_dir`;
3. the Notarius application fallback source; and
4. PromptKit's embedded built-in catalog.
Only an absent ID falls through to the next source. A matching profile is a
complete definition: fields are not merged with a lower-precedence definition,
and a malformed matching operator profile fails rather than silently selecting
the application fallback.
Production, development, and local deployments should normally provide
different complete definitions for the same `dnd-extraction` ID. An operator
source is optional because the application fallback keeps the maintained D&D
workflow usable, but a deployment that needs an intentional model or backend
policy should configure its own definition.
### Domain Ownership And Asset Assembly
The D&D fallback profile remains under `internal/modules/dnd` and is registered
by the D&D registrar, consistent with ADR-0004. Generic LLM plumbing knows how
to collect and flatten application fallback profile filesystems but contains no
D&D model or policy knowledge.
The shared asset registry detects invalid roots, unreadable sources, and
duplicate flattened paths. PromptKit remains responsible for strict profile
YAML parsing, duplicate profile-ID detection, source precedence, and effective
target resolution. The same assembled fallback source is supplied to runtime
execution and CLI profile inspection.
### Explicit Module Execution Metadata
Every registered input, chunk, extract, merge, normalize, and output module
declares one required execution class: `deterministic` or `llm_backed`.
Validator registrations continue to declare the same distinction through their
validator specifications.
The registered specification is authoritative for configuration resolution.
Current production classifications are:
- the D&D scene chunker, all D&D extractors, and the D&D NPC normalizer are
LLM-backed;
- the Seriatim input adapter, generic chunker, all current mergers, all other
current normalizers, and the JSON output encoder are deterministic; and
- current validators retain their declared classifications.
Missing or unsupported execution metadata is a registration error. Explicitly
assigning `llm_profile` to a deterministic module or validator is a pipeline
resolution error. The framework does not infer execution class by inspecting
domain package names or concrete implementation types at runtime.
The module specification replaces the chunk runner's special runtime
execution-class probe. Effective resolved bindings already express the result:
only LLM-backed bindings may retain a non-empty profile.
### Pipeline-Level Profile Default
Configuration version 4 gains one optional non-empty pipeline field:
```yaml
pipelines:
dnd-session:
llm_profile: dnd-extraction
```
No configuration-version increment is required because the field is additive
and existing files remain valid. An explicitly present blank value is invalid.
For every selected LLM-backed module and validator, the effective profile uses
this precedence:
1. non-empty run-wide `--llm-profile` override;
2. binding-specific `llm_profile`;
3. pipeline-level `llm_profile`; and
4. the prompt definition's `default_profile`, represented by an empty effective
Notarius binding profile.
The run-wide override and inherited pipeline default never attach to a
deterministic binding. Binding-specific exceptions remain available when one
operation needs a different cost, latency, quality, backend, or reasoning
policy.
Inheritance is resolved after module and validator selection, including
`--only` lane filtering, but before effective-pipeline validation, digest
construction, explicit-profile inspection, checkpoint construction,
preparation, execution, or provenance capture. Only profiles used by selected
LLM-backed bindings are inspected. An unused pipeline default in a pipeline
with no selected LLM-backed work does not require an otherwise unused profile
to exist.
The resolved pipeline contains effective binding profiles rather than a second
runtime inheritance mechanism. Two pipelines that differ only by spelling the
same effective policy once as a pipeline default and once on every LLM-backed
binding have the same semantic resolved digest. Changing an effective profile
changes the digest and applicable checkpoint identity.
### Provenance And Checkpoints
The PromptKit profile-source checkpoint fingerprint covers:
- the PromptKit v0.5.0 built-in profile catalog identity;
- exact application fallback profile asset content; and
- exact configured operator profile YAML content, when present.
The existing local-backend target fingerprint remains separate and continues
to exclude scheduling-only concurrency limits. Fingerprints contain hashes and
stable markers, not profile contents, filesystem paths, endpoints, credentials,
or other secrets.
Changing the PromptKit version, application fallback, operator profile, or
effective pipeline profile makes incompatible LLM checkpoints ineligible for
reuse. The dependency upgrade is expected to invalidate checkpoints produced
under v0.3.0.
Successful run manifests continue to record only profiles actually selected by
PromptKit, including their effective model, backend, and reasoning metadata.
Debug output reports the same effective execution snapshot used for generation.
### Operator Documentation And Examples
Canonical documentation clearly distinguishes:
- Notarius prompt and schema assets embedded in the application;
- Notarius application fallback profiles embedded in the application;
- PromptKit's own embedded built-in profiles; and
- operator profile files on the deployment filesystem.
The configuration reference owns the pipeline field, profile-source fields,
validation rules, and precedence. Operations owns deployment layout, working
directory behavior, credentials, and environment-specific profile management.
The PromptKit integration document owns the pinned upstream contract and
source-precedence boundary. Internal documents describe asset registration,
resolution, inspection, prepared execution, fingerprinting, and tests without
duplicating user-facing field definitions.
The maintained examples continue to include only the minimal and complete D&D
configurations. They use the stable `dnd-extraction` policy, and one maintained
PromptKit profile file under `examples/` demonstrates an operator override.
Examples remain secret-free and are validated without live provider calls.
## Out Of Scope
- Implementing the separate deterministic prompt-session identity roadmap
item.
- Changing the default `concurrency.total_llm` value; PromptKit's retained
OpenRouter capacity of 16 remains relevant to that separate item.
- Adding model evaluation as a deterministic or CI correctness gate.
- Automatically selecting production, development, or local environments.
Deployment configuration chooses the operator profile source.
- Profile inheritance, partial profile merging, or cross-profile aliases.
- Exposing PromptKit types to modules, validators, durable output contracts, or
public configuration structures.
- Adopting `InspectPrompt` without a separately justified prompt-interface
validation contract.
## Acceptance Criteria
- Notarius builds and its offline test suite passes with PromptKit v0.5.0.
- Every structured completion executes the exact snapshot used for safe debug
prompt details.
- Profile preflight uses profile inspection and no synthetic prompt.
- The embedded `dnd-extraction` fallback resolves without an operator source,
and a matching valid operator profile replaces it completely.
- Every production module has explicit, correct execution metadata.
- Pipeline, binding, CLI, and prompt-default precedence behaves as defined for
modules and validators, while deterministic bindings remain profile-free.
- Effective profiles participate in pipeline digests, profile inspection,
checkpoint identity, debug records, and run provenance at the appropriate
boundaries.
- The dependency and application fallback changes invalidate incompatible old
checkpoints without exposing profile or credential content.
- Canonical documentation and maintained examples accurately describe and
exercise the implemented operator workflow.
- Default tests remain deterministic, offline, credential-free, and focused on
Notarius-owned behavior rather than duplicating PromptKit's upstream suite.

View File

@@ -1,22 +0,0 @@
version: 3
output:
directory: ./notarius-output
cache:
chunk_plans:
mode: bypass
directory: ./notarius-cache/chunk-plans
checkpoints:
enabled: false
directory: ./notarius-cache/checkpoints
debug:
directory: ./notarius-debug
pipelines:
dnd-combat:
input: seriatim
chunk: generic
artifacts:
combat:
extract:
module: dnd/combat-turns
retries: 2
normalize: dnd/combat-turns

View File

@@ -0,0 +1,85 @@
{
"metadata": {
"id": "session-ravenfall",
"title": "The Ravenfall Watchtower"
},
"segments": [
{
"id": 1,
"start": 0,
"end": 14,
"speaker": "DM",
"text": "Recap: last session, the party learned that Elder Rowan vanished near the Ravenfall watchtower."
},
{
"id": 2,
"start": 14,
"end": 25,
"speaker": "Player",
"text": "Out of character, we agree to investigate the watchtower before the next game."
},
{
"id": 3,
"start": 25,
"end": 39,
"speaker": "DM",
"text": "Aria and Borin arrive at the ruined Ravenfall watchtower as dusk settles over the road."
},
{
"id": 4,
"start": 39,
"end": 55,
"speaker": "Mira Thorn",
"text": "Mira Thorn steps from the doorway and says, \"Elder Rowan warned me that Kesh would return for the relic.\""
},
{
"id": 5,
"start": 55,
"end": 70,
"speaker": "DM",
"text": "Mira leads the party to a hidden cache. The party discovers a moonblade and acquires 20 silver pieces."
},
{
"id": 6,
"start": 70,
"end": 83,
"speaker": "Aria",
"text": "Aria hands her healing potion to Borin so he can carry it into the tower."
},
{
"id": 7,
"start": 83,
"end": 96,
"speaker": "DM",
"text": "Kesh, the goblin captain, orders the raiders to attack. Roll initiative."
},
{
"id": 8,
"start": 96,
"end": 110,
"speaker": "DM",
"text": "On Kesh's turn, he strikes Borin with his scimitar. Borin drinks the healing potion on his turn."
},
{
"id": 9,
"start": 110,
"end": 124,
"speaker": "Aria",
"text": "Aria casts Cure Wounds on Borin, then invokes Aegis of Emberfall as Kesh closes in."
},
{
"id": 10,
"start": 124,
"end": 137,
"speaker": "DM",
"text": "Kesh casts Shield as a reaction against Borin's counterattack, but the party drives the raiders away."
},
{
"id": 11,
"start": 137,
"end": 150,
"speaker": "DM",
"text": "After the battle, Aria pays 5 silver pieces to repair the watchtower gate."
}
]
}

View File

@@ -0,0 +1,103 @@
version: 4
promptkit:
profile_file: ./examples/profiles/dnd-extraction.yml
concurrency:
total_llm: 2
stage_workers:
extract: 2
output:
directory: ./notarius-output
cache:
chunk_plans:
mode: auto
directory: ./notarius-cache/chunk-plans
checkpoints:
enabled: true
directory: ./notarius-cache/checkpoints
debug:
directory: ./notarius-debug
pipelines:
dnd-session:
llm_profile: dnd-extraction
input: seriatim
# Stable campaign context is shared by every module that accepts these slots.
references:
party: ./dnd-party.txt
glossary: ./dnd-glossary.txt
chunk:
module: dnd/scenes
retries: 2
output:
module: json
options:
include_chunk_map: true
evidence_context:
enabled: true
window_units: 3
lanes:
- item-events
- npcs
- spells
- combat-turns
- npc-interactions
steps:
# Establish session-wide reference artifacts alongside independent item events.
- id: describe-session
artifacts:
item-events:
extract:
module: dnd/item-events
retries: 2
merge: appendorder
normalize: dnd/item-events
npcs:
extract:
module: dnd/npcs
retries: 2
merge: appendorder
normalize:
module: dnd/npcs
retries: 2
scene-descriptions:
extract:
module: dnd/scene-descriptions
retries: 2
merge: appendorder
normalize: dnd/scene-descriptions
- id: extract-events
# Accepted NPC grounding and scene-description eligibility artifacts are
# supplied in memory to their compatible consumers in this step.
references:
npcs:
artifact:
step: describe-session
lane: npcs
scene_descriptions:
artifact:
step: describe-session
lane: scene-descriptions
artifacts:
spells:
extract:
module: dnd/spells
retries: 2
references:
spell_catalog: ./dnd-spell-catalog.json
merge: appendorder
# Stage-local file references are intentionally bound at each stage.
normalize:
module: dnd/spells
references:
spell_catalog: ./dnd-spell-catalog.json
combat-turns:
extract:
module: dnd/combat-turns
retries: 2
merge: appendorder
normalize: dnd/combat-turns
npc-interactions:
extract:
module: dnd/npc-interactions
retries: 2
merge: appendorder
normalize: dnd/npc-interactions

View File

@@ -0,0 +1,8 @@
Ravenfall watchtower: a ruined watchtower near the party's current route.
Mira Thorn: the watchtower's keeper.
Elder Rowan: a missing local scholar.
Kesh: a goblin captain leading raiders.
Moonblade: a blade found in the watchtower's hidden cache.
Cure Wounds: a healing spell.
Shield: a defensive reaction spell.
Aegis of Emberfall: a campaign spell recorded in the supplied catalog overlay.

View File

@@ -0,0 +1,9 @@
version: 4
pipelines:
dnd-session:
llm_profile: dnd-extraction
input: seriatim
artifacts:
spells:
extract: dnd/spells
normalize: dnd/spells

View File

@@ -1,37 +0,0 @@
version: 3
output:
directory: ./notarius-output
cache:
chunk_plans:
mode: bypass
checkpoints:
enabled: false
directory: ""
debug:
directory: ./notarius-debug
pipelines:
dnd-npc-grounded:
input: seriatim
steps:
- id: identify-npcs
artifacts:
npcs:
extract:
module: dnd/npcs
retries: 2
normalize: dnd/npcs
- id: grounded-events
references:
npcs:
artifact:
step: identify-npcs
lane: npcs
artifacts:
spells:
extract: dnd/spells
normalize: dnd/spells
combat:
extract:
module: dnd/combat-turns
retries: 2
normalize: dnd/combat-turns

View File

@@ -1,34 +0,0 @@
version: 3
output:
directory: ./notarius-output
cache:
chunk_plans:
mode: bypass
checkpoints:
enabled: false
directory: ""
debug:
directory: ./notarius-debug
pipelines:
dnd-npc-interactions:
input: seriatim
steps:
- id: identify-npcs
artifacts:
npcs:
extract:
module: dnd/npcs
retries: 2
normalize: dnd/npcs
- id: extract-interactions
references:
npcs:
artifact:
step: identify-npcs
lane: npcs
artifacts:
interactions:
extract:
module: dnd/npc-interactions
retries: 2
normalize: dnd/npc-interactions

View File

@@ -1,21 +0,0 @@
version: 3
output:
directory: ./notarius-output
cache:
chunk_plans:
mode: bypass
checkpoints:
enabled: false
directory: ""
debug:
directory: ./notarius-debug
pipelines:
dnd-session:
input: seriatim
chunk: generic
artifacts:
npcs:
extract:
module: dnd/npcs
retries: 2
normalize: dnd/npcs

View File

@@ -1,3 +1,2 @@
Aria: party cleric and recurring healer.
Borin: fighter ally.
Bandit mage: hostile spellcaster.

View File

@@ -1,23 +0,0 @@
version: 3
output:
directory: ./notarius-output
cache:
chunk_plans:
mode: bypass
checkpoints:
enabled: false
directory: ""
debug:
directory: ./notarius-debug
pipelines:
dnd-scene-chunk-map:
input: seriatim
chunk: dnd/scenes
output:
module: json
options:
include_chunk_map: true
artifacts:
spells:
extract: dnd/spells
normalize: dnd/spells

View File

@@ -1,20 +0,0 @@
version: 3
output:
directory: ./notarius-output
cache:
chunk_plans:
mode: bypass
directory: ./notarius-cache/chunk-plans
checkpoints:
enabled: false
directory: ./notarius-cache/checkpoints
debug:
directory: ./notarius-debug
pipelines:
dnd-scene-descriptions:
input: seriatim
chunk: dnd/scenes
artifacts:
scene-descriptions:
extract: dnd/scene-descriptions
normalize: dnd/scene-descriptions

View File

@@ -1,2 +0,0 @@
Cure Wounds: healing spell cast by touch.
Shield: defensive reaction spell.

View File

@@ -1,39 +0,0 @@
version: 3
concurrency:
total_llm: 1
stage_workers:
extract: 1
output:
directory: ./notarius-output
cache:
chunk_plans:
directory: /var/cache/notarius/chunk-plans
mode: auto
checkpoints:
enabled: false
directory: /var/cache/notarius/checkpoints
debug:
directory: ./notarius-debug
pipelines:
dnd-session:
input: seriatim
references:
party: ./dnd-spells-roster.txt
glossary: ./dnd-spells-glossary.txt
chunk:
module: generic
options:
max_units: 50
artifacts:
spells:
extract:
module: dnd/spells
retries: 2
# Overlay behavior binds the same catalog independently at each stage.
references:
spell_catalog: ./dnd-spells-catalog.json
normalize:
module: dnd/spells
# Normalize-stage references are local and must be bound explicitly.
references:
spell_catalog: ./dnd-spells-catalog.json

View File

@@ -1,19 +0,0 @@
version: 3
output:
directory: ./notarius-output
cache:
chunk_plans:
mode: bypass
checkpoints:
enabled: false
directory: ""
debug:
directory: ./notarius-debug
pipelines:
dnd-session:
input: seriatim
artifacts:
spells:
extract: dnd/spells
# Base-only behavior: normalization uses the embedded SRD catalog.
normalize: dnd/spells

View File

@@ -0,0 +1,5 @@
id: dnd-extraction
backend: openrouter
model: openai/gpt-5.6-luna
timeout_seconds: 240
service_tier: flex

2
go.mod
View File

@@ -3,7 +3,7 @@ module gitea.maximumdirect.net/eric/notarius
go 1.25.5
require (
gitea.maximumdirect.net/eric/scriptorium v0.11.1
gitea.maximumdirect.net/eric/promptkit v0.5.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1
)

8
go.sum
View File

@@ -1,13 +1,9 @@
gitea.maximumdirect.net/eric/scriptorium v0.11.0 h1:rjvbt9FTaWHxYlHq7QlUzmMVUt3QdbTmeCkmH81N//o=
gitea.maximumdirect.net/eric/scriptorium v0.11.0/go.mod h1:FQ5lEuNxmrQyNgIomkpZdxvfTC0jWjbXYuq3tbJWF64=
gitea.maximumdirect.net/eric/scriptorium v0.11.1 h1:zBKtB3+fP8FcHGI8DJD99CiTL6crAGitBhWtE+xYJHc=
gitea.maximumdirect.net/eric/scriptorium v0.11.1/go.mod h1:FQ5lEuNxmrQyNgIomkpZdxvfTC0jWjbXYuq3tbJWF64=
gitea.maximumdirect.net/eric/promptkit v0.5.0 h1:jnpazLyyNhWrB2xzwwtUkNUfktkTdkENTwuSPnKiYrc=
gitea.maximumdirect.net/eric/promptkit v0.5.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

View File

@@ -213,11 +213,12 @@ func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions)
components := productionTestComponents(t)
extractor := &assembledSpellExtractor{unknownSpell: options.unknownSpell}
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
Key: assembledSpellExtractorKey,
Stage: pipeline.StageExtract,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"dnd.spell_casts"},
ArtifactKind: dnd.SpellListKind,
Key: assembledSpellExtractorKey,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassDeterministic,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"dnd.spell_casts"},
ArtifactKind: dnd.SpellListKind,
}, func() (contracts.Extractor[dnd.SpellList], error) {
return extractor, nil
}); err != nil {

View File

@@ -21,15 +21,16 @@ type productionComponents struct {
func newProductionComponents() (productionComponents, error) {
registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(),
Chunkers: pipeline.NewChunkerRegistry(),
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(),
Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(),
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
Inputs: pipeline.NewInputAdapterRegistry(),
Chunkers: pipeline.NewChunkerRegistry(),
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
ArtifactEvidence: pipeline.NewArtifactEvidenceRegistry(),
Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(),
Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(),
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
}
assets := llm.NewAssetRegistry()
registrars := []struct {
@@ -88,29 +89,31 @@ func effectiveRegistries(opts Options) (pipeline.Registries, error) {
func catalogFromRegistries(registries pipeline.Registries) pipeline.ModuleCatalog {
return pipeline.ModuleCatalog{
Inputs: registries.Inputs,
Chunkers: registries.Chunkers,
ArtifactCodecs: registries.ArtifactCodecs,
Extractors: registries.Extractors,
Mergers: registries.Mergers,
Normalizers: registries.Normalizers,
Validators: registries.Validators,
ValidatorChains: registries.ValidatorChains,
Outputs: registries.Outputs,
Inputs: registries.Inputs,
Chunkers: registries.Chunkers,
ArtifactCodecs: registries.ArtifactCodecs,
ArtifactEvidence: registries.ArtifactEvidence,
Extractors: registries.Extractors,
Mergers: registries.Mergers,
Normalizers: registries.Normalizers,
Validators: registries.Validators,
ValidatorChains: registries.ValidatorChains,
Outputs: registries.Outputs,
}
}
func registriesFromCatalog(catalog pipeline.ModuleCatalog) pipeline.Registries {
return pipeline.Registries{
Inputs: catalog.Inputs,
Chunkers: catalog.Chunkers,
ArtifactCodecs: catalog.ArtifactCodecs,
Extractors: catalog.Extractors,
Mergers: catalog.Mergers,
Normalizers: catalog.Normalizers,
Validators: catalog.Validators,
ValidatorChains: catalog.ValidatorChains,
Outputs: catalog.Outputs,
Inputs: catalog.Inputs,
Chunkers: catalog.Chunkers,
ArtifactCodecs: catalog.ArtifactCodecs,
ArtifactEvidence: catalog.ArtifactEvidence,
Extractors: catalog.Extractors,
Mergers: catalog.Mergers,
Normalizers: catalog.Normalizers,
Validators: catalog.Validators,
ValidatorChains: catalog.ValidatorChains,
Outputs: catalog.Outputs,
}
}
@@ -118,6 +121,7 @@ func isEmptyCatalog(catalog pipeline.ModuleCatalog) bool {
return catalog.Inputs == nil &&
catalog.Chunkers == nil &&
catalog.ArtifactCodecs == nil &&
catalog.ArtifactEvidence == nil &&
catalog.Extractors == nil &&
catalog.Mergers == nil &&
catalog.Normalizers == nil &&
@@ -130,6 +134,7 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
return registries.Inputs == nil &&
registries.Chunkers == nil &&
registries.ArtifactCodecs == nil &&
registries.ArtifactEvidence == nil &&
registries.Extractors == nil &&
registries.Mergers == nil &&
registries.Normalizers == nil &&
@@ -138,24 +143,13 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
registries.Outputs == nil
}
func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
assets, err := productionPromptAssets()
if err != nil {
return nil, nil, err
}
return buildProductionLLMClient(ctx, cfg, profileID, assets)
}
func productionLLMClientFactoryWithAssets(assets *llm.AssetRegistry) LLMClientFactory {
return func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return buildProductionLLMClient(ctx, cfg, profileID, assets)
return func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return buildProductionLLMClient(ctx, cfg, profileID, overrides, assets)
}
}
func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
@@ -163,14 +157,16 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID
return nil, nil, fmt.Errorf("production asset registry must not be nil")
}
recorder := llm.NewLLMProfileRecorder()
client, err := llm.NewScriptoriumClient(llm.ScriptoriumClientConfig{
ProfileDir: cfg.Scriptorium.ProfileDir,
ProfileFile: cfg.Scriptorium.ProfileFile,
Assets: assets,
Recorder: recorder,
client, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{
ProfileDir: cfg.PromptKit.ProfileDir,
ProfileFile: cfg.PromptKit.ProfileFile,
LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend),
Assets: assets,
Recorder: recorder,
ReasoningEffort: overrides.ReasoningEffort,
})
if err != nil {
return nil, nil, fmt.Errorf("create Scriptorium-backed LLM client: %w", err)
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)
}
scheduler, err := llm.NewScheduler(cfg.Concurrency.TotalLLM)
if err != nil {

View File

@@ -154,6 +154,23 @@ func TestConfigValidateResolvesPipelineAndChecksSelection(t *testing.T) {
}
}
func TestConfigValidatePipelineDefaultProfileIsOffline(t *testing.T) {
configPath := writeCommandConfigContent(t, `version: 4
pipelines:
demo:
llm_profile: dnd-extraction
input: seriatim
artifacts:
spells:
extract: dnd/spells
`)
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo"}, &stdout, &stderr, Options{})
if code != 0 || !strings.Contains(stdout.String(), "valid for pipeline \"demo\"") || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
}
func TestPipelinesListSortsNormalizedIDsInTextAndJSON(t *testing.T) {
configPath := writeCommandConfig(t, " zeta ", "alpha")
options := commandContractOptions(t)
@@ -214,13 +231,13 @@ func commandContractOptionsWithLookup(t *testing.T, lookup func(string) (string,
func writeCommandConfig(t *testing.T, firstID, secondID string) string {
t.Helper()
content := fmt.Sprintf("version: 3\npipelines:\n %q:\n input: seriatim\n %q:\n input: seriatim\n", firstID, secondID)
content := fmt.Sprintf("version: 4\npipelines:\n %q:\n input: seriatim\n %q:\n input: seriatim\n", firstID, secondID)
return writeCommandConfigContent(t, content)
}
func writeResolvableCommandConfig(t *testing.T) string {
t.Helper()
return writeCommandConfigContent(t, `version: 3
return writeCommandConfigContent(t, `version: 4
pipelines:
demo:
input: seriatim

View File

@@ -15,8 +15,7 @@ import (
func TestProductionCombatConfigurationResolvesTypedLane(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-combat-turns.config.yml")
cfg := loadMaintainedExample(t, configPath)
cfg := productionCombatContractConfig()
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-combat", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
@@ -49,8 +48,12 @@ func TestProductionCombatConfigurationResolvesTypedLane(t *testing.T) {
if !ok || codecSpec.Schema.ID != "notarius.dnd.combat_turns" || codecSpec.Schema.Version != "v1" {
t.Fatalf("combat codec spec = %#v, want compatible durable schema", codecSpec)
}
if !hasReferenceSlot(extractSpec.ReferenceSlots, "npcs") || !hasReferenceSlot(normalizeSpec.ReferenceSlots, "npcs") {
t.Fatalf("combat reference slots = %#v / %#v, want stage-local NPC slots", extractSpec.ReferenceSlots, normalizeSpec.ReferenceSlots)
if !hasReferenceSlot(extractSpec.ReferenceSlots, "npcs") || !hasReferenceSlot(extractSpec.ReferenceSlots, "scene_descriptions") || !hasReferenceSlot(normalizeSpec.ReferenceSlots, "npcs") {
t.Fatalf("combat reference slots = %#v / %#v, want extraction scene and NPC slots plus normalization NPC slot", extractSpec.ReferenceSlots, normalizeSpec.ReferenceSlots)
}
sceneSlot := referenceSlot(extractSpec.ReferenceSlots, "scene_descriptions")
if !sceneSlot.Required || !reflect.DeepEqual(sceneSlot.AcceptedMediaTypes, []string{"application/json"}) || !reflect.DeepEqual(sceneSlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.SceneDescriptionListKind}) || sceneSlot.MaxBytes != 1048576 {
t.Fatalf("scene description slot = %#v, want required approved scene artifact", sceneSlot)
}
wantExtractChain := []pipeline.ModuleBinding{
@@ -90,16 +93,26 @@ func TestProductionCombatConfigurationResolvesTypedLane(t *testing.T) {
t.Fatalf("Resolve(bound references) error = %v, want nil", err)
}
boundLane := bound.ResolvedPipeline.Steps[0].ArtifactLanes[0]
if len(boundLane.ExtractReferences.Bindings) != 1 || len(boundLane.NormalizeReferences.Bindings) != 1 || boundLane.ExtractReferences.Bindings[0].SlotName != "npcs" || boundLane.NormalizeReferences.Bindings[0].SlotName != "npcs" {
t.Fatalf("bound combat references = %#v / %#v, want one independent NPC binding per stage", boundLane.ExtractReferences, boundLane.NormalizeReferences)
if len(boundLane.ExtractReferences.Bindings) != 2 || len(boundLane.NormalizeReferences.Bindings) != 1 || !hasReferenceBinding(boundLane.ExtractReferences.Bindings, "npcs") || !hasReferenceBinding(boundLane.ExtractReferences.Bindings, "scene_descriptions") || !hasReferenceBinding(boundLane.NormalizeReferences.Bindings, "npcs") {
t.Fatalf("bound combat references = %#v / %#v, want extraction scene and NPC bindings plus normalization NPC binding", boundLane.ExtractReferences, boundLane.NormalizeReferences)
}
}
func TestProductionCombatConfigurationRequiresSceneDescriptions(t *testing.T) {
components := productionTestComponents(t)
cfg := productionCombatContractConfig()
profile := cfg.Pipelines["dnd-combat"]
profile.References = nil
cfg.Pipelines["dnd-combat"] = profile
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-combat", Catalog: catalogFromRegistries(components.registries)}); err == nil || !strings.Contains(err.Error(), "scene_descriptions") || !strings.Contains(err.Error(), "required") {
t.Fatalf("Resolve() error = %v, want required scene reference failure", err)
}
}
func TestProductionCombatConfigurationRejectsLooseOptionsAndLaneValidators(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-combat-turns.config.yml")
resolve := func(mutate func(*pipeline.PipelineProfile)) error {
cfg := loadMaintainedExample(t, configPath)
cfg := productionCombatContractConfig()
profile := cfg.Pipelines["dnd-combat"]
mutate(&profile)
cfg.Pipelines["dnd-combat"] = profile
@@ -131,7 +144,7 @@ func TestProductionCombatConfigurationRejectsLooseOptionsAndLaneValidators(t *te
func TestProductionCombatConfigurationResolvesTypedUnconditionalValidators(t *testing.T) {
components := productionTestComponents(t)
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-combat-turns.config.yml"))
cfg := productionCombatContractConfig()
profile := cfg.Pipelines["dnd-combat"]
lane := profile.Artifacts["combat"]
lane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{pipeline.Binding("generic/always_accept")}}
@@ -150,6 +163,23 @@ func TestProductionCombatConfigurationResolvesTypedUnconditionalValidators(t *te
}
}
func productionCombatContractConfig() config.Config {
cfg := config.Default()
cfg.Pipelines["dnd-combat"] = pipeline.PipelineProfile{
ID: "dnd-combat",
Input: pipeline.Binding("seriatim"),
Chunk: pipeline.Binding(pipeline.DefaultChunkModule),
References: map[string]pipeline.ReferenceSource{"scene_descriptions": pipeline.ExternalReference("scenes.json")},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"combat": {
Extract: pipeline.ModuleBinding{Module: combatextract.Key, Retries: 2},
Normalize: pipeline.Binding(combatnormalize.Key),
},
},
}
return cfg
}
func hasReferenceSlot(slots []contracts.ReferenceSlot, name string) bool {
for _, slot := range slots {
if slot.Name == name {
@@ -158,3 +188,21 @@ func hasReferenceSlot(slots []contracts.ReferenceSlot, name string) bool {
}
return false
}
func hasReferenceBinding(bindings []pipeline.ReferenceBinding, name string) bool {
for _, binding := range bindings {
if binding.SlotName == name {
return true
}
}
return false
}
func referenceSlot(slots []contracts.ReferenceSlot, name string) contracts.ReferenceSlot {
for _, slot := range slots {
if slot.Name == name {
return slot
}
}
return contracts.ReferenceSlot{}
}

View File

@@ -16,8 +16,7 @@ import (
func TestProductionNPCConfigurationResolvesTypedLane(t *testing.T) {
components := productionTestComponents(t)
catalog := catalogFromRegistries(components.registries)
configPath := repositoryPath("examples", "dnd-npcs.config.yml")
cfg := loadMaintainedExample(t, configPath)
cfg := productionNPCContractConfig()
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
@@ -81,9 +80,8 @@ func TestProductionNPCConfigurationResolvesTypedLane(t *testing.T) {
func TestProductionNPCConfigurationValidatesOptionsReferencesAndPlacement(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-npcs.config.yml")
resolve := func(mutate func(*pipeline.PipelineProfile)) error {
cfg := loadMaintainedExample(t, configPath)
cfg := productionNPCContractConfig()
profile := cfg.Pipelines["dnd-session"]
mutate(&profile)
cfg.Pipelines["dnd-session"] = profile
@@ -123,6 +121,22 @@ func TestProductionNPCConfigurationValidatesOptionsReferencesAndPlacement(t *tes
}
}
func productionNPCContractConfig() config.Config {
cfg := config.Default()
cfg.Pipelines["dnd-session"] = pipeline.PipelineProfile{
ID: "dnd-session",
Input: pipeline.Binding("seriatim"),
Chunk: pipeline.Binding(pipeline.DefaultChunkModule),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"npcs": {
Extract: pipeline.ModuleBinding{Module: npcextract.Key, Retries: 2},
Normalize: pipeline.Binding(npcnormalize.Key),
},
},
}
return cfg
}
func validatorChain(resolved pipeline.ResolvedPipeline, stage pipeline.ModuleStage, module string) []pipeline.ModuleBinding {
for _, chain := range resolved.ValidatorChains {
if chain.Stage == stage && chain.ModuleKey == module {

View File

@@ -6,8 +6,10 @@ import (
"fmt"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -28,7 +30,7 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
Output: pipeline.Binding("json"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"scene-descriptions": {
Extract: pipeline.Binding(sceneextract.Key),
Extract: pipeline.ModuleBinding{Module: sceneextract.Key, LLMProfile: "scene-description-profile"},
Normalize: pipeline.Binding(scenenormalize.Key),
},
},
@@ -45,7 +47,8 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
t.Fatalf("resolved references = %#v / %#v, want no generated or required references", lane.ExtractReferences, lane.NormalizeReferences)
}
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: sceneDescriptionLLM{}})
llmClient := &sceneDescriptionLLM{}
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
@@ -60,6 +63,14 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
t.Fatalf("run output = %#v, want one approved normalized artifact", output)
}
wantProfiles := []artifacts.LLMProfileManifest{{
ID: "scene-description-profile",
Provider: "promptkit",
Model: "deterministic",
}}
if !reflect.DeepEqual(output.Manifest.LLMProfiles, wantProfiles) {
t.Fatalf("manifest LLM profiles = %#v, want %#v", output.Manifest.LLMProfiles, wantProfiles)
}
normalizedOutput := output.NormalizeOutputs[0]
if normalizedOutput.NormalizerKey != scenenormalize.Key || normalizedOutput.Artifact.Kind != dnd.SceneDescriptionListKind || normalizedOutput.Artifact.Schema.ID != scenecodec.SchemaID || normalizedOutput.Artifact.Schema.Name != scenecodec.SchemaName || normalizedOutput.Artifact.Schema.Version != scenecodec.SchemaVersion {
t.Fatalf("normalized output = %#v, want registered durable scene-description schema", normalizedOutput)
@@ -85,9 +96,12 @@ func TestProductionSceneDescriptionWorkflow(t *testing.T) {
}
}
type sceneDescriptionLLM struct{}
type sceneDescriptionLLM struct {
mu sync.Mutex
profile *artifacts.LLMProfileManifest
}
func (sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
func (client *sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
@@ -107,5 +121,27 @@ func (sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts
if err := json.Unmarshal([]byte(content), out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured response: %w", err)
}
return contracts.StructuredCompletionResponse{Content: []byte(content), Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
profile := artifacts.LLMProfileManifest{
ID: req.ProfileID,
Provider: "promptkit",
Model: "deterministic",
}
client.mu.Lock()
client.profile = &profile
client.mu.Unlock()
return contracts.StructuredCompletionResponse{
Content: []byte(content),
Provider: profile.Provider,
Model: profile.Model,
ProfileID: profile.ID,
}, nil
}
func (client *sceneDescriptionLLM) LLMProfileManifests() []artifacts.LLMProfileManifest {
client.mu.Lock()
defer client.mu.Unlock()
if client.profile == nil {
return nil
}
return []artifacts.LLMProfileManifest{*client.profile}
}

View File

@@ -1,18 +1,22 @@
package cli
import (
"context"
"encoding/json"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
)
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
@@ -20,6 +24,20 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
for _, example := range maintainedExampleFiles(t) {
t.Run(example.name, func(t *testing.T) {
cfg := loadMaintainedExample(t, example.path)
raw, err := os.ReadFile(example.transcriptPath)
if err != nil {
t.Fatalf("read maintained transcript %q: %v", example.transcriptPath, err)
}
document, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{
Path: example.transcriptPath,
Raw: raw,
})
if err != nil {
t.Fatalf("parse maintained transcript %q: %v", example.transcriptPath, err)
}
if len(document.Units) == 0 {
t.Fatalf("maintained transcript %q has no parsed units", example.transcriptPath)
}
for _, pipelineID := range example.pipelineIDs {
effective, err := cfg.Resolve(resolveInputForMaintainedExample(components, pipelineID))
if err != nil {
@@ -32,11 +50,23 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
if err != nil {
t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err)
}
if example.name == "production" {
if len(materialized.Steps[0].ArtifactLanes) != 1 ||
len(materialized.Steps[0].ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 ||
len(materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 {
t.Fatalf("production spell catalog reference was not materialized: %#v", materialized.Steps[0].ArtifactLanes)
if example.name == "complete" {
if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-events,npcs,scene-descriptions|extract-events:combat-turns,npc-interactions,spells" {
t.Fatalf("complete example steps and lanes = %v, want every D&D extractor in the documented two-step composition", got)
}
spellLane := referenceContractLane(t, materialized, "spells")
if len(spellLane.ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 ||
len(spellLane.NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 {
t.Fatalf("complete example spell catalog reference was not materialized: %#v", spellLane)
}
itemEventLane := referenceContractLane(t, materialized, "item-events")
for _, references := range []pipeline.ResolvedReferenceTarget{itemEventLane.ExtractReferences, itemEventLane.NormalizeReferences} {
if _, found := references.ReferenceSet.Slots["npcs"]; found {
t.Fatalf("item event lane unexpectedly depends on generated NPCs: %#v", itemEventLane)
}
if _, found := references.ReferenceSet.Slots["scene_descriptions"]; found {
t.Fatalf("item event lane unexpectedly depends on generated scene descriptions: %#v", itemEventLane)
}
}
}
}
@@ -49,6 +79,67 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
}
}
func TestMaintainedConfigurationExampleSet(t *testing.T) {
entries, err := os.ReadDir(repositoryPath("examples"))
if err != nil {
t.Fatal(err)
}
var names []string
for _, entry := range entries {
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".config.yml") {
names = append(names, entry.Name())
}
}
sort.Strings(names)
if got := strings.Join(names, ","); got != "dnd-complete.config.yml,dnd-minimal.config.yml" {
t.Fatalf("maintained configuration examples = %q, want only the minimal and complete D&D examples", got)
}
profileEntries, err := os.ReadDir(repositoryPath("examples", "profiles"))
if err != nil {
t.Fatal(err)
}
names = names[:0]
for _, entry := range profileEntries {
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".yml") {
names = append(names, entry.Name())
}
}
sort.Strings(names)
if got := strings.Join(names, ","); got != "dnd-extraction.yml" {
t.Fatalf("maintained operator profiles = %q, want dnd-extraction.yml", got)
}
}
func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
t.Chdir(repositoryPath())
t.Setenv("OPENROUTER_API_KEY", "")
for _, example := range maintainedExampleFiles(t) {
t.Run(example.name, func(t *testing.T) {
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"config", "validate", "--config", example.path, "--pipeline", "dnd-session",
}, &stdout, &stderr, Options{})
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `valid for pipeline "dnd-session"`) {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
}
func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
result := make([]string, 0, len(resolved.Steps))
for _, step := range resolved.Steps {
laneIDs := make([]string, 0, len(step.ArtifactLanes))
for _, lane := range step.ArtifactLanes {
laneIDs = append(laneIDs, lane.ID)
}
sort.Strings(laneIDs)
result = append(result, step.ID+":"+strings.Join(laneIDs, ","))
}
return result
}
func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
outputRoot := filepath.Join(t.TempDir(), "output")
fake := &productionFakeLLMClient{}
@@ -56,7 +147,7 @@ func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session",
"--config", repositoryPath("examples", "dnd-spells.config.yml"),
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
}, &stdout, &stderr, options)
@@ -130,7 +221,7 @@ func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing
options := productionRunOptions(t, &productionFakeLLMClient{})
args := []string{
"run", "dnd-session",
"--config", repositoryPath("examples", "dnd-spells.config.yml"),
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
"--input", malformed, "--chunk_cache", "bypass", "--output-dir", outputRoot,
}
if debug {

View File

@@ -3,6 +3,7 @@ package cli
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
@@ -22,9 +23,24 @@ func TestOversizedNPCRegistryFailsBeforeRuntimeAndCheckpointConstruction(t *test
t.Fatal(err)
}
checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
content := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml"))
content = replaceRequiredOnce(t, content, " extract: dnd/spells", " extract:\n module: dnd/spells\n references:\n npcs: "+npcPath)
content = replaceRequiredOnce(t, content, " enabled: false\n directory: \"\"", " enabled: true\n directory: "+checkpointRoot)
content := fmt.Sprintf(`version: 4
cache:
chunk_plans:
mode: bypass
checkpoints:
enabled: true
directory: %q
pipelines:
dnd-session:
input: seriatim
artifacts:
spells:
extract:
module: dnd/spells
references:
npcs: %q
normalize: dnd/spells
`, checkpointRoot, npcPath)
configPath := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(configPath, []byte(content), 0o600); err != nil {
t.Fatal(err)
@@ -35,7 +51,7 @@ func TestOversizedNPCRegistryFailsBeforeRuntimeAndCheckpointConstruction(t *test
options := Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
llmConstructed = true
return nil, nil, errors.New("LLM client must not be constructed")
},

View File

@@ -6,6 +6,8 @@ import (
"errors"
"fmt"
"io/fs"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
@@ -13,23 +15,30 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
itemeventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemevents"
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
itemeventextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/itemevents"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
combatnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/combatturns"
itemeventnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/itemevents"
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
@@ -38,9 +47,9 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
assertProductionContains(t, "inputs", registries.Inputs.RegisteredKeys(), []string{"seriatim"})
assertProductionContains(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes", "generic"})
assertProductionContains(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells", "dnd/npcs", combatextract.Key})
assertProductionContains(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells", "dnd/npcs", combatextract.Key, itemeventextract.Key})
assertProductionContains(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"})
assertProductionContains(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop", spellnormalize.Key, "dnd/npcs", combatnormalize.Key})
assertProductionContains(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop", spellnormalize.Key, "dnd/npcs", combatnormalize.Key, itemeventnormalize.Key})
assertProductionContains(t, "outputs", registries.Outputs.RegisteredKeys(), []string{"json"})
assertProductionContains(t, "validators", registries.Validators.RegisteredKeys(), []string{
"extract/dnd/spells/catalog",
@@ -51,16 +60,21 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
"extract/dnd/combat-turns/source_refs",
"extract/dnd/combat-turns/source_relatedness",
"normalize/dnd/combat-turns/invariants",
"extract/dnd/item-events/shape",
"extract/dnd/item-events/source_refs",
"extract/dnd/item-events/source_relatedness",
"normalize/dnd/item-events/invariants",
"generic/always_accept",
"generic/always_reject",
"generic/valid_json",
"generic/valid_json_schema",
})
assertProductionContains(t, "artifact codec kinds", registries.ArtifactCodecs.RegisteredKinds(), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind})
assertProductionContains(t, "merger variants", registries.Mergers.RegisteredArtifactKinds(pipeline.DefaultMergeModule), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind})
assertProductionContains(t, "normalizer variants", registries.Normalizers.RegisteredArtifactKinds(pipeline.DefaultNormalizeModule), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind})
assertProductionContains(t, "artifact codec kinds", registries.ArtifactCodecs.RegisteredKinds(), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind, dnd.ItemEventListKind})
assertProductionContains(t, "merger variants", registries.Mergers.RegisteredArtifactKinds(pipeline.DefaultMergeModule), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind, dnd.ItemEventListKind})
assertProductionContains(t, "normalizer variants", registries.Normalizers.RegisteredArtifactKinds(pipeline.DefaultNormalizeModule), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind, dnd.ItemEventListKind})
assertProductionContains(t, "spell normalizer variants", registries.Normalizers.RegisteredArtifactKinds(spellnormalize.Key), []contracts.ArtifactKind{dnd.SpellListKind})
assertProductionContains(t, "combat normalizer variants", registries.Normalizers.RegisteredArtifactKinds(combatnormalize.Key), []contracts.ArtifactKind{dnd.CombatTurnListKind})
assertProductionContains(t, "item event normalizer variants", registries.Normalizers.RegisteredArtifactKinds(itemeventnormalize.Key), []contracts.ArtifactKind{dnd.ItemEventListKind})
wantChain := []pipeline.ModuleBinding{
pipeline.Binding("generic/valid_json"),
@@ -97,6 +111,27 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, combatnormalize.Key); !reflect.DeepEqual(got, combatNormalizeChain) {
t.Fatalf("combat normalize validator chain = %#v, want %#v", got, combatNormalizeChain)
}
itemEventExtractChain := []pipeline.ModuleBinding{
pipeline.Binding("generic/valid_json"),
pipeline.Binding("extract/dnd/item-events/shape"),
pipeline.Binding("extract/dnd/item-events/source_refs"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/item-events/source_relatedness"),
}
itemEventNormalizeChain := []pipeline.ModuleBinding{
pipeline.Binding("generic/valid_json"),
pipeline.Binding("extract/dnd/item-events/shape"),
pipeline.Binding("normalize/dnd/item-events/invariants"),
pipeline.Binding("extract/dnd/item-events/source_refs"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/item-events/source_relatedness"),
}
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, itemeventextract.Key); !reflect.DeepEqual(got, itemEventExtractChain) {
t.Fatalf("item event extract validator chain = %#v, want %#v", got, itemEventExtractChain)
}
if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, itemeventnormalize.Key); !reflect.DeepEqual(got, itemEventNormalizeChain) {
t.Fatalf("item event normalize validator chain = %#v, want %#v", got, itemEventNormalizeChain)
}
assetNames := productionAssetNames(t, components.assets.PromptFS)
requiredAssets := []string{
@@ -119,13 +154,50 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
"dnd.combat_turns/sharedassets/common-dnd-system.md",
"dnd.combat_turns/sharedassets/common-dnd-transcript.md",
"dnd.combat_turns/task.md",
"dnd.item_events/dnd.item_events.yaml",
"dnd.item_events/instructions.md",
"dnd.item_events/sharedassets/common-dnd-extraction-evidence.md",
"dnd.item_events/sharedassets/common-dnd-identity.md",
"dnd.item_events/sharedassets/common-dnd-references.md",
"dnd.item_events/sharedassets/common-dnd-system.md",
"dnd.item_events/sharedassets/common-dnd-transcript.md",
"dnd.item_events/task.md",
}
assertProductionContains(t, "production prompt assets", assetNames, requiredAssets)
catalog := catalogFromRegistries(registries)
for _, test := range []struct {
stage pipeline.ModuleStage
key string
want contracts.ExecutionClass
}{
{stage: pipeline.StageInput, key: "seriatim", want: contracts.ExecutionClassDeterministic},
{stage: pipeline.StageChunk, key: "generic", want: contracts.ExecutionClassDeterministic},
{stage: pipeline.StageChunk, key: "dnd/scenes", want: contracts.ExecutionClassLLMBacked},
{stage: pipeline.StageExtract, key: "dnd/spells", want: contracts.ExecutionClassLLMBacked},
{stage: pipeline.StageExtract, key: "dnd/npcs", want: contracts.ExecutionClassLLMBacked},
{stage: pipeline.StageExtract, key: "dnd/combat-turns", want: contracts.ExecutionClassLLMBacked},
{stage: pipeline.StageExtract, key: "dnd/item-events", want: contracts.ExecutionClassLLMBacked},
{stage: pipeline.StageExtract, key: "dnd/npc-interactions", want: contracts.ExecutionClassLLMBacked},
{stage: pipeline.StageExtract, key: "dnd/scene-descriptions", want: contracts.ExecutionClassLLMBacked},
{stage: pipeline.StageMerge, key: "appendorder", want: contracts.ExecutionClassDeterministic},
{stage: pipeline.StageNormalize, key: "noop", want: contracts.ExecutionClassDeterministic},
{stage: pipeline.StageNormalize, key: "dnd/spells", want: contracts.ExecutionClassDeterministic},
{stage: pipeline.StageNormalize, key: "dnd/npcs", want: contracts.ExecutionClassLLMBacked},
{stage: pipeline.StageNormalize, key: "dnd/combat-turns", want: contracts.ExecutionClassDeterministic},
{stage: pipeline.StageNormalize, key: "dnd/item-events", want: contracts.ExecutionClassDeterministic},
{stage: pipeline.StageNormalize, key: "dnd/npc-interactions", want: contracts.ExecutionClassDeterministic},
{stage: pipeline.StageNormalize, key: "dnd/scene-descriptions", want: contracts.ExecutionClassDeterministic},
{stage: pipeline.StageOutput, key: "json", want: contracts.ExecutionClassDeterministic},
} {
got, ok := catalog.ExecutionClass(test.stage, test.key)
if !ok || got != test.want {
t.Fatalf("production execution class for %s/%s = %q, %t; want %q, true", test.stage, test.key, got, ok, test.want)
}
}
converted := registriesFromCatalog(catalog)
if converted.ArtifactCodecs != registries.ArtifactCodecs || converted.ValidatorChains != registries.ValidatorChains {
t.Fatal("catalog/registry conversion did not preserve codec and validator-chain registries")
if converted.ArtifactCodecs != registries.ArtifactCodecs || converted.ArtifactEvidence != registries.ArtifactEvidence || converted.ValidatorChains != registries.ValidatorChains {
t.Fatal("catalog/registry conversion did not preserve artifact and validator registries")
}
codecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.SpellListKind)
if !ok || codecSpec.Kind != dnd.SpellListKind || codecSpec.Schema.ID != spellcodec.SchemaID {
@@ -135,6 +207,10 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
if !ok || combatCodecSpec.Kind != dnd.CombatTurnListKind || combatCodecSpec.Schema.ID != combatcodec.SchemaID {
t.Fatalf("combat codec spec = %#v, ok=%t, want typed D&D combat codec", combatCodecSpec, ok)
}
itemEventCodecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.ItemEventListKind)
if !ok || itemEventCodecSpec.Kind != dnd.ItemEventListKind || itemEventCodecSpec.Schema.ID != itemeventcodec.SchemaID {
t.Fatalf("item event codec spec = %#v, ok=%t, want typed D&D item-event codec", itemEventCodecSpec, ok)
}
if got := catalog.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
t.Fatalf("catalog validator chain = %#v, want %#v", got, wantChain)
}
@@ -147,13 +223,83 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
func TestDefaultCLICompositionValidatesRepresentativeConfiguration(t *testing.T) {
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"config", "validate", "--config", repositoryPath("examples", "dnd-spells.config.yml"), "--pipeline", "dnd-session",
"config", "validate", "--config", repositoryPath("examples", "dnd-minimal.config.yml"), "--pipeline", "dnd-session",
}, &stdout, &stderr, Options{})
if code != 0 || stderr.Len() != 0 {
t.Fatalf("validate representative config with default composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
}
func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
components := productionTestComponents(t)
newEngine := func(profileFile string) (*promptkit.Engine, error) {
t.Helper()
options, err := components.assets.PromptKitOptions()
if err != nil {
return nil, err
}
if profileFile != "" {
options = append(options, promptkit.WithProfileFile(profileFile))
}
return promptkit.NewEngine(promptkit.Config{}, options...)
}
t.Run("fallback", func(t *testing.T) {
engine, err := newEngine("")
if err != nil {
t.Fatal(err)
}
inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction")
if err != nil {
t.Fatalf("InspectProfile() error = %v, want fallback profile", err)
}
params := inspection.EffectiveModelParams
if params.BackendID != "openrouter" || params.Model != "openai/gpt-5.6-luna" || params.TimeoutSeconds != 240 || params.ServiceTier != "flex" {
t.Fatalf("fallback profile parameters = %#v", params)
}
if params.ReasoningEffort != "" || params.Temperature != 0 || params.MaxTokens != 0 || params.TopP != 0 {
t.Fatalf("fallback profile selected optional provider controls: %#v", params)
}
})
t.Run("valid operator profile wins", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
if err := os.WriteFile(profilePath, []byte(`id: dnd-extraction
endpoint: http://operator.example.test/v1
model: operator-model
timeout_seconds: 75
`), 0o600); err != nil {
t.Fatal(err)
}
engine, err := newEngine(profilePath)
if err != nil {
t.Fatal(err)
}
inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction")
if err != nil {
t.Fatalf("InspectProfile() error = %v, want operator profile", err)
}
params := inspection.EffectiveModelParams
if params.BackendID != "" || params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "" {
t.Fatalf("operator profile parameters = %#v, want complete replacement", params)
}
})
t.Run("invalid operator profile does not fall through", func(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
if err := os.WriteFile(profilePath, []byte("id: dnd-extraction\nendpoint: http://operator.example.test/v1\nmodel: operator-model\nunknown: value\n"), 0o600); err != nil {
t.Fatal(err)
}
engine, err := newEngine(profilePath)
if err == nil {
_, err = engine.InspectProfile(context.Background(), "dnd-extraction")
}
if err == nil {
t.Fatal("operator profile error = nil, want failure instead of fallback")
}
})
}
func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
components := productionTestComponents(t)
cfg := config.Default()
@@ -176,7 +322,7 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
configPath := writeProductionSpellCatalogContractConfig(t)
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatalf("resolve production spell configuration: %v", err)
@@ -203,7 +349,7 @@ func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
configPath := writeProductionSpellCatalogContractConfig(t)
resolve := func(t *testing.T) pipeline.ResolvedPipeline {
t.Helper()
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
@@ -261,12 +407,15 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
t.Fatal(err)
}
checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
content := string(readRepositoryFile(t, "examples", "dnd-spells-production.config.yml"))
content = replaceRequiredOnce(t, content, "./dnd-spells-roster.txt", repositoryPath("examples", "dnd-spells-roster.txt"))
content = replaceRequiredOnce(t, content, "./dnd-spells-glossary.txt", repositoryPath("examples", "dnd-spells-glossary.txt"))
content = strings.Replace(content, "./dnd-spells-catalog.json", repositoryPath("examples", "dnd-spells-catalog.json"), 1)
content = replaceRequiredOnce(t, content, "./dnd-spells-catalog.json", catalogPath)
content = replaceRequiredOnce(t, content, " enabled: false\n directory: /var/cache/notarius/checkpoints", " enabled: true\n directory: "+checkpointRoot)
content := productionSpellCatalogContractConfig(t)
catalogSource := repositoryPath("examples", "dnd-spell-catalog.json")
if count := strings.Count(content, catalogSource); count != 2 {
t.Fatalf("spell catalog source occurs %d times, want extract and normalize bindings", count)
}
content = strings.Replace(content, catalogSource, "__extract_catalog__", 1)
content = replaceRequiredOnce(t, content, catalogSource, catalogPath)
content = replaceRequiredOnce(t, content, "__extract_catalog__", catalogSource)
content = replaceRequiredOnce(t, content, " enabled: false\n directory: \"\"", " enabled: true\n directory: "+checkpointRoot)
configFile := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(configFile, []byte(content), 0o600); err != nil {
t.Fatal(err)
@@ -277,7 +426,7 @@ func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution
options := Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
llmConstructed = true
return nil, nil, errors.New("LLM client must not be constructed")
},
@@ -325,31 +474,150 @@ func setNormalizeSpellCatalogSource(t *testing.T, resolved *pipeline.ResolvedPip
resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings = bindings
}
func TestProductionLLMClientFactoriesBuildOfflineRuntime(t *testing.T) {
func TestProductionLLMClientFactoryBuildsOfflineRuntime(t *testing.T) {
components := productionTestComponents(t)
factories := []struct {
name string
factory LLMClientFactory
}{
{name: "default production assets", factory: productionLLMClientFactory},
{name: "provided production assets", factory: productionLLMClientFactoryWithAssets(components.assets)},
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
if err != nil {
t.Fatalf("build production LLM runtime: %v", err)
}
for _, tt := range factories {
t.Run(tt.name, func(t *testing.T) {
client, manifests, err := tt.factory(context.Background(), config.Default(), "test-profile")
if err != nil {
t.Fatalf("build production LLM runtime: %v", err)
}
if client == nil {
t.Fatal("production LLM runtime returned a nil client")
}
if len(manifests) != 0 {
t.Fatalf("eager profile manifests = %#v, want none", manifests)
}
if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
t.Fatalf("production LLM client %T does not provide profile manifests", client)
}
})
if client == nil {
t.Fatal("production LLM runtime returned a nil client")
}
if len(manifests) != 0 {
t.Fatalf("eager profile manifests = %#v, want none", manifests)
}
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
if !ok {
t.Fatalf("production LLM client %T does not provide one profile-source checkpoint fingerprint", client)
}
fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
if err != nil || len(fingerprints) != 1 {
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one profile-source identity", fingerprints, err)
}
if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
t.Fatalf("production LLM client %T does not provide profile manifests", client)
}
}
func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *testing.T) {
opts, err := normalizeOptions(Options{
Catalog: pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry()},
})
if err != nil {
t.Fatal(err)
}
if opts.promptKitAssets == nil || opts.LLMClientFactory == nil {
t.Fatalf("normalized options = %#v, want shared profile assets and default runtime factory", opts)
}
if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"dnd-extraction"}, opts.promptKitAssets); err != nil {
t.Fatalf("inspect application fallback profile: %v", err)
}
client, _, err := opts.LLMClientFactory(context.Background(), config.Default(), "dnd-extraction", LLMRuntimeOverrides{})
if err != nil {
t.Fatalf("build default runtime: %v", err)
}
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
if !ok {
t.Fatalf("default runtime client %T does not provide checkpoint fingerprints", client)
}
runtimeFingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
if err != nil {
t.Fatal(err)
}
directClient, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{Assets: opts.promptKitAssets})
if err != nil {
t.Fatal(err)
}
inspectionFingerprints, err := directClient.LLMCheckpointFingerprints()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) {
t.Fatalf("runtime profile fingerprints = %#v, inspection profile fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
}
}
func TestProductionLLMClientFactoryUsesConfiguredLocalBackend(t *testing.T) {
var providerCalls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
providerCalls.Add(1)
if r.URL.Path != "/v1/chat/completions" {
t.Errorf("provider path = %q, want /v1/chat/completions", r.URL.Path)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"choices": [{"message": {"role": "assistant", "content": "{\"ok\":true}"}}],
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
}`))
}))
defer server.Close()
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
if err := os.WriteFile(profilePath, []byte(`id: local-profile
backend: local
model: local-model
`), 0o600); err != nil {
t.Fatal(err)
}
assets := llm.NewAssetRegistry()
if err := assets.RegisterPromptFS(fstest.MapFS{
"production.local.yaml": {Data: []byte(`id: production.local
version: "v1"
inputs:
- name: transcript
required: true
messages:
- role: user
content: '{{ input "transcript" }}'
output:
format: json
validation_mode: json
`)},
}, "."); err != nil {
t.Fatalf("register prompt assets: %v", err)
}
cfg := config.Default()
cfg.PromptKit.ProfileFile = profilePath
cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
Endpoint: server.URL + "/v1",
ConcurrencyLimit: 2,
}
client, manifests, err := productionLLMClientFactoryWithAssets(assets)(
context.Background(),
cfg,
"local-profile",
LLMRuntimeOverrides{},
)
if err != nil {
t.Fatalf("build production LLM runtime: %v", err)
}
if len(manifests) != 0 {
t.Fatalf("eager profile manifests = %#v, want none", manifests)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "production.local",
ProfileID: "local-profile",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "text/plain", []byte("local request"), "", ""),
},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", err)
}
if providerCalls.Load() != 1 {
t.Fatalf("provider calls = %d, want 1", providerCalls.Load())
}
provider, ok := client.(contracts.LLMProfileManifestProvider)
if !ok {
t.Fatalf("production client %T does not provide profile manifests", client)
}
recorded := provider.LLMProfileManifests()
if len(recorded) != 1 || recorded[0].BackendID != promptkit.BackendLocal {
t.Fatalf("production profile manifests = %#v, want local backend", recorded)
}
}
@@ -357,14 +625,15 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
client, manifests, err := productionLLMClientFactory(ctx, config.Default(), "test-profile")
components := productionTestComponents(t)
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(ctx, config.Default(), "test-profile", LLMRuntimeOverrides{})
if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err)
}
})
t.Run("nil assets", func(t *testing.T) {
client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile")
client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{})
if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err)
}
@@ -374,7 +643,7 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
components := productionTestComponents(t)
cfg := config.Default()
cfg.Concurrency.TotalLLM = 0
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile")
client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile", LLMRuntimeOverrides{})
if err == nil || !strings.Contains(err.Error(), "create LLM scheduler") || !strings.Contains(err.Error(), "greater than zero") || client != nil || len(manifests) != 0 {
t.Fatalf("client=%T manifests=%#v error=%v, want scheduler-construction failure", client, manifests, err)
}
@@ -382,7 +651,7 @@ func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
}
func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) {
base := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml"))
base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml"))
validPath := writeProductionContractConfig(t, base)
options := productionCLIOptions(t)
var stdout, stderr strings.Builder
@@ -438,7 +707,7 @@ func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T)
}
func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) {
base := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml"))
base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml"))
content := replaceRequiredOnce(t, base, " normalize: dnd/spells\n", " normalize:\n module: dnd/spells\n validators:\n - module: generic/always_accept\n - module: generic/valid_json\n")
path := writeProductionContractConfig(t, content)
components := productionTestComponents(t)
@@ -517,31 +786,60 @@ func TestProductionSceneRunRecordsAnnotationFreeChunkPlanAndProvenance(t *testin
if len(warnings.Warnings) != 0 {
t.Fatalf("warnings = %#v, want none", warnings.Warnings)
}
if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 {
t.Fatalf("fake prompt requests = %#v, want one scene and one spell request", fake.requestPrompts())
if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemeventextract.PromptID)) != 1 {
t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-event request", fake.requestPrompts())
}
}
type maintainedExample struct {
name string
path string
pipelineIDs []string
name string
path string
transcriptPath string
pipelineIDs []string
}
func maintainedExampleFiles(t *testing.T) []maintainedExample {
t.Helper()
return []maintainedExample{
{name: "minimal", path: repositoryPath("examples", "dnd-spells.config.yml"), pipelineIDs: []string{"dnd-session"}},
{name: "production", path: repositoryPath("examples", "dnd-spells-production.config.yml"), pipelineIDs: []string{"dnd-session"}},
{name: "npcs", path: repositoryPath("examples", "dnd-npcs.config.yml"), pipelineIDs: []string{"dnd-session"}},
{name: "combat", path: repositoryPath("examples", "dnd-combat-turns.config.yml"), pipelineIDs: []string{"dnd-combat"}},
{name: "npc-grounded", path: repositoryPath("examples", "dnd-npc-grounded.config.yml"), pipelineIDs: []string{"dnd-npc-grounded"}},
{name: "npc-interactions", path: repositoryPath("examples", "dnd-npc-interactions.config.yml"), pipelineIDs: []string{"dnd-npc-interactions"}},
{name: "scene-descriptions", path: repositoryPath("examples", "dnd-scene-descriptions.config.yml"), pipelineIDs: []string{"dnd-scene-descriptions"}},
{name: "scene-chunk-map", path: repositoryPath("examples", "dnd-scene-chunk-map.config.yml"), pipelineIDs: []string{"dnd-scene-chunk-map"}},
{name: "minimal", path: repositoryPath("examples", "dnd-minimal.config.yml"), transcriptPath: repositoryPath("examples", "seriatim-minimal-transcript.json"), pipelineIDs: []string{"dnd-session"}},
{name: "complete", path: repositoryPath("examples", "dnd-complete.config.yml"), transcriptPath: repositoryPath("examples", "dnd-complete-transcript.json"), pipelineIDs: []string{"dnd-session"}},
}
}
func productionSpellCatalogContractConfig(t *testing.T) string {
t.Helper()
return fmt.Sprintf(`version: 4
cache:
chunk_plans:
mode: bypass
checkpoints:
enabled: false
directory: ""
pipelines:
dnd-session:
input: seriatim
references:
party: %q
glossary: %q
artifacts:
spells:
extract:
module: dnd/spells
retries: 2
references:
spell_catalog: %q
normalize:
module: dnd/spells
references:
spell_catalog: %q
`, repositoryPath("examples", "dnd-party.txt"), repositoryPath("examples", "dnd-glossary.txt"), repositoryPath("examples", "dnd-spell-catalog.json"), repositoryPath("examples", "dnd-spell-catalog.json"))
}
func writeProductionSpellCatalogContractConfig(t *testing.T) string {
t.Helper()
return writeProductionContractConfig(t, productionSpellCatalogContractConfig(t))
}
func loadMaintainedExample(t *testing.T, path string) config.Config {
t.Helper()
fileConfig, err := config.LoadFileConfig(path)
@@ -575,9 +873,10 @@ func productionCLIOptions(t *testing.T) Options {
func productionOptionsFromComponents(components productionComponents) Options {
return Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LookupEnv: emptyLookup,
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LookupEnv: emptyLookup,
promptKitAssets: components.assets,
}
}
@@ -600,14 +899,14 @@ func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options {
options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() }
options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil }
options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") }
options.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
options.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return fake, nil, nil
}
return options
}
func productionRunConfig(outputRoot, chunkModule string) string {
return fmt.Sprintf(`version: 3
return fmt.Sprintf(`version: 4
output:
directory: %q
cache:
@@ -627,6 +926,8 @@ pipelines:
artifacts:
spells:
extract: dnd/spells
item-events:
extract: dnd/item-events
`, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule)
}
@@ -725,6 +1026,8 @@ func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, r
} else {
content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`)
}
case itemeventextract.PromptID:
content = []byte(`{"events":[{"name":"Cure Wounds","kind":"acquired","to":"party","source_refs":[{"start_segment":1,"end_segment":1}]}]}`)
default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
}

View File

@@ -0,0 +1,46 @@
package cli
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string, assets *llm.AssetRegistry) error {
if len(profileIDs) == 0 {
return nil
}
inspector, err := llm.NewPromptKitProfileInspector(llm.PromptKitProfileInspectorConfig{
Source: promptKitProfileSourceConfig(cfg),
Assets: assets,
})
if err != nil {
return fmt.Errorf("load PromptKit profiles: %w", err)
}
for _, profileID := range profileIDs {
if _, err := inspector.InspectProfile(ctx, profileID); err != nil {
return err
}
}
return nil
}
func promptKitProfileSourceConfig(cfg config.Config) llm.PromptKitProfileSourceConfig {
return llm.PromptKitProfileSourceConfig{
ProfileDir: cfg.PromptKit.ProfileDir,
ProfileFile: cfg.PromptKit.ProfileFile,
LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend),
}
}
func mapPromptKitLocalBackend(cfg *config.PromptKitLocalBackendConfig) *llm.PromptKitLocalBackendConfig {
if cfg == nil {
return nil
}
return &llm.PromptKitLocalBackendConfig{
Endpoint: cfg.Endpoint,
ConcurrencyLimit: cfg.ConcurrencyLimit,
}
}

View File

@@ -0,0 +1,159 @@
package cli
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) {
var providerCalls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
providerCalls.Add(1)
}))
defer server.Close()
writeProfile := func(t *testing.T, name, content string) string {
t.Helper()
profilePath := filepath.Join(t.TempDir(), name+".yaml")
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return profilePath
}
localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n"
credentialProfile := `id: credential-profile
endpoint: ` + server.URL + `/v1
model: credential-model
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
`
t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")
tests := []struct {
name string
profilePath string
profileID string
profileDir bool
localBackend bool
canceled bool
wantErr []string
rejectErr []string
}{
{
name: "configured local backend",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
profileDir: true,
localBackend: true,
},
{
name: "missing local backend registration",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
wantErr: []string{`PromptKit profile "local-profile" is invalid or unreadable`},
},
{
name: "absent profile",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "absent-profile",
localBackend: true,
wantErr: []string{`PromptKit profile "absent-profile" is not configured`},
},
{
name: "malformed profile",
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
profileID: "malformed-profile",
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
rejectErr: []string{"malformed-profile.yaml", "backend: ["},
},
{
name: "invalid profile source",
profilePath: filepath.Join(t.TempDir(), "missing-profile.yaml"),
profileID: "missing-profile",
wantErr: []string{"load PromptKit profiles", "profile configuration is invalid or unreadable"},
},
{
name: "credential environment intentionally unset",
profilePath: writeProfile(t, "credential-profile", credentialProfile),
profileID: "credential-profile",
},
{
name: "canceled inspection",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
localBackend: true,
canceled: true,
wantErr: []string{"context canceled"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := config.Default()
if tt.profileDir {
cfg.PromptKit.ProfileDir = filepath.Dir(tt.profilePath)
} else {
cfg.PromptKit.ProfileFile = tt.profilePath
}
if tt.localBackend {
cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
Endpoint: server.URL + "/v1",
ConcurrencyLimit: 2,
}
}
ctx := context.Background()
if tt.canceled {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
cancel()
}
err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID}, nil)
if len(tt.wantErr) == 0 {
if err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
}
return
}
if err == nil {
t.Fatal("validateExplicitPromptKitProfiles() error = nil, want failure")
}
if tt.canceled && !errors.Is(err, context.Canceled) {
t.Fatalf("canceled inspection error = %v, want context canceled", err)
}
for _, want := range tt.wantErr {
if !strings.Contains(err.Error(), want) {
t.Fatalf("validation error = %q, want %q", err, want)
}
}
for _, rejected := range append(tt.rejectErr, tt.profilePath) {
if rejected != "" && strings.Contains(err.Error(), rejected) {
t.Fatalf("validation error = %q, must not expose %q", err, rejected)
}
}
})
}
if providerCalls.Load() != 0 {
t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load())
}
}
func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
assets := llm.NewAssetRegistry()
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/fallback.yaml": {Data: []byte("id: fallback-profile\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
}, "profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"fallback-profile"}, assets); err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
}
}

View File

@@ -116,7 +116,7 @@ func (h *recomputeTestHarness) options() Options {
for _, key := range []string{"test/extract/producer", "test/extract/unrelated", "test/extract/middle", "test/extract/dependent"} {
moduleKey := key
spec := pipeline.ModuleSpec{
Key: moduleKey, Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind,
Key: moduleKey, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind,
ReferenceSlots: []contracts.ReferenceSlot{{Name: "upstream", AcceptedMediaTypes: []string{"application/json"}, AcceptedArtifactKinds: []contracts.ArtifactKind{stateTestArtifactKind}}},
}
if err := pipeline.RegisterExtractor(opts.Registries.Extractors, spec, func() (contracts.Extractor[stateTestArtifact], error) {
@@ -125,7 +125,7 @@ func (h *recomputeTestHarness) options() Options {
panic(err)
}
}
if err := opts.Registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/recompute-output", Stage: pipeline.StageOutput, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
if err := opts.Registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/recompute-output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
return recomputeTestOutput{}, nil
}); err != nil {
panic(err)
@@ -194,7 +194,7 @@ func (recomputeTestOutput) Encode(_ context.Context, req contracts.OutputRequest
func newRecomputeTestRoots(t *testing.T) stateTestRoots {
t.Helper()
roots := newStateTestRoots(t)
config := fmt.Sprintf(`version: 3
config := fmt.Sprintf(`version: 4
output:
directory: %q
cache:

View File

@@ -217,7 +217,7 @@ func TestReferenceMaterializationSeparatesCLIAndConfigPathOrigins(t *testing.T)
workingDir := t.TempDir()
cfg := referenceContractConfig()
configPath := filepath.Join(configDir, "config.yml")
if err := os.WriteFile(configPath, []byte("version: 3\n"), 0o600); err != nil {
if err := os.WriteFile(configPath, []byte("version: 4\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(configDir, "required.txt"), []byte("config reference"), 0o600); err != nil {
@@ -364,21 +364,21 @@ func referenceContractCatalog(t *testing.T, includeBetaMerger, includeBetaNormal
t.Fatal(err)
}
}
register(registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "reference/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }))
register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) { return stateTestChunker{}, nil }))
register(registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "reference/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }))
register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) { return stateTestChunker{}, nil }))
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecA{}))
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecB{}))
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
if includeBetaMerger {
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
}
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil }))
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil }))
if includeBetaNormalizer {
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil }))
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil }))
}
register(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/output", Stage: pipeline.StageOutput, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) { return stateTestOutput{}, nil }))
register(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) { return stateTestOutput{}, nil }))
return catalogFromRegistries(registries)
}
@@ -418,9 +418,11 @@ func (referenceContractCodecB) Decode([]byte) (stateTestArtifact, error) {
func referenceContractLane(t *testing.T, resolved pipeline.ResolvedPipeline, id string) pipeline.ResolvedArtifactLane {
t.Helper()
for _, lane := range resolved.Steps[0].ArtifactLanes {
if lane.ID == id {
return lane
for _, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
if lane.ID == id {
return lane
}
}
}
t.Fatalf("lane %q not found", id)

View File

@@ -5,6 +5,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
@@ -22,13 +23,14 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
const usage = `Usage:
notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--output-dir path] [--chunk_cache auto|bypass|refresh] [--resume] [--recompute-step step-id] [--debug [--debug-dir path]] [--only lane-a,lane-b] [--session-id id] [--reference selector=path] [--without-reference selector]
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json]
`
@@ -44,9 +46,14 @@ type Options struct {
ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
promptKitAssets *frameworkllm.AssetRegistry
}
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
type LLMRuntimeOverrides struct {
ReasoningEffort *string
}
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
// Run executes the command-line interface and returns a process exit code.
func Run(args []string, stdout, stderr io.Writer) int {
@@ -115,12 +122,17 @@ func normalizeOptions(opts Options) (Options, error) {
}
opts.Registries = components.registries
opts.Catalog = catalogFromRegistries(components.registries)
if opts.LLMClientFactory == nil {
opts.LLMClientFactory = productionLLMClientFactoryWithAssets(components.assets)
}
opts.promptKitAssets = components.assets
}
if opts.LLMClientFactory == nil {
opts.LLMClientFactory = productionLLMClientFactory
if opts.promptKitAssets == nil {
assets, err := productionPromptAssets()
if err != nil {
return Options{}, err
}
opts.promptKitAssets = assets
}
opts.LLMClientFactory = productionLLMClientFactoryWithAssets(opts.promptKitAssets)
}
return opts, nil
}
@@ -132,16 +144,20 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
inputPath := fs.String("input", "", "source input file path")
onlyRaw := fs.String("only", "", "comma-separated artifact lanes")
outputDir := fs.String("output-dir", "", "output directory")
machineOutput := fs.Bool("json", false, "write the successful run result as JSON")
debug := fs.Bool("debug", false, "write a debug bundle")
debugDir := fs.String("debug-dir", "", "debug bundle directory")
llmProfile := fs.String("llm-profile", "", "LLM profile override")
reasoningEffort := singleValueFlag{name: "--reasoning-effort"}
clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort")
resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints")
recomputeStep := singleValueFlag{}
recomputeStep := singleValueFlag{name: "--recompute-step"}
chunkCache := chunkCacheFlag{}
sessionID := sessionIDFlag{}
referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{}
fs.Var(&sessionID, "session-id", "prompt session identifier")
fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
@@ -187,6 +203,22 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintln(stderr, "notarius: --session-id must not be empty")
return 2
}
if reasoningEffort.set && *clearReasoningEffort {
fmt.Fprintln(stderr, "notarius: --reasoning-effort cannot be combined with --clear-reasoning-effort")
return 2
}
if reasoningEffort.set && strings.TrimSpace(reasoningEffort.value) == "" {
fmt.Fprintln(stderr, "notarius: --reasoning-effort must not be empty")
return 2
}
runtimeOverrides := LLMRuntimeOverrides{}
if reasoningEffort.set {
value := strings.TrimSpace(reasoningEffort.value)
runtimeOverrides.ReasoningEffort = &value
} else if *clearReasoningEffort {
value := ""
runtimeOverrides.ReasoningEffort = &value
}
only, err := parseOnly(*onlyRaw)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
@@ -277,17 +309,18 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
debugRecorder = pipeline.SynchronizedDebugRecorder(debugRecorder)
}
invocation := debugbundle.Invocation{
Operation: "run",
PipelineID: pipelineID,
InputPath: strings.TrimSpace(*inputPath),
ConfigPath: loadedConfigPath,
ConfigSource: configSource(*configPath),
OnlyLanes: append([]string(nil), only...),
ChunkCacheOverride: chunkCache.explicitValue(),
Resume: *resume,
RecomputeStep: strings.TrimSpace(recomputeStep.value),
RunID: runID,
StartedAt: startedAt,
Operation: "run",
PipelineID: pipelineID,
InputPath: strings.TrimSpace(*inputPath),
ConfigPath: loadedConfigPath,
ConfigSource: configSource(*configPath),
OnlyLanes: append([]string(nil), only...),
ChunkCacheOverride: chunkCache.explicitValue(),
ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
Resume: *resume,
RecomputeStep: strings.TrimSpace(recomputeStep.value),
RunID: runID,
StartedAt: startedAt,
}
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
@@ -313,7 +346,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, profileIDs); err != nil {
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs, opts.promptKitAssets); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
workingDir, err := os.Getwd()
@@ -361,10 +394,17 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if len(profileIDs) == 1 {
factoryProfileID = profileIDs[0]
}
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID)
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, runtimeOverrides)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
var llmFingerprints []checkpoint.Fingerprint
if effective.Config.Cache.Checkpoints.Enabled {
llmFingerprints, err = llmCheckpointFingerprints(llmClient)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare LLM checkpoint identity: %w", err))
}
}
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
if err != nil {
@@ -378,7 +418,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), runtimeOverrides, *resume)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
@@ -415,6 +455,17 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err := writePartialSummary(summary, output); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err))
}
var encodedResult []byte
if *machineOutput {
result, err := newRunResult(effective.ResolvedPipeline, output, runOutputDir, debugPath)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
encodedResult, err = encodeRunResult(result)
if err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
}
if err := writeOutputFiles(runOutputDir, output.OutputFiles); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
@@ -422,9 +473,15 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return writePipelineCommandFailure(stderr, commandState, primaryErr, persistenceErr)
}
fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir)
if debugPath != "" {
fmt.Fprintf(stdout, "debug=%s\n", debugPath)
if *machineOutput {
if err := writeRunResult(stdout, encodedResult); err != nil {
return writePipelineCommandFailure(stderr, commandState, errors.New("write run result"), nil)
}
} else {
fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir)
if debugPath != "" {
fmt.Fprintf(stdout, "debug=%s\n", debugPath)
}
}
if len(output.Warnings) > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings))
@@ -462,11 +519,13 @@ func checkpointHandlersForRun(
opts Options,
resolved pipeline.ResolvedPipeline,
componentFingerprints []pipeline.CheckpointFingerprint,
llmFingerprints []checkpoint.Fingerprint,
rawInput []byte,
only []string,
llmProfiles []artifacts.LLMProfileManifest,
llmProfileOverride string,
sessionID string,
runtimeOverrides LLMRuntimeOverrides,
resume bool,
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
if !settings.Enabled {
@@ -476,13 +535,17 @@ func checkpointHandlersForRun(
return pipeline.NoopCheckpointRecorder(), pipeline.NoopCheckpointLoader(), nil
}
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: resolved,
InputKey: resolved.Input.Module,
RawInputDigest: rawInputDigest(rawInput),
SelectedLanes: only,
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID),
References: pipeline.ReferenceProvenance(resolved),
ProvenanceFingerprints: append(llmProfileFingerprints(llmProfiles), checkpointIdentityFingerprints(componentFingerprints)...),
Pipeline: resolved,
InputKey: resolved.Input.Module,
RawInputDigest: rawInputDigest(rawInput),
SelectedLanes: only,
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID, runtimeOverrides),
References: pipeline.ReferenceProvenance(resolved),
ProvenanceFingerprints: combineCheckpointFingerprints(
llmProfileFingerprints(llmProfiles),
llmFingerprints,
checkpointIdentityFingerprints(componentFingerprints),
),
})
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
@@ -508,6 +571,30 @@ func checkpointHandlersForRun(
return recorder, loader, nil
}
func llmCheckpointFingerprints(client contracts.StructuredLLMClient) ([]checkpoint.Fingerprint, error) {
provider, ok := client.(frameworkllm.CheckpointFingerprintProvider)
if !ok {
return nil, nil
}
values, err := provider.LLMCheckpointFingerprints()
if err != nil {
return nil, err
}
out := make([]checkpoint.Fingerprint, 0, len(values))
for _, value := range values {
out = append(out, checkpoint.Fingerprint{Name: value.Name, Value: value.Value})
}
return out, nil
}
func combineCheckpointFingerprints(sources ...[]checkpoint.Fingerprint) []checkpoint.Fingerprint {
var out []checkpoint.Fingerprint
for _, source := range sources {
out = append(out, source...)
}
return out
}
func recomputePolicy(resolved pipeline.ResolvedPipeline, requestedStep string) (pipeline.CheckpointExecutionPolicy, error) {
requestedStep = strings.TrimSpace(requestedStep)
if requestedStep == "" {
@@ -613,7 +700,7 @@ func rawInputDigest(data []byte) string {
return "sha256:" + hex.EncodeToString(sum[:])
}
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) []checkpoint.Fingerprint {
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string, runtimeOverrides LLMRuntimeOverrides) []checkpoint.Fingerprint {
var values []checkpoint.Fingerprint
if strings.TrimSpace(llmProfileOverride) != "" {
values = append(values, checkpoint.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
@@ -621,6 +708,13 @@ func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) []
if strings.TrimSpace(sessionID) != "" {
values = append(values, checkpoint.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
}
if runtimeOverrides.ReasoningEffort != nil {
value := strings.TrimSpace(*runtimeOverrides.ReasoningEffort)
if value == "" {
value = "<cleared>"
}
values = append(values, checkpoint.Fingerprint{Name: "reasoning_effort_override", Value: value})
}
return values
}
@@ -777,7 +871,7 @@ func reorderRunArgs(args []string) []string {
func runFlagTakesValue(arg string) bool {
switch arg {
case "--config", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
case "--config", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
return true
default:
return false
@@ -837,11 +931,11 @@ func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeli
func validateRunFlagValues(args []string) error {
for i, arg := range args {
if arg != "--session-id" {
if arg != "--session-id" && arg != "--reasoning-effort" {
continue
}
if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") {
return fmt.Errorf("flag needs an argument: --session-id")
return fmt.Errorf("flag needs an argument: %s", arg)
}
}
return nil
@@ -864,11 +958,22 @@ func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
seen[id] = struct{}{}
}
}
add(resolved.Chunk)
if resolved.InputExecutionClass == contracts.ExecutionClassLLMBacked {
add(resolved.Input)
}
if resolved.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
add(resolved.Chunk)
}
for _, lane := range resolved.AllArtifactLanes() {
add(lane.Extract)
add(lane.Merge)
add(lane.Normalize)
if lane.ExtractExecutionClass == contracts.ExecutionClassLLMBacked {
add(lane.Extract)
}
if lane.MergeExecutionClass == contracts.ExecutionClassLLMBacked {
add(lane.Merge)
}
if lane.NormalizeExecutionClass == contracts.ExecutionClassLLMBacked {
add(lane.Normalize)
}
}
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
@@ -877,6 +982,9 @@ func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
}
}
}
if resolved.OutputExecutionClass == contracts.ExecutionClassLLMBacked {
add(resolved.Output)
}
ids := make([]string, 0, len(seen))
for id := range seen {
ids = append(ids, id)
@@ -960,7 +1068,7 @@ func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) in
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline)); err != nil {
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline), opts.promptKitAssets); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
@@ -1125,6 +1233,7 @@ type sessionIDFlag struct {
}
type singleValueFlag struct {
name string
value string
set bool
}
@@ -1138,7 +1247,7 @@ func (flag *singleValueFlag) String() string {
func (flag *singleValueFlag) Set(value string) error {
if flag.set {
return fmt.Errorf("--recompute-step may be specified only once")
return fmt.Errorf("%s may be specified only once", flag.name)
}
flag.value = value
flag.set = true

View File

@@ -12,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -241,12 +242,14 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
t.Run("one effective profile reaches the factory and modules", func(t *testing.T) {
roots := newStateTestRoots(t)
profileDir := writeRunContractProfiles(t, "override-profile")
prependRunContractConfig(t, roots, fmt.Sprintf("scriptorium:\n profile_dir: %q\n", profileDir))
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
harness := newStateTestHarness()
var factoryProfiles []string
opts := harness.options()
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
var factoryOverrides []LLMRuntimeOverrides
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryProfiles = append(factoryProfiles, profileID)
factoryOverrides = append(factoryOverrides, overrides)
return nil, nil, nil
}
var stdout, stderr bytes.Buffer
@@ -257,6 +260,9 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
}
if len(factoryOverrides) != 1 || factoryOverrides[0].ReasoningEffort != nil {
t.Fatalf("factory overrides = %#v, want inherited reasoning", factoryOverrides)
}
harness.mu.Lock()
profiles := append([]string(nil), harness.moduleProfiles...)
harness.mu.Unlock()
@@ -270,16 +276,16 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
}
})
t.Run("validator profile remains distinct", func(t *testing.T) {
t.Run("runtime override applies to validators", func(t *testing.T) {
roots := newStateTestRoots(t)
profileDir := writeRunContractProfiles(t, "override-profile", "validator-profile")
prependRunContractConfig(t, roots, fmt.Sprintf("scriptorium:\n profile_dir: %q\n", profileDir))
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
harness := newStateTestHarness()
var validatorProfiles []string
opts := harness.options()
registerRunContractValidator(t, &opts, &validatorProfiles)
factoryProfiles := []string{}
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, _ LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryProfiles = append(factoryProfiles, profileID)
return nil, nil, nil
}
@@ -288,21 +294,21 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if len(factoryProfiles) != 1 || factoryProfiles[0] != "" {
t.Fatalf("factory profiles = %#v, want one call without a unique profile", factoryProfiles)
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
}
if len(validatorProfiles) != 1 || validatorProfiles[0] != "validator-profile" {
t.Fatalf("validator profiles = %#v, want configured validator profile", validatorProfiles)
if len(validatorProfiles) != 1 || validatorProfiles[0] != "override-profile" {
t.Fatalf("validator profiles = %#v, want runtime override", validatorProfiles)
}
})
t.Run("unknown profile is rejected without factory access", func(t *testing.T) {
roots := newStateTestRoots(t)
profileDir := writeRunContractProfiles(t, "override-profile")
prependRunContractConfig(t, roots, fmt.Sprintf("scriptorium:\n profile_dir: %q\n", profileDir))
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
factoryCalls := 0
opts := newStateTestHarness().options()
opts.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryCalls++
return nil, nil, nil
}
@@ -312,28 +318,175 @@ func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls)
}
})
t.Run("pipeline default is rejected before factory access", func(t *testing.T) {
roots := newStateTestRoots(t)
profileDir := writeRunContractProfiles(t, "configured-profile")
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
replaceStateTestConfigLine(t, roots.config, " sample:\n", " sample:\n llm_profile: missing-profile\n")
factoryCalls := 0
opts := newStateTestHarness().options()
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryCalls++
return nil, nil, nil
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls)
}
})
}
func TestRunReasoningEffortOverrideReachesFactory(t *testing.T) {
tests := []struct {
name string
flags []string
wantValue string
wantSet bool
}{
{name: "inherit"},
{name: "replace", flags: []string{"--reasoning-effort", " focused "}, wantValue: "focused", wantSet: true},
{name: "clear", flags: []string{"--clear-reasoning-effort"}, wantSet: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
var got []LLMRuntimeOverrides
opts.LLMClientFactory = func(_ context.Context, _ config.Config, _ string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
got = append(got, overrides)
return nil, nil, nil
}
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.flags...)
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if len(got) != 1 {
t.Fatalf("factory overrides = %#v, want one call", got)
}
if !tt.wantSet {
if got[0].ReasoningEffort != nil {
t.Fatalf("reasoning effort = %q, want inherit", *got[0].ReasoningEffort)
}
return
}
if got[0].ReasoningEffort == nil || *got[0].ReasoningEffort != tt.wantValue {
t.Fatalf("reasoning effort = %#v, want %q", got[0].ReasoningEffort, tt.wantValue)
}
})
}
}
func TestRunReasoningEffortOverrideRejectsInvalidSyntax(t *testing.T) {
tests := []struct {
name string
flags []string
wantError string
}{
{
name: "mutually exclusive controls",
flags: []string{"--reasoning-effort", "focused", "--clear-reasoning-effort"},
wantError: "cannot be combined",
},
{
name: "empty replacement",
flags: []string{"--reasoning-effort", " "},
wantError: "must not be empty",
},
{
name: "duplicate replacement",
flags: []string{"--reasoning-effort", "low", "--reasoning-effort", "high"},
wantError: "may be specified only once",
},
{
name: "missing replacement",
flags: []string{"--reasoning-effort"},
wantError: "flag needs an argument",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, tt.flags...)
var stdout, stderr bytes.Buffer
code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertNoRunState(t, roots)
})
}
}
func TestReasoningEffortOverrideSeparatesCheckpointIdentities(t *testing.T) {
replacement := " focused "
cleared := ""
states := []struct {
name string
overrides LLMRuntimeOverrides
wantValue string
wantSet bool
}{
{name: "inherit"},
{name: "replace", overrides: LLMRuntimeOverrides{ReasoningEffort: &replacement}, wantValue: "focused", wantSet: true},
{name: "clear", overrides: LLMRuntimeOverrides{ReasoningEffort: &cleared}, wantValue: "<cleared>", wantSet: true},
}
digests := make(map[string]string, len(states))
for _, state := range states {
fingerprints := runtimeOverrideFingerprints("", "", state.overrides)
var value string
var found bool
for _, fingerprint := range fingerprints {
if fingerprint.Name == "reasoning_effort_override" {
value, found = fingerprint.Value, true
}
}
if found != state.wantSet || (found && value != state.wantValue) {
t.Fatalf("%s fingerprint found=%t value=%q, want found=%t value=%q", state.name, found, value, state.wantSet, state.wantValue)
}
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: pipeline.ResolvedPipeline{ID: "sample", Digest: "sha256:pipeline", Input: pipeline.Binding("test/input")},
RawInputDigest: "sha256:input",
RuntimeOverrides: fingerprints,
})
if err != nil {
t.Fatal(err)
}
digests[state.name] = identity.Digest
}
if digests["inherit"] == digests["replace"] || digests["inherit"] == digests["clear"] || digests["replace"] == digests["clear"] {
t.Fatalf("checkpoint identity digests are not distinct: %#v", digests)
}
}
func TestEffectiveLLMProfileIDsAreSortedDeduplicatedAndLLMOnly(t *testing.T) {
resolved := pipeline.ResolvedPipeline{
Input: pipeline.ModuleBinding{LLMProfile: "input-profile"},
Chunk: pipeline.ModuleBinding{LLMProfile: " zeta "},
Input: pipeline.ModuleBinding{LLMProfile: "input-profile"},
InputExecutionClass: contracts.ExecutionClassLLMBacked,
Chunk: pipeline.ModuleBinding{LLMProfile: " zeta "},
ChunkExecutionClass: contracts.ExecutionClassLLMBacked,
Steps: []pipeline.ResolvedPipelineStep{{
ID: "default",
ArtifactLanes: []pipeline.ResolvedArtifactLane{{
Extract: pipeline.ModuleBinding{LLMProfile: "alpha"},
Merge: pipeline.ModuleBinding{LLMProfile: "zeta"},
Normalize: pipeline.ModuleBinding{LLMProfile: " gamma "},
Extract: pipeline.ModuleBinding{LLMProfile: "alpha"},
ExtractExecutionClass: contracts.ExecutionClassLLMBacked,
Merge: pipeline.ModuleBinding{LLMProfile: "deterministic-merge"},
MergeExecutionClass: contracts.ExecutionClassDeterministic,
Normalize: pipeline.ModuleBinding{LLMProfile: " gamma "},
NormalizeExecutionClass: contracts.ExecutionClassLLMBacked,
}},
}},
ValidatorChains: []pipeline.ResolvedValidatorChain{{Validators: []pipeline.ResolvedValidator{
{Binding: pipeline.ModuleBinding{LLMProfile: "deterministic-profile"}, ExecutionClass: contracts.ExecutionClassDeterministic},
{Binding: pipeline.ModuleBinding{LLMProfile: "beta"}, ExecutionClass: contracts.ExecutionClassLLMBacked},
}}},
Output: pipeline.ModuleBinding{LLMProfile: "output-profile"},
Output: pipeline.ModuleBinding{LLMProfile: "output-profile"},
OutputExecutionClass: contracts.ExecutionClassLLMBacked,
}
got := effectiveLLMProfileIDs(resolved)
want := []string{"alpha", "beta", "gamma", "zeta"}
want := []string{"alpha", "beta", "gamma", "input-profile", "output-profile", "zeta"}
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("effective profiles = %#v, want %#v", got, want)
}
@@ -376,7 +529,7 @@ func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) {
t.Run("LLM factory", func(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
opts.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return nil, nil, errors.New("injected LLM factory failure")
}
var stdout, stderr bytes.Buffer
@@ -397,7 +550,7 @@ func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) {
t.Fatal(err)
}
opts := newStateTestHarness().options()
if err := pipeline.RegisterExtractorBuilder(opts.Registries.Extractors, pipeline.ModuleSpec{Key: "test/failing-extract", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Extractor[stateTestArtifact], error) {
if err := pipeline.RegisterExtractorBuilder(opts.Registries.Extractors, pipeline.ModuleSpec{Key: "test/failing-extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Extractor[stateTestArtifact], error) {
return nil, errors.New("injected extractor construction failure")
}); err != nil {
t.Fatal(err)

111
internal/cli/run_result.go Normal file
View File

@@ -0,0 +1,111 @@
package cli
import (
"encoding/json"
"fmt"
"io"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const runResultSchemaVersion = "notarius.run-result.v1"
type runResult struct {
SchemaVersion string `json:"schema_version"`
RunID string `json:"run_id"`
PipelineID string `json:"pipeline_id"`
OutputDirectory string `json:"output_directory"`
IndexFile string `json:"index_file,omitempty"`
NormalizedOutputCount int `json:"normalized_output_count"`
RejectedOutputCount int `json:"rejected_output_count"`
WarningCount int `json:"warning_count"`
ValidationStatus string `json:"validation_status"`
DebugDirectory string `json:"debug_directory,omitempty"`
}
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
if strings.TrimSpace(output.Manifest.RunID) == "" {
return runResult{}, fmt.Errorf("run result requires a run ID")
}
if strings.TrimSpace(resolved.ID) == "" {
return runResult{}, fmt.Errorf("run result requires a resolved pipeline ID")
}
if strings.TrimSpace(output.Manifest.PipelineID) == "" {
return runResult{}, fmt.Errorf("run result requires a manifest pipeline ID")
}
if output.Manifest.PipelineID != resolved.ID {
return runResult{}, fmt.Errorf("run result pipeline ID does not match resolved pipeline")
}
if strings.TrimSpace(output.Manifest.ValidationStatus) == "" {
return runResult{}, fmt.Errorf("run result requires a validation status")
}
if strings.TrimSpace(outputDirectory) == "" {
return runResult{}, fmt.Errorf("run result requires an output directory")
}
absOutputDirectory, err := filepath.Abs(outputDirectory)
if err != nil {
return runResult{}, fmt.Errorf("make output directory absolute: %w", err)
}
result := runResult{
SchemaVersion: runResultSchemaVersion,
RunID: output.Manifest.RunID,
PipelineID: output.Manifest.PipelineID,
OutputDirectory: absOutputDirectory,
NormalizedOutputCount: len(output.NormalizeOutputs),
RejectedOutputCount: len(output.Rejected),
WarningCount: len(output.Warnings),
ValidationStatus: output.Manifest.ValidationStatus,
}
if strings.TrimSpace(debugDirectory) != "" {
absDebugDirectory, err := filepath.Abs(debugDirectory)
if err != nil {
return runResult{}, fmt.Errorf("make debug directory absolute: %w", err)
}
result.DebugDirectory = absDebugDirectory
}
if resolved.Output.Module == pipeline.DefaultOutputModule {
indexCount := 0
for _, file := range output.OutputFiles {
if file.Name == "index.json" {
indexCount++
}
}
if indexCount != 1 {
return runResult{}, fmt.Errorf("production JSON output must contain exactly one index.json file")
}
result.IndexFile = "index.json"
}
return result, nil
}
func encodeRunResult(result runResult) ([]byte, error) {
encoded, err := json.Marshal(result)
if err != nil {
return nil, fmt.Errorf("encode run result: %w", err)
}
return append(encoded, '\n'), nil
}
func writeRunResult(writer io.Writer, content []byte) error {
for len(content) > 0 {
written, err := writer.Write(content)
if written < 0 || written > len(content) {
return io.ErrShortWrite
}
content = content[written:]
if err != nil {
return err
}
if written == 0 {
return io.ErrShortWrite
}
}
return nil
}

View File

@@ -0,0 +1,199 @@
package cli
import (
"bytes"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
)
func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
outputRoot := filepath.Join(t.TempDir(), "output")
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session",
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json",
}, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{}))
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["schema_version"]; got != "notarius.run-result.v1" {
t.Fatalf("schema_version = %q", got)
}
if got := receipt["run_id"]; got != productionRunID {
t.Fatalf("run_id = %q", got)
}
if got := receipt["pipeline_id"]; got != "dnd-session" {
t.Fatalf("pipeline_id = %q", got)
}
if got := receipt["index_file"]; got != "index.json" {
t.Fatalf("index_file = %q", got)
}
if got := receipt["normalized_output_count"]; got != float64(1) {
t.Fatalf("normalized_output_count = %v", got)
}
if got := receipt["rejected_output_count"]; got != float64(0) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := receipt["warning_count"]; got != float64(0) {
t.Fatalf("warning_count = %v", got)
}
if got := receipt["validation_status"]; got != "approved" {
t.Fatalf("validation_status = %q", got)
}
outputDirectory, ok := receipt["output_directory"].(string)
if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) {
t.Fatalf("output_directory = %q", receipt["output_directory"])
}
indexFile := receipt["index_file"].(string)
assertFile(t, filepath.Join(outputDirectory, indexFile))
}
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json",
}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["warning_count"]; got != float64(1) {
t.Fatalf("warning_count = %v", got)
}
debugDirectory, ok := receipt["debug_directory"].(string)
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
t.Fatalf("debug_directory = %q", receipt["debug_directory"])
}
if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") {
t.Fatalf("machine stdout contains human reporting: %q", stdout.String())
}
}
func TestRunResultReportsSuccessfulRejection(t *testing.T) {
roots := newStateTestRoots(t)
configBytes, err := os.ReadFile(roots.config)
if err != nil {
t.Fatal(err)
}
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n"))
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
t.Fatal(err)
}
harness := newStateTestHarness()
opts := harness.options()
if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil {
t.Fatal(err)
}
opts.Catalog = catalogFromRegistries(opts.Registries)
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--json",
}, &stdout, &stderr, opts)
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
receipt := decodeRunResultDocument(t, stdout.String())
if got := receipt["normalized_output_count"]; got != float64(0) {
t.Fatalf("normalized_output_count = %v", got)
}
if got := receipt["rejected_output_count"]; got != float64(1) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := receipt["validation_status"]; got != "rejected" {
t.Fatalf("validation_status = %q", got)
}
}
func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) {
t.Run("syntax", func(t *testing.T) {
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options())
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
t.Run("runtime", func(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.extractErr = errors.New("injected extraction failure")
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--json",
}, &stdout, &stderr, harness.options())
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) {
roots := newStateTestRoots(t)
writerErr := errors.New("result writer sentinel")
stdout := &resultDeliveryWriter{err: writerErr}
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--input", roots.input,
"--chunk_cache", "bypass", "--debug", "--json",
}, stdout, &stderr, newStateTestHarness().options())
if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) {
t.Fatalf("code=%d stderr=%q", code, stderr.String())
}
if stdout.accepted.Len() != 0 {
t.Fatalf("accepted stdout = %q", stdout.accepted.String())
}
assertStateTestOutput(t, roots.output)
debugBundle := onlyChildDir(t, roots.debug)
report := readStateTestRunReport(t, debugBundle)
if !report.Succeeded {
t.Fatalf("debug report = %#v, want successful persisted run", report)
}
if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) {
t.Fatalf("debug bundle contains result writer error")
}
}
func decodeRunResultDocument(t *testing.T, stdout string) map[string]any {
t.Helper()
if strings.Count(stdout, "\n") != 1 {
t.Fatalf("stdout = %q, want one JSON document", stdout)
}
var receipt map[string]any
if err := json.Unmarshal([]byte(stdout), &receipt); err != nil {
t.Fatalf("decode run result: %v; stdout=%q", err, stdout)
}
return receipt
}
type resultDeliveryWriter struct {
err error
accepted bytes.Buffer
}
func (w *resultDeliveryWriter) Write(content []byte) (int, error) {
if w.err != nil {
return 0, w.err
}
return w.accepted.Write(content)
}

View File

@@ -0,0 +1,200 @@
package cli
import (
"bytes"
"encoding/json"
"errors"
"io"
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), testRunOutput(), "relative-output", "relative-debug")
if err != nil {
t.Fatal(err)
}
encoded, err := encodeRunResult(result)
if err != nil {
t.Fatal(err)
}
if encoded[len(encoded)-1] != '\n' || bytes.Count(encoded, []byte{'\n'}) != 1 {
t.Fatalf("encoded result is not one newline-terminated object: %q", encoded)
}
var decoded map[string]any
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
if got := decoded["schema_version"]; got != runResultSchemaVersion {
t.Fatalf("schema_version = %q", got)
}
if got := decoded["run_id"]; got != "run-123" {
t.Fatalf("run_id = %q", got)
}
if got := decoded["pipeline_id"]; got != "sample" {
t.Fatalf("pipeline_id = %q", got)
}
if got := decoded["validation_status"]; got != "rejected" {
t.Fatalf("validation_status = %q", got)
}
if got := decoded["index_file"]; got != "index.json" {
t.Fatalf("index_file = %q", got)
}
if got := decoded["normalized_output_count"]; got != float64(2) {
t.Fatalf("normalized_output_count = %v", got)
}
if got := decoded["rejected_output_count"]; got != float64(1) {
t.Fatalf("rejected_output_count = %v", got)
}
if got := decoded["warning_count"]; got != float64(1) {
t.Fatalf("warning_count = %v", got)
}
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
t.Fatalf("output_directory = %q", got)
}
if got := decoded["debug_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-debug") {
t.Fatalf("debug_directory = %q", got)
}
}
func TestRunResultRejectsInvalidRequiredValues(t *testing.T) {
tests := []struct {
name string
resolved pipeline.ResolvedPipeline
output pipeline.RunOutput
directory string
}{
{name: "blank run ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.RunID = " " }), directory: "output"},
{name: "blank resolved pipeline ID", resolved: pipeline.ResolvedPipeline{Output: pipeline.ModuleBinding{Module: pipeline.DefaultOutputModule}}, output: testRunOutput(), directory: "output"},
{name: "blank manifest pipeline ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "" }), directory: "output"},
{name: "mismatched pipeline IDs", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "other" }), directory: "output"},
{name: "blank validation status", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.ValidationStatus = " " }), directory: "output"},
{name: "blank output directory", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutput(), directory: " "},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if _, err := newRunResult(tt.resolved, tt.output, tt.directory, ""); err == nil {
t.Fatal("newRunResult() succeeded")
}
})
}
}
func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
result, err := newRunResult(testResolvedPipeline("test/output"), testRunOutputWithout(func(output *pipeline.RunOutput) {
output.OutputFiles = nil
}), "output", "")
if err != nil {
t.Fatal(err)
}
if result.IndexFile != "" {
t.Fatalf("index_file = %q", result.IndexFile)
}
encoded, err := encodeRunResult(result)
if err != nil {
t.Fatal(err)
}
var decoded map[string]any
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
if _, ok := decoded["index_file"]; ok {
t.Fatalf("encoded non-JSON result contains index_file: %s", encoded)
}
if _, ok := decoded["debug_directory"]; ok {
t.Fatalf("encoded result without debug capture contains debug_directory: %s", encoded)
}
}
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
tests := []struct {
name string
files []contracts.OutputFile
}{
{name: "missing", files: nil},
{name: "duplicate", files: []contracts.OutputFile{{Name: "index.json"}, {Name: "index.json"}}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
output := testRunOutput()
output.OutputFiles = tt.files
if _, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", ""); err == nil {
t.Fatal("newRunResult() succeeded")
}
})
}
}
func TestWriteRunResultCompletesAndReportsWriterFailure(t *testing.T) {
content := []byte("result\n")
var target bytes.Buffer
if err := writeRunResult(partialResultWriter{writer: &target, limit: 2}, content); err != nil {
t.Fatal(err)
}
if got := target.String(); got != string(content) {
t.Fatalf("written result = %q", got)
}
writerErr := errors.New("result writer failed")
if err := writeRunResult(failingResultWriter{err: writerErr}, content); !errors.Is(err, writerErr) {
t.Fatalf("writeRunResult() error = %v", err)
}
if err := writeRunResult(zeroResultWriter{}, content); !errors.Is(err, io.ErrShortWrite) {
t.Fatalf("zero-progress error = %v", err)
}
}
func testResolvedPipeline(outputModule string) pipeline.ResolvedPipeline {
return pipeline.ResolvedPipeline{ID: "sample", Output: pipeline.ModuleBinding{Module: outputModule}}
}
func testRunOutput() pipeline.RunOutput {
return pipeline.RunOutput{
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
Rejected: []contracts.RejectedOutput{{}},
Warnings: []contracts.Warning{{}},
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
}
}
func testRunOutputWithout(change func(*pipeline.RunOutput)) pipeline.RunOutput {
output := testRunOutput()
change(&output)
return output
}
func mustWorkingDirectory(t *testing.T) string {
t.Helper()
workingDirectory, err := filepath.Abs(".")
if err != nil {
t.Fatal(err)
}
return workingDirectory
}
type partialResultWriter struct {
writer io.Writer
limit int
}
func (w partialResultWriter) Write(content []byte) (int, error) {
if len(content) > w.limit {
content = content[:w.limit]
}
return w.writer.Write(content)
}
type failingResultWriter struct{ err error }
func (w failingResultWriter) Write([]byte) (int, error) { return 0, w.err }
type zeroResultWriter struct{}
func (zeroResultWriter) Write([]byte) (int, error) { return 0, nil }

View File

@@ -1,68 +0,0 @@
package cli
import (
"context"
"errors"
"fmt"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/scriptorium"
)
const profileCheckPromptID = "notarius.profile.check"
var profileCheckPromptFS = fstest.MapFS{
"prompts/profile-check.yaml": &fstest.MapFile{Data: []byte(`id: notarius.profile.check
version: "1.0.0"
default_profile: mistral-small-3
inputs:
- name: transcript
required: true
messages:
- role: user
content: "{{input \"transcript\"}}"
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
}
func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
if len(profileIDs) == 0 {
return nil
}
engine, err := newProfileValidationEngine(cfg)
if err != nil {
return fmt.Errorf("load Scriptorium profiles: %w", err)
}
for _, profileID := range profileIDs {
if _, err := engine.Prepare(ctx, scriptorium.RunRequest{
PromptID: profileCheckPromptID,
ProfileID: profileID,
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("profile check"),
},
}); err != nil {
if errors.Is(err, scriptorium.ErrProfileNotFound) {
return fmt.Errorf("Scriptorium profile %q is not configured", profileID)
}
return fmt.Errorf("validate Scriptorium profile %q: %w", profileID, err)
}
}
return nil
}
func newProfileValidationEngine(cfg config.Config) (*scriptorium.Engine, error) {
opts := []scriptorium.Option{
scriptorium.WithPromptFS(profileCheckPromptFS, "prompts"),
}
if cfg.Scriptorium.ProfileFile != "" {
opts = append(opts, scriptorium.WithProfileFile(cfg.Scriptorium.ProfileFile))
}
return scriptorium.NewEngine(scriptorium.Config{
PromptDir: "unused",
ProfileDir: cfg.Scriptorium.ProfileDir,
}, opts...)
}

View File

@@ -24,7 +24,7 @@ import (
func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
configPath := writeProductionSpellCatalogContractConfig(t)
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatalf("resolve production configuration: %v", err)
@@ -108,8 +108,8 @@ func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testin
}
func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) {
base := string(readRepositoryFile(t, "examples", "dnd-spells-production.config.yml"))
changed := strings.Replace(base, "./dnd-spells-catalog.json", "./alternate-spell-catalog.json", 1)
base := productionSpellCatalogContractConfig(t)
changed := strings.Replace(base, repositoryPath("examples", "dnd-spell-catalog.json"), filepath.Join(t.TempDir(), "alternate-spell-catalog.json"), 1)
if changed == base {
t.Fatal("production configuration did not contain the maintained catalog binding")
}
@@ -138,7 +138,7 @@ func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing
func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
configPath := writeProductionSpellCatalogContractConfig(t)
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatal(err)
@@ -200,7 +200,7 @@ func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenc
func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
configPath := writeProductionSpellCatalogContractConfig(t)
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
if err != nil {
t.Fatal(err)
@@ -214,8 +214,9 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
t.Fatal(err)
}
fingerprints := prepared.CheckpointFingerprints()
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}}
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", false)
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
if err != nil {
t.Fatal(err)
}
@@ -241,7 +242,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
t.Fatal(err)
}
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true)
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
@@ -253,7 +254,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
}
changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true)
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
@@ -263,12 +264,37 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision)
}
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v3")
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName())
_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
if _, decision := mappingLoader.Source(materialized.Input.Module); decision.Reused {
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
}
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}}
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
t.Fatalf("changed PromptKit profile source decision = %#v, want cold miss", decision)
}
if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
t.Fatalf("changed PromptKit profile normalize decision = %#v, want cold miss", decision)
}
}
func normalizeSpellCatalogFingerprintName() string {
return "normalize:spells:" + spellnormalize.Key + ":effective_catalog"
}
func extractSpellMappingFingerprintName() string {
return "extract:spells:" + spells.Key + ":mapping_policy"
}
func replaceCheckpointFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, value string) []pipeline.CheckpointFingerprint {
t.Helper()
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
@@ -315,7 +341,7 @@ func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t
var stdout, stderr strings.Builder
code := RunWithOptions([]string{
"run", "dnd-session",
"--config", repositoryPath("examples", "dnd-spells-production.config.yml"),
"--config", writeProductionSpellCatalogContractConfig(t),
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
}, &stdout, &stderr, options)
@@ -360,12 +386,12 @@ func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t
if len(catalogProvenances) != 2 {
t.Fatalf("manifest references = %#v, want independently materialized extract and normalize catalog provenance", manifest.References)
}
overlayBytes := readRepositoryFile(t, "examples", "dnd-spells-catalog.json")
overlayBytes := readRepositoryFile(t, "examples", "dnd-spell-catalog.json")
for _, catalogProvenance := range catalogProvenances {
if catalogProvenance.Stage != "extract" && catalogProvenance.Stage != "normalize" {
t.Fatalf("catalog provenance = %#v, want extract or normalize scope", catalogProvenance)
}
if catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spells-catalog.json") {
if catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spell-catalog.json") {
t.Fatalf("catalog provenance = %#v, want raw overlay provenance in both scopes", catalogProvenance)
}
}

View File

@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"sync"
"testing"
@@ -50,7 +51,7 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
components := productionTestComponents(t)
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
configPath := writeProductionSpellCatalogContractConfig(t)
cfg := loadMaintainedExample(t, configPath)
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
if err != nil {
@@ -58,7 +59,7 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
}
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
ConfigPath: configPath,
WorkingDir: repositoryPath("examples"),
WorkingDir: filepath.Dir(configPath),
})
if err != nil {
t.Fatalf("materialize production references: %v", err)

View File

@@ -635,7 +635,7 @@ func newStateTestRoots(t *testing.T) stateTestRoots {
t.Fatal(err)
}
roots.config = filepath.Join(base, "config.yml")
config := fmt.Sprintf("version: 3\noutput:\n directory: %q\ncache:\n chunk_plans:\n directory: %q\n mode: auto\n checkpoints:\n enabled: true\n directory: %q\ndebug:\n directory: %q\npipelines:\n sample:\n input: test/input\n chunk: test/chunk\n artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n output: test/output\n", roots.output, roots.plans, roots.checkpoints, roots.debug)
config := fmt.Sprintf("version: 4\noutput:\n directory: %q\ncache:\n chunk_plans:\n directory: %q\n mode: auto\n checkpoints:\n enabled: true\n directory: %q\ndebug:\n directory: %q\npipelines:\n sample:\n input: test/input\n chunk: test/chunk\n artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n output: test/output\n", roots.output, roots.plans, roots.checkpoints, roots.debug)
if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil {
t.Fatal(err)
}
@@ -823,22 +823,22 @@ func (h *stateTestHarness) options() Options {
if err := pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, stateTestCodec{}); err != nil {
panic(err)
}
if err := registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }); err != nil {
if err := registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }); err != nil {
panic(err)
}
if err := registries.Chunkers.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/chunk", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "cache-reference"}}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Chunker, error) { return stateTestChunker{h}, nil }); err != nil {
if err := registries.Chunkers.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "cache-reference"}}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Chunker, error) { return stateTestChunker{h}, nil }); err != nil {
panic(err)
}
if err := pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "test/extract", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{h}, nil }); err != nil {
if err := pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "test/extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{h}, nil }); err != nil {
panic(err)
}
if err := pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "test/merge", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{harness: h}, nil }); err != nil {
if err := pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "test/merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{harness: h}, nil }); err != nil {
panic(err)
}
if err := pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "test/normalize", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{harness: h}, nil }); err != nil {
if err := pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "test/normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{harness: h}, nil }); err != nil {
panic(err)
}
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
return stateTestOutput{harness: h, includeWarnings: h.includeWarnings}, nil
}); err != nil {
panic(err)
@@ -848,7 +848,7 @@ func (h *stateTestHarness) options() Options {
defer h.mu.Unlock()
h.runIDCalls++
return fmt.Sprintf("run-%d-%032x", startedAt.UnixNano(), h.runIDCalls), nil
}, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
}, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return nil, nil, nil
}}
}

View File

@@ -1,6 +1,7 @@
package artifacts
import (
"strings"
"time"
)
@@ -26,9 +27,32 @@ type ValidatorManifest struct {
}
type LLMProfileManifest struct {
ID string `json:"id"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ID string `json:"id"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
BackendID string `json:"backend_id,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
}
// Normalized returns the canonical representation used for manifest identity
// and publication.
func (profile LLMProfileManifest) Normalized() LLMProfileManifest {
profile.ID = strings.TrimSpace(profile.ID)
profile.Provider = strings.TrimSpace(profile.Provider)
profile.Model = strings.TrimSpace(profile.Model)
profile.BackendID = strings.TrimSpace(profile.BackendID)
profile.ReasoningEffort = strings.TrimSpace(profile.ReasoningEffort)
return profile
}
// IdentityKey returns an opaque, deterministic key for the effective profile.
func (profile LLMProfileManifest) IdentityKey() string {
profile = profile.Normalized()
return profile.ID + "\x00" +
profile.Provider + "\x00" +
profile.Model + "\x00" +
profile.BackendID + "\x00" +
profile.ReasoningEffort
}
type ReferenceProvenance struct {

View File

@@ -53,7 +53,13 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
PipelineID: "pipeline-1",
PipelineDigest: "sha256:abc123",
LLMProfiles: []LLMProfileManifest{
{ID: "default", Provider: "scriptorium", Model: "model-a"},
{
ID: "default",
Provider: "promptkit",
Model: "model-a",
BackendID: "openrouter",
ReasoningEffort: "high",
},
},
ArtifactLanes: []ArtifactLaneManifest{
{
@@ -101,7 +107,13 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
if !ok {
t.Fatalf("llm_profiles[0] = %#v, want object", profiles[0])
}
assertHasKeys(t, profile, "id", "provider", "model")
assertHasKeys(t, profile, "id", "provider", "model", "backend_id", "reasoning_effort")
if profile["provider"] != "promptkit" {
t.Fatalf("llm_profiles[0].provider = %#v, want promptkit", profile["provider"])
}
if profile["backend_id"] != "openrouter" || profile["reasoning_effort"] != "high" {
t.Fatalf("llm_profiles[0] = %#v, want backend and reasoning provenance", profile)
}
lanes, ok := got["artifact_lanes"].([]any)
if !ok {

View File

@@ -4,10 +4,10 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const SupportedFileConfigVersion = 3
const SupportedFileConfigVersion = 4
type Config struct {
Scriptorium ScriptoriumConfig `json:"scriptorium,omitempty"`
PromptKit PromptKitConfig `json:"promptkit,omitempty"`
Pipelines map[string]pipeline.PipelineProfile `json:"pipelines"`
Concurrency ConcurrencyConfig `json:"concurrency"`
Output OutputConfig `json:"output"`
@@ -15,9 +15,15 @@ type Config struct {
Debug DebugConfig `json:"debug"`
}
type ScriptoriumConfig struct {
ProfileDir string `json:"profile_dir,omitempty"`
ProfileFile string `json:"profile_file,omitempty"`
type PromptKitConfig struct {
ProfileDir string `json:"profile_dir,omitempty"`
ProfileFile string `json:"profile_file,omitempty"`
LocalBackend *PromptKitLocalBackendConfig `json:"local_backend,omitempty"`
}
type PromptKitLocalBackendConfig struct {
Endpoint string `json:"endpoint"`
ConcurrencyLimit int `json:"concurrency_limit"`
}
type ConcurrencyConfig struct {
@@ -66,6 +72,10 @@ func Default() Config {
func cloneConfig(in Config) Config {
out := in
if in.PromptKit.LocalBackend != nil {
localBackend := *in.PromptKit.LocalBackend
out.PromptKit.LocalBackend = &localBackend
}
out.Concurrency.StageWorkers = cloneIntMap(in.Concurrency.StageWorkers)
out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines))
for key, profile := range in.Pipelines {

View File

@@ -42,12 +42,10 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
}
profile = clonePipelineProfile(profile)
profile.ID = pipelineID
if override := strings.TrimSpace(input.LLMProfileOverride); override != "" {
applyLLMProfileOverride(&profile, override)
}
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{
Only: input.Only,
LLMProfileOverride: input.LLMProfileOverride,
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
}, input.Catalog)
@@ -65,22 +63,6 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
}, nil
}
func applyLLMProfileOverride(profile *pipeline.PipelineProfile, profileID string) {
profile.Chunk.LLMProfile = profileID
apply := func(artifacts map[string]pipeline.ArtifactLaneProfile) {
for laneID, lane := range artifacts {
lane.Extract.LLMProfile = profileID
lane.Merge.LLMProfile = profileID
lane.Normalize.LLMProfile = profileID
artifacts[laneID] = lane
}
}
apply(profile.Artifacts)
for index := range profile.Steps {
apply(profile.Steps[index].Artifacts)
}
}
func lookupPipelineProfile(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
pipelineID = strings.TrimSpace(pipelineID)
for rawID, profile := range profiles {

View File

@@ -84,6 +84,56 @@ func TestEffectiveConfigMaterializesDefaultBindingsThroughCatalog(t *testing.T)
}
}
func TestEffectiveConfigPreservesPromptKitProfileSource(t *testing.T) {
tests := []struct {
name string
profileSource PromptKitConfig
}{
{name: "profile directory", profileSource: PromptKitConfig{ProfileDir: "./profiles"}},
{name: "profile file", profileSource: PromptKitConfig{ProfileFile: "./profiles.yml"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := configForEffectiveTests(t, effectiveProfile())
cfg.PromptKit = tt.profileSource
effective, err := cfg.Resolve(ResolveInput{PipelineID: "main", Catalog: effectiveCatalog(t)})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
if effective.Config.PromptKit != cfg.PromptKit {
t.Fatalf("effective PromptKit config = %#v, want %#v", effective.Config.PromptKit, cfg.PromptKit)
}
})
}
}
func TestEffectiveConfigOwnsPromptKitLocalBackend(t *testing.T) {
cfg := configForEffectiveTests(t, effectiveProfile())
cfg.PromptKit.LocalBackend = &PromptKitLocalBackendConfig{
Endpoint: "http://localhost:8000/v1",
ConcurrencyLimit: 2,
}
effective, err := cfg.Resolve(ResolveInput{PipelineID: "main", Catalog: effectiveCatalog(t)})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
if effective.Config.PromptKit.LocalBackend == nil {
t.Fatal("effective local backend = nil")
}
if effective.Config.PromptKit.LocalBackend == cfg.PromptKit.LocalBackend {
t.Fatal("effective local backend aliases input config")
}
cfg.PromptKit.LocalBackend.Endpoint = "http://changed-input.example/v1"
if effective.Config.PromptKit.LocalBackend.Endpoint != "http://localhost:8000/v1" {
t.Fatalf("input mutation changed effective config: %#v", effective.Config.PromptKit.LocalBackend)
}
effective.Config.PromptKit.LocalBackend.ConcurrencyLimit = 9
if cfg.PromptKit.LocalBackend.ConcurrencyLimit != 2 {
t.Fatalf("effective mutation changed input config: %#v", cfg.PromptKit.LocalBackend)
}
}
func TestEffectiveConfigResolutionFailuresRetainContext(t *testing.T) {
tests := []struct {
name string
@@ -154,7 +204,7 @@ func TestEffectiveConfigResolutionFailuresRetainContext(t *testing.T) {
}
}
func TestEffectiveConfigLLMProfileOverrideChangesDigestWithoutOverridingValidators(t *testing.T) {
func TestEffectiveConfigLLMProfileOverrideChangesDigestAndOverridesValidators(t *testing.T) {
profile := effectiveProfile()
profile.Chunk.LLMProfile = "chunk-profile"
lane := profile.Artifacts["lane"]
@@ -187,8 +237,27 @@ func TestEffectiveConfigLLMProfileOverrideChangesDigestWithoutOverridingValidato
t.Fatalf("pipeline profile override was not applied: %#v", resolved)
}
validators := findEffectiveValidatorChain(resolved, pipeline.StageExtract, "lane")
if len(validators.Validators) != 1 || validators.Validators[0].Binding.LLMProfile != "validator-profile" {
t.Fatalf("validator profile was overridden: %#v", validators)
if len(validators.Validators) != 1 || validators.Validators[0].Binding.LLMProfile != "override-profile" {
t.Fatalf("validator profile = %#v, want runtime override", validators)
}
}
func TestEffectiveConfigPipelineLLMProfileIsInheritedWithoutMutatingConfig(t *testing.T) {
profile := effectiveProfile()
profile.LLMProfile = " configured-profile "
effective, err := resolveEffectiveProfile(t, profile, ResolveInput{})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
if got := effective.Config.Pipelines["main"].LLMProfile; got != " configured-profile " {
t.Fatalf("effective config pipeline llm profile = %q, want preserved programmatic value", got)
}
resolved := effective.ResolvedPipeline
if got := resolved.Chunk.LLMProfile; got != "configured-profile" {
t.Fatalf("resolved chunk profile = %q, want inherited profile", got)
}
if got := resolved.Steps[0].ArtifactLanes[0].Extract.LLMProfile; got != "configured-profile" {
t.Fatalf("resolved extract profile = %q, want inherited profile", got)
}
}
@@ -435,7 +504,7 @@ func effectiveCatalog(t *testing.T) pipeline.ModuleCatalog {
if err := pipeline.RegisterArtifactCodec(catalog.ArtifactCodecs, effectiveCodec{}); err != nil {
t.Fatal(err)
}
if err := catalog.Inputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func() (contracts.InputAdapter, error) {
if err := catalog.Inputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func() (contracts.InputAdapter, error) {
return effectiveInput{key: "input"}, nil
}); err != nil {
t.Fatal(err)
@@ -443,6 +512,7 @@ func effectiveCatalog(t *testing.T) pipeline.ModuleCatalog {
chunkSpec := pipeline.ModuleSpec{
Key: pipeline.DefaultChunkModule,
Stage: pipeline.StageChunk,
ExecutionClass: contracts.ExecutionClassLLMBacked,
Requires: []string{"source"},
Provides: []string{"chunk"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-ref"}},
@@ -453,32 +523,32 @@ func effectiveCatalog(t *testing.T) pipeline.ModuleCatalog {
}); err != nil {
t.Fatal(err)
}
if err := catalog.Chunkers.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "needs-capability", Stage: pipeline.StageChunk, Requires: []string{"missing"}}, chunkOptions, func(pipeline.BuildRequest) (contracts.Chunker, error) {
if err := catalog.Chunkers.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "needs-capability", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"missing"}}, chunkOptions, func(pipeline.BuildRequest) (contracts.Chunker, error) {
return effectiveChunker{key: "needs-capability"}, nil
}); err != nil {
t.Fatal(err)
}
if err := pipeline.RegisterExtractor(catalog.Extractors, pipeline.ModuleSpec{Key: "extract", Stage: pipeline.StageExtract, ArtifactKind: effectiveArtifactKind, Requires: []string{"chunk"}, Provides: []string{"candidate"}}, func() (contracts.Extractor[effectiveArtifact], error) {
if err := pipeline.RegisterExtractor(catalog.Extractors, pipeline.ModuleSpec{Key: "extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: effectiveArtifactKind, Requires: []string{"chunk"}, Provides: []string{"candidate"}}, func() (contracts.Extractor[effectiveArtifact], error) {
return effectiveExtractor{key: "extract"}, nil
}); err != nil {
t.Fatal(err)
}
if err := pipeline.RegisterMerger(catalog.Mergers, pipeline.ModuleSpec{Key: pipeline.DefaultMergeModule, Stage: pipeline.StageMerge, ArtifactKind: effectiveArtifactKind, Requires: []string{"candidate"}, Provides: []string{"merged"}}, func() (contracts.Merger[effectiveArtifact], error) {
if err := pipeline.RegisterMerger(catalog.Mergers, pipeline.ModuleSpec{Key: pipeline.DefaultMergeModule, Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: effectiveArtifactKind, Requires: []string{"candidate"}, Provides: []string{"merged"}}, func() (contracts.Merger[effectiveArtifact], error) {
return effectiveMerger{key: pipeline.DefaultMergeModule}, nil
}); err != nil {
t.Fatal(err)
}
if err := pipeline.RegisterMerger(catalog.Mergers, pipeline.ModuleSpec{Key: "other-merge", Stage: pipeline.StageMerge, ArtifactKind: "other-kind", Requires: []string{"candidate"}, Provides: []string{"merged"}}, func() (contracts.Merger[effectiveArtifact], error) {
if err := pipeline.RegisterMerger(catalog.Mergers, pipeline.ModuleSpec{Key: "other-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: "other-kind", Requires: []string{"candidate"}, Provides: []string{"merged"}}, func() (contracts.Merger[effectiveArtifact], error) {
return effectiveMerger{key: "other-merge"}, nil
}); err != nil {
t.Fatal(err)
}
if err := pipeline.RegisterNormalizer(catalog.Normalizers, pipeline.ModuleSpec{Key: pipeline.DefaultNormalizeModule, Stage: pipeline.StageNormalize, ArtifactKind: effectiveArtifactKind, Requires: []string{"merged"}, Provides: []string{"normalized"}}, func() (contracts.Normalizer[effectiveArtifact], error) {
if err := pipeline.RegisterNormalizer(catalog.Normalizers, pipeline.ModuleSpec{Key: pipeline.DefaultNormalizeModule, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: effectiveArtifactKind, Requires: []string{"merged"}, Provides: []string{"normalized"}}, func() (contracts.Normalizer[effectiveArtifact], error) {
return effectiveNormalizer{key: pipeline.DefaultNormalizeModule}, nil
}); err != nil {
t.Fatal(err)
}
if err := catalog.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, Requires: []string{"normalized"}}, func() (contracts.OutputEncoder, error) {
if err := catalog.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}}, func() (contracts.OutputEncoder, error) {
return effectiveOutput{key: pipeline.DefaultOutputModule}, nil
}); err != nil {
t.Fatal(err)

View File

@@ -10,7 +10,7 @@ import (
)
func TestPrecedenceFileValuesOverrideBuiltInDefaults(t *testing.T) {
cfg := applyFileConfig(t, `version: 3
cfg := applyFileConfig(t, `version: 4
concurrency:
total_llm: 4
stage_workers:
@@ -35,7 +35,7 @@ debug:
}
func TestPrecedenceOperationalEnvironmentOverridesFileValues(t *testing.T) {
cfg := applyFileConfig(t, `version: 3
cfg := applyFileConfig(t, `version: 4
concurrency:
total_llm: 2
stage_workers:
@@ -81,21 +81,21 @@ func TestPrecedenceExtractWorkersFollowEffectiveConcurrencyUnlessExplicit(t *tes
}{
{
name: "default follows environment total",
file: "version: 3\n",
file: "version: 4\n",
env: map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "5"},
wantTotal: 5,
wantWorker: 5,
},
{
name: "file worker is retained",
file: "version: 3\nconcurrency:\n total_llm: 3\n stage_workers:\n extract: 2\n",
file: "version: 4\nconcurrency:\n total_llm: 3\n stage_workers:\n extract: 2\n",
env: map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6"},
wantTotal: 6,
wantWorker: 2,
},
{
name: "environment worker is retained",
file: "version: 3\nconcurrency:\n total_llm: 2\n",
file: "version: 4\nconcurrency:\n total_llm: 2\n",
env: map[string]string{
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6",
"NOTARIUS_STAGE_WORKERS_EXTRACT": "4",
@@ -118,7 +118,7 @@ func TestPrecedenceExtractWorkersFollowEffectiveConcurrencyUnlessExplicit(t *tes
}
func TestPrecedenceEmptyFileCacheDirectoriesDeferPerUserResolution(t *testing.T) {
cfg := applyFileConfig(t, `version: 3
cfg := applyFileConfig(t, `version: 4
cache:
chunk_plans:
directory: ""

View File

@@ -14,7 +14,7 @@ import (
type FileConfig struct {
Version int `yaml:"version"`
Scriptorium *FileScriptoriumConfig `yaml:"scriptorium,omitempty"`
PromptKit *FilePromptKitConfig `yaml:"promptkit,omitempty"`
Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"`
Output *FileOutputConfig `yaml:"output,omitempty"`
@@ -22,27 +22,35 @@ type FileConfig struct {
Debug *FileDebugConfig `yaml:"debug,omitempty"`
}
type FileScriptoriumConfig struct {
ProfileDir *string `yaml:"profile_dir,omitempty"`
ProfileFile *string `yaml:"profile_file,omitempty"`
type FilePromptKitConfig struct {
ProfileDir *string `yaml:"profile_dir,omitempty"`
ProfileFile *string `yaml:"profile_file,omitempty"`
LocalBackend *FilePromptKitLocalBackendConfig `yaml:"local_backend,omitempty"`
}
type FilePromptKitLocalBackendConfig struct {
Endpoint *string `yaml:"endpoint,omitempty"`
ConcurrencyLimit *int `yaml:"concurrency_limit,omitempty"`
}
type FilePipelineProfile struct {
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Steps []FilePipelineStepProfile `yaml:"steps,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]fileReferenceSource `yaml:"references,omitempty"`
artifactsSet bool `yaml:"-"`
stepsSet bool `yaml:"-"`
LLMProfile *string `yaml:"llm_profile,omitempty"`
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Steps []FilePipelineStepProfile `yaml:"steps,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]fileReferenceSource `yaml:"references,omitempty"`
artifactsSet bool `yaml:"-"`
stepsSet bool `yaml:"-"`
llmProfileSet bool `yaml:"-"`
}
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
type plainFilePipelineProfile FilePipelineProfile
var decoded plainFilePipelineProfile
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
"input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
"llm_profile": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
}, "pipeline profile")
if err != nil {
return err
@@ -50,6 +58,7 @@ func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
*p = FilePipelineProfile(decoded)
_, p.artifactsSet = seen["artifacts"]
_, p.stepsSet = seen["steps"]
_, p.llmProfileSet = seen["llm_profile"]
return nil
}
@@ -251,6 +260,9 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err
}
b.LLMProfile = strings.TrimSpace(llmProfile)
if b.LLMProfile == "" {
return fmt.Errorf("llm_profile must not be empty when set")
}
case "retries":
var retries int
if err := valueNode.Decode(&retries); err != nil {
@@ -325,6 +337,9 @@ func ParseFileConfigYAML(data []byte) (FileConfig, error) {
if header.Version == 2 {
return FileConfig{}, fmt.Errorf("config version 2 is no longer supported; migrate the file using the version 2-to-3 migration in docs/config.md")
}
if header.Version == 3 {
return FileConfig{}, fmt.Errorf("config version 3 is no longer supported; change \"version: 3\" to \"version: 4\" and rename \"scriptorium:\" to \"promptkit:\"")
}
if header.Version != SupportedFileConfigVersion {
return FileConfig{}, fmt.Errorf("unsupported config version %d (supported version is %d)", header.Version, SupportedFileConfigVersion)
}
@@ -450,25 +465,46 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
}
}
if fileCfg.Scriptorium != nil {
if fileCfg.Scriptorium.ProfileDir != nil {
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileDir)
if fileCfg.PromptKit != nil {
if fileCfg.PromptKit.ProfileDir != nil {
value := strings.TrimSpace(*fileCfg.PromptKit.ProfileDir)
if value == "" {
return fmt.Errorf("scriptorium.profile_dir must not be empty when set")
return fmt.Errorf("promptkit.profile_dir must not be empty when set")
}
c.Scriptorium.ProfileDir = value
c.PromptKit.ProfileDir = value
}
if fileCfg.Scriptorium.ProfileFile != nil {
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileFile)
if fileCfg.PromptKit.ProfileFile != nil {
value := strings.TrimSpace(*fileCfg.PromptKit.ProfileFile)
if value == "" {
return fmt.Errorf("scriptorium.profile_file must not be empty when set")
return fmt.Errorf("promptkit.profile_file must not be empty when set")
}
c.Scriptorium.ProfileFile = value
c.PromptKit.ProfileFile = value
}
if fileCfg.PromptKit.LocalBackend != nil {
if fileCfg.PromptKit.LocalBackend.Endpoint == nil {
return fmt.Errorf("promptkit.local_backend.endpoint must not be empty when set")
}
endpoint := strings.TrimSpace(*fileCfg.PromptKit.LocalBackend.Endpoint)
if endpoint == "" {
return fmt.Errorf("promptkit.local_backend.endpoint must not be empty when set")
}
localBackend := PromptKitLocalBackendConfig{Endpoint: endpoint}
if fileCfg.PromptKit.LocalBackend.ConcurrencyLimit != nil {
localBackend.ConcurrencyLimit = *fileCfg.PromptKit.LocalBackend.ConcurrencyLimit
}
c.PromptKit.LocalBackend = &localBackend
}
}
for _, pipelineID := range pipelineIDs {
filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]]
llmProfile := ""
if filePipeline.llmProfileSet || filePipeline.LLMProfile != nil {
if filePipeline.LLMProfile == nil || strings.TrimSpace(*filePipeline.LLMProfile) == "" {
return fmt.Errorf("pipeline %q llm_profile must not be empty when set", pipelineID)
}
llmProfile = strings.TrimSpace(*filePipeline.LLMProfile)
}
hasSteps := filePipeline.stepsSet || filePipeline.Steps != nil
laneIDs, rawLaneIDs, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID))
if err != nil {
@@ -476,6 +512,7 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
}
profile := pipeline.PipelineProfile{
ID: pipelineID,
LLMProfile: llmProfile,
Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: fileReferenceSourcesToPipeline(filePipeline.References),

View File

@@ -1,6 +1,8 @@
package config
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
@@ -34,8 +36,8 @@ func TestDefaultReturnsDocumentedValuesAndIndependentMaps(t *testing.T) {
}
}
func TestFileConfigMinimalVersion3AppliesOverDefaults(t *testing.T) {
file := parseFileConfig(t, "version: 3\n")
func TestFileConfigMinimalVersion4AppliesOverDefaults(t *testing.T) {
file := parseFileConfig(t, "version: 4\n")
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
@@ -48,6 +50,259 @@ func TestFileConfigMinimalVersion3AppliesOverDefaults(t *testing.T) {
}
}
func TestFilePipelineLLMProfileIsPresenceAwareAndDetached(t *testing.T) {
const pipelineYAML = `version: 4
pipelines:
main:
%s
input: input
artifacts:
lane:
extract: extract
`
t.Run("omitted", func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, ""))
if file.Pipelines["main"].LLMProfile != nil || file.Pipelines["main"].llmProfileSet {
t.Fatalf("parsed pipeline profile = %#v, want omitted llm profile", file.Pipelines["main"])
}
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
if got := cfg.Pipelines["main"].LLMProfile; got != "" {
t.Fatalf("pipeline llm profile = %q, want empty", got)
}
})
t.Run("trimmed and detached", func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "llm_profile: ' configured-profile '"))
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
if got := cfg.Pipelines["main"].LLMProfile; got != "configured-profile" {
t.Fatalf("pipeline llm profile = %q, want trimmed value", got)
}
*file.Pipelines["main"].LLMProfile = "changed-profile"
if got := cfg.Pipelines["main"].LLMProfile; got != "configured-profile" {
t.Fatalf("effective config aliases parsed file: %q", got)
}
if got := cloneConfig(cfg).Pipelines["main"].LLMProfile; got != "configured-profile" {
t.Fatalf("cloned pipeline llm profile = %q", got)
}
data, err := json.Marshal(cfg)
if err != nil {
t.Fatal(err)
}
var roundTripped Config
if err := json.Unmarshal(data, &roundTripped); err != nil {
t.Fatal(err)
}
if got := roundTripped.Pipelines["main"].LLMProfile; got != "configured-profile" {
t.Fatalf("round-tripped pipeline llm profile = %q", got)
}
})
for _, value := range []string{"''", "' '", "null"} {
t.Run("explicit empty "+value, func(t *testing.T) {
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "llm_profile: "+value))
cfg := Default()
err := cfg.ApplyFileConfig(file)
if err == nil || !strings.Contains(err.Error(), `pipeline "main" llm_profile must not be empty`) {
t.Fatalf("ApplyFileConfig() error = %v, want explicit-empty rejection", err)
}
})
}
}
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
const configYAML = `version: 4
pipelines:
main:
input:
module: input
llm_profile: %s
artifacts:
lane:
extract: extract
`
for _, value := range []string{"''", "' '", "null"} {
t.Run(value, func(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(configYAML, value)))
if err == nil || !strings.Contains(err.Error(), "llm_profile must not be empty when set") {
t.Fatalf("ParseFileConfigYAML() error = %v, want explicit-empty binding profile rejection", err)
}
})
}
}
func TestFilePromptKitProfileSourcesSurviveConfigBoundaries(t *testing.T) {
tests := []struct {
name string
yaml string
want PromptKitConfig
}{
{
name: "profile directory",
yaml: "version: 4\npromptkit:\n profile_dir: ' ./profiles '\n",
want: PromptKitConfig{ProfileDir: "./profiles"},
},
{
name: "profile file",
yaml: "version: 4\npromptkit:\n profile_file: ' ./profiles.yml '\n",
want: PromptKitConfig{ProfileFile: "./profiles.yml"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := applyFileConfig(t, tt.yaml)
if cfg.PromptKit != tt.want {
t.Fatalf("PromptKit config = %#v, want %#v", cfg.PromptKit, tt.want)
}
if got := cloneConfig(cfg).PromptKit; got != tt.want {
t.Fatalf("cloned PromptKit config = %#v, want %#v", got, tt.want)
}
if got := cfg.Redacted().PromptKit; got != tt.want {
t.Fatalf("redacted PromptKit config = %#v, want %#v", got, tt.want)
}
data, err := json.Marshal(cfg)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var payload map[string]json.RawMessage
if err := json.Unmarshal(data, &payload); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if _, ok := payload["promptkit"]; !ok {
t.Fatalf("runtime JSON keys = %v, want promptkit", payload)
}
if _, ok := payload["scriptorium"]; ok {
t.Fatalf("runtime JSON keys = %v, must not contain removed section", payload)
}
})
}
}
func TestFilePromptKitLocalBackendSurvivesConfigBoundaries(t *testing.T) {
tests := []struct {
name string
concurrencyYAML string
wantConcurrency int
}{
{name: "omitted concurrency defaults to zero"},
{name: "positive concurrency is preserved", concurrencyYAML: " concurrency_limit: 2\n", wantConcurrency: 2},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
file := parseFileConfig(t, "version: 4\npromptkit:\n local_backend:\n endpoint: ' http://localhost:8000/v1 '\n"+tt.concurrencyYAML)
cfg := Default()
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatalf("ApplyFileConfig() error = %v", err)
}
want := PromptKitLocalBackendConfig{
Endpoint: "http://localhost:8000/v1",
ConcurrencyLimit: tt.wantConcurrency,
}
if cfg.PromptKit.LocalBackend == nil || *cfg.PromptKit.LocalBackend != want {
t.Fatalf("local backend config = %#v, want %#v", cfg.PromptKit.LocalBackend, want)
}
*file.PromptKit.LocalBackend.Endpoint = "http://changed.example/v1"
if file.PromptKit.LocalBackend.ConcurrencyLimit != nil {
*file.PromptKit.LocalBackend.ConcurrencyLimit = 99
}
if *cfg.PromptKit.LocalBackend != want {
t.Fatalf("effective config aliases parsed file model: %#v", cfg.PromptKit.LocalBackend)
}
cloned := cloneConfig(cfg)
if cloned.PromptKit.LocalBackend == cfg.PromptKit.LocalBackend || *cloned.PromptKit.LocalBackend != want {
t.Fatalf("cloned local backend = %#v, want detached %#v", cloned.PromptKit.LocalBackend, want)
}
cloned.PromptKit.LocalBackend.Endpoint = "http://clone.example/v1"
if *cfg.PromptKit.LocalBackend != want {
t.Fatalf("mutating clone changed source config: %#v", cfg.PromptKit.LocalBackend)
}
redacted := cfg.Redacted()
if redacted.PromptKit.LocalBackend == cfg.PromptKit.LocalBackend || *redacted.PromptKit.LocalBackend != want {
t.Fatalf("redacted local backend = %#v, want detached %#v", redacted.PromptKit.LocalBackend, want)
}
data, err := json.Marshal(cfg)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var payload struct {
PromptKit map[string]json.RawMessage `json:"promptkit"`
}
if err := json.Unmarshal(data, &payload); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
localJSON, ok := payload.PromptKit["local_backend"]
if !ok {
t.Fatalf("runtime PromptKit JSON keys = %v, want local_backend", payload.PromptKit)
}
var localPayload map[string]json.RawMessage
if err := json.Unmarshal(localJSON, &localPayload); err != nil {
t.Fatalf("unmarshal local_backend JSON: %v", err)
}
if _, ok := localPayload["endpoint"]; !ok {
t.Fatalf("runtime local_backend JSON keys = %v, want endpoint", localPayload)
}
if _, ok := localPayload["concurrency_limit"]; !ok {
t.Fatalf("runtime local_backend JSON keys = %v, want concurrency_limit", localPayload)
}
})
}
}
func TestFilePromptKitLocalBackendRequiresEndpoint(t *testing.T) {
for _, tt := range []struct {
name string
yaml string
}{
{name: "missing", yaml: "version: 4\npromptkit:\n local_backend: {}\n"},
{name: "empty", yaml: "version: 4\npromptkit:\n local_backend:\n endpoint: ''\n"},
{name: "blank", yaml: "version: 4\npromptkit:\n local_backend:\n endpoint: ' '\n"},
} {
t.Run(tt.name, func(t *testing.T) {
file := parseFileConfig(t, tt.yaml)
cfg := Default()
err := cfg.ApplyFileConfig(file)
if err == nil || !strings.Contains(err.Error(), "promptkit.local_backend.endpoint") {
t.Fatalf("ApplyFileConfig() error = %v, want endpoint field context", err)
}
})
}
}
func TestFilePromptKitExplicitEmptyProfileSourcesAreRejected(t *testing.T) {
for _, field := range []string{"profile_dir", "profile_file"} {
t.Run(field, func(t *testing.T) {
file := parseFileConfig(t, "version: 4\npromptkit:\n "+field+": ''\n")
cfg := Default()
err := cfg.ApplyFileConfig(file)
if err == nil || !strings.Contains(err.Error(), "promptkit."+field+" must not be empty") {
t.Fatalf("ApplyFileConfig() error = %v, want explicit-empty rejection", err)
}
})
}
}
func TestFilePromptKitProfileSourcesRemainMutuallyExclusive(t *testing.T) {
cfg := applyFileConfig(t, `version: 4
promptkit:
profile_dir: ./profiles
profile_file: ./profiles.yml
`)
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "promptkit profile_dir and profile_file are mutually exclusive") {
t.Fatalf("Validate() error = %v, want mutually exclusive profile sources", err)
}
}
func TestFileConfigMissingVersionIsReportedBeforeFieldDecoding(t *testing.T) {
_, err := ParseFileConfigYAML([]byte("workspace:\n directory: /tmp/old\n"))
if err == nil || !strings.Contains(err.Error(), "config version is required") {
@@ -55,6 +310,15 @@ func TestFileConfigMissingVersionIsReportedBeforeFieldDecoding(t *testing.T) {
}
}
func TestFileConfigVersion3ReportsPromptKitMigration(t *testing.T) {
_, err := ParseFileConfigYAML([]byte("version: 3\nscriptorium:\n profile_dir: ./profiles\n"))
if err == nil ||
!strings.Contains(err.Error(), `change "version: 3" to "version: 4"`) ||
!strings.Contains(err.Error(), `rename "scriptorium:" to "promptkit:"`) {
t.Fatalf("version 3 error = %v, want actionable version and section migration", err)
}
}
func TestFileConfigRejectsUnknownCurrentAndRemovedFields(t *testing.T) {
tests := []struct {
name string
@@ -63,14 +327,19 @@ func TestFileConfigRejectsUnknownCurrentAndRemovedFields(t *testing.T) {
}{
{
name: "removed diagnostics",
yaml: "version: 3\ndiagnostics: {}\n",
yaml: "version: 4\ndiagnostics: {}\n",
want: "field diagnostics not found",
},
{
name: "removed llm profiles",
yaml: "version: 3\nllm_profiles: {}\n",
yaml: "version: 4\nllm_profiles: {}\n",
want: "field llm_profiles not found",
},
{
name: "removed scriptorium section",
yaml: "version: 4\nscriptorium: {}\n",
want: "field scriptorium not found",
},
{
name: "version 2 migration",
yaml: "version: 2\nworkspace:\n directory: /tmp/old\n",
@@ -78,29 +347,34 @@ func TestFileConfigRejectsUnknownCurrentAndRemovedFields(t *testing.T) {
},
{
name: "pipeline field",
yaml: "version: 3\npipelines:\n main:\n unknown: true\n",
yaml: "version: 4\npipelines:\n main:\n unknown: true\n",
want: "field unknown not found",
},
{
name: "lane field",
yaml: "version: 3\npipelines:\n main:\n artifacts:\n spells:\n unknown: true\n",
yaml: "version: 4\npipelines:\n main:\n artifacts:\n spells:\n unknown: true\n",
want: "field unknown not found",
},
{
name: "module binding field",
yaml: "version: 3\npipelines:\n main:\n input:\n module: seriatim\n unknown: true\n",
yaml: "version: 4\npipelines:\n main:\n input:\n module: seriatim\n unknown: true\n",
want: "field unknown not found in module binding",
},
{
name: "checkpoint field",
yaml: "version: 3\ncache:\n checkpoints:\n unknown: true\n",
yaml: "version: 4\ncache:\n checkpoints:\n unknown: true\n",
want: "field unknown not found",
},
{
name: "checkpoint enabled type",
yaml: "version: 3\ncache:\n checkpoints:\n enabled: definitely\n",
yaml: "version: 4\ncache:\n checkpoints:\n enabled: definitely\n",
want: "cannot unmarshal",
},
{
name: "local backend field",
yaml: "version: 4\npromptkit:\n local_backend:\n endpoint: http://localhost:8000/v1\n unknown: true\n",
want: "field unknown not found",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -113,7 +387,7 @@ func TestFileConfigRejectsUnknownCurrentAndRemovedFields(t *testing.T) {
}
func TestFileConfigModuleBindingsPreserveFormsAndValidatorPresence(t *testing.T) {
cfg := applyFileConfig(t, `version: 3
cfg := applyFileConfig(t, `version: 4
pipelines:
main:
input: seriatim
@@ -160,7 +434,7 @@ pipelines:
}
func TestFileConfigReferencePrecedenceIsRetained(t *testing.T) {
cfg := applyFileConfig(t, `version: 3
cfg := applyFileConfig(t, `version: 4
pipelines:
main:
input: seriatim
@@ -224,7 +498,7 @@ pipelines:
}
func TestFileConfigStageLocalValidatorsPreserveOrderAndFields(t *testing.T) {
cfg := applyFileConfig(t, `version: 3
cfg := applyFileConfig(t, `version: 4
pipelines:
main:
input: seriatim
@@ -277,8 +551,8 @@ pipelines:
}
func TestFileConfigStateSectionsApplyIndependently(t *testing.T) {
cfg := applyFileConfig(t, `version: 3
scriptorium:
cfg := applyFileConfig(t, `version: 4
promptkit:
profile_dir: ./profiles
concurrency:
total_llm: 7
@@ -294,15 +568,15 @@ cache:
debug:
directory: ./debug
`)
if cfg.Scriptorium.ProfileDir != "./profiles" || cfg.Scriptorium.ProfileFile != "" {
t.Fatalf("scriptorium = %#v", cfg.Scriptorium)
if cfg.PromptKit.ProfileDir != "./profiles" || cfg.PromptKit.ProfileFile != "" {
t.Fatalf("promptkit = %#v", cfg.PromptKit)
}
if cfg.Concurrency.TotalLLM != 7 || cfg.Concurrency.StageWorkers["extract"] != 7 {
t.Fatalf("concurrency = %#v", cfg.Concurrency)
}
if cfg.Output.Directory != "./output" || cfg.Cache.ChunkPlans.Directory != "plans" || cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheBypass ||
!cfg.Cache.Checkpoints.Enabled || cfg.Cache.Checkpoints.Directory != "checkpoints" || cfg.Debug.Directory != "./debug" {
t.Fatalf("state sections = %#v, %#v, %#v, %#v", cfg.Output, cfg.Cache, cfg.Debug, cfg.Scriptorium)
t.Fatalf("state sections = %#v, %#v, %#v, %#v", cfg.Output, cfg.Cache, cfg.Debug, cfg.PromptKit)
}
if cfg.Output.Directory == cfg.Cache.ChunkPlans.Directory || cfg.Cache.ChunkPlans.Directory == cfg.Cache.Checkpoints.Directory || cfg.Cache.Checkpoints.Directory == cfg.Debug.Directory {
t.Fatal("state roots were coupled")
@@ -310,11 +584,11 @@ debug:
}
func TestFileConfigCheckpointEnabledCanBeExplicitlyDisabled(t *testing.T) {
cfg := applyFileConfig(t, "version: 3\ncache:\n checkpoints:\n enabled: true\n")
cfg := applyFileConfig(t, "version: 4\ncache:\n checkpoints:\n enabled: true\n")
if !cfg.Cache.Checkpoints.Enabled || !cloneConfig(cfg).Cache.Checkpoints.Enabled {
t.Fatalf("enabled checkpoint config was not retained: %#v", cfg.Cache.Checkpoints)
}
file := parseFileConfig(t, "version: 3\ncache:\n checkpoints:\n enabled: false\n")
file := parseFileConfig(t, "version: 4\ncache:\n checkpoints:\n enabled: false\n")
if err := cfg.ApplyFileConfig(file); err != nil {
t.Fatal(err)
}
@@ -331,17 +605,17 @@ func TestFileConfigRejectsTrimmedKeyCollisions(t *testing.T) {
}{
{
name: "pipeline ids",
yaml: "version: 3\npipelines:\n main: {}\n ' main ': {}\n",
yaml: "version: 4\npipelines:\n main: {}\n ' main ': {}\n",
want: "pipeline id \"main\" is duplicated after trimming",
},
{
name: "lane ids",
yaml: "version: 3\npipelines:\n main:\n artifacts:\n spells: {}\n ' spells ': {}\n",
yaml: "version: 4\npipelines:\n main:\n artifacts:\n spells: {}\n ' spells ': {}\n",
want: "artifact lane id \"spells\" is duplicated after trimming",
},
{
name: "reference slots",
yaml: "version: 3\npipelines:\n main:\n references:\n slot: ./one.txt\n ' slot ': ./two.txt\n",
yaml: "version: 4\npipelines:\n main:\n references:\n slot: ./one.txt\n ' slot ': ./two.txt\n",
want: "reference slot \"slot\" is duplicated after trimming",
},
}
@@ -358,7 +632,7 @@ func TestFileConfigRejectsTrimmedKeyCollisions(t *testing.T) {
}
func TestFileConfigParsesOrderedStepsAndReferenceSources(t *testing.T) {
file := parseFileConfig(t, `version: 3
file := parseFileConfig(t, `version: 4
pipelines:
session:
input: seriatim
@@ -397,7 +671,7 @@ func TestFileConfigRejectsAmbiguousReferenceSourceForms(t *testing.T) {
"artifact: {step: 1, lane: b}",
"1",
} {
_, err := ParseFileConfigYAML([]byte("version: 3\npipelines:\n p:\n input: text\n references:\n slot: " + source + "\n"))
_, err := ParseFileConfigYAML([]byte("version: 4\npipelines:\n p:\n input: text\n references:\n slot: " + source + "\n"))
if err == nil {
t.Fatalf("ParseFileConfigYAML(%q) error = nil", source)
}
@@ -412,12 +686,12 @@ func TestFileConfigRejectsEmptyAndAmbiguousPipelineShapes(t *testing.T) {
}{
{
name: "empty steps",
yaml: "version: 3\npipelines:\n p:\n input: text\n steps: []\n",
yaml: "version: 4\npipelines:\n p:\n input: text\n steps: []\n",
want: "at least one ordered step",
},
{
name: "both forms",
yaml: "version: 3\npipelines:\n p:\n input: text\n artifacts: {}\n steps: []\n",
yaml: "version: 4\npipelines:\n p:\n input: text\n artifacts: {}\n steps: []\n",
want: "both artifacts and steps",
},
}

View File

@@ -19,23 +19,28 @@ func TestRedactedResolvedPipelinePayloadRedactsEveryBinding(t *testing.T) {
}
resolved := pipeline.ResolvedPipeline{
ID: "redaction-test",
Digest: "sha256:safe-digest",
Input: bindings["input"],
Chunk: bindings["chunk"],
ChunkReferences: redactionTestReferenceTarget(pipeline.StageChunk, "", "chunk-reference-content"),
ID: "redaction-test",
Digest: "sha256:safe-digest",
Input: bindings["input"],
InputExecutionClass: contracts.ExecutionClassDeterministic,
Chunk: bindings["chunk"],
ChunkExecutionClass: contracts.ExecutionClassLLMBacked,
ChunkReferences: redactionTestReferenceTarget(pipeline.StageChunk, "", "chunk-reference-content"),
Steps: []pipeline.ResolvedPipelineStep{{
ID: "default",
ArtifactLanes: []pipeline.ResolvedArtifactLane{{
ID: "safe-lane",
ArtifactKind: "safe/artifact",
Extract: bindings["extract"],
Merge: bindings["merge"],
Normalize: bindings["normalize"],
Validators: []pipeline.ModuleBinding{bindings["lane-validator"]},
ExtractReferences: redactionTestReferenceTarget(pipeline.StageExtract, "safe-lane", "extract-reference-content"),
MergeReferences: redactionTestReferenceTarget(pipeline.StageMerge, "safe-lane", "merge-reference-content"),
NormalizeReferences: redactionTestReferenceTarget(pipeline.StageNormalize, "safe-lane", "normalize-reference-content"),
ID: "safe-lane",
ArtifactKind: "safe/artifact",
Extract: bindings["extract"],
ExtractExecutionClass: contracts.ExecutionClassLLMBacked,
Merge: bindings["merge"],
MergeExecutionClass: contracts.ExecutionClassDeterministic,
Normalize: bindings["normalize"],
NormalizeExecutionClass: contracts.ExecutionClassLLMBacked,
Validators: []pipeline.ModuleBinding{bindings["lane-validator"]},
ExtractReferences: redactionTestReferenceTarget(pipeline.StageExtract, "safe-lane", "extract-reference-content"),
MergeReferences: redactionTestReferenceTarget(pipeline.StageMerge, "safe-lane", "merge-reference-content"),
NormalizeReferences: redactionTestReferenceTarget(pipeline.StageNormalize, "safe-lane", "normalize-reference-content"),
}},
}},
ValidatorChains: []pipeline.ResolvedValidatorChain{{
@@ -49,7 +54,8 @@ func TestRedactedResolvedPipelinePayloadRedactsEveryBinding(t *testing.T) {
ArtifactKind: "safe/artifact",
}},
}},
Output: bindings["output"],
Output: bindings["output"],
OutputExecutionClass: contracts.ExecutionClassDeterministic,
}
effective := EffectiveConfig{
Config: Config{Pipelines: map[string]pipeline.PipelineProfile{
@@ -88,6 +94,11 @@ func TestRedactedResolvedPipelinePayloadRedactsEveryBinding(t *testing.T) {
t.Fatalf("resolved pipeline summary does not retain %q: %s", safe, text)
}
}
for _, executionClass := range []string{"input_execution_class\":\"deterministic", "chunk_execution_class\":\"llm_backed", "extract_execution_class\":\"llm_backed", "merge_execution_class\":\"deterministic", "normalize_execution_class\":\"llm_backed", "output_execution_class\":\"deterministic"} {
if !strings.Contains(text, executionClass) {
t.Fatalf("resolved pipeline summary does not retain %q: %s", executionClass, text)
}
}
payload.Input.Options["safe"] = "mutated"
nested := payload.Input.Options["nested"].([]any)[0].([]any)[0].(map[string]any)

View File

@@ -2,6 +2,7 @@ package config
import (
"fmt"
"net/url"
"sort"
"strings"
@@ -10,7 +11,7 @@ import (
func (c Config) Validate() error {
c.Concurrency.recomputeStageWorkerDefaults()
if err := validateScriptorium(c.Scriptorium); err != nil {
if err := validatePromptKit(c.PromptKit); err != nil {
return err
}
if err := validateStateSurfaces(c); err != nil {
@@ -49,9 +50,30 @@ func validateStageWorkers(cfg ConcurrencyConfig) error {
return nil
}
func validateScriptorium(cfg ScriptoriumConfig) error {
func validatePromptKit(cfg PromptKitConfig) error {
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
return fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")
return fmt.Errorf("promptkit profile_dir and profile_file are mutually exclusive")
}
if cfg.LocalBackend == nil {
return nil
}
endpoint := strings.TrimSpace(cfg.LocalBackend.Endpoint)
if endpoint == "" {
return fmt.Errorf("promptkit.local_backend.endpoint must not be empty when set")
}
parsed, err := url.Parse(endpoint)
if err != nil ||
(!strings.EqualFold(parsed.Scheme, "http") && !strings.EqualFold(parsed.Scheme, "https")) ||
!parsed.IsAbs() ||
parsed.Hostname() == "" ||
parsed.User != nil ||
parsed.RawQuery != "" ||
parsed.ForceQuery ||
strings.Contains(endpoint, "#") {
return fmt.Errorf("promptkit.local_backend.endpoint must be an absolute HTTP or HTTPS URL with a host and no user information, query, or fragment")
}
if cfg.LocalBackend.ConcurrencyLimit < 0 {
return fmt.Errorf("promptkit.local_backend.concurrency_limit must not be negative")
}
return nil
}

View File

@@ -87,10 +87,77 @@ func TestValidateConcurrencyRules(t *testing.T) {
}
}
func TestValidateScriptoriumSourcesAreMutuallyExclusive(t *testing.T) {
func TestValidatePromptKitSourcesAreMutuallyExclusive(t *testing.T) {
cfg := Default()
cfg.Scriptorium = ScriptoriumConfig{ProfileDir: "./profiles", ProfileFile: "./profile.yml"}
assertValidationContains(t, cfg, "scriptorium profile_dir and profile_file are mutually exclusive")
cfg.PromptKit = PromptKitConfig{ProfileDir: "./profiles", ProfileFile: "./profile.yml"}
assertValidationContains(t, cfg, "promptkit profile_dir and profile_file are mutually exclusive")
}
func TestValidatePromptKitLocalBackendEndpoints(t *testing.T) {
tests := []struct {
name string
endpoint string
profileSource PromptKitConfig
}{
{
name: "HTTP endpoint with path and profile directory",
endpoint: "http://localhost:8000/v1",
profileSource: PromptKitConfig{ProfileDir: "./profiles"},
},
{
name: "case-insensitive HTTPS endpoint and profile file",
endpoint: "HTTPS://inference.example.test/api",
profileSource: PromptKitConfig{ProfileFile: "./profiles.yml"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Default()
cfg.PromptKit = tt.profileSource
cfg.PromptKit.LocalBackend = &PromptKitLocalBackendConfig{
Endpoint: tt.endpoint,
ConcurrencyLimit: 2,
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
}
}
func TestValidatePromptKitLocalBackendRejectsInvalidValues(t *testing.T) {
tests := []struct {
name string
endpoint string
concurrencyLimit int
want string
}{
{name: "blank endpoint", endpoint: " ", want: "promptkit.local_backend.endpoint"},
{name: "relative URL", endpoint: "localhost:8000/v1", want: "promptkit.local_backend.endpoint"},
{name: "unsupported scheme", endpoint: "ftp://localhost/model", want: "promptkit.local_backend.endpoint"},
{name: "missing host", endpoint: "http:///v1", want: "promptkit.local_backend.endpoint"},
{name: "user information", endpoint: "http://user:secret@localhost/v1", want: "promptkit.local_backend.endpoint"},
{name: "query", endpoint: "http://localhost/v1?model=example", want: "promptkit.local_backend.endpoint"},
{name: "empty query", endpoint: "http://localhost/v1?", want: "promptkit.local_backend.endpoint"},
{name: "fragment", endpoint: "http://localhost/v1#model", want: "promptkit.local_backend.endpoint"},
{name: "empty fragment", endpoint: "http://localhost/v1#", want: "promptkit.local_backend.endpoint"},
{
name: "negative concurrency",
endpoint: "http://localhost:8000/v1",
concurrencyLimit: -1,
want: "promptkit.local_backend.concurrency_limit",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Default()
cfg.PromptKit.LocalBackend = &PromptKitLocalBackendConfig{
Endpoint: tt.endpoint,
ConcurrencyLimit: tt.concurrencyLimit,
}
assertValidationContains(t, cfg, tt.want)
})
}
}
func TestValidateStateSurfaceRules(t *testing.T) {

View File

@@ -2,6 +2,7 @@ package debugbundle
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
@@ -137,6 +138,47 @@ func TestSummaryWriterWritesEverySummaryArtifact(t *testing.T) {
}
}
}
func TestWriteInvocationPreservesReasoningEffortOverrideStates(t *testing.T) {
replacement := "focused"
cleared := ""
tests := []struct {
name string
override *string
wantValue string
wantSet bool
}{
{name: "inherit"},
{name: "replace", override: &replacement, wantValue: "focused", wantSet: true},
{name: "clear", override: &cleared, wantSet: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
bundle, err := Allocate(t.TempDir(), testBundleRunID, time.Unix(0, 42))
if err != nil {
t.Fatal(err)
}
if err := bundle.Summary().WriteInvocation(Invocation{
Operation: "run",
ReasoningEffortOverride: tt.override,
}); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(filepath.Join(bundle.SummaryRoot(), ArtifactInvocationMetadata))
if err != nil {
t.Fatal(err)
}
var payload map[string]any
if err := json.Unmarshal(data, &payload); err != nil {
t.Fatal(err)
}
value, found := payload["reasoning_effort_override"]
if found != tt.wantSet || (found && value != tt.wantValue) {
t.Fatalf("reasoning override found=%t value=%#v, want found=%t value=%q; JSON=%s", found, value, tt.wantSet, tt.wantValue, data)
}
})
}
}
func TestSummaryWriterInternalWritesConfineArtifacts(t *testing.T) {
bundle, err := Allocate(t.TempDir(), testBundleRunID, time.Unix(0, 42))
if err != nil {

View File

@@ -29,18 +29,19 @@ type RedactedResolvedPipelinePayload interface {
RedactedResolvedPipelinePayload() pipeline.ResolvedPipeline
}
type Invocation struct {
Operation string `json:"operation"`
PipelineID string `json:"pipeline_id,omitempty"`
PipelineDigest string `json:"pipeline_digest,omitempty"`
Resume bool `json:"resume,omitempty"`
RecomputeStep string `json:"recompute_step,omitempty"`
InputPath string `json:"input_path,omitempty"`
ConfigPath string `json:"config_path,omitempty"`
ConfigSource string `json:"config_source,omitempty"`
OnlyLanes []string `json:"only_lanes,omitempty"`
ChunkCacheOverride string `json:"chunk_cache_override,omitempty"`
RunID string `json:"run_id"`
StartedAt time.Time `json:"started_at"`
Operation string `json:"operation"`
PipelineID string `json:"pipeline_id,omitempty"`
PipelineDigest string `json:"pipeline_digest,omitempty"`
Resume bool `json:"resume,omitempty"`
RecomputeStep string `json:"recompute_step,omitempty"`
InputPath string `json:"input_path,omitempty"`
ConfigPath string `json:"config_path,omitempty"`
ConfigSource string `json:"config_source,omitempty"`
OnlyLanes []string `json:"only_lanes,omitempty"`
ChunkCacheOverride string `json:"chunk_cache_override,omitempty"`
ReasoningEffortOverride *string `json:"reasoning_effort_override,omitempty"`
RunID string `json:"run_id"`
StartedAt time.Time `json:"started_at"`
}
type RunReport struct {
RunID string `json:"run_id"`
@@ -68,6 +69,10 @@ func (w *SummaryWriter) WriteInvocation(payload Invocation) error {
if payload.StartedAt.IsZero() {
payload.StartedAt = w.createdAt
}
if payload.ReasoningEffortOverride != nil {
value := *payload.ReasoningEffortOverride
payload.ReasoningEffortOverride = &value
}
return w.writeJSON(ArtifactInvocationMetadata, payload)
}
func (w *SummaryWriter) WriteRedactedEffectiveConfig(payload RedactedSummaryPayload) error {

View File

@@ -249,6 +249,9 @@ func TestValidateRefValid(t *testing.T) {
if err := ValidateRef(doc, ref); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
if err := NewDocumentIndex(doc).ValidateRef(ref); err != nil {
t.Fatalf("DocumentIndex.ValidateRef() error = %v, want nil", err)
}
}
func TestValidateRefRejectsMalformedReferences(t *testing.T) {
@@ -301,13 +304,62 @@ func TestValidateRefRejectsMalformedReferences(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ValidateRef(validDocument(), tt.ref)
requireErrorFragments(t, err, tt.fragments...)
doc := validDocument()
validators := []struct {
name string
validate func(SourceRef) error
}{
{name: "document", validate: func(ref SourceRef) error { return ValidateRef(doc, ref) }},
{name: "index", validate: NewDocumentIndex(doc).ValidateRef},
}
for _, validator := range validators {
t.Run(validator.name, func(t *testing.T) {
requireErrorFragments(t, validator.validate(tt.ref), tt.fragments...)
})
}
})
}
}
func TestDocumentIndexSnapshotsIdentityAndUnitPositions(t *testing.T) {
doc := &SourceDocument{
ID: "source-1",
Units: []SourceUnit{
{ID: 30},
{ID: 10},
{ID: 30},
},
}
index := NewDocumentIndex(doc)
doc.ID = "changed"
doc.Units[0].ID = 99
if documentID, ok := index.DocumentID(); !ok || documentID != "source-1" {
t.Fatalf("DocumentID() = %q, %t, want source-1, true", documentID, ok)
}
if position, ok := index.Position(30); !ok || position != 0 {
t.Fatalf("Position(30) = %d, %t, want 0, true", position, ok)
}
if position, ok := index.Position(10); !ok || position != 1 {
t.Fatalf("Position(10) = %d, %t, want 1, true", position, ok)
}
ref := SourceRef{SourceID: "source-1", StartUnitID: 30, EndUnitID: 10}
if err := index.ValidateRef(ref); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
}
func TestZeroDocumentIndexIsSafe(t *testing.T) {
var index DocumentIndex
if documentID, ok := index.DocumentID(); ok || documentID != "" {
t.Fatalf("DocumentID() = %q, %t, want empty, false", documentID, ok)
}
if position, ok := index.Position(1); ok || position != 0 {
t.Fatalf("Position(1) = %d, %t, want 0, false", position, ok)
}
requireErrorFragments(t, index.ValidateRef(SourceRef{}), "source document must not be nil")
}
func TestUnitIndex(t *testing.T) {
doc := validDocument()

View File

@@ -5,6 +5,54 @@ import (
"strings"
)
// DocumentIndex is an immutable snapshot of a source document's identity and
// unit positions for repeated source-reference operations.
type DocumentIndex struct {
documentID string
positions map[int]int
hasDocument bool
}
// NewDocumentIndex snapshots doc without retaining or mutating it.
func NewDocumentIndex(doc *SourceDocument) DocumentIndex {
if doc == nil {
return DocumentIndex{}
}
positions := make(map[int]int, len(doc.Units))
for position, unit := range doc.Units {
if _, exists := positions[unit.ID]; !exists {
positions[unit.ID] = position
}
}
return DocumentIndex{
documentID: doc.ID,
positions: positions,
hasDocument: true,
}
}
// DocumentID returns the indexed document identity.
func (i DocumentIndex) DocumentID() (string, bool) {
if !i.hasDocument {
return "", false
}
return i.documentID, true
}
// Position returns the indexed document position for unitID.
func (i DocumentIndex) Position(unitID int) (int, bool) {
position, ok := i.positions[unitID]
return position, ok
}
// ValidateRef validates ref against the indexed document snapshot.
func (i DocumentIndex) ValidateRef(ref SourceRef) error {
if !i.hasDocument {
return fmt.Errorf("source document must not be nil")
}
return validateRef(i.documentID, i.Position, ref)
}
func ValidateDocument(doc *SourceDocument) error {
if doc == nil {
return fmt.Errorf("source document must not be nil")
@@ -60,6 +108,12 @@ func ValidateRef(doc *SourceDocument, ref SourceRef) error {
if doc == nil {
return fmt.Errorf("source document must not be nil")
}
return validateRef(doc.ID, func(unitID int) (int, bool) {
return UnitIndex(doc, unitID)
}, ref)
}
func validateRef(documentID string, position func(int) (int, bool), ref SourceRef) error {
if isBlank(ref.SourceID) {
return fmt.Errorf("source ref source_id must not be empty")
}
@@ -72,15 +126,15 @@ func ValidateRef(doc *SourceDocument, ref SourceRef) error {
if ref.EndUnitID <= 0 {
return fmt.Errorf("source ref end_unit_id must be positive")
}
if ref.SourceID != doc.ID {
return fmt.Errorf("source ref source_id %q does not match document id %q", ref.SourceID, doc.ID)
if ref.SourceID != documentID {
return fmt.Errorf("source ref source_id %q does not match document id %q", ref.SourceID, documentID)
}
startIndex, ok := UnitIndex(doc, ref.StartUnitID)
startIndex, ok := position(ref.StartUnitID)
if !ok {
return fmt.Errorf("source ref start_unit_id %d was not found", ref.StartUnitID)
}
endIndex, ok := UnitIndex(doc, ref.EndUnitID)
endIndex, ok := position(ref.EndUnitID)
if !ok {
return fmt.Errorf("source ref end_unit_id %d was not found", ref.EndUnitID)
}

View File

@@ -43,6 +43,7 @@ type LLMDebugPrompt struct {
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id,omitempty"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
Messages []LLMDebugMessage `json:"messages,omitempty"`
@@ -266,10 +267,6 @@ const (
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
)
type ChunkExecutionClassProvider interface {
ExecutionClass() ExecutionClass
}
type ValidationResult struct {
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code,omitempty"`
@@ -292,6 +289,7 @@ type OutputRequest struct {
LLMProfile string `json:"llm_profile,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
ChunkMap *SerializedArtifact `json:"chunk_map,omitempty"`
EvidenceContext *SerializedArtifact `json:"evidence_context,omitempty"`
}
type OutputFile struct {

View File

@@ -0,0 +1,11 @@
package contracts
import "errors"
// ErrInvalidStructuredOutput identifies a provider response that cannot satisfy
// the caller's declared structured-output contract.
var ErrInvalidStructuredOutput = errors.New("invalid structured output")
// ErrLLMCapacityExceeded identifies backend admission exhaustion before model
// generation begins.
var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded")

View File

@@ -90,6 +90,22 @@ type TypedNormalizeRequest[T any] struct {
type TypedNormalizeResult[T any] struct {
Value T
Warnings []Warning
Retry *NormalizeRetry
}
// Normalize retry diagnostic limits bound module-provided values before the
// framework persists them in debug artifacts.
const (
MaxNormalizeRetryReasonCodeBytes = 128
MaxNormalizeRetryMessageBytes = 4096
)
// NormalizeRetry asks the framework to retry normalization while retaining a
// safe candidate for acceptance if the retry budget is exhausted.
type NormalizeRetry struct {
ReasonCode string
Message string
FallbackWarnings []Warning
}
type Normalizer[T any] interface {

View File

@@ -0,0 +1,66 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "notarius.source.evidence_context",
"title": "notarius_source_evidence_context_v1",
"type": "object",
"additionalProperties": false,
"required": ["source_id", "source_digest", "window_units", "selected_lanes", "contexts"],
"properties": {
"source_id": {"type": "string", "minLength": 1},
"source_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
"window_units": {"type": "integer", "minimum": 0},
"selected_lanes": {
"type": "array",
"minItems": 1,
"uniqueItems": true,
"items": {"type": "string", "minLength": 1}
},
"contexts": {
"type": "array",
"items": {"$ref": "#/$defs/context"}
}
},
"$defs": {
"source_ref": {
"type": "object",
"additionalProperties": false,
"required": ["source_id", "start_unit_id", "end_unit_id"],
"properties": {
"source_id": {"type": "string", "minLength": 1},
"start_unit_id": {"type": "integer", "minimum": 1},
"end_unit_id": {"type": "integer", "minimum": 1}
}
},
"unit": {
"type": "object",
"additionalProperties": false,
"required": ["id", "kind", "text", "ref"],
"properties": {
"id": {"type": "integer", "minimum": 1},
"kind": {"type": "string", "minLength": 1},
"text": {"type": "string", "minLength": 1},
"ref": {"$ref": "#/$defs/source_ref"},
"metadata": {"type": "object", "additionalProperties": true}
}
},
"evidence_ref": {
"type": "object",
"additionalProperties": false,
"required": ["lane_id", "source_ref"],
"properties": {
"lane_id": {"type": "string", "minLength": 1},
"source_ref": {"$ref": "#/$defs/source_ref"}
}
},
"context": {
"type": "object",
"additionalProperties": false,
"required": ["context_ref", "evidence_refs", "units"],
"properties": {
"context_ref": {"$ref": "#/$defs/source_ref"},
"evidence_refs": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/evidence_ref"}},
"units": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/unit"}}
}
}
}
}

View File

@@ -0,0 +1,196 @@
package evidencecontext
import (
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
type contribution struct {
laneID string
ref source.SourceRef
startPos int
endPos int
}
type expandedRange struct {
startPos int
endPos int
contributions []contribution
}
// Build validates accepted direct references, expands them by source-document
// position, and returns their deterministic context union.
func Build(request BuildRequest) (Document, error) {
if request.WindowUnits < 0 {
return Document{}, fmt.Errorf("window_units must not be negative")
}
lanes, err := normalizeSelectedLanes(request.SelectedLanes)
if err != nil {
return Document{}, err
}
if err := source.ValidateDocument(request.Source); err != nil {
return Document{}, fmt.Errorf("validate source document: %w", err)
}
digest, err := source.DigestDocument(request.Source)
if err != nil {
return Document{}, fmt.Errorf("digest source document: %w", err)
}
if digest != request.Source.Digest {
return Document{}, fmt.Errorf("source digest does not match source document digest")
}
selected := make(map[string]struct{}, len(lanes))
for _, laneID := range lanes {
selected[laneID] = struct{}{}
}
index := source.NewDocumentIndex(request.Source)
seen := make(map[evidenceKey]struct{})
contributions := make([]contribution, 0)
for laneIndex, laneEvidence := range request.LaneEvidence {
laneID := strings.TrimSpace(laneEvidence.LaneID)
if _, ok := selected[laneID]; !ok {
return Document{}, fmt.Errorf("lane evidence[%d] lane %q is not selected", laneIndex, laneID)
}
for refIndex, ref := range laneEvidence.SourceRefs {
if err := index.ValidateRef(ref); err != nil {
return Document{}, fmt.Errorf("lane %q source reference[%d]: %w", laneID, refIndex, err)
}
key := evidenceKey{laneID: laneID, ref: ref}
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
startPos, _ := index.Position(ref.StartUnitID)
endPos, _ := index.Position(ref.EndUnitID)
contributions = append(contributions, contribution{laneID: laneID, ref: ref, startPos: expandStart(startPos, request.WindowUnits), endPos: expandEnd(endPos, len(request.Source.Units), request.WindowUnits)})
}
}
sort.Slice(contributions, func(i, j int) bool { return lessContribution(contributions[i], contributions[j]) })
document := Document{
SourceID: request.Source.ID,
SourceDigest: digest,
WindowUnits: request.WindowUnits,
SelectedLanes: lanes,
Contexts: make([]Context, 0),
}
for _, rangeValue := range mergeRanges(contributions) {
context, err := buildContext(request.Source, rangeValue)
if err != nil {
return Document{}, err
}
document.Contexts = append(document.Contexts, context)
}
canonical, err := canonicalize(document)
if err != nil {
return Document{}, fmt.Errorf("validate evidence context: %w", err)
}
return clone(canonical)
}
type evidenceKey struct {
laneID string
ref source.SourceRef
}
func normalizeSelectedLanes(values []string) ([]string, error) {
if len(values) == 0 {
return nil, fmt.Errorf("selected_lanes must not be empty")
}
seen := make(map[string]struct{}, len(values))
lanes := make([]string, 0, len(values))
for index, raw := range values {
laneID := strings.TrimSpace(raw)
if laneID == "" {
return nil, fmt.Errorf("selected_lanes[%d] must not be empty", index)
}
if _, exists := seen[laneID]; exists {
return nil, fmt.Errorf("selected_lanes lane %q is duplicated", laneID)
}
seen[laneID] = struct{}{}
lanes = append(lanes, laneID)
}
sort.Strings(lanes)
return lanes, nil
}
func expandStart(position, window int) int {
if window > position {
return 0
}
return position - window
}
func expandEnd(position, length, window int) int {
last := length - 1
if window > last-position {
return last
}
return position + window
}
func lessContribution(left, right contribution) bool {
if left.startPos != right.startPos {
return left.startPos < right.startPos
}
if left.endPos != right.endPos {
return left.endPos < right.endPos
}
return lessEvidenceRef(EvidenceRef{LaneID: left.laneID, SourceRef: left.ref}, EvidenceRef{LaneID: right.laneID, SourceRef: right.ref})
}
func mergeRanges(values []contribution) []expandedRange {
if len(values) == 0 {
return nil
}
ranges := make([]expandedRange, 0, len(values))
for _, value := range values {
if len(ranges) == 0 || value.startPos > ranges[len(ranges)-1].endPos+1 {
ranges = append(ranges, expandedRange{startPos: value.startPos, endPos: value.endPos, contributions: []contribution{value}})
continue
}
current := &ranges[len(ranges)-1]
if value.endPos > current.endPos {
current.endPos = value.endPos
}
current.contributions = append(current.contributions, value)
}
return ranges
}
func buildContext(document *source.SourceDocument, value expandedRange) (Context, error) {
evidenceRefs := make([]EvidenceRef, 0, len(value.contributions))
for _, contribution := range value.contributions {
evidenceRefs = append(evidenceRefs, EvidenceRef{LaneID: contribution.laneID, SourceRef: contribution.ref})
}
sort.Slice(evidenceRefs, func(i, j int) bool { return lessEvidenceRef(evidenceRefs[i], evidenceRefs[j]) })
units := make([]source.SourceUnit, 0, value.endPos-value.startPos+1)
for position := value.startPos; position <= value.endPos; position++ {
unit, err := cloneSourceUnit(document.Units[position])
if err != nil {
return Context{}, fmt.Errorf("clone source unit at position %d: %w", position, err)
}
units = append(units, unit)
}
return Context{
ContextRef: source.SourceRef{SourceID: document.ID, StartUnitID: units[0].ID, EndUnitID: units[len(units)-1].ID},
EvidenceRefs: evidenceRefs,
Units: units,
}, nil
}
func lessEvidenceRef(left, right EvidenceRef) bool {
if left.LaneID != right.LaneID {
return left.LaneID < right.LaneID
}
if left.SourceRef.SourceID != right.SourceRef.SourceID {
return left.SourceRef.SourceID < right.SourceRef.SourceID
}
if left.SourceRef.StartUnitID != right.SourceRef.StartUnitID {
return left.SourceRef.StartUnitID < right.SourceRef.StartUnitID
}
return left.SourceRef.EndUnitID < right.SourceRef.EndUnitID
}

View File

@@ -0,0 +1,329 @@
package evidencecontext
import (
"bytes"
"embed"
"encoding/json"
"fmt"
"io"
"regexp"
"strings"
"sync"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"github.com/santhosh-tekuri/jsonschema/v6"
)
//go:embed assets/schemas/source_evidence_context.v1.json
var schemaAssets embed.FS
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
var (
loadSchemaOnce sync.Once
loadedSchema []byte
compiledSchema *jsonschema.Schema
loadSchemaErr error
)
// Codec owns strict serialization for the durable evidence-context contract.
type Codec struct{}
func New() *Codec { return &Codec{} }
func (c *Codec) Kind() contracts.ArtifactKind { return ArtifactKind }
func (c *Codec) Schema() contracts.ArtifactSchema {
raw, err := c.schemaBytes()
if err != nil {
return contracts.ArtifactSchema{}
}
return contracts.ArtifactSchema{ID: SchemaID, Name: SchemaName, Version: SchemaVersion, JSONSchema: raw}
}
func (c *Codec) MediaType() string { return MediaType }
// Serialize builds and encodes the framework-owned serialized artifact.
func Serialize(request BuildRequest) (contracts.SerializedArtifact, error) {
value, err := Build(request)
if err != nil {
return contracts.SerializedArtifact{}, err
}
codec := New()
content, err := codec.Encode(value)
if err != nil {
return contracts.SerializedArtifact{}, err
}
return contracts.SerializedArtifact{Kind: ArtifactKind, Schema: codec.Schema(), MediaType: MediaType, Content: content}, nil
}
func (c *Codec) Encode(value Document) ([]byte, error) {
if _, err := c.schemaBytes(); err != nil {
return nil, err
}
canonical, err := canonicalize(value)
if err != nil {
return nil, fmt.Errorf("encode evidence context: %w", err)
}
content, err := json.Marshal(canonical)
if err != nil {
return nil, fmt.Errorf("encode evidence context: %w", err)
}
if err := validateSchemaInstance(content); err != nil {
return nil, fmt.Errorf("encode evidence context: %w", err)
}
return content, nil
}
func (c *Codec) Decode(content []byte) (Document, error) {
if _, err := c.schemaBytes(); err != nil {
return Document{}, err
}
if err := validateSchemaInstance(content); err != nil {
return Document{}, fmt.Errorf("decode evidence context: %w", err)
}
decoder := json.NewDecoder(bytes.NewReader(content))
decoder.DisallowUnknownFields()
var value Document
if err := decoder.Decode(&value); err != nil {
return Document{}, fmt.Errorf("decode evidence context: %w", err)
}
var trailing any
if err := decoder.Decode(&trailing); err != io.EOF {
return Document{}, fmt.Errorf("decode evidence context: multiple JSON values")
}
canonical, err := canonicalize(value)
if err != nil {
return Document{}, fmt.Errorf("decode evidence context: %w", err)
}
return clone(canonical)
}
func (c *Codec) schemaBytes() ([]byte, error) {
loadSchemaOnce.Do(loadAndCompileSchema)
if loadSchemaErr != nil {
return nil, loadSchemaErr
}
return append([]byte(nil), loadedSchema...), nil
}
func loadAndCompileSchema() {
raw, err := schemaAssets.ReadFile("assets/schemas/source_evidence_context.v1.json")
if err != nil {
loadSchemaErr = fmt.Errorf("read source evidence context schema: %w", err)
return
}
var identity struct {
ID string `json:"$id"`
Title string `json:"title"`
Type string `json:"type"`
Required []string `json:"required"`
}
if err := json.Unmarshal(raw, &identity); err != nil {
loadSchemaErr = fmt.Errorf("decode source evidence context schema: %w", err)
return
}
if identity.ID != SchemaID || identity.Title != SchemaName || identity.Type != "object" || !hasRequiredFields(identity.Required) {
loadSchemaErr = fmt.Errorf("source evidence context schema identity or required fields are invalid")
return
}
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(raw))
if err != nil {
loadSchemaErr = fmt.Errorf("parse source evidence context schema: %w", err)
return
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("source-evidence-context-schema.json", schemaDocument); err != nil {
loadSchemaErr = fmt.Errorf("load source evidence context schema: %w", err)
return
}
compiled, err := compiler.Compile("source-evidence-context-schema.json")
if err != nil {
loadSchemaErr = fmt.Errorf("compile source evidence context schema: %w", err)
return
}
loadedSchema = append([]byte(nil), raw...)
compiledSchema = compiled
}
func validateSchemaInstance(content []byte) error {
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(content))
if err != nil {
return fmt.Errorf("payload is not valid JSON: %w", err)
}
if err := compiledSchema.Validate(instance); err != nil {
return fmt.Errorf("payload does not conform to source evidence context schema: %w", err)
}
return nil
}
func hasRequiredFields(required []string) bool {
want := map[string]bool{"source_id": true, "source_digest": true, "window_units": true, "selected_lanes": true, "contexts": true}
for _, field := range required {
delete(want, field)
}
return len(want) == 0
}
func canonicalize(value Document) (Document, error) {
owned, err := clone(value)
if err != nil {
return Document{}, err
}
value = owned
if err := requireIdentity("source_id", value.SourceID); err != nil {
return Document{}, err
}
if !digestPattern.MatchString(value.SourceDigest) {
return Document{}, fmt.Errorf("source_digest must be a sha256 digest")
}
if value.WindowUnits < 0 {
return Document{}, fmt.Errorf("window_units must not be negative")
}
if err := validateSelectedLanes(value.SelectedLanes); err != nil {
return Document{}, err
}
if value.Contexts == nil {
value.Contexts = make([]Context, 0)
}
selected := make(map[string]struct{}, len(value.SelectedLanes))
for _, laneID := range value.SelectedLanes {
selected[laneID] = struct{}{}
}
seenUnits := make(map[int]struct{})
for contextIndex := range value.Contexts {
context, err := canonicalizeContext(value.SourceID, selected, seenUnits, value.Contexts[contextIndex], contextIndex)
if err != nil {
return Document{}, err
}
value.Contexts[contextIndex] = context
}
return value, nil
}
func validateSelectedLanes(lanes []string) error {
if len(lanes) == 0 {
return fmt.Errorf("selected_lanes must not be empty")
}
for index, laneID := range lanes {
if err := requireIdentity(fmt.Sprintf("selected_lanes[%d]", index), laneID); err != nil {
return err
}
if index > 0 && lanes[index-1] >= laneID {
return fmt.Errorf("selected_lanes must be unique and in lexical order")
}
}
return nil
}
func canonicalizeContext(sourceID string, selected map[string]struct{}, seenUnits map[int]struct{}, value Context, contextIndex int) (Context, error) {
prefix := fmt.Sprintf("contexts[%d]", contextIndex)
if len(value.EvidenceRefs) == 0 {
return Context{}, fmt.Errorf("%s.evidence_refs must not be empty", prefix)
}
if len(value.Units) == 0 {
return Context{}, fmt.Errorf("%s.units must not be empty", prefix)
}
if err := validateRefIdentity(sourceID, value.ContextRef, prefix+".context_ref"); err != nil {
return Context{}, err
}
positions := make(map[int]int, len(value.Units))
for unitIndex := range value.Units {
unit, err := cloneSourceUnit(value.Units[unitIndex])
if err != nil {
return Context{}, fmt.Errorf("%s.units[%d]: %w", prefix, unitIndex, err)
}
if unit.ID <= 0 || strings.TrimSpace(unit.Kind) == "" || strings.TrimSpace(unit.Text) == "" {
return Context{}, fmt.Errorf("%s.units[%d] has invalid required fields", prefix, unitIndex)
}
if err := validateRefIdentity(sourceID, unit.Ref, fmt.Sprintf("%s.units[%d].ref", prefix, unitIndex)); err != nil {
return Context{}, err
}
if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
return Context{}, fmt.Errorf("%s.units[%d].ref must identify unit id %d", prefix, unitIndex, unit.ID)
}
if _, exists := positions[unit.ID]; exists {
return Context{}, fmt.Errorf("%s.units contains duplicate unit id %d", prefix, unit.ID)
}
if _, exists := seenUnits[unit.ID]; exists {
return Context{}, fmt.Errorf("contexts contain duplicate unit id %d", unit.ID)
}
positions[unit.ID] = unitIndex
seenUnits[unit.ID] = struct{}{}
value.Units[unitIndex] = unit
}
if value.ContextRef.StartUnitID != value.Units[0].ID || value.ContextRef.EndUnitID != value.Units[len(value.Units)-1].ID {
return Context{}, fmt.Errorf("%s.context_ref must identify the first and last units", prefix)
}
for evidenceIndex := range value.EvidenceRefs {
evidence := value.EvidenceRefs[evidenceIndex]
if _, ok := selected[evidence.LaneID]; !ok {
return Context{}, fmt.Errorf("%s.evidence_refs[%d].lane_id is not selected", prefix, evidenceIndex)
}
if err := requireIdentity(fmt.Sprintf("%s.evidence_refs[%d].lane_id", prefix, evidenceIndex), evidence.LaneID); err != nil {
return Context{}, err
}
if err := validateRefIdentity(sourceID, evidence.SourceRef, fmt.Sprintf("%s.evidence_refs[%d].source_ref", prefix, evidenceIndex)); err != nil {
return Context{}, err
}
start, startOK := positions[evidence.SourceRef.StartUnitID]
end, endOK := positions[evidence.SourceRef.EndUnitID]
if !startOK || !endOK || start > end {
return Context{}, fmt.Errorf("%s.evidence_refs[%d].source_ref is outside context units", prefix, evidenceIndex)
}
if evidenceIndex > 0 && !lessEvidenceRef(value.EvidenceRefs[evidenceIndex-1], evidence) {
return Context{}, fmt.Errorf("%s.evidence_refs must be unique and in deterministic order", prefix)
}
}
return value, nil
}
func validateRefIdentity(sourceID string, ref source.SourceRef, field string) error {
if ref.SourceID != sourceID {
return fmt.Errorf("%s.source_id does not match source_id", field)
}
if ref.StartUnitID <= 0 || ref.EndUnitID <= 0 {
return fmt.Errorf("%s endpoints must be positive", field)
}
return nil
}
func requireIdentity(field, value string) error {
if strings.TrimSpace(value) == "" || strings.TrimSpace(value) != value {
return fmt.Errorf("%s must be a non-empty trimmed string", field)
}
return nil
}
func clone(value Document) (Document, error) {
value.SelectedLanes = append([]string(nil), value.SelectedLanes...)
if value.Contexts == nil {
value.Contexts = make([]Context, 0)
} else {
contexts := make([]Context, len(value.Contexts))
for contextIndex, context := range value.Contexts {
contexts[contextIndex].ContextRef = context.ContextRef
contexts[contextIndex].EvidenceRefs = append([]EvidenceRef(nil), context.EvidenceRefs...)
contexts[contextIndex].Units = make([]source.SourceUnit, len(context.Units))
for unitIndex, unit := range context.Units {
cloned, err := cloneSourceUnit(unit)
if err != nil {
return Document{}, fmt.Errorf("clone contexts[%d].units[%d]: %w", contextIndex, unitIndex, err)
}
contexts[contextIndex].Units[unitIndex] = cloned
}
}
value.Contexts = contexts
}
return value, nil
}
func cloneSourceUnit(unit source.SourceUnit) (source.SourceUnit, error) {
metadata, err := source.CloneMetadata(unit.Metadata)
if err != nil {
return source.SourceUnit{}, fmt.Errorf("clone metadata: %w", err)
}
unit.Metadata = metadata
return unit, nil
}

View File

@@ -0,0 +1,350 @@
package evidencecontext
import (
"bytes"
"encoding/json"
"math"
"os"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
func TestBuildExpandsAndMergesEvidenceByDocumentPosition(t *testing.T) {
for _, test := range []struct {
name string
window int
evidence []LaneEvidence
wantUnits [][]int
wantRefs [][]EvidenceRef
}{
{
name: "zero window",
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}},
wantUnits: [][]int{{3}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}}},
},
{
name: "non monotonic ids use positions and clip boundaries",
window: 1,
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}},
wantUnits: [][]int{{10, 3, 30}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}}},
},
{
name: "separate gaps stay separate",
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(10, 10), ref(50, 50)}}},
wantUnits: [][]int{{10}, {50}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(10, 10)}}, {{LaneID: "npcs", SourceRef: ref(50, 50)}}},
},
{
name: "overlapping windows merge",
window: 1,
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3), ref(30, 30)}}},
wantUnits: [][]int{{10, 3, 30, 7}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(3, 3)}, {LaneID: "npcs", SourceRef: ref(30, 30)}}},
},
{
name: "contiguous windows merge",
window: 1,
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(10, 10), ref(7, 7)}}},
wantUnits: [][]int{{10, 3, 30, 7, 50}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(7, 7)}, {LaneID: "npcs", SourceRef: ref(10, 10)}}},
},
{
name: "duplicate contributions retain unique lane attribution",
evidence: []LaneEvidence{
{LaneID: "spells", SourceRefs: []source.SourceRef{ref(30, 30), ref(30, 30)}},
{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(30, 30)}},
},
wantUnits: [][]int{{30}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(30, 30)}, {LaneID: "spells", SourceRef: ref(30, 30)}}},
},
{
name: "empty contributions retain explicit empty contexts",
evidence: []LaneEvidence{{LaneID: "npcs"}},
wantUnits: [][]int{},
wantRefs: [][]EvidenceRef{},
},
{
name: "largest window clips without overflow",
window: math.MaxInt,
evidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(30, 30)}}},
wantUnits: [][]int{{10, 3, 30, 7, 50}},
wantRefs: [][]EvidenceRef{{{LaneID: "npcs", SourceRef: ref(30, 30)}}},
},
} {
t.Run(test.name, func(t *testing.T) {
document := testDocument(t)
got, err := Build(BuildRequest{Source: document, WindowUnits: test.window, SelectedLanes: []string{"spells", "npcs"}, LaneEvidence: test.evidence})
if err != nil {
t.Fatalf("Build() error = %v", err)
}
if want := []string{"npcs", "spells"}; !reflect.DeepEqual(got.SelectedLanes, want) {
t.Fatalf("SelectedLanes = %#v, want %#v", got.SelectedLanes, want)
}
if got.WindowUnits != test.window || got.SourceID != document.ID || got.SourceDigest != document.Digest {
t.Fatalf("Build() identity = %#v, want source and window identity", got)
}
if actual := contextUnitIDs(got.Contexts); !reflect.DeepEqual(actual, test.wantUnits) {
t.Fatalf("context unit ids = %#v, want %#v", actual, test.wantUnits)
}
if actual := contextEvidenceRefs(got.Contexts); !reflect.DeepEqual(actual, test.wantRefs) {
t.Fatalf("context evidence refs = %#v, want %#v", actual, test.wantRefs)
}
})
}
}
func TestBuildIsStableAndOwnsSourceAndInputs(t *testing.T) {
document := testDocument(t)
refs := []source.SourceRef{ref(30, 30), ref(3, 3)}
request := BuildRequest{
Source: document,
WindowUnits: 1,
SelectedLanes: []string{"spells", "npcs"},
LaneEvidence: []LaneEvidence{{LaneID: "spells", SourceRefs: refs}, {LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}},
}
first, err := Build(request)
if err != nil {
t.Fatal(err)
}
secondRequest := request
secondRequest.LaneEvidence = []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}, {LaneID: "spells", SourceRefs: []source.SourceRef{ref(3, 3), ref(30, 30)}}}
second, err := Build(secondRequest)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(first, second) {
t.Fatalf("Build() order differs:\nfirst: %#v\nsecond: %#v", first, second)
}
first.SelectedLanes[0] = "changed"
first.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] = "changed"
if document.Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
t.Fatal("Build() returned metadata aliases to source document")
}
document.Units[0].Metadata["nested"].(map[string]any)["value"] = "later"
if second.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
t.Fatal("Build() retained metadata aliases to source document")
}
refs[0].StartUnitID = 999
if !containsEvidenceRef(second.Contexts[0].EvidenceRefs, ref(30, 30)) {
t.Fatal("Build() retained source-reference input aliases")
}
}
func TestBuildRejectsInvalidInputs(t *testing.T) {
for _, test := range []struct {
name string
mutate func(*BuildRequest)
want string
}{
{name: "negative window", mutate: func(request *BuildRequest) { request.WindowUnits = -1 }, want: "window_units"},
{name: "blank selected lane", mutate: func(request *BuildRequest) { request.SelectedLanes = []string{" "} }, want: "selected_lanes"},
{name: "duplicate selected lane", mutate: func(request *BuildRequest) { request.SelectedLanes = []string{"npcs", " npcs "} }, want: "duplicated"},
{name: "unselected contribution", mutate: func(request *BuildRequest) {
request.LaneEvidence = []LaneEvidence{{LaneID: "other", SourceRefs: []source.SourceRef{ref(3, 3)}}}
}, want: "not selected"},
{name: "source digest mismatch", mutate: func(request *BuildRequest) { request.Source.Digest = "sha256:" + strings.Repeat("0", 64) }, want: "does not match"},
{name: "invalid reference", mutate: func(request *BuildRequest) {
request.LaneEvidence = []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(99, 99)}}}
}, want: "source reference[0]"},
} {
t.Run(test.name, func(t *testing.T) {
request := BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}}
test.mutate(&request)
if _, err := Build(request); err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Build() error = %v, want %q", err, test.want)
}
})
}
}
func TestCodecRoundTripsCompactFixtureAndOwnsDecodedValues(t *testing.T) {
fixture, err := os.ReadFile("testdata/source_evidence_context.v1.json")
if err != nil {
t.Fatal(err)
}
codec := New()
value, err := codec.Decode(fixture)
if err != nil {
t.Fatalf("Decode(fixture) error = %v", err)
}
encoded, err := codec.Encode(value)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(encoded, bytes.TrimSpace(fixture)) {
t.Fatalf("fixture does not use canonical encoding\nwant: %s\n got: %s", fixture, encoded)
}
value.Contexts[0].Units[0].Text = "changed"
decoded, err := codec.Decode(fixture)
if err != nil {
t.Fatal(err)
}
if decoded.Contexts[0].Units[0].Text != "The party meets Rowan." {
t.Fatal("Decode() reused mutable document storage")
}
built, err := Build(BuildRequest{Source: testDocument(t), WindowUnits: 1, SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}})
if err != nil {
t.Fatal(err)
}
content, err := codec.Encode(built)
if err != nil {
t.Fatal(err)
}
first, err := codec.Decode(content)
if err != nil {
t.Fatal(err)
}
second, err := codec.Decode(content)
if err != nil {
t.Fatal(err)
}
first.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] = "changed"
if second.Contexts[0].Units[0].Metadata["nested"].(map[string]any)["value"] != "original" {
t.Fatal("Decode() returned metadata aliases")
}
}
func TestCodecRejectsInvalidDurableBoundaries(t *testing.T) {
value, err := Build(BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}, LaneEvidence: []LaneEvidence{{LaneID: "npcs", SourceRefs: []source.SourceRef{ref(3, 3)}}}})
if err != nil {
t.Fatal(err)
}
for _, test := range []struct {
name string
mutate func(*Document)
}{
{name: "unsorted lanes", mutate: func(value *Document) { value.SelectedLanes = []string{"z", "a"} }},
{name: "context range mismatch", mutate: func(value *Document) { value.Contexts[0].ContextRef.EndUnitID = 999 }},
{name: "mismatched evidence source", mutate: func(value *Document) { value.Contexts[0].EvidenceRefs[0].SourceRef.SourceID = "other" }},
{name: "invalid evidence range", mutate: func(value *Document) {
value.Contexts[0].EvidenceRefs[0].SourceRef.StartUnitID = 10
}},
{name: "duplicate context unit", mutate: func(value *Document) { value.Contexts = append(value.Contexts, value.Contexts[0]) }},
} {
t.Run(test.name, func(t *testing.T) {
candidate, err := clone(value)
if err != nil {
t.Fatal(err)
}
test.mutate(&candidate)
if _, err := New().Encode(candidate); err == nil {
t.Fatal("Encode() error = nil, want durable model rejection")
}
})
}
content, err := New().Encode(value)
if err != nil {
t.Fatal(err)
}
for _, test := range []struct {
name string
mutate func(map[string]any)
}{
{name: "missing contexts", mutate: func(value map[string]any) { delete(value, "contexts") }},
{name: "null contexts", mutate: func(value map[string]any) { value["contexts"] = nil }},
{name: "unknown fixed field", mutate: func(value map[string]any) { value["unknown"] = true }},
{name: "missing units", mutate: func(value map[string]any) { delete(contextObject(value, 0), "units") }},
{name: "null evidence refs", mutate: func(value map[string]any) { contextObject(value, 0)["evidence_refs"] = nil }},
} {
t.Run(test.name, func(t *testing.T) {
raw := decodeJSON(t, content)
test.mutate(raw)
mutated, err := json.Marshal(raw)
if err != nil {
t.Fatal(err)
}
if _, err := New().Decode(mutated); err == nil {
t.Fatal("Decode() error = nil, want strict payload rejection")
}
})
}
if _, err := New().Decode(append(content, []byte(" {}")...)); err == nil {
t.Fatal("Decode() error = nil, want trailing JSON rejection")
}
}
func TestSerializeUsesFixedArtifactIdentity(t *testing.T) {
artifact, err := Serialize(BuildRequest{Source: testDocument(t), SelectedLanes: []string{"npcs"}})
if err != nil {
t.Fatal(err)
}
if artifact.Kind != ArtifactKind || artifact.MediaType != MediaType || artifact.Schema.ID != SchemaID || artifact.Schema.Name != SchemaName || artifact.Schema.Version != SchemaVersion {
t.Fatalf("Serialize() = %#v, want fixed artifact identity", artifact)
}
decoded, err := New().Decode(artifact.Content)
if err != nil || len(decoded.Contexts) != 0 || decoded.Contexts == nil {
t.Fatalf("Decode(Serialize()) = %#v, %v; want explicit empty contexts", decoded, err)
}
}
func testDocument(t *testing.T) *source.SourceDocument {
t.Helper()
document := &source.SourceDocument{
ID: "session", Kind: "transcript", Format: "application/json",
Units: []source.SourceUnit{
{ID: 10, Kind: "segment", Text: "first", Ref: ref(10, 10), Metadata: map[string]any{"nested": map[string]any{"value": "original"}}},
{ID: 3, Kind: "segment", Text: "second", Ref: ref(3, 3)},
{ID: 30, Kind: "segment", Text: "third", Ref: ref(30, 30)},
{ID: 7, Kind: "segment", Text: "fourth", Ref: ref(7, 7)},
{ID: 50, Kind: "segment", Text: "fifth", Ref: ref(50, 50)},
},
}
digest, err := source.DigestDocument(document)
if err != nil {
t.Fatal(err)
}
document.Digest = digest
return document
}
func ref(start, end int) source.SourceRef {
return source.SourceRef{SourceID: "session", StartUnitID: start, EndUnitID: end}
}
func contextUnitIDs(contexts []Context) [][]int {
values := make([][]int, len(contexts))
for index, context := range contexts {
values[index] = make([]int, len(context.Units))
for unitIndex, unit := range context.Units {
values[index][unitIndex] = unit.ID
}
}
return values
}
func contextEvidenceRefs(contexts []Context) [][]EvidenceRef {
values := make([][]EvidenceRef, len(contexts))
for index, context := range contexts {
values[index] = append([]EvidenceRef(nil), context.EvidenceRefs...)
}
return values
}
func containsEvidenceRef(values []EvidenceRef, want source.SourceRef) bool {
for _, value := range values {
if value.SourceRef == want {
return true
}
}
return false
}
func decodeJSON(t *testing.T, content []byte) map[string]any {
t.Helper()
decoder := json.NewDecoder(bytes.NewReader(content))
decoder.UseNumber()
var value map[string]any
if err := decoder.Decode(&value); err != nil {
t.Fatal(err)
}
return value
}
func contextObject(value map[string]any, index int) map[string]any {
return value["contexts"].([]any)[index].(map[string]any)
}

View File

@@ -0,0 +1,50 @@
// Package evidencecontext owns the durable source evidence-context contract.
package evidencecontext
import (
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
const (
ArtifactKind contracts.ArtifactKind = "source/evidence-context"
SchemaID = "notarius.source.evidence_context"
SchemaName = "notarius_source_evidence_context_v1"
SchemaVersion = "v1"
MediaType = "application/json"
)
// Document is the durable union of direct evidence and surrounding source
// context selected for one accepted source document.
type Document struct {
SourceID string `json:"source_id"`
SourceDigest string `json:"source_digest"`
WindowUnits int `json:"window_units"`
SelectedLanes []string `json:"selected_lanes"`
Contexts []Context `json:"contexts"`
}
type Context struct {
ContextRef source.SourceRef `json:"context_ref"`
EvidenceRefs []EvidenceRef `json:"evidence_refs"`
Units []source.SourceUnit `json:"units"`
}
type EvidenceRef struct {
LaneID string `json:"lane_id"`
SourceRef source.SourceRef `json:"source_ref"`
}
// LaneEvidence attributes direct source references to one selected lane.
type LaneEvidence struct {
LaneID string `json:"lane_id"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
// BuildRequest supplies accepted source material and direct lane evidence.
type BuildRequest struct {
Source *source.SourceDocument
WindowUnits int
SelectedLanes []string
LaneEvidence []LaneEvidence
}

View File

@@ -0,0 +1 @@
{"source_id":"session-alpha","source_digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","window_units":0,"selected_lanes":["npcs"],"contexts":[{"context_ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13},"evidence_refs":[{"lane_id":"npcs","source_ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}],"units":[{"id":13,"kind":"transcript_segment","text":"The party meets Rowan.","ref":{"source_id":"session-alpha","start_unit_id":13,"end_unit_id":13}}]}]}

View File

@@ -12,7 +12,7 @@ import (
"strings"
"time"
"gitea.maximumdirect.net/eric/scriptorium"
"gitea.maximumdirect.net/eric/promptkit"
)
type AssetSource struct {
@@ -21,8 +21,9 @@ type AssetSource struct {
}
type AssetRegistry struct {
prompts []AssetSource
schemas []AssetSource
prompts []AssetSource
schemas []AssetSource
fallbackProfiles []AssetSource
}
type AssetHashPart struct {
@@ -58,6 +59,20 @@ func (r *AssetRegistry) RegisterSchemaFS(fsys fs.FS, root string) error {
return nil
}
// RegisterFallbackProfileFS registers profile assets that PromptKit uses only
// when an operator-configured source does not provide a matching profile.
func (r *AssetRegistry) RegisterFallbackProfileFS(fsys fs.FS, root string) error {
if r == nil {
return fmt.Errorf("asset registry must not be nil")
}
source, err := newAssetSource(fsys, root)
if err != nil {
return fmt.Errorf("register fallback profile assets: %w", err)
}
r.fallbackProfiles = append(r.fallbackProfiles, source)
return nil
}
func (r *AssetRegistry) PromptFS() (fs.FS, error) {
if r == nil {
return nil, fmt.Errorf("asset registry must not be nil")
@@ -72,19 +87,76 @@ func (r *AssetRegistry) SchemaFS() (fs.FS, error) {
return flattenAssetSources(r.schemas)
}
func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
func (r *AssetRegistry) FallbackProfileFS() (fs.FS, error) {
if r == nil {
return nil, fmt.Errorf("asset registry must not be nil")
}
return flattenAssetSources(r.fallbackProfiles)
}
// FallbackProfileDigest returns a deterministic, non-secret identity for the
// flattened fallback profile assets.
func (r *AssetRegistry) FallbackProfileDigest() (string, error) {
_, digest, _, err := r.fallbackProfileAssets()
return digest, err
}
func (r *AssetRegistry) PromptKitOptions() ([]promptkit.Option, error) {
options, _, err := r.promptKitOptions()
return options, err
}
func (r *AssetRegistry) promptKitOptions() ([]promptkit.Option, string, error) {
promptFS, err := r.PromptFS()
if err != nil {
return nil, fmt.Errorf("prepare prompt assets: %w", err)
return nil, "", fmt.Errorf("prepare prompt assets: %w", err)
}
schemaFS, err := r.SchemaFS()
if err != nil {
return nil, fmt.Errorf("prepare schema assets: %w", err)
return nil, "", fmt.Errorf("prepare schema assets: %w", err)
}
return []scriptorium.Option{
scriptorium.WithPromptFS(promptFS, "."),
scriptorium.WithSchemaFS(schemaFS, "."),
}, nil
options := []promptkit.Option{
promptkit.WithPromptFS(promptFS, "."),
promptkit.WithSchemaFS(schemaFS, "."),
}
fallbackFS, fallbackDigest, hasFallback, err := r.fallbackProfileAssets()
if err != nil {
return nil, "", err
}
if hasFallback {
options = append(options, promptkit.WithFallbackProfileFS(fallbackFS, "."))
}
return options, fallbackDigest, nil
}
func (r *AssetRegistry) promptKitFallbackProfileOption() (promptkit.Option, bool, error) {
fallbackFS, _, hasFallback, err := r.fallbackProfileAssets()
if err != nil {
return nil, false, err
}
if !hasFallback {
return nil, false, nil
}
return promptkit.WithFallbackProfileFS(fallbackFS, "."), true, nil
}
func (r *AssetRegistry) fallbackProfileAssets() (fs.FS, string, bool, error) {
if r == nil {
return nil, "", false, fmt.Errorf("asset registry must not be nil")
}
if len(r.fallbackProfiles) == 0 {
empty := sha256.Sum256([]byte("notarius:fallback-profile-assets:empty"))
return nil, "sha256:" + hex.EncodeToString(empty[:]), false, nil
}
fallbackFS, err := r.FallbackProfileFS()
if err != nil {
return nil, "", false, fmt.Errorf("prepare fallback profile assets: %w", err)
}
digest, err := hashAssetFilesystem(fallbackFS)
if err != nil {
return nil, "", false, err
}
return fallbackFS, digest, true, nil
}
func HashAssets(parts []AssetHashPart) (string, error) {
@@ -117,6 +189,28 @@ func HashAssets(parts []AssetHashPart) (string, error) {
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil
}
func hashAssetFilesystem(fsys fs.FS) (string, error) {
var parts []AssetHashPart
err := fs.WalkDir(fsys, ".", func(name string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
return nil
}
parts = append(parts, AssetHashPart{FS: fsys, Path: name})
return nil
})
if err != nil {
return "", fmt.Errorf("walk assets for digest: %w", err)
}
if len(parts) > 0 {
return HashAssets(parts)
}
empty := sha256.Sum256([]byte("notarius:fallback-profile-assets:empty"))
return "sha256:" + hex.EncodeToString(empty[:]), nil
}
func newAssetSource(fsys fs.FS, root string) (AssetSource, error) {
if fsys == nil {
return AssetSource{}, fmt.Errorf("filesystem must not be nil")
@@ -248,6 +342,9 @@ func (m assetMapFS) dirEntries(dir string) []fs.DirEntry {
children[childName] = entry
}
if len(children) == 0 {
if dir == "." {
return []fs.DirEntry{}
}
return nil
}
names := make([]string, 0, len(children))

Some files were not shown because too many files have changed in this diff Show More