Compare commits
247 Commits
96a49bb7cd
...
v0.2.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 39388e96d4 | |||
| 12ac25bd63 | |||
| 394278e1f2 | |||
| 5cd7f8e737 | |||
| bf3fadf9ae | |||
| 58815aaf33 | |||
| ce857966f1 | |||
| a3bd0c1867 | |||
| b05634ee86 | |||
| 4829f94157 | |||
| 67b315099d | |||
| b5c86de4d7 | |||
| 2eeca2ed5a | |||
| b5aaeb1c78 | |||
| 9171b66a41 | |||
| b4363b3b73 | |||
| 241e9d2a89 | |||
| 715fff7b72 | |||
| d627b91b4f | |||
| a67b3aa76d | |||
| a16dcdfa52 | |||
| 46e4466d28 | |||
| 71a004bfc8 | |||
| f8333f2c15 | |||
| f603f7ac64 | |||
| 7a00e7049c | |||
| 2a9db9a957 | |||
| de046a8f13 | |||
| f1a6574013 | |||
| 4bca6d3103 | |||
| 7c569a3d8c | |||
| 8e04ef9e2b | |||
| 53a330587b | |||
| 7cfab8ada0 | |||
| 5c82b62856 | |||
| de8ed41b34 | |||
| d1eaec4dad | |||
| c0ec068f53 | |||
| 5cbd9e56e4 | |||
| 53490cdb59 | |||
| 7c94b5eeed | |||
| 4f2864fc96 | |||
| 893b03fccf | |||
| 256cc98ddb | |||
| e61e522662 | |||
| a4c7eca87b | |||
| 224a8292c4 | |||
| 64d461fc18 | |||
| fb1134e591 | |||
| 1da29e6788 | |||
| 0d947549fb | |||
| 950fba17ce | |||
| 678d2c6099 | |||
| db8db5ffc5 | |||
| 94b3eafb1a | |||
| f6981e2264 | |||
| 2f506f4985 | |||
| fdf8c4afd4 | |||
| 74c793e6a1 | |||
| b5835fbc37 | |||
| fd3f7b85cc | |||
| d86b74f485 | |||
| 59cbf1eb27 | |||
| ee43add75c | |||
| 46761706a2 | |||
| 5a968b64eb | |||
| 7b077c269d | |||
| 80ec939383 | |||
| d3c4d6f133 | |||
| 5ad661f95f | |||
| d63e5c6852 | |||
| fbb8e0d241 | |||
| 8d9a496935 | |||
| d1c48db4bc | |||
| 6bd781d344 | |||
| 8a12c56971 | |||
| 26bd59a5a2 | |||
| 927a7beb88 | |||
| f7059607af | |||
| 63de44c347 | |||
| f0ede9dacc | |||
| 5711f8b9e3 | |||
| da83510234 | |||
| f51b22bea7 | |||
| f320c2fcee | |||
| 4ba1e50a89 | |||
| 2a7e025251 | |||
| 3da20e9d6a | |||
| b1c0faa748 | |||
| 989f2c220b | |||
| 7e35915b3e | |||
| 24238d249e | |||
| 9614469b45 | |||
| 29ee68824d | |||
| aeaaf44ae0 | |||
| d752c51aec | |||
| 8199d95dc1 | |||
| 97cdb01357 | |||
| 7a66095912 | |||
| 9d1356a20e | |||
| e4471fc300 | |||
| 84a2854b5e | |||
| a1b76093ce | |||
| 1aa30a73db | |||
| dc7c0e2f9e | |||
| e2cb0d901a | |||
| 8e0b029f5f | |||
| 9bbf2535dd | |||
| 83fde83a58 | |||
| bef3d1359d | |||
| 1ff449435f | |||
| 6e21c83fd8 | |||
| 6dc9d522b1 | |||
| 8adcf6840d | |||
| f5ed30e455 | |||
| cacf3f24e7 | |||
| f08ca4ddfa | |||
| e1c2f3c202 | |||
| 9614eb540d | |||
| 1b46596a39 | |||
| e043d61a99 | |||
| ad89782c9b | |||
| cd29265d5d | |||
| 2f36b7c3b6 | |||
| b8b3f3abfa | |||
| b490297cde | |||
| 06148074a2 | |||
| 16a998055c | |||
| 97c9a8e5ce | |||
| 66415fd1fa | |||
| bfe25609a7 | |||
| 36e0512454 | |||
| b02f667107 | |||
| ed2b6f4580 | |||
| cb7f145c76 | |||
| 2b9d2eaeaa | |||
| 61016671ab | |||
| b2c076946b | |||
| 250c5c22b8 | |||
| 4b0b166143 | |||
| 90481a0e4b | |||
| 2cbaf20e55 | |||
| a263a0840c | |||
| 14991cf58b | |||
| ab0b4e350c | |||
| 7b2fb0880d | |||
| 748e02db80 | |||
| 23c55f8925 | |||
| 906d97b391 | |||
| f15fd4f9c1 | |||
| 7aadb088a6 | |||
| 9de399432e | |||
| 5bdd56cfb1 | |||
| 64ea23c21f | |||
| 7071102ab7 | |||
| 9184072839 | |||
| c437682407 | |||
| 22d4f29670 | |||
| afb7ed3cf1 | |||
| f846f252c0 | |||
| f5618d1f0c | |||
| f94ab0a6bf | |||
| 41b52aae74 | |||
| ed36f7d7fd | |||
| 3ba2bfd7f6 | |||
| b344d16dc1 | |||
| 6e12c09952 | |||
| 07460341e3 | |||
| 732b13669f | |||
| 3a8a82ebc9 | |||
| 1c9819f08e | |||
| 447c4f73f9 | |||
| e01b8d1b6d | |||
| 3d70920f3d | |||
| 110593ece1 | |||
| a1f5dce405 | |||
| 50aa60e0b8 | |||
| 2fbb3813aa | |||
| 6dd695611c | |||
| 92acb45775 | |||
| d3e171aa82 | |||
| c6f330eb06 | |||
| 20cfbfd311 | |||
| fb043325e1 | |||
| 06c0259788 | |||
| 3d5fd9dc05 | |||
| 3ba2c62cc1 | |||
| e2ab01f9d2 | |||
| 5186e061a8 | |||
| c4c907d421 | |||
| fa5076f5f1 | |||
| 8b5a4e0efd | |||
| 3eb68baca6 | |||
| ae97adb8b0 | |||
| 79b9fffcaf | |||
| be22852daa | |||
| f5107045c3 | |||
| 2c98763b9b | |||
| d2eb763b9b | |||
| 0f25e7339f | |||
| 87c57681f6 | |||
| 3d0d79360e | |||
| f08b407b72 | |||
| 4ff2c7795f | |||
| 3bfe05ab56 | |||
| 7806dba509 | |||
| ac53f83ac8 | |||
| 385e4593f4 | |||
| f64bb7c883 | |||
| 9d3175d36a | |||
| 4f96abf42c | |||
| d88bcb6070 | |||
| 0cca3b1f5d | |||
| bbc83ab042 | |||
| 2cba6d4512 | |||
| e70450c401 | |||
| 4d3351c774 | |||
| a586257d5e | |||
| b8163091cc | |||
| c7b3af82b4 | |||
| a42b06ba20 | |||
| 8d62973627 | |||
| 8cdefc72a1 | |||
| d3a8dc7930 | |||
| 86ebb62f84 | |||
| 50191ee694 | |||
| 3c35124db4 | |||
| e4ec521bed | |||
| 2111e01142 | |||
| a39eea7ed6 | |||
| 7bcce9953e | |||
| 9746a42e04 | |||
| 47bacc7abb | |||
| 8824948910 | |||
| 8cb11e60e4 | |||
| 26142f0e05 | |||
| 1542a12497 | |||
| a9250206d5 | |||
| 5bd0ba7a72 | |||
| 286fb9dce7 | |||
| 9fa9154dda | |||
| 604c7a7945 | |||
| a0f5e6e2b9 | |||
| 561d65a505 | |||
| 205e2a9908 | |||
| 8c59b6af14 | |||
| b3328b93e5 |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -1,6 +1,8 @@
|
||||
# build and testing artifacts
|
||||
notarius
|
||||
notarius-output
|
||||
workspace/
|
||||
.codebase-memory/
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
@@ -72,4 +74,3 @@ Icon
|
||||
Network Trash Folder
|
||||
Temporary Items
|
||||
.apdisk
|
||||
|
||||
|
||||
63
README.md
63
README.md
@@ -1,32 +1,45 @@
|
||||
# Notarius
|
||||
|
||||
Notarius is a Go CLI for extracting structured artifacts from source material
|
||||
with explicit, configurable pipeline modules.
|
||||
Notarius is a Go CLI for turning source material into structured artifacts with
|
||||
configured extraction pipelines. The implemented D&D workflow reads Seriatim
|
||||
transcript JSON and can produce scene descriptions, item and currency events,
|
||||
NPC identities, combat turns, NPC interactions, and spell casts.
|
||||
|
||||
The current implementation reads Seriatim transcript JSON, chunks the source
|
||||
units, extracts D&D spell-cast artifacts with a Scriptorium-backed LLM runtime,
|
||||
and writes JSON output plus diagnostics when enabled.
|
||||
## Quickstart
|
||||
|
||||
```sh
|
||||
OPENROUTER_API_KEY=... \
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json
|
||||
```
|
||||
Provide an OpenRouter API key through the environment, then run the maintained
|
||||
minimal example:
|
||||
|
||||
This invocation uses the maintained example configuration and input. See the
|
||||
configuration and operations references for profile selection, credentials, and
|
||||
run artifacts.
|
||||
~~~
|
||||
OPENROUTER_API_KEY=your-api-key \
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-minimal.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json
|
||||
~~~
|
||||
|
||||
Useful references:
|
||||
The command publishes a JSON output bundle. Its command syntax and exit
|
||||
behavior are documented in the [CLI reference](docs/cli.md); configuration,
|
||||
credentials, and module selection are owned by the
|
||||
[configuration reference](docs/config.md).
|
||||
|
||||
- [CLI reference](docs/cli.md)
|
||||
- [Configuration reference](docs/config.md)
|
||||
- [Operations](docs/operations.md)
|
||||
- [Seriatim input contract](docs/integrations/seriatim.md)
|
||||
- [JSON output contract](docs/integrations/json-output.md)
|
||||
- [D&D spell artifact contract](docs/integrations/dnd-spell-artifacts.md)
|
||||
- [Developer guide](docs/development.md)
|
||||
- [Internal implementation docs](docs/internal/overview.md)
|
||||
- [Maintained example config](examples/dnd-spells.config.yml)
|
||||
- [Maintained example input](examples/seriatim-minimal-transcript.json)
|
||||
For the complete ordered D&D workflow, use
|
||||
[the complete configuration](examples/dnd-complete.config.yml) with
|
||||
[its synthetic transcript](examples/dnd-complete-transcript.json). It
|
||||
demonstrates all implemented D&D lanes and the supporting campaign references.
|
||||
|
||||
## Documentation
|
||||
|
||||
- [CLI reference](docs/cli.md) — commands, flags, output streams, and exits.
|
||||
- [Configuration reference](docs/config.md) — configuration files, profiles,
|
||||
validation, and module selection.
|
||||
- [Operations](docs/operations.md) — output, state, recovery, and debug
|
||||
handling.
|
||||
- [Integration contracts](docs/integrations/) — Seriatim input and published
|
||||
artifact formats.
|
||||
- [Subprocess consumer guide](docs/consumers/subprocess.md) — invoke Notarius
|
||||
from an orchestrator and consume a published result.
|
||||
- [Internal overview](docs/internal/overview.md) — implemented component map
|
||||
for maintainers.
|
||||
- [Developer guide](docs/development.md) — contributor orientation and
|
||||
validation guidance.
|
||||
- [Future work](docs/roadmap/future.md) — unimplemented ideas and priorities.
|
||||
|
||||
114
docs/adr/0006-separate-output-cache-and-debug-state.md
Normal file
114
docs/adr/0006-separate-output-cache-and-debug-state.md
Normal file
@@ -0,0 +1,114 @@
|
||||
# ADR-0006: Separate output, cache, and debug state
|
||||
|
||||
**Status:** Superseded by [ADR-0007](0007-separate-checkpoint-recording-from-reuse.md)
|
||||
**Date:** 2026-07-17
|
||||
|
||||
## Context
|
||||
|
||||
Notarius currently exposes a workspace as a shared parent for checkpoints,
|
||||
debug artifacts, and preferred diagnostics settings. Diagnostics are a second
|
||||
inspection surface with their own enablement, directory, retention, and legacy
|
||||
configuration. Durable output uses a separate CLI-selected root, while the
|
||||
canonical chunk-plan cache introduced by ADR-0005 correctly uses an independent
|
||||
cache root.
|
||||
|
||||
These concepts reflect implementation history more than operator intent. A user
|
||||
must understand differences among workspace state, diagnostics, debug artifacts,
|
||||
checkpoints, and chunk plans before deciding where Notarius may write. Some of
|
||||
those distinctions are important internally: a redacted run summary has a
|
||||
different sensitivity from a trace containing source material, prompts, and
|
||||
model responses. They do not require separate public filesystem categories.
|
||||
|
||||
Notarius needs a smaller state model that communicates why data exists, how it
|
||||
may be treated, and whether it is reconstructible.
|
||||
|
||||
## Decision
|
||||
|
||||
Notarius exposes three filesystem surfaces: output, cache, and debug. The
|
||||
public workspace concept and diagnostics as a separate output surface are
|
||||
removed.
|
||||
|
||||
### Output
|
||||
|
||||
Output is the durable result of a run and the only surface intended for normal
|
||||
consumption. It contains the logical files produced by the output stage,
|
||||
including the maintained result, manifest, warning, and rejection contracts.
|
||||
Output is not cache or inspection state.
|
||||
|
||||
### Cache
|
||||
|
||||
Cache contains reconstructible state used to avoid repeated work or resume an
|
||||
interrupted workflow. Canonical chunk plans and invocation checkpoints are
|
||||
distinct cache families with independent identities, compatibility rules,
|
||||
enablement policies, locations, and cleanup lifecycles.
|
||||
|
||||
ADR-0005 continues to govern canonical chunk-plan selection and reuse. Grouping
|
||||
chunk plans and checkpoints under the public cache category does not permit a
|
||||
checkpoint to compete with canonical plan reuse or couple their storage roots.
|
||||
|
||||
Checkpointing is an invocation policy rather than a prerequisite hidden in
|
||||
persistent workspace configuration. An explicit resume invocation may read
|
||||
compatible checkpoints and record replacement checkpoint state for work it
|
||||
executes. Runs that do not request resume perform no checkpoint I/O.
|
||||
|
||||
### Debug
|
||||
|
||||
Debug is an explicitly requested per-run inspection bundle intended for
|
||||
developers and troubleshooting. It is off by default. When enabled, one bundle
|
||||
contains both redacted run summaries and detailed stage and LLM traces. The
|
||||
internal distinction between a safe summary and a sensitive trace remains, but
|
||||
there is one public enablement and location model.
|
||||
|
||||
Debug data is never a cache input and has no automatic retention policy.
|
||||
Notarius does not create a debug directory unless debug is requested, and it
|
||||
does not automatically delete a requested bundle. Credentials remain redacted
|
||||
at every level, while the bundle as a whole is treated as potentially sensitive
|
||||
because traces may contain source, reference, prompt, model-response, and
|
||||
intermediate artifact content.
|
||||
|
||||
Concise progress, warnings, and failures continue to use stdout or stderr. A
|
||||
run without debug may fail without producing a filesystem inspection record.
|
||||
|
||||
Exact configuration fields, CLI flags, default paths, layouts, compatibility
|
||||
handling, and migration mechanics are configuration and operational contracts
|
||||
rather than part of this decision.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- Keep workspace, diagnostics, checkpoints, debug, and chunk-plan cache as
|
||||
separate public concepts. This preserves compatibility and the current safe
|
||||
default-on failure records, but retains overlapping configuration and asks
|
||||
operators to reason about implementation-specific categories.
|
||||
- Keep diagnostics as an always-available redacted operational surface and use
|
||||
debug only for sensitive traces. This distinction is useful for a daemon or
|
||||
managed service with an operational logging contract, but the current CLI can
|
||||
report concise failures on stderr and provide inspection data when explicitly
|
||||
requested.
|
||||
- Put all non-output state beneath one physical root. This minimizes path
|
||||
configuration, but couples reconstructible caches to per-run inspection data
|
||||
and couples cache families whose identity, sensitivity, and cleanup policies
|
||||
differ.
|
||||
- Treat checkpoints as durable run state rather than cache. This emphasizes
|
||||
resumability, but checkpoints are derived, compatibility-checked data that may
|
||||
be deleted and recomputed. Cache more accurately describes their lifecycle.
|
||||
|
||||
## Consequences
|
||||
|
||||
The operator model becomes smaller: normal runs produce output and may use
|
||||
cache; developers explicitly request debug. Public configuration no longer
|
||||
exposes a workspace or overlapping diagnostics and debug systems.
|
||||
|
||||
The implementation retains separate collaborators and serializers where their
|
||||
security or lifecycle boundaries differ. Redacted summaries remain useful as
|
||||
the index to a debug bundle, and chunk plans and checkpoints retain separate
|
||||
stores even though both are cache.
|
||||
|
||||
Existing configuration, environment variables, flags, examples, and
|
||||
documentation require a deliberate compatibility transition. Default-on
|
||||
diagnostic directories disappear, so failures without debug are inspectable
|
||||
only through stderr and any durable output completed before the failure.
|
||||
|
||||
Debug becomes easier to request and substantially more complete, but enabling
|
||||
it creates sensitive files that the operator must protect and remove. Cache
|
||||
cleanup is recoverable but may repeat expensive work, while deleting output is
|
||||
data loss from the user's perspective.
|
||||
50
docs/adr/0007-separate-checkpoint-recording-from-reuse.md
Normal file
50
docs/adr/0007-separate-checkpoint-recording-from-reuse.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# ADR-0007: Separate checkpoint recording from reuse
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-07-19
|
||||
|
||||
## Context
|
||||
|
||||
ADR-0006 made checkpoint I/O conditional on an explicit `--resume` invocation.
|
||||
That policy requires an operator to anticipate the need for recovery before a
|
||||
run begins. A failed ordinary run cannot reuse completed work because it did not
|
||||
record checkpoints.
|
||||
|
||||
Recording reconstructible state and authorizing reuse are separate operational
|
||||
decisions. Recording consumes storage and retains sensitive derived application
|
||||
data, while reuse may change which module operations execute during a run.
|
||||
|
||||
## Decision
|
||||
|
||||
ADR-0006's separation of output, cache, and debug surfaces remains in effect;
|
||||
this decision supersedes only its checkpoint invocation policy.
|
||||
|
||||
Checkpoint recording is controlled by an explicit persistent Boolean
|
||||
configuration setting and remains disabled by default. When recording is
|
||||
enabled, every run records checkpoint transitions and reusable approved stage
|
||||
results.
|
||||
|
||||
Checkpoint loading remains an invocation policy. Only a run with `--resume`
|
||||
loads and reuses compatible completed work. A recording-enabled run without
|
||||
`--resume` executes every stage normally and never loads checkpoints. A resume
|
||||
request while recording is disabled is rejected.
|
||||
|
||||
The existing checkpoint identities, compatibility rules, payload format,
|
||||
filesystem root behavior, and pipeline collaborator contracts remain unchanged.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- Continue coupling reads and writes to `--resume`. This is safe by default but
|
||||
prevents recovery unless resume was anticipated on the earlier run.
|
||||
- Always record checkpoints. This maximizes recovery but creates potentially
|
||||
sensitive state without explicit operator consent.
|
||||
- Add a multi-value recording policy. This preserves the old behavior as an
|
||||
option but adds configuration complexity without a current need.
|
||||
|
||||
## Consequences
|
||||
|
||||
Operators can opt into recovery-ready runs while keeping checkpoint reuse
|
||||
explicit. Enabled successful, rejected, and failed runs may all leave sensitive
|
||||
checkpoint state, so operators remain responsible for access and retention.
|
||||
Disabled configurations perform no checkpoint I/O, and `--resume` requires the
|
||||
operator to enable recording first.
|
||||
50
docs/adr/0008-ordered-pipeline-steps.md
Normal file
50
docs/adr/0008-ordered-pipeline-steps.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# ADR-0008: Bounded ordered pipeline steps and explicit artifact references
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-07-21
|
||||
|
||||
## Context
|
||||
|
||||
Notarius currently models one pipeline-wide input, chunking plan, artifact
|
||||
lanes, and output boundary. Some workflows need a deterministic handoff from
|
||||
one set of normalized artifacts to a later set of artifacts, such as using
|
||||
extracted NPC records while grounding later combat events. The workflow needs
|
||||
an explicit topology without turning the pipeline into a general-purpose
|
||||
workflow engine.
|
||||
|
||||
## Decision
|
||||
|
||||
Add an ordered collection of pipeline steps. Each step owns one or more
|
||||
artifact lanes, and lanes within a step retain the existing independent
|
||||
execution model. The pipeline continues to have one input, chunk plan, output,
|
||||
and failure boundary. Steps are barriers: a later step may consume only
|
||||
normalized artifacts from an earlier step.
|
||||
|
||||
Generated references use an explicit step-and-lane selector. Reference slots
|
||||
declare the generated artifact kinds and media types they accept. The resolver
|
||||
validates the topology, ordering, lane identity, artifact kind, schema, and
|
||||
codec compatibility before execution. External references remain supported as
|
||||
path sources, and the legacy top-level artifact map is interpreted as an
|
||||
implicit `default` step.
|
||||
|
||||
Pipeline-level references may not select generated artifacts. General DAGs,
|
||||
branches, loops, conditional execution, joins, and inferred dependencies are
|
||||
not part of this model.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- A general DAG would provide more flexibility but would also require a new
|
||||
scheduler, lifecycle model, failure semantics, and provenance model.
|
||||
- Separate pipeline runs connected through filesystem paths would lose the
|
||||
static topology and typed compatibility checks.
|
||||
- Inferring dependencies from module or lane names would make ordering and
|
||||
configuration errors difficult to detect reliably.
|
||||
|
||||
## Consequences
|
||||
|
||||
The resolved pipeline has a deterministic, inspectable topology and can
|
||||
include it in its identity digest. Configuration validation can reject invalid
|
||||
generated bindings before any work begins. Existing single-step profiles keep
|
||||
their behavior through the implicit `default` step. Execution handoff and
|
||||
multi-step scheduling require follow-up work in the runner and checkpoint
|
||||
layers.
|
||||
@@ -0,0 +1,98 @@
|
||||
# ADR-0009: Prefer minimal evidence-grounded extraction artifacts
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-07-22
|
||||
|
||||
## Context
|
||||
|
||||
Notarius is intended to extract structured facts from source material. Several
|
||||
early D&D artifacts grew to include descriptive prose, inferred relationships,
|
||||
immediate outcomes, summaries, and other enrichment alongside the facts that
|
||||
identify an event or entity. Those fields make one model call responsible for
|
||||
both extraction and synthesis.
|
||||
|
||||
In practice, the richer contracts have produced overlapping or weakly grounded
|
||||
fields and have made structurally valid, semantically coherent output harder for
|
||||
cost-effective smaller models. They also increase prompt size, validation and
|
||||
normalization policy, durable schema surface, downstream coupling, and the
|
||||
number of claims whose provenance must be evaluated.
|
||||
|
||||
The application needs a consistent rule for deciding what belongs in an
|
||||
extractor before redesigning the current D&D spell, NPC, and combat-turn
|
||||
contracts or adding new artifact families.
|
||||
|
||||
## Decision
|
||||
|
||||
An extraction module answers one narrowly stated question and returns the
|
||||
smallest durable structured artifact that usefully answers it.
|
||||
|
||||
Every model-produced field in an extraction artifact must:
|
||||
|
||||
- be necessary to answer the extractor's stated question or serve a known
|
||||
downstream consumer;
|
||||
- represent a fact or bounded classification that can be supported directly by
|
||||
cited source ranges;
|
||||
- remain independently meaningful without model-generated explanatory prose;
|
||||
and
|
||||
- justify the additional prompt, schema, validation, normalization, and
|
||||
compatibility surface it creates.
|
||||
|
||||
Source references are required provenance for extracted records. Auxiliary
|
||||
references may disambiguate identities or canonical names, but they do not
|
||||
establish source facts and are not copied into evidence.
|
||||
|
||||
Extraction artifacts do not include narrative summaries, general analysis,
|
||||
speculative enrichment, inferred biography or relationships, or redundant
|
||||
free-text descriptions by default. When such output has a demonstrated use, it
|
||||
belongs in an explicitly named extraction, classification, enrichment, or
|
||||
analysis module with its own contract and evidence policy.
|
||||
|
||||
Occurrence-level facts are not forced into entity-level attributes. A fact
|
||||
that can change between encounters, such as an NPC's role in a scene, belongs
|
||||
on an occurrence artifact rather than as one scalar property of a normalized
|
||||
NPC registry entry.
|
||||
|
||||
Deterministic mapping and normalization may assign application-owned
|
||||
identifiers, canonicalize known catalog values, order and deduplicate evidence,
|
||||
and collapse records under an explicit identity rule. They must not manufacture
|
||||
removed descriptive fields or synthesize missing claims to satisfy an older
|
||||
contract.
|
||||
|
||||
This is a default design rule, not a prohibition on rich artifacts. A richer
|
||||
field is appropriate when its consumer, evidence semantics, and ownership are
|
||||
explicit.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- Keep rich schemas and improve prompts or use larger models. This retains
|
||||
potentially convenient prose but does not resolve overlapping field
|
||||
responsibilities, weak provenance, higher cost, or unnecessary downstream
|
||||
coupling.
|
||||
- Make enrichment fields optional. This reduces rejection pressure but leaves
|
||||
ambiguous artifact semantics and inconsistent records, and many strict
|
||||
structured-output providers still require nullable placeholders.
|
||||
- Keep minimal private LLM schemas while preserving rich durable artifacts.
|
||||
Deterministic code would have to invent, default, or separately derive the
|
||||
missing fields, hiding synthesis behind the extraction boundary.
|
||||
- Use one broad session-analysis module. This reduces the number of lanes but
|
||||
couples unrelated facts, schemas, retries, evaluation, and downstream
|
||||
consumers into one model call.
|
||||
|
||||
## Consequences
|
||||
|
||||
Extraction prompts and response schemas become smaller, more focused, and more
|
||||
suitable for lower-cost models. Artifacts carry fewer unsupported claims, and
|
||||
their evidence and validation policies become easier to explain and evaluate.
|
||||
Independent extractors can evolve, retry, and be consumed without requiring
|
||||
unrelated enrichment.
|
||||
|
||||
Some descriptive convenience fields will disappear from primary artifacts.
|
||||
Consumers that genuinely need them may require a separate module and explicit
|
||||
pipeline step. Entity registries may no longer resolve aliases or relationships
|
||||
unless a dedicated, evidence-grounded capability supplies them.
|
||||
|
||||
Removing durable fields is a schema compatibility change. Each affected
|
||||
artifact requires an explicit version and reference policy; private prompt
|
||||
changes alone are insufficient. Current-behavior integration and internal
|
||||
documentation must change with implementation, while the roadmap owns the
|
||||
proposed contract until then.
|
||||
49
docs/adr/0010-workload-oriented-llm-profile-defaults.md
Normal file
49
docs/adr/0010-workload-oriented-llm-profile-defaults.md
Normal file
@@ -0,0 +1,49 @@
|
||||
# ADR-0010: Use workload-oriented LLM profile defaults
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-08-03
|
||||
|
||||
## Context
|
||||
|
||||
LLM-backed D&D operations share an execution-policy choice, but repeating a
|
||||
provider or model-named profile on every module binding ties pipeline structure
|
||||
to a deployment decision. Different environments may require different model,
|
||||
backend, timeout, or reasoning settings while retaining the same workload.
|
||||
|
||||
Notarius also needs a usable default for maintained D&D prompts without making
|
||||
an operator profile mandatory. That default must remain owned by the D&D
|
||||
family, while generic LLM infrastructure stays unaware of domain-specific
|
||||
policy.
|
||||
|
||||
## Decision
|
||||
|
||||
Pipelines may name one workload-oriented default profile, inherited only by
|
||||
selected LLM-backed bindings and validators. Binding-level profile IDs remain
|
||||
intentional exceptions, and the run-wide CLI profile override has highest
|
||||
precedence.
|
||||
|
||||
The D&D family owns an embedded fallback profile named `dnd-extraction`.
|
||||
Operators may provide a complete profile with the same ID through a PromptKit
|
||||
filesystem source. PromptKit selects the higher-precedence matching definition;
|
||||
Notarius does not merge profile documents. Production, development, and local
|
||||
deployments can therefore use different execution policy behind one unchanged
|
||||
pipeline ID.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- Repeat a model-named profile on every binding. This makes routine deployment
|
||||
policy changes noisy and obscures the shared workload intent.
|
||||
- Require every deployment to install a profile file. This adds configuration
|
||||
friction and leaves maintained D&D prompts without an application-owned
|
||||
fallback.
|
||||
- Put D&D profile policy in generic LLM infrastructure. This breaks domain
|
||||
ownership and makes generic code depend on one workload.
|
||||
|
||||
## Consequences
|
||||
|
||||
Pipeline configuration expresses workload intent rather than a specific
|
||||
provider or model. Operators can replace the complete execution policy without
|
||||
editing bindings, while binding-level and run-wide exceptions remain available.
|
||||
Profile changes affect resolved pipeline and checkpoint identity, so they may
|
||||
intentionally cause work to be recomputed. The D&D fallback becomes a
|
||||
maintained application execution-policy asset.
|
||||
293
docs/cli.md
293
docs/cli.md
@@ -1,212 +1,165 @@
|
||||
# CLI Reference
|
||||
|
||||
This is the canonical reference for the implemented Notarius command-line
|
||||
interface.
|
||||
interface. For the shortest successful run, see the [README](../README.md).
|
||||
Configuration fields, discovery rules, and selectable module keys are defined
|
||||
in [Configuration](config.md); runtime state and recovery procedures are
|
||||
defined in [Operations](operations.md).
|
||||
|
||||
For the minimal end-to-end invocation, see the [README](../README.md).
|
||||
## Command Summary
|
||||
|
||||
## Commands
|
||||
|
||||
```text
|
||||
~~~
|
||||
notarius help
|
||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--chunk_cache auto|bypass|refresh] [--output-dir path] [--diagnostics-dir path] [--llm-profile id] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
|
||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||
notarius pipelines list [--config path/to/config.yml] [--json]
|
||||
```
|
||||
~~~
|
||||
|
||||
Running `notarius` with no arguments, `notarius help`, `notarius --help`, or
|
||||
`notarius -h` prints usage and exits successfully.
|
||||
Running Notarius without arguments, or with **help**, **--help**, or **-h**,
|
||||
writes the command summary to standard output and exits with status 0.
|
||||
|
||||
## `run`
|
||||
## run
|
||||
|
||||
`notarius run <pipeline-id>` executes a configured pipeline against one input
|
||||
file.
|
||||
~~~
|
||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||
~~~
|
||||
|
||||
Flags:
|
||||
The **run** command executes the named pipeline for one input file. The
|
||||
pipeline ID and **--input** are required.
|
||||
|
||||
- `--input path`: required source input file.
|
||||
- `--config path`: config file path. If omitted, Notarius uses the discovery
|
||||
rules in [Configuration](config.md#discovery).
|
||||
- `--only lane-a,lane-b`: run only the named artifact lanes. Values are
|
||||
comma-separated and must be non-empty.
|
||||
- `--resume`: request checkpoint reuse for this invocation. See
|
||||
[Operations](operations.md#checkpoints) for prerequisites and reuse behavior.
|
||||
- `--chunk_cache auto|bypass|refresh`: select chunk-plan reuse for this
|
||||
invocation. `auto` reuses a valid plan by canonical source digest, `bypass`
|
||||
performs no plan-cache I/O, and `refresh` regenerates and replaces a valid
|
||||
plan only after chunk validation succeeds. See
|
||||
[Configuration](config.md#workspace) for the persistent setting, precedence,
|
||||
and cache-root selection.
|
||||
- `--output-dir path`: output root. Defaults to `./notarius-output`.
|
||||
- `--diagnostics-dir path`: diagnostics work directory override for this
|
||||
invocation. It does not change the workspace directory.
|
||||
- `--llm-profile id`: override every effective LLM-capable pipeline module
|
||||
binding with one Scriptorium profile ID. Validator-specific profiles are not
|
||||
overridden.
|
||||
- `--session-id id`: pass a stable prompt session identifier through LLM-backed
|
||||
module calls.
|
||||
- `--reference selector=path`: bind a reference path to a chunk, extractor,
|
||||
merger, or normalizer reference slot. Repeatable.
|
||||
- `--without-reference selector`: remove a configured optional reference binding.
|
||||
Repeatable. It accepts the same selector forms as `--reference`, without
|
||||
`=path`.
|
||||
| Flag | Meaning |
|
||||
| --- | --- |
|
||||
| **--config path** | Use this configuration file. When omitted, configuration discovery applies; see [Configuration](config.md). |
|
||||
| **--input path** | Source input file to process. Required. |
|
||||
| **--output-dir path** | Override the configured output root for this run. |
|
||||
| **--json** | Write the successful run-result receipt as JSON to standard output. |
|
||||
| **--chunk_cache auto\|bypass\|refresh** | Override chunk-plan cache handling for this run. |
|
||||
| **--resume** | Reuse compatible recorded checkpoints when checkpoint recording is enabled. |
|
||||
| **--recompute-step step-id** | With **--resume**, recompute the selected ordered step and its dependent lanes. It cannot be combined with **--only**. |
|
||||
| **--debug** | Retain a debug bundle for this run. |
|
||||
| **--debug-dir path** | Override the debug-bundle root. Requires **--debug**. |
|
||||
| **--only lane-a,lane-b** | Run only the selected comma-separated artifact lanes when that selection is valid for the configured pipeline. |
|
||||
| **--llm-profile id** | Highest-precedence configured profile for selected LLM-backed bindings and validators; it replaces binding and [pipeline](config.md#pipelines) defaults. |
|
||||
| **--session-id id** | Supply a non-empty prompt session identifier to LLM-backed module calls. |
|
||||
| **--reasoning-effort value** | Replace the selected PromptKit profile's reasoning effort for every LLM-backed call in this run. The value must be non-empty and the flag may be specified only once. |
|
||||
| **--clear-reasoning-effort** | Clear reasoning effort inherited from the selected PromptKit profile for every LLM-backed call in this run. |
|
||||
| **--reference selector=path** | Add or replace a file reference binding. Repeatable. |
|
||||
| **--without-reference selector** | Remove a configured optional reference binding. Repeatable. |
|
||||
|
||||
On success, the command prints the completed pipeline ID, normalized output and
|
||||
rejected output counts, and the output directory. If the run completes with warnings,
|
||||
the warning count is printed to stderr.
|
||||
**--chunk_cache** accepts only **auto**, **bypass**, or **refresh**.
|
||||
**--debug-dir**, **--output-dir**, **--session-id**, and
|
||||
**--reasoning-effort**, and **--recompute-step** reject explicit empty values.
|
||||
**--reasoning-effort** and **--clear-reasoning-effort** are mutually exclusive.
|
||||
When neither is present, reasoning effort comes from the selected PromptKit
|
||||
profile. These controls apply to the shared run client, including retries and
|
||||
LLM-backed validators, and do not modify configuration or profile files.
|
||||
Persistent reasoning settings remain a PromptKit profile concern.
|
||||
**--recompute-step** requires **--resume**; checkpoint requirements and reuse
|
||||
behavior are documented in [Operations](operations.md).
|
||||
|
||||
Reference flags are resolved against selected chunk, extractor, merger, and
|
||||
normalizer targets before the run starts. Flat slot names are accepted only
|
||||
when exactly one selected target declares that slot. For configured reference
|
||||
bindings, precedence, path resolution, and validation, see
|
||||
[Configuration](config.md#pipelines).
|
||||
### Reference selectors
|
||||
|
||||
`--reference` binds or replaces one slot for one selected target. Selectors are:
|
||||
Use **--reference** only for a reference slot declared by the selected
|
||||
configured target. The accepted selector forms are:
|
||||
|
||||
- `slot=path`: valid when exactly one selected target declares `slot`;
|
||||
- `chunk.slot=path`: target the chunker;
|
||||
- `merge.slot=path`: valid when exactly one selected merger declares `slot`;
|
||||
- `lane.slot=path`: valid when exactly one selected extractor, merger, or
|
||||
normalizer in that lane declares `slot`;
|
||||
- `lane.extract.slot=path`: target a lane extractor;
|
||||
- `lane.merge.slot=path`: target a lane merger;
|
||||
- `lane.normalize.slot=path`: target a lane normalizer.
|
||||
| Form | Target |
|
||||
| --- | --- |
|
||||
| slot=path | The unique selected target that declares slot. |
|
||||
| chunk.slot=path | The chunker. |
|
||||
| merge.slot=path | The unique selected merger that declares slot. |
|
||||
| lane.slot=path | The unique extractor, merger, or normalizer in lane that declares slot. |
|
||||
| lane.extract.slot=path | The extractor in lane. |
|
||||
| lane.merge.slot=path | The merger in lane. |
|
||||
| lane.normalize.slot=path | The normalizer in lane. |
|
||||
|
||||
Use `slot=path` when the selected targets declare the slot unambiguously:
|
||||
**--without-reference** uses the same selector forms without =path. Slot
|
||||
names, requiredness, and configured bindings are part of the
|
||||
[configuration contract](config.md).
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--reference roster=./campaign-roster.txt
|
||||
```
|
||||
### Run output
|
||||
|
||||
Use an explicit selector when multiple selected targets declare the same slot or
|
||||
when you want to target a specific target:
|
||||
Without **--json**, standard output contains the completed pipeline ID, counts
|
||||
of normalized and rejected outputs, and the output directory. A debug-enabled
|
||||
run also prints its debug-bundle path to standard output. A successful run with
|
||||
warnings reports the warning count to standard error. The published JSON bundle
|
||||
is defined by the [JSON output contract](integrations/json-output.md).
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--reference spells.extract.glossary=./campaign-glossary.txt
|
||||
```
|
||||
With **--json**, successful standard output is exactly one
|
||||
`notarius.run-result.v1` JSON document followed by a newline, with no
|
||||
human-oriented status or debug-path line. Its fields and compatibility policy
|
||||
are defined by the [run-result contract](integrations/run-result.md). A caller
|
||||
must check for exit status 0 before decoding this output; a failed write can
|
||||
leave incomplete standard-output bytes that are not a result document.
|
||||
|
||||
The same grammar can target chunk, merge, and normalize slots when the configured
|
||||
modules declare them:
|
||||
Example:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config path/to/config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--reference chunk.scene_guide=./campaign-scenes.txt \
|
||||
--reference spells.merge.merge_notes=./merge-notes.txt \
|
||||
--reference spells.normalize.normalization_notes=./normalization-notes.txt
|
||||
```
|
||||
~~~
|
||||
OPENROUTER_API_KEY=your-api-key \
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-minimal.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json
|
||||
~~~
|
||||
|
||||
Use `--without-reference` to remove a configured optional binding for a run:
|
||||
## config validate
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--without-reference glossary
|
||||
```
|
||||
~~~
|
||||
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||
~~~
|
||||
|
||||
Use `--session-id` when an external orchestrator needs all prompt calls from one
|
||||
run to share an identifier:
|
||||
This command loads and validates a configuration. With **--pipeline**, it also
|
||||
resolves that pipeline against the production module catalog. **--only** selects
|
||||
lanes during that resolution and requires **--pipeline**.
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--session-id campaign-17-session-04
|
||||
```
|
||||
|
||||
The resume flag can be added to an otherwise identical run invocation:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--resume
|
||||
```
|
||||
|
||||
Use `refresh` when intentionally replacing the cached plan for the same source:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--chunk_cache refresh
|
||||
```
|
||||
|
||||
Use `bypass` for a one-off run that must not inspect or create plan-cache state:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--chunk_cache bypass
|
||||
```
|
||||
|
||||
For checkpoint behavior, durable output, diagnostics, retention, and failure
|
||||
inspection, see [Operations](operations.md).
|
||||
|
||||
## `config validate`
|
||||
|
||||
`notarius config validate` loads and validates configuration.
|
||||
|
||||
Flags:
|
||||
|
||||
- `--config path`: config file path. If omitted, Notarius uses the discovery
|
||||
rules in [Configuration](config.md#discovery).
|
||||
- `--pipeline pipeline-id`: additionally resolve one configured pipeline against
|
||||
the production module catalog.
|
||||
- `--only lane-a,lane-b`: validate resolution for selected artifact lanes. This
|
||||
flag requires `--pipeline`.
|
||||
Success is written to standard output as either config "<path>" is valid or
|
||||
config "<path>" is valid for pipeline "<pipeline-id>".
|
||||
|
||||
Examples:
|
||||
|
||||
```sh
|
||||
~~~
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-spells.config.yml
|
||||
--config examples/dnd-minimal.config.yml \
|
||||
--pipeline dnd-session
|
||||
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--pipeline dnd-session \
|
||||
--only spells
|
||||
```
|
||||
OPENROUTER_API_KEY=validation-placeholder \
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-complete.config.yml \
|
||||
--pipeline dnd-session
|
||||
~~~
|
||||
|
||||
## `pipelines list`
|
||||
The placeholder in the second command is sufficient only for offline
|
||||
validation; it cannot run a provider-backed pipeline.
|
||||
|
||||
`notarius pipelines list` prints configured pipeline IDs in sorted order.
|
||||
## pipelines list
|
||||
|
||||
Flags:
|
||||
~~~
|
||||
notarius pipelines list [--config path/to/config.yml] [--json]
|
||||
~~~
|
||||
|
||||
- `--config path`: config file path. If omitted, Notarius uses the discovery
|
||||
rules in [Configuration](config.md#discovery).
|
||||
- `--json`: print `{"pipelines":[...]}` instead of one ID per line.
|
||||
This command lists configured pipeline IDs in sorted order. By default, it
|
||||
writes one ID per line to standard output. **--json** writes an object shaped as
|
||||
{"pipelines":[...]} instead.
|
||||
|
||||
Examples:
|
||||
|
||||
```sh
|
||||
~~~
|
||||
go run ./cmd/notarius pipelines list \
|
||||
--config examples/dnd-spells.config.yml
|
||||
--config examples/dnd-minimal.config.yml
|
||||
~~~
|
||||
|
||||
go run ./cmd/notarius pipelines list \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--json
|
||||
```
|
||||
## Output Streams And Exit Statuses
|
||||
|
||||
## Exit Codes
|
||||
Successful commands write their primary result to standard output. Warnings and
|
||||
errors are written to standard error.
|
||||
|
||||
- `0`: command succeeded.
|
||||
- `1`: command syntax was valid, but loading config, resolving modules, running
|
||||
the pipeline, calling the provider, writing output, or writing diagnostics
|
||||
failed.
|
||||
- `2`: command syntax was invalid, a command was unknown, a required argument
|
||||
was missing, or a flag value was malformed.
|
||||
For **run --json**, warnings remain on standard error and standard output is a
|
||||
machine-readable success result only. Syntax and runtime diagnostics remain on
|
||||
standard error. Parse the result only after the process exits with status 0.
|
||||
|
||||
For YAML structure, defaults, Scriptorium profile sources, environment
|
||||
overrides, and selectable module and validator keys, see
|
||||
[Configuration](config.md).
|
||||
| Status | Meaning |
|
||||
| --- | --- |
|
||||
| 0 | The command completed successfully, including root help. |
|
||||
| 1 | Command syntax was valid but configuration loading or validation, pipeline resolution or execution, provider use, output, or requested debug handling failed. |
|
||||
| 2 | The command or flag syntax was invalid, including unknown commands, missing required arguments, invalid flag values, or invalid flag combinations. |
|
||||
|
||||
The root help spellings are the supported help path. Invoking **--help** on
|
||||
**run**, **config validate**, or **pipelines list** is handled by the flag
|
||||
parser as a usage error: it writes an error to standard error and exits with
|
||||
status 2.
|
||||
|
||||
740
docs/config.md
740
docs/config.md
@@ -1,438 +1,446 @@
|
||||
# Configuration
|
||||
|
||||
This is the canonical reference for implemented Notarius configuration.
|
||||
This is the canonical reference for Notarius configuration. Configuration files
|
||||
are YAML and must declare version 4. They select pipelines and their modules;
|
||||
the [CLI reference](cli.md) owns invocation syntax, and
|
||||
[Operations](operations.md) owns run-state procedures.
|
||||
|
||||
Notarius reads YAML config files with `version: 2`. File config is applied over
|
||||
built-in defaults, then environment overrides are applied.
|
||||
## Configuration Discovery And Precedence
|
||||
|
||||
## Discovery
|
||||
Commands that load configuration choose a file in this order:
|
||||
|
||||
Commands that load configuration use this order:
|
||||
1. a non-empty **--config** CLI value;
|
||||
2. a non-empty **NOTARIUS_CONFIG** environment value;
|
||||
3. the installed default file at **/usr/local/etc/notarius/config.yml**, when
|
||||
it exists.
|
||||
|
||||
1. an explicit path supplied through the CLI, when provided;
|
||||
2. `NOTARIUS_CONFIG`, when set to a non-empty path;
|
||||
3. `/usr/local/etc/notarius/config.yml`.
|
||||
The command fails if none of these paths provides a configuration file.
|
||||
|
||||
If none is available, the command fails with a config file not found error.
|
||||
The explicit-path option is defined in the [CLI reference](cli.md).
|
||||
For configuration values, precedence is:
|
||||
|
||||
1. built-in defaults;
|
||||
2. the selected YAML file;
|
||||
3. supported operational environment variables; and
|
||||
4. the CLI run overrides that apply to a command.
|
||||
|
||||
Environment variables do not provide a second configuration schema. They only
|
||||
override the fields listed below.
|
||||
|
||||
## Maintained Examples
|
||||
|
||||
- [Minimal D&D spell configuration](../examples/dnd-spells.config.yml)
|
||||
- [Production-oriented D&D spell configuration](../examples/dnd-spells-production.config.yml)
|
||||
- [Minimal D&D configuration](../examples/dnd-minimal.config.yml) is a
|
||||
single-lane Seriatim-to-spell pipeline.
|
||||
- [Complete D&D configuration](../examples/dnd-complete.config.yml) uses
|
||||
ordered steps, all implemented D&D lanes, generated references, state
|
||||
settings, bounded LLM concurrency, and the maintained
|
||||
[operator profile](../examples/profiles/dnd-extraction.yml).
|
||||
|
||||
Both complete files are validated by the CLI test suite. The fragments below
|
||||
illustrate individual fields and are not alternate complete configurations.
|
||||
Use these complete files as starting points rather than combining the
|
||||
illustrative fragments in this reference.
|
||||
|
||||
## Top-Level Fields
|
||||
## File Shape And Defaults
|
||||
|
||||
- `version`: required. The only supported value is `2`.
|
||||
- `scriptorium`: optional Scriptorium profile source settings.
|
||||
- `pipelines`: optional map of pipeline IDs to pipeline definitions.
|
||||
- `concurrency`: optional global concurrency settings.
|
||||
- `workspace`: optional workspace settings for Notarius-owned local state.
|
||||
- `diagnostics`: optional diagnostics settings.
|
||||
Unknown fields, duplicate mapping keys, empty identifiers, and identifiers that
|
||||
become duplicates after trimming whitespace are rejected. Every top-level field
|
||||
other than **version** is optional.
|
||||
|
||||
Unknown YAML fields are rejected. The removed top-level `llm_profiles` field is
|
||||
rejected; execution profiles now come from Scriptorium.
|
||||
| Field | Type | Default | Rules |
|
||||
| --- | --- | --- | --- |
|
||||
| **version** | integer | none | Required; must be 4. |
|
||||
| **promptkit** | object | none | Profile source and optional local-backend configuration. |
|
||||
| **pipelines** | map | empty | Maps pipeline IDs to pipeline definitions. |
|
||||
| **concurrency** | object | see below | Global LLM and extraction limits. |
|
||||
| **output** | object | see below | Published output settings. |
|
||||
| **cache** | object | see below | Chunk-plan and checkpoint settings. |
|
||||
| **debug** | object | see below | Debug-bundle root only; it does not enable capture. |
|
||||
|
||||
## Defaults
|
||||
Built-in defaults are:
|
||||
|
||||
Built-in defaults:
|
||||
| Field | Default |
|
||||
| --- | --- |
|
||||
| **concurrency.total_llm** | 1 |
|
||||
| **concurrency.stage_workers.extract** | Effective **total_llm** |
|
||||
| **output.directory** | **./notarius-output** |
|
||||
| **cache.chunk_plans.mode** | **auto** |
|
||||
| **cache.chunk_plans.directory** | Empty, selecting the per-user chunk-plan root |
|
||||
| **cache.checkpoints.enabled** | false |
|
||||
| **cache.checkpoints.directory** | Empty, selecting the per-user checkpoint root |
|
||||
| **debug.directory** | **./notarius-debug** |
|
||||
|
||||
- `concurrency.total_llm`: `1`
|
||||
- `concurrency.stage_workers.extract`: effective `concurrency.total_llm`
|
||||
- `diagnostics.work_dir`: `/tmp/notarius`
|
||||
- `diagnostics.retention`: `auto`
|
||||
- `workspace.directory`: unset
|
||||
- `workspace.diagnostics.enabled`: `true`
|
||||
- `workspace.resume.enabled`: `false`
|
||||
- `workspace.debug.enabled`: `false`
|
||||
- `workspace.chunk_cache.mode`: `auto`
|
||||
- `workspace.chunk_cache.directory`: unset
|
||||
An empty cache directory in YAML deliberately selects the corresponding
|
||||
per-user root. An explicit empty output or debug directory is invalid.
|
||||
|
||||
No pipelines are built in. A run requires a configured pipeline.
|
||||
## PromptKit Profiles
|
||||
|
||||
If `scriptorium` is omitted, Notarius uses Scriptorium's built-in profile
|
||||
catalog. Prompt definitions may also name default profile IDs. The current D&D
|
||||
scene and spell prompts default to the built-in `mistral-small-3` profile when a
|
||||
module binding does not set `llm_profile`. That built-in profile reads its
|
||||
credential from `OPENROUTER_API_KEY`.
|
||||
The optional **promptkit** object selects one source of profile definitions and
|
||||
may register one conventional local OpenAI-compatible backend:
|
||||
|
||||
## Scriptorium Profiles
|
||||
~~~yaml
|
||||
version: 4
|
||||
|
||||
`scriptorium` fields:
|
||||
promptkit:
|
||||
profile_dir: ./profiles
|
||||
# profile_file: ./profiles.yml
|
||||
local_backend:
|
||||
endpoint: http://localhost:8000/v1
|
||||
concurrency_limit: 2
|
||||
~~~
|
||||
|
||||
- `profile_dir`: optional directory containing Scriptorium profile YAML files.
|
||||
- `profile_file`: optional Scriptorium profile YAML file.
|
||||
| Field | Type | Rules |
|
||||
| --- | --- | --- |
|
||||
| **profile_dir** | string | Non-empty directory containing profile files. |
|
||||
| **profile_file** | string | Non-empty profile file. |
|
||||
| **local_backend** | object | Optional registration for the conventional PromptKit backend ID **local**. |
|
||||
| **local_backend.endpoint** | string | Required when **local_backend** is present; absolute HTTP or HTTPS URL with a host. |
|
||||
| **local_backend.concurrency_limit** | integer | Optional non-negative limit; defaults to 0. |
|
||||
|
||||
`profile_dir` and `profile_file` are mutually exclusive. Custom profiles
|
||||
overlay Scriptorium built-in profiles by profile ID.
|
||||
Set at most one of **profile_dir** and **profile_file**. Relative values use
|
||||
the process working directory, not the configuration file's directory. The
|
||||
complete example's `./examples/profiles/dnd-extraction.yml` value is therefore
|
||||
valid when Notarius is launched from the repository root; use an absolute path
|
||||
for services and containers.
|
||||
|
||||
Scriptorium profile files use Scriptorium's profile schema. A minimal profile
|
||||
looks like:
|
||||
An operator source is optional. For a requested ID, PromptKit checks the
|
||||
configured operator source first, then Notarius's embedded fallback profiles,
|
||||
then its own built-in catalog. A matching profile is complete: it replaces a
|
||||
lower-precedence definition rather than merging with it. The maintained
|
||||
[`dnd-extraction` operator profile](../examples/profiles/dnd-extraction.yml)
|
||||
is a secret-free deployment artifact; production, development, and local
|
||||
deployments can each provide a complete definition with that same workload ID.
|
||||
Use workload-oriented IDs for new profiles instead of model names.
|
||||
[Operations](operations.md#promptkit-profile-deployment) owns the deployment
|
||||
workflow and credential-handling guidance.
|
||||
|
||||
```yaml
|
||||
id: local-fast
|
||||
endpoint: http://127.0.0.1:8080/v1
|
||||
model: your-model
|
||||
api_key_env: SCRIPTORIUM_API_KEY
|
||||
timeout_seconds: 180
|
||||
```
|
||||
When **local_backend** is present, its endpoint is trimmed and must use HTTP or
|
||||
HTTPS case-insensitively, be absolute, and have a non-empty host. URL paths are
|
||||
allowed. User information, queries, and fragments are rejected. A zero
|
||||
**concurrency_limit** leaves the local backend unrestricted inside PromptKit;
|
||||
a positive value limits simultaneous local generations. The application-wide
|
||||
**concurrency.total_llm** limit still applies in both cases. Neither local
|
||||
backend field has an environment override. Omitting **local_backend** registers
|
||||
nothing and preserves existing built-in and endpoint-only profile behavior.
|
||||
|
||||
Notarius does not accept raw API keys in Notarius config. For file-backed
|
||||
Scriptorium profiles, store the environment variable name in `api_key_env` and
|
||||
set that variable in the run environment. Scriptorium rejects raw `api_key`
|
||||
fields in profile YAML.
|
||||
A file-backed PromptKit profile selects the registration by its case-sensitive
|
||||
backend ID:
|
||||
|
||||
## Environment Overrides
|
||||
~~~yaml
|
||||
id: local-summary
|
||||
backend: local
|
||||
model: example-model
|
||||
~~~
|
||||
|
||||
These environment variables are applied after the config file:
|
||||
Keep credentials out of the local-backend object. A PromptKit profile may name
|
||||
its credential environment variable through `api_key_env`; set that variable
|
||||
only in the run environment. PromptKit owns the
|
||||
[pinned profile-file format](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md).
|
||||
The [PromptKit upstream boundary](integrations/pkg-promptkit.md) identifies the
|
||||
supported package API, and [Operations](operations.md#operational-limits)
|
||||
describes the effective concurrency layers.
|
||||
|
||||
- `NOTARIUS_CONFIG`: config discovery path.
|
||||
- `NOTARIUS_TOTAL_LLM_CONCURRENCY`: integer global LLM concurrency.
|
||||
- `NOTARIUS_STAGE_WORKERS_EXTRACT`: integer extract worker limit.
|
||||
- `NOTARIUS_WORKSPACE_DIR`: workspace directory.
|
||||
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED`: boolean diagnostics enablement.
|
||||
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION`: workspace diagnostics retention
|
||||
mode.
|
||||
- `NOTARIUS_WORKSPACE_RESUME_ENABLED`: boolean resume checkpointing
|
||||
enablement.
|
||||
- `NOTARIUS_WORKSPACE_DEBUG_ENABLED`: boolean debug artifact enablement.
|
||||
- `NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE`: chunk-plan cache mode.
|
||||
- `NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR`: chunk-plan cache root.
|
||||
- `NOTARIUS_WORK_DIR`: deprecated diagnostics work directory compatibility
|
||||
override.
|
||||
- `NOTARIUS_DIAGNOSTICS_RETENTION`: deprecated diagnostics retention
|
||||
compatibility override.
|
||||
`notarius config validate --pipeline <id>` resolves the selected pipeline and
|
||||
inspects every explicit effective profile without contacting a provider or
|
||||
requiring credential values. It rejects absent, malformed, or incompatible
|
||||
profiles before a run prepares modules. Credential availability is checked only
|
||||
when a generation is prepared.
|
||||
|
||||
Integer environment values must parse as base-10 integers. Boolean environment
|
||||
values must parse as Go booleans such as `true`, `false`, `1`, or `0`.
|
||||
## Migrating Version 3 Configuration
|
||||
|
||||
The removed `NOTARIUS_LLM_DEFAULT_*` variables are not read. Configure provider
|
||||
endpoint, model, and credential environment variable names through Scriptorium
|
||||
profiles.
|
||||
Version 3 files are not decoded or rewritten. Change **version: 3** to
|
||||
**version: 4** and rename the top-level **scriptorium:** section to
|
||||
**promptkit:**. Version 4 decoding is strict, so a remaining **scriptorium**
|
||||
field is rejected as unknown.
|
||||
|
||||
## Concurrency
|
||||
## Operational Environment Variables
|
||||
|
||||
`concurrency` fields:
|
||||
These variables are applied after YAML values:
|
||||
|
||||
- `total_llm`: positive integer ceiling on concurrent provider calls.
|
||||
- `stage_workers`: optional map of framework worker limits. The only supported
|
||||
key is `extract`.
|
||||
| Variable | Overrides | Rules |
|
||||
| --- | --- | --- |
|
||||
| **NOTARIUS_TOTAL_LLM_CONCURRENCY** | **concurrency.total_llm** | Integer. |
|
||||
| **NOTARIUS_STAGE_WORKERS_EXTRACT** | **concurrency.stage_workers.extract** | Integer. |
|
||||
| **NOTARIUS_OUTPUT_DIR** | **output.directory** | Non-empty path. |
|
||||
| **NOTARIUS_CACHE_CHUNK_PLANS_MODE** | **cache.chunk_plans.mode** | **auto**, **bypass**, or **refresh**. |
|
||||
| **NOTARIUS_CACHE_CHUNK_PLANS_DIR** | **cache.chunk_plans.directory** | Non-empty path. |
|
||||
| **NOTARIUS_CACHE_CHECKPOINTS_DIR** | **cache.checkpoints.directory** | Non-empty path. |
|
||||
| **NOTARIUS_DEBUG_DIR** | **debug.directory** | Non-empty path. |
|
||||
|
||||
`stage_workers.extract` defaults to the effective `total_llm` value after file
|
||||
and environment precedence. It must be between `1` and `total_llm`, inclusive.
|
||||
Unknown or empty stage-worker keys are rejected. The environment override
|
||||
`NOTARIUS_STAGE_WORKERS_EXTRACT` takes precedence over the file value, as does
|
||||
`NOTARIUS_TOTAL_LLM_CONCURRENCY` for the global ceiling.
|
||||
Integer values are trimmed then parsed as base-10 integers. Directory and
|
||||
output values reject NUL characters. **NOTARIUS_CONFIG** participates only in
|
||||
configuration discovery.
|
||||
|
||||
The worker value is present in effective and redacted configuration. It bounds
|
||||
the fixed run-wide extract pool and its bounded dispatch queue. Extract jobs are
|
||||
submitted by source chunk and then resolved lane; `total_llm` independently
|
||||
bounds actual provider calls made by extracts, retries, and validators.
|
||||
## Concurrency, Output, Cache, And Debug
|
||||
|
||||
~~~yaml
|
||||
concurrency:
|
||||
total_llm: 2
|
||||
stage_workers:
|
||||
extract: 2
|
||||
output:
|
||||
directory: ./notarius-output
|
||||
cache:
|
||||
chunk_plans:
|
||||
mode: auto
|
||||
directory: ./notarius-cache/chunk-plans
|
||||
checkpoints:
|
||||
enabled: true
|
||||
directory: ./notarius-cache/checkpoints
|
||||
debug:
|
||||
directory: ./notarius-debug
|
||||
~~~
|
||||
|
||||
**concurrency.total_llm** must be greater than zero. The only supported
|
||||
**concurrency.stage_workers** key is **extract**; its value must be from 1
|
||||
through **total_llm**. When omitted, it is recalculated from the effective
|
||||
**total_llm** after YAML and environment precedence.
|
||||
|
||||
**cache.chunk_plans.mode** accepts **auto**, **bypass**, or **refresh**.
|
||||
**cache.checkpoints.enabled** is a boolean. The CLI can override the output
|
||||
directory and chunk-plan mode for one run; see [CLI reference](cli.md#run).
|
||||
|
||||
## Pipelines
|
||||
|
||||
A pipeline selects implementations for the fixed workflow defined by
|
||||
[Architecture](policy/architecture.md#system-shape).
|
||||
Each **pipelines** entry has a unique, non-empty ID and the following shape:
|
||||
|
||||
Pipeline fields:
|
||||
|
||||
- `input`: required module binding.
|
||||
- `chunk`: optional module binding. Default module is `generic`.
|
||||
- `artifacts`: required for pipeline resolution. It maps artifact lane IDs to
|
||||
lane definitions.
|
||||
- `output`: optional module binding. Default module is `json`.
|
||||
- `references`: optional map of reference slot names to reference paths. These
|
||||
bindings are defaults for eligible pipeline targets that declare the matching
|
||||
slot.
|
||||
|
||||
Artifact lane fields:
|
||||
|
||||
- `extract`: required module binding.
|
||||
- `merge`: optional module binding. Default module is `appendorder`.
|
||||
- `normalize`: optional module binding. Default module is `noop`.
|
||||
- `validators`: deprecated lane-level validator list. Non-empty lists are
|
||||
rejected; use `extract.validators`, `merge.validators`, or
|
||||
`normalize.validators`.
|
||||
- `references`: optional compatibility alias for extractor reference bindings.
|
||||
Lane bindings override pipeline-level bindings for the same slot.
|
||||
|
||||
Commands that resolve a pipeline fail for unknown or incompatible module keys.
|
||||
See [CLI Reference](cli.md) for command syntax.
|
||||
|
||||
Reference bindings are validated against reference slots declared by eligible
|
||||
chunk, extract, merge, and normalize targets during pipeline resolution. Required slots
|
||||
must be bound after config defaults, target-local references, lane-level
|
||||
compatibility bindings, and command-line reference overrides are applied.
|
||||
Config-relative paths are resolved relative to the config file; command-line
|
||||
reference paths are resolved relative to the current working directory. Bound
|
||||
files must be UTF-8 text. Reference media types are inferred from file
|
||||
extensions and checked when a module restricts accepted types; unknown
|
||||
extensions use `application/octet-stream`. See [CLI Reference](cli.md#run) for
|
||||
command-line selectors and [Operations](operations.md) for recorded provenance
|
||||
and sensitive-data handling.
|
||||
|
||||
Pipeline-level `references` are defaults. They are valid when at least one
|
||||
eligible target in the full configured pipeline declares the slot, including
|
||||
chunk, extractor, merger, and normalizer targets. During a run, they apply only
|
||||
to the selected targets that declare the slot:
|
||||
|
||||
```yaml
|
||||
~~~yaml
|
||||
pipelines:
|
||||
dnd-session:
|
||||
llm_profile: dnd-extraction
|
||||
input: seriatim
|
||||
references:
|
||||
players: ./campaign/players.txt
|
||||
party: ./campaign/party-roster.txt
|
||||
glossary: ./campaign/glossary.txt
|
||||
chunk: generic
|
||||
output: json
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
```
|
||||
merge: appendorder
|
||||
normalize: dnd/spells
|
||||
~~~
|
||||
|
||||
Extractor binding `references` are the canonical lane-local location. The
|
||||
legacy lane-level `references` field remains supported as an alias; when both
|
||||
bind the same slot, `extract.references` wins:
|
||||
| Field | Type | Default | Rules |
|
||||
| --- | --- | --- | --- |
|
||||
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
|
||||
| **input** | module binding | none | Required. |
|
||||
| **chunk** | module binding | **generic** | Optional. |
|
||||
| **output** | module binding | **json** | Optional. |
|
||||
| **artifacts** | map | none | Compact single-step lane map. |
|
||||
| **steps** | list | none | Ordered lane definitions. Mutually exclusive with **artifacts**. |
|
||||
| **references** | map | none | External reference defaults for eligible targets. |
|
||||
|
||||
```yaml
|
||||
pipelines:
|
||||
dnd-session:
|
||||
input: seriatim
|
||||
Use either **artifacts** or **steps**. The compact **artifacts** form is an
|
||||
implicit single step. An explicit **steps** list must be non-empty; every step
|
||||
needs a unique non-empty **id**, an **artifacts** map, and may have
|
||||
**references**. A lane ID must not appear more than once in a pipeline,
|
||||
including across explicit steps.
|
||||
|
||||
For each selected LLM-backed binding or validator, profile selection occurs
|
||||
after module, validator, and `--only` lane selection. It uses the
|
||||
run-level **--llm-profile** value first, then the binding's **llm_profile**,
|
||||
then the pipeline's **llm_profile**, and finally the PromptKit default.
|
||||
Deterministic bindings do not receive these defaults or run overrides.
|
||||
|
||||
A lane has these fields:
|
||||
|
||||
| Field | Type | Default | Rules |
|
||||
| --- | --- | --- | --- |
|
||||
| **extract** | module binding | none | Required. |
|
||||
| **merge** | module binding | **appendorder** | Optional. |
|
||||
| **normalize** | module binding | **noop** | Optional. |
|
||||
| **references** | map | none | Supported compatibility alias for **extract.references**. |
|
||||
| **validators** | list | none | Non-empty lane-level lists are rejected. Set validator overrides on a binding instead. |
|
||||
|
||||
The lane-level **references** alias remains accepted. When the alias and
|
||||
**extract.references** bind the same slot, **extract.references** wins. Use
|
||||
the binding-local form in new configurations.
|
||||
|
||||
## Module Bindings And Validators
|
||||
|
||||
Use a module key directly when no other binding fields are needed:
|
||||
|
||||
~~~yaml
|
||||
input: seriatim
|
||||
~~~
|
||||
|
||||
Use an object for fields:
|
||||
|
||||
~~~yaml
|
||||
extract:
|
||||
module: dnd/spells
|
||||
llm_profile: dnd-extraction
|
||||
retries: 2
|
||||
references:
|
||||
spell_catalog: ./dnd-spell-catalog.json
|
||||
~~~
|
||||
|
||||
| Binding field | Type | Default | Rules |
|
||||
| --- | --- | --- | --- |
|
||||
| **module** | string | none | Required for an object binding. Must be a registered compatible key. |
|
||||
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
|
||||
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. |
|
||||
| **options** | object | none | Must satisfy the selected module. |
|
||||
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
|
||||
| **validators** | list | production chain | Valid only on chunk, extract, merge, and normalize bindings. |
|
||||
|
||||
Omitting **validators** uses the registered chain. **validators: []** selects
|
||||
an empty chain; a non-empty list replaces the chain in the listed order.
|
||||
Validator bindings accept only **module**, **llm_profile**, and **options**.
|
||||
They reject **references**, **retries**, and nested **validators**. Deterministic
|
||||
validators reject an explicit **llm_profile**. Deterministic module bindings
|
||||
also reject an explicit **llm_profile**.
|
||||
|
||||
The **json** output module accepts optional **include_chunk_map** and
|
||||
**evidence_context** settings:
|
||||
|
||||
~~~yaml
|
||||
output:
|
||||
module: json
|
||||
options:
|
||||
include_chunk_map: true
|
||||
evidence_context:
|
||||
enabled: true
|
||||
window_units: 3
|
||||
lanes:
|
||||
- npcs
|
||||
- spells
|
||||
~~~
|
||||
|
||||
**include_chunk_map** is a boolean and defaults to false. It adds the accepted
|
||||
chunk map when one exists; its wire format is defined in the
|
||||
[chunk-map contract](integrations/chunk-map.md).
|
||||
|
||||
Omitting **evidence_context** disables evidence publication. When present, it
|
||||
is an object with these strict fields:
|
||||
|
||||
| Field | Type | Rules |
|
||||
| --- | --- | --- |
|
||||
| **enabled** | boolean | Required. `false` permits no other evidence fields. |
|
||||
| **lanes** | array of strings | Required and non-empty when enabled. Each value is trimmed and must be unique; every value must name a configured pipeline lane. |
|
||||
| **window_units** | non-negative integer | Optional when enabled; defaults to 3. Zero retains only directly cited units. |
|
||||
|
||||
Unknown outer or nested option fields are rejected, as are incompatible YAML
|
||||
types. The allowlist remains valid when a run uses lane filtering: a configured
|
||||
lane that is not active for that invocation simply contributes no evidence.
|
||||
Evidence publication is opt-in because it can persist source text and metadata.
|
||||
Its payload contract is [Published Evidence Context](integrations/evidence-context.md).
|
||||
|
||||
## References And Ordered Handoffs
|
||||
|
||||
Reference maps bind named slots that the selected target declares. A scalar is
|
||||
an external path. Pipeline-level maps accept only external paths; step-local
|
||||
and binding-local maps may also select a normalized artifact from an earlier
|
||||
step:
|
||||
|
||||
~~~yaml
|
||||
steps:
|
||||
- id: describe-session
|
||||
artifacts:
|
||||
npcs:
|
||||
extract: dnd/npcs
|
||||
normalize: dnd/npcs
|
||||
- id: extract-events
|
||||
references:
|
||||
glossary: ./campaign/glossary.txt
|
||||
npcs:
|
||||
artifact:
|
||||
step: describe-session
|
||||
lane: npcs
|
||||
artifacts:
|
||||
spells:
|
||||
references:
|
||||
roster: ./campaign/legacy-roster.txt
|
||||
extract:
|
||||
module: dnd/spells
|
||||
references:
|
||||
party: ./campaign/session-party.txt
|
||||
```
|
||||
extract: dnd/spells
|
||||
normalize: dnd/spells
|
||||
~~~
|
||||
|
||||
`chunk.references`, `merge.references`, and `normalize.references` are accepted
|
||||
in object-form bindings. They override pipeline-level defaults for slots
|
||||
declared by that target module. Extractor-local references apply only to the
|
||||
extractor, merger-local references apply only to the merger, and
|
||||
normalizer-local references apply only to the normalizer.
|
||||
An artifact selector contains only **step** and **lane**. The producer must be
|
||||
an earlier step and the selected artifact must be compatible with the consumer
|
||||
slot. A generated binding supplies one accepted normalized artifact; it does
|
||||
not name a file. A configured generated dependency remains required even when
|
||||
that consumer slot is otherwise optional.
|
||||
|
||||
Target-local reference fields use the same map shape at:
|
||||
Pipeline references are defaults. A matching step-local or binding-local
|
||||
external path overrides a pipeline default. Required slots must be bound after
|
||||
these configuration values and any CLI reference overrides are applied.
|
||||
Reference paths in YAML are resolved relative to the configuration file.
|
||||
|
||||
- `pipelines.<id>.chunk.references`
|
||||
- `pipelines.<id>.artifacts.<lane>.extract.references`
|
||||
- `pipelines.<id>.artifacts.<lane>.merge.references`
|
||||
- `pipelines.<id>.artifacts.<lane>.normalize.references`
|
||||
### D&D Reference Slots
|
||||
|
||||
Each binding is valid only when that target module declares the slot.
|
||||
The following slot names are accepted by the implemented D&D modules when the
|
||||
selected target declares them:
|
||||
|
||||
## Module Bindings
|
||||
| Slot | Source and use |
|
||||
| --- | --- |
|
||||
| **party** | Optional text campaign context. This is the canonical party-roster spelling. |
|
||||
| **roster** | Accepted compatibility alias for **party**. |
|
||||
| **players** | Optional text player context. |
|
||||
| **glossary** | Optional text campaign glossary. |
|
||||
| **spell_catalog** | Optional JSON spell-catalog overlay for spell extraction and normalization. See [spell-catalog overlays](integrations/dnd-spell-catalog-overlays.md). |
|
||||
| **npcs** | Normalized NPC registry. Optional for spells and combat turns; required for NPC interactions. |
|
||||
| **scene_descriptions** | Required normalized scene-description artifact for combat-turn extraction. |
|
||||
|
||||
Every module binding may use shorthand:
|
||||
Scene descriptions accept **party**, **players**, and **glossary**, but not
|
||||
**roster**. NPC interactions require **npcs** for both extraction and
|
||||
normalization. Combat turns require **scene_descriptions** for extraction; the
|
||||
normalized combat-turn module may use optional **npcs**. The complete example
|
||||
shows generated **npcs** and **scene_descriptions** bindings.
|
||||
|
||||
```yaml
|
||||
input: seriatim
|
||||
```
|
||||
## Production Module Keys
|
||||
|
||||
or object form:
|
||||
| Kind | Keys |
|
||||
| --- | --- |
|
||||
| Input | **seriatim** |
|
||||
| Chunk | **generic**, **dnd/scenes** |
|
||||
| Extract | **dnd/spells**, **dnd/npcs**, **dnd/combat-turns**, **dnd/item-events**, **dnd/npc-interactions**, **dnd/scene-descriptions** |
|
||||
| Merge | **appendorder** |
|
||||
| Normalize | **noop**, **dnd/spells**, **dnd/npcs**, **dnd/combat-turns**, **dnd/item-events**, **dnd/npc-interactions**, **dnd/scene-descriptions** |
|
||||
| Output | **json** |
|
||||
|
||||
```yaml
|
||||
chunk:
|
||||
module: dnd/scenes
|
||||
llm_profile: local-fast
|
||||
```
|
||||
The D&D artifact contracts define each emitted schema:
|
||||
[spells](integrations/dnd-spell-artifacts.md),
|
||||
[NPCs](integrations/dnd-npc-artifacts.md),
|
||||
[NPC interactions](integrations/dnd-npc-interaction-artifacts.md),
|
||||
[combat turns](integrations/dnd-combat-turn-artifacts.md),
|
||||
[item events](integrations/dnd-item-event-artifacts.md), and
|
||||
[scene descriptions](integrations/dnd-scene-description-artifacts.md).
|
||||
|
||||
Binding fields:
|
||||
## Production Validator Keys And Default Chains
|
||||
|
||||
- `module`: module key.
|
||||
- `llm_profile`: optional Scriptorium profile ID. Empty or omitted lets the
|
||||
Scriptorium prompt default select the profile.
|
||||
- `retries`: non-negative retry count for extra runtime attempts after the
|
||||
first attempt. Default: `0`. Supported on `chunk`, `extract`, `merge`, and
|
||||
`normalize` bindings.
|
||||
- `options`: optional module-specific settings.
|
||||
- `references`: optional reference bindings. Supported only for `chunk`,
|
||||
`extract`, `merge`, and `normalize` bindings. `input` and `output` bindings
|
||||
reject this field during validation.
|
||||
- `validators`: optional stage-local validator chain override. Supported only
|
||||
for `chunk`, `extract`, `merge`, and `normalize` bindings. Omit the field to
|
||||
use the production default chain; set `validators: []` to force an empty
|
||||
chain; set a non-empty list to use exactly those validators in configured
|
||||
order.
|
||||
Available validator keys are:
|
||||
|
||||
During resolution, each selected module's registered option validator runs.
|
||||
Production input, chunk, and output bindings reject unknown or invalid options
|
||||
with the affected binding context.
|
||||
| Family | Keys |
|
||||
| --- | --- |
|
||||
| Generic | **generic/always_accept**, **generic/always_reject**, **generic/valid_json**, **generic/valid_json_schema** |
|
||||
| Spells | **extract/dnd/spells/shape**, **extract/dnd/spells/catalog**, **extract/dnd/spells/source_refs**, **extract/dnd/spells/source_relatedness** |
|
||||
| NPCs | **extract/dnd/npcs/shape**, **extract/dnd/npcs/source_refs**, **extract/dnd/npcs/source_relatedness**, **normalize/dnd/npcs/identity** |
|
||||
| Combat turns | **extract/dnd/combat-turns/shape**, **extract/dnd/combat-turns/source_refs**, **extract/dnd/combat-turns/source_relatedness**, **normalize/dnd/combat-turns/invariants** |
|
||||
| Item events | **extract/dnd/item-events/shape**, **extract/dnd/item-events/source_refs**, **extract/dnd/item-events/source_relatedness**, **normalize/dnd/item-events/invariants** |
|
||||
| NPC interactions | **extract/dnd/npc-interactions/shape**, **extract/dnd/npc-interactions/registry**, **extract/dnd/npc-interactions/source_refs**, **extract/dnd/npc-interactions/source_relatedness**, **normalize/dnd/npc-interactions/invariants** |
|
||||
| Scene descriptions | **extract/dnd/scene-descriptions/shape**, **extract/dnd/scene-descriptions/source_refs**, **extract/dnd/scene-descriptions/source_relatedness**, **normalize/dnd/scene-descriptions/invariants** |
|
||||
|
||||
Validator bindings use the same shorthand or object module-binding form, but
|
||||
only these fields are supported:
|
||||
When no override is configured, production D&D bindings use the following
|
||||
ordered chains. Each row lists extract then normalize; spell chains are the
|
||||
same at both stages.
|
||||
|
||||
- `module`: validator key.
|
||||
- `llm_profile`: optional Scriptorium profile ID for LLM-backed validators.
|
||||
- `options`: optional validator-specific settings.
|
||||
|
||||
Validator bindings reject `references`, `retries`, and nested `validators`.
|
||||
During resolution, deterministic validators reject explicit `llm_profile`
|
||||
values.
|
||||
|
||||
Configured LLM-backed validators with explicit `llm_profile` values are
|
||||
validated against the configured Scriptorium profile source. Deterministic
|
||||
production validators do not call the LLM and must not set `llm_profile`.
|
||||
|
||||
## Implemented Production Modules
|
||||
|
||||
| Slot | Key | Notes |
|
||||
| Lane | Extract | Normalize |
|
||||
| --- | --- | --- |
|
||||
| input | `seriatim` | Reads Seriatim transcript JSON. |
|
||||
| chunk | `generic` | Splits source units into ordered chunks. |
|
||||
| chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. |
|
||||
| extract | `dnd/spells` | Extracts typed D&D spell-list artifacts. |
|
||||
| merge | `appendorder` | Combines typed artifacts in chunk order. |
|
||||
| normalize | `noop` | Passes merged typed artifacts through unchanged. |
|
||||
| output | `json` | Produces JSON output files for normalized `application/json` lanes. |
|
||||
| Spells | generic/valid_json, extract/dnd/spells/shape, extract/dnd/spells/catalog, extract/dnd/spells/source_refs, generic/valid_json_schema, extract/dnd/spells/source_relatedness | Same as extract |
|
||||
| NPCs | generic/valid_json, extract/dnd/npcs/shape, extract/dnd/npcs/source_refs, generic/valid_json_schema, extract/dnd/npcs/source_relatedness | generic/valid_json, extract/dnd/npcs/shape, normalize/dnd/npcs/identity, extract/dnd/npcs/source_refs, generic/valid_json_schema, extract/dnd/npcs/source_relatedness |
|
||||
| Combat turns | generic/valid_json, extract/dnd/combat-turns/shape, extract/dnd/combat-turns/source_refs, generic/valid_json_schema, extract/dnd/combat-turns/source_relatedness | generic/valid_json, extract/dnd/combat-turns/shape, normalize/dnd/combat-turns/invariants, extract/dnd/combat-turns/source_refs, generic/valid_json_schema, extract/dnd/combat-turns/source_relatedness |
|
||||
| Item events | generic/valid_json, extract/dnd/item-events/shape, extract/dnd/item-events/source_refs, generic/valid_json_schema, extract/dnd/item-events/source_relatedness | generic/valid_json, extract/dnd/item-events/shape, normalize/dnd/item-events/invariants, extract/dnd/item-events/source_refs, generic/valid_json_schema, extract/dnd/item-events/source_relatedness |
|
||||
| NPC interactions | generic/valid_json, extract/dnd/npc-interactions/shape, extract/dnd/npc-interactions/registry, extract/dnd/npc-interactions/source_refs, generic/valid_json_schema, extract/dnd/npc-interactions/source_relatedness | generic/valid_json, extract/dnd/npc-interactions/shape, extract/dnd/npc-interactions/registry, normalize/dnd/npc-interactions/invariants, extract/dnd/npc-interactions/source_refs, generic/valid_json_schema, extract/dnd/npc-interactions/source_relatedness |
|
||||
| Scene descriptions | generic/valid_json, extract/dnd/scene-descriptions/shape, extract/dnd/scene-descriptions/source_refs, generic/valid_json_schema, extract/dnd/scene-descriptions/source_relatedness | generic/valid_json, extract/dnd/scene-descriptions/shape, normalize/dnd/scene-descriptions/invariants, extract/dnd/scene-descriptions/source_refs, generic/valid_json_schema, extract/dnd/scene-descriptions/source_relatedness |
|
||||
|
||||
## Implemented Production Validators
|
||||
|
||||
| Key | Execution | Notes |
|
||||
| --- | --- | --- |
|
||||
| `generic/always_accept` | deterministic | Accepts returned module output. |
|
||||
| `generic/always_reject` | deterministic | Rejects returned module output with reason `always_reject`. |
|
||||
| `generic/valid_json` | deterministic | Rejects payloads that are not syntactically valid JSON. |
|
||||
| `generic/valid_json_schema` | deterministic | Rejects invalid JSON or JSON that does not conform to the module response schema. |
|
||||
| `extract/dnd/spells/shape` | deterministic | Rejects malformed D&D spell-list artifacts. |
|
||||
| `extract/dnd/spells/source_refs` | deterministic | Rejects missing or invalid D&D spell source references. |
|
||||
| `extract/dnd/spells/source_relatedness` | deterministic | Emits warnings when a spell name is not found near its cited source text. |
|
||||
|
||||
The production default chain for the `dnd/spells` extractor is:
|
||||
|
||||
```yaml
|
||||
validators:
|
||||
- generic/valid_json
|
||||
- generic/valid_json_schema
|
||||
- extract/dnd/spells/shape
|
||||
- extract/dnd/spells/source_refs
|
||||
- extract/dnd/spells/source_relatedness
|
||||
```
|
||||
|
||||
No other production module currently has a default validator chain. Empty
|
||||
chains approve output by default.
|
||||
|
||||
The `generic` chunker accepts:
|
||||
|
||||
- `max_units`: positive integer, default `50`;
|
||||
- `overlap_units`: non-negative integer, default `0`, and must be less than
|
||||
`max_units`.
|
||||
|
||||
The `dnd/scenes` chunker requires transcript source capabilities, calls the
|
||||
configured structured LLM runtime, and does not accept module options. It
|
||||
declares optional `players`, `party`, and `glossary` references for scene
|
||||
disambiguation, and accepts `roster` as a deprecated compatibility alias for
|
||||
`party`.
|
||||
|
||||
The `dnd/spells` extractor declares optional reference slots:
|
||||
|
||||
- `players`
|
||||
- `party`
|
||||
- `glossary`
|
||||
- `roster` as a deprecated compatibility alias for `party`
|
||||
|
||||
Both modules accept UTF-8 plain text, Markdown, YAML, or JSON reference files.
|
||||
The extractor uses references only as supporting disambiguation material; spell
|
||||
casts still must be present in the source transcript.
|
||||
|
||||
## Workspace
|
||||
|
||||
`workspace` fields:
|
||||
|
||||
- `directory`: optional workspace root for Notarius-owned local state.
|
||||
- `resume.enabled`: boolean resume checkpointing setting.
|
||||
- `debug.enabled`: boolean debug artifact setting.
|
||||
- `chunk_cache.mode`: persistent chunk-plan cache mode: `auto`, `bypass`, or
|
||||
`refresh`. The default is `auto`.
|
||||
- `chunk_cache.directory`: optional chunk-plan cache root. This value is the
|
||||
root itself; Notarius does not append `chunk-plans` to it.
|
||||
- `diagnostics`: optional diagnostics settings defined below.
|
||||
|
||||
`workspace.resume.enabled`, `workspace.debug.enabled`, and
|
||||
`workspace.chunk_cache` are independent. `workspace.directory` does not affect
|
||||
chunk-plan placement. For directory layout, state lifecycle, permissions, and
|
||||
sensitive content, see [Operations](operations.md).
|
||||
|
||||
`chunk_cache.mode` accepts only `auto`, `bypass`, and `refresh`. In `auto`, a
|
||||
valid source-addressed plan is reused and a missing or invalid record is
|
||||
regenerated and published after chunk validation. `bypass` neither reads nor
|
||||
writes plan-cache state. `refresh` always generates a new plan and publishes it
|
||||
only after validation succeeds.
|
||||
|
||||
Configuration values are applied in file then environment order; an explicit
|
||||
`--chunk_cache` CLI value has highest precedence for the mode. The cache root
|
||||
is selected from environment, file, then the per-user default; there is no CLI
|
||||
root override. Every supplied value is parsed strictly even when a higher
|
||||
precedence value wins, so malformed configuration is still an error.
|
||||
|
||||
When `chunk_cache.directory` is unset, the root is
|
||||
`<os.UserCacheDir>/notarius/chunk-plans`. On Unix this is ordinarily
|
||||
`$XDG_CACHE_HOME/notarius/chunk-plans` when `XDG_CACHE_HOME` is an absolute
|
||||
path, or `$HOME/.cache/notarius/chunk-plans` when it is unset. A relative
|
||||
`XDG_CACHE_HOME` is rejected by `os.UserCacheDir`; Notarius reports that as a
|
||||
configuration error and does not fall back to another directory.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Preferred workspace diagnostics fields:
|
||||
|
||||
- `workspace.diagnostics.enabled`: set to `false` to skip creating diagnostics
|
||||
run directories and diagnostics artifacts.
|
||||
- `workspace.diagnostics.retention`: `auto`, `always`, or `never`.
|
||||
|
||||
Defaults for workspace and diagnostics fields are listed in
|
||||
[Defaults](#defaults).
|
||||
|
||||
`workspace.diagnostics.retention` overrides legacy diagnostics retention when
|
||||
set.
|
||||
|
||||
`diagnostics` fields:
|
||||
|
||||
- `work_dir`: deprecated compatibility directory for per-run diagnostics.
|
||||
- `retention`: deprecated compatibility retention mode. `auto`, `always`, or
|
||||
`never`.
|
||||
|
||||
Existing `diagnostics.work_dir`, `diagnostics.retention`, `NOTARIUS_WORK_DIR`,
|
||||
and `NOTARIUS_DIAGNOSTICS_RETENTION` inputs remain supported for compatibility.
|
||||
New configuration should use `workspace.directory` and
|
||||
`workspace.diagnostics.retention` instead.
|
||||
|
||||
For retention behavior and the physical diagnostics layout, see
|
||||
[Operations](operations.md#retention). For the invocation-specific diagnostics
|
||||
override, see [CLI Reference](cli.md#run).
|
||||
Chains are only registered for the D&D extract and normalize modules shown
|
||||
above; select an explicit override when a different compatible chain is
|
||||
required.
|
||||
|
||||
## Validation
|
||||
|
||||
Configuration validation checks:
|
||||
Validate a file and one pipeline before running it:
|
||||
|
||||
- supported config version and known YAML fields;
|
||||
- mutually exclusive `scriptorium.profile_dir` and `scriptorium.profile_file`;
|
||||
- non-empty, non-duplicated IDs after trimming;
|
||||
- positive global LLM concurrency;
|
||||
- supported stage-worker keys and an effective extract worker count in the
|
||||
inclusive range `1..concurrency.total_llm`;
|
||||
- supported diagnostics retention and non-empty work directory;
|
||||
- a supported chunk-cache mode and a chunk-cache directory without NUL bytes;
|
||||
- stale removed fields such as `llm_profiles`.
|
||||
~~~sh
|
||||
go run ./cmd/notarius config validate \
|
||||
--config examples/dnd-minimal.config.yml \
|
||||
--pipeline dnd-session
|
||||
~~~
|
||||
|
||||
Pipeline resolution additionally checks:
|
||||
|
||||
- the pipeline ID exists;
|
||||
- at least one artifact lane is declared and selected;
|
||||
- lanes selected through the CLI exist in the resolved pipeline;
|
||||
- required module keys are present;
|
||||
- module keys are registered for the expected slot;
|
||||
- module capability requirements are satisfied;
|
||||
- non-empty validator overrides reference registered validator keys;
|
||||
- deterministic validators do not set `llm_profile`;
|
||||
- LLM-backed validators with explicit `llm_profile` values reference configured
|
||||
Scriptorium profile IDs;
|
||||
- bound reference slots are declared by selected chunk, extractor, merger, or
|
||||
normalizer targets;
|
||||
- required reference slots are bound for selected targets.
|
||||
Configuration validation rejects invalid YAML, unsupported fields, invalid
|
||||
defaults or environment overrides, incompatible module keys, unknown options,
|
||||
invalid reference bindings, missing required reference slots, invalid validator
|
||||
overrides, and incompatible generated artifact handoffs. Use
|
||||
[pipelines list](cli.md#pipelines-list) to inspect configured IDs.
|
||||
|
||||
74
docs/consumers/subprocess.md
Normal file
74
docs/consumers/subprocess.md
Normal file
@@ -0,0 +1,74 @@
|
||||
# Using Notarius As A Subprocess
|
||||
|
||||
Use this workflow when an orchestrator runs Notarius and consumes its published
|
||||
artifacts. The [CLI reference](../cli.md) owns invocation syntax and exit
|
||||
statuses, while the [run-result receipt](../integrations/run-result.md) and
|
||||
[Published JSON Output contract](../integrations/json-output.md) own the
|
||||
durable result formats.
|
||||
|
||||
## Run And Check The Process
|
||||
|
||||
Optionally preflight a selected configuration and pipeline before work starts:
|
||||
|
||||
```sh
|
||||
notarius config validate --config /path/to/notarius.yml --pipeline pipeline-id
|
||||
```
|
||||
|
||||
Invoke the run with explicit paths and machine-readable output. Capture
|
||||
standard output and standard error separately; do not combine them before
|
||||
processing the result.
|
||||
|
||||
```sh
|
||||
notarius run pipeline-id \
|
||||
--config /path/to/notarius.yml \
|
||||
--input /path/to/source.json \
|
||||
--output-dir /path/to/output-root \
|
||||
--json
|
||||
```
|
||||
|
||||
Use absolute paths for supplied input, configuration, output-root, and
|
||||
reference files. When a stable prompt session identifier or references are
|
||||
needed, pass the supported CLI flags. Supply credentials through Notarius's
|
||||
documented configuration and environment mechanisms, never as command-line
|
||||
arguments or generated secret-bearing configuration.
|
||||
|
||||
Wait for the process before interpreting standard output. Only an exit status
|
||||
of 0 permits decoding the receipt. On a nonzero exit, retain standard error for
|
||||
diagnosis and ignore all standard-output bytes: a failed receipt write may have
|
||||
left a partial document.
|
||||
|
||||
## Discover Required Artifacts
|
||||
|
||||
Decode the successful receipt and accept the schema versions supported by the
|
||||
caller. Use its `output_directory` as the bundle root. For the production JSON
|
||||
output, resolve `index_file` under that root with a confinement check and reject
|
||||
an absolute path or a result that escapes the root.
|
||||
|
||||
Read the resulting `index.json` and locate each artifact by `lane_id`, not by a
|
||||
guessed filename. Before decoding a selected payload, verify its descriptor's
|
||||
media type and schema identity against the relevant published artifact
|
||||
contract. The JSON bundle contract links to the available lane contracts.
|
||||
|
||||
If `index.json` has an `evidence_context` descriptor, treat it as a
|
||||
pipeline-wide artifact rather than a lane entry. Verify its six descriptor
|
||||
fields before decoding the linked file according to the [Published Evidence
|
||||
Context contract](../integrations/evidence-context.md). Use each
|
||||
`evidence_refs` entry as the citation to source material. Its surrounding
|
||||
context range and included units explain the citation, but do not widen or
|
||||
replace the cited source reference.
|
||||
|
||||
A zero exit status may still report rejected outputs, warnings, or absent
|
||||
lanes. The caller decides which lane IDs are required for its own work and
|
||||
which are optional; it should make that decision explicitly rather than infer
|
||||
failure from the receipt counts alone.
|
||||
|
||||
## Preserve Provenance And Handle Data Carefully
|
||||
|
||||
Keep the receipt with the published `manifest.json`, and retain
|
||||
`rejected.json` and `warnings.json` when review or later provenance requires
|
||||
them. Treat the input, output bundle, cache, debug bundle, and captured process
|
||||
logs as potentially sensitive data. Apply the caller's access controls and
|
||||
retention policy, and avoid copying secrets into arguments, logs, or
|
||||
provenance records. An evidence-context artifact contains source-unit text and
|
||||
metadata, and selected lanes can cover most of an input; preserve and share it
|
||||
only when that source content is authorized for the recipient.
|
||||
@@ -16,11 +16,14 @@ implemented component map.
|
||||
| Finding the package or component that owns current behavior | [Internal Overview](internal/overview.md) | It is the implemented component inventory and routes to focused internals. |
|
||||
| Application shape, package boundaries, contracts, dependency direction, runtime guarantees, or safety properties | [Architecture](policy/architecture.md) and relevant [ADRs](adr/) | Architecture defines the intended system and its invariants; ADRs preserve significant decision rationale. |
|
||||
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
|
||||
| Adding, changing, reviewing, or deleting tests | [Testing Policy](policy/testing.md) | It defines risk-based sufficiency, durable test boundaries, test-double guidance, and criteria for retaining tests. |
|
||||
| CLI composition or command behavior | [CLI Internals](internal/cli.md) and [CLI Reference](cli.md) | The internal guide owns composition and command flow; the reference owns public syntax. |
|
||||
| Building a subprocess caller or changing its result protocol | [Subprocess Consumer Guide](consumers/subprocess.md), [Run Result Receipt](integrations/run-result.md), and [CLI Internals](internal/cli.md) | These separate caller workflow, durable receipt contract, and CLI implementation behavior. |
|
||||
| Configuration loading, resolution, or user-visible configuration behavior | [Configuration Internals](internal/configuration.md) and [Configuration](config.md) | The internal guide owns loading and resolution mechanics; the reference owns the configuration contract. |
|
||||
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
|
||||
| Production modules or validators | [Module Internals](internal/modules.md) | It documents implemented module contracts, capabilities, assets, and registration. |
|
||||
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and Scriptorium integration. |
|
||||
| Diagnostics, workspace state, resume, or debug artifacts | [Diagnostics Internals](internal/diagnostics.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
|
||||
| CLI or user-visible configuration behavior | [CLI Reference](cli.md) and [Configuration](config.md) | These are the canonical user and operator references. |
|
||||
| Production modules or validators | [Module Internals](internal/modules.md), [D&D Module Internals](internal/dnd.md), and [D&D integration contracts](integrations/) | The generic guide owns extension mechanics, the D&D guide owns shared family conventions, and the contracts own durable output shapes. |
|
||||
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
|
||||
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
|
||||
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
|
||||
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
|
||||
|
||||
|
||||
81
docs/integrations/chunk-map.md
Normal file
81
docs/integrations/chunk-map.md
Normal file
@@ -0,0 +1,81 @@
|
||||
# Accepted Chunk Map
|
||||
|
||||
This document defines the optional durable `chunk-map.json` artifact in a
|
||||
[published JSON bundle](json-output.md). It describes the accepted,
|
||||
materialized chunk plan used by one run. It is not a lane payload and is never
|
||||
an input to a later pipeline step.
|
||||
|
||||
## Contract Identity
|
||||
|
||||
| Property | Value |
|
||||
| --- | --- |
|
||||
| Artifact kind | `source/chunk-map` |
|
||||
| Logical file | `chunk-map.json` |
|
||||
| Media type | `application/json` |
|
||||
| Schema ID | `notarius.source.chunk_map` |
|
||||
| Schema name | `notarius_source_chunk_map_v1` |
|
||||
| Schema version | `v1` |
|
||||
|
||||
The optional `chunk_map` descriptor in `index.json` identifies this artifact.
|
||||
Export is controlled by the JSON output binding described in
|
||||
[Configuration](../config.md#module-bindings-and-validators).
|
||||
|
||||
## Wire Shape
|
||||
|
||||
Every payload has these required fields:
|
||||
|
||||
| Field | Meaning |
|
||||
| --- | --- |
|
||||
| `source_id` | Accepted source-document identity. |
|
||||
| `source_digest` | Lower-case `sha256:` digest of that source document. |
|
||||
| `plan_digest` | Lower-case `sha256:` digest of the logical chunk plan. |
|
||||
| `requested_chunker` | Chunk module selected by the resolved pipeline. |
|
||||
| `producer` | Original accepted-plan producer. `input_module` and `chunk_module` are required; `llm_profile` is optional. |
|
||||
| `plan_annotations` | Plan-level annotation namespace map; `{}` when none are present. |
|
||||
| `chunks` | Non-empty execution-order chunk collection. |
|
||||
|
||||
Each `chunks` entry contains non-empty `id`, zero-based `index`, `source_ref`,
|
||||
positive `unit_count`, and an explicit `annotations` map. `source_ref` contains
|
||||
the same `source_id` as the top-level value plus positive inclusive
|
||||
`start_unit_id` and `end_unit_id` values. Endpoints identify source units; their
|
||||
numeric values do not by themselves establish source-document order.
|
||||
|
||||
Annotation namespaces are non-empty trimmed strings. Their values are arbitrary
|
||||
valid JSON and are retained without interpreting a module-specific namespace.
|
||||
|
||||
## Ordering And Validation
|
||||
|
||||
`chunks` are in execution order. Their indexes are contiguous, start at zero,
|
||||
and equal their array positions; chunk IDs are unique. The emitted map is built
|
||||
only after the selected plan has been accepted and materialized against the
|
||||
source document, so its ranges, unit counts, annotations, and digests describe
|
||||
that exact plan.
|
||||
|
||||
The codec rejects malformed JSON, trailing content, unknown fixed-object
|
||||
fields, invalid identities or digests, invalid annotations, duplicate chunk
|
||||
IDs, non-contiguous indexes, and a `plan_digest` that does not match the
|
||||
reconstructed logical plan. The checked-in
|
||||
[schema](../../internal/framework/chunkmap/assets/schemas/source_chunk_map.v1.json)
|
||||
defines the strict JSON shape.
|
||||
|
||||
## Valid Example
|
||||
|
||||
The compact
|
||||
[source chunk-map fixture](../../internal/framework/chunkmap/testdata/source_chunk_map.v1.json)
|
||||
is decoded by the production codec and demonstrates an accepted map with
|
||||
annotations, producer identity, and ordered chunks.
|
||||
|
||||
## Publication And Compatibility
|
||||
|
||||
The map is present only when a chunk plan was accepted and its export is
|
||||
enabled. It remains publishable if a later lane is rejected, but is absent when
|
||||
chunk-plan validation rejects the plan. `requested_chunker` identifies the
|
||||
current pipeline selection, while `producer` identifies the component that
|
||||
originally produced the accepted plan; they may differ when an accepted plan is
|
||||
reused.
|
||||
|
||||
The map contains structure rather than source content: it excludes transcript
|
||||
bytes, source-unit metadata, chunk text, private model output, reference
|
||||
content, debug data, and filesystem paths. Treat the exported map with the
|
||||
same care as other published output. Publication location and retention are
|
||||
defined in [Operations](../operations.md#output-bundles).
|
||||
69
docs/integrations/dnd-combat-turn-artifacts.md
Normal file
69
docs/integrations/dnd-combat-turn-artifacts.md
Normal file
@@ -0,0 +1,69 @@
|
||||
# D&D Combat-Turn Artifact
|
||||
|
||||
This contract defines the durable combat-action occurrence list produced by
|
||||
`dnd/combat-turns`. It records source-grounded turns and actions; it is not a
|
||||
complete initiative tracker, combat summary, or state model.
|
||||
|
||||
## Identity and compatibility
|
||||
|
||||
| Property | Value |
|
||||
| --- | --- |
|
||||
| Artifact kind | `dnd/combat-turn-list` |
|
||||
| Schema ID | `notarius.dnd.combat_turns` |
|
||||
| Schema name | `notarius_dnd_combat_turns_v1` |
|
||||
| Schema version | `v1` |
|
||||
| Media type | `application/json` |
|
||||
|
||||
`v1` is a strict JSON object with required `combat_turns`; the array may be
|
||||
empty. Turn and source-reference objects reject unknown fields. An incompatible
|
||||
shape change requires a new schema version.
|
||||
|
||||
## Wire shape
|
||||
|
||||
Each combat turn has these required fields:
|
||||
|
||||
| Field | Contract |
|
||||
| --- | --- |
|
||||
| `actor` | Non-empty acting character or creature name. |
|
||||
| `turn_kind` | `turn`, `reaction`, `legendary_action`, `lair_action`, or `other`. |
|
||||
| `source_refs` | One or more transcript evidence ranges. |
|
||||
|
||||
Each source reference has exactly `source_id`, `start_unit_id`, and
|
||||
`end_unit_id`. It identifies an inclusive current-transcript range; unit IDs
|
||||
are positive and the start may not follow the end.
|
||||
|
||||
```json
|
||||
{
|
||||
"combat_turns": [
|
||||
{
|
||||
"actor": "Mira Thorn",
|
||||
"turn_kind": "turn",
|
||||
"source_refs": [
|
||||
{"source_id": "session-7", "start_unit_id": 31, "end_unit_id": 32}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Eligibility, evidence, and normalized form
|
||||
|
||||
The extractor requires an approved [scene-description artifact](dnd-scene-description-artifacts.md).
|
||||
It emits combat turns only for a chunk with an exact matching scene classified
|
||||
`combat`; an exact non-combat scene produces an accepted empty list. The scene
|
||||
record controls eligibility only: its title, summary, and reference do not
|
||||
become turn evidence. No exact matching scene also produces an empty list and
|
||||
the `scene_classification_unavailable` warning.
|
||||
|
||||
An optional normalized [NPC artifact](dnd-npc-artifacts.md) can ground an
|
||||
actor name. Its registry references are provenance, never combat evidence.
|
||||
Normalization trims and, where possible, canonicalizes actor names; orders and
|
||||
deduplicates exact source references; orders valid-evidence turns by source
|
||||
chronology; and collapses only duplicates with the same actor identity, turn
|
||||
kind, and complete valid evidence. It does not infer turns, initiative, or
|
||||
actions from registry or scene data.
|
||||
|
||||
The [NPC-interaction artifact](dnd-npc-interaction-artifacts.md) records
|
||||
broader NPC occurrences. The [JSON output contract](json-output.md) defines
|
||||
publication, and [D&D module internals](../internal/dnd.md) describes routing
|
||||
and validation mechanics.
|
||||
78
docs/integrations/dnd-item-event-artifacts.md
Normal file
78
docs/integrations/dnd-item-event-artifacts.md
Normal file
@@ -0,0 +1,78 @@
|
||||
# D&D Item-Event Artifact
|
||||
|
||||
This contract defines the durable item and currency occurrence list produced by
|
||||
`dnd/item-events`. It records source-grounded discoveries and possession
|
||||
changes; it does not maintain an inventory, balance, or ledger.
|
||||
|
||||
## Identity and compatibility
|
||||
|
||||
| Property | Value |
|
||||
| --- | --- |
|
||||
| Artifact kind | `dnd/item-event-list` |
|
||||
| Schema ID | `notarius.dnd.item_events` |
|
||||
| Schema name | `notarius_dnd_item_events_v1` |
|
||||
| Schema version | `v1` |
|
||||
| Media type | `application/json` |
|
||||
|
||||
`v1` is a strict JSON object with required `events`; the array may be empty.
|
||||
Event and source-reference objects reject unknown fields. An incompatible
|
||||
shape change requires a new schema version.
|
||||
|
||||
## Wire shape
|
||||
|
||||
Every event has required `name`, `kind`, and `source_refs`. `quantity`, `from`,
|
||||
and `to` are optional where the event kind permits them.
|
||||
|
||||
| Field | Contract |
|
||||
| --- | --- |
|
||||
| `name` | Non-empty item or currency display name. |
|
||||
| `kind` | `discovered`, `acquired`, `lost`, `consumed`, or `transferred`. |
|
||||
| `quantity` | Optional positive integer; omit it when no count is established. |
|
||||
| `from` | Optional non-empty losing holder, when allowed by `kind`. |
|
||||
| `to` | Optional non-empty gaining holder, when allowed by `kind`. |
|
||||
| `source_refs` | One or more transcript evidence ranges. |
|
||||
|
||||
Each source reference has exactly `source_id`, `start_unit_id`, and
|
||||
`end_unit_id`. It identifies an inclusive current-transcript range; unit IDs
|
||||
are positive and the start may not follow the end.
|
||||
|
||||
```json
|
||||
{
|
||||
"events": [
|
||||
{
|
||||
"name": "Silver Pieces",
|
||||
"kind": "acquired",
|
||||
"quantity": 20,
|
||||
"to": "party",
|
||||
"source_refs": [
|
||||
{"source_id": "session-7", "start_unit_id": 2, "end_unit_id": 2}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Holder rules and minimal extraction
|
||||
|
||||
`discovered` has neither holder; `acquired` requires `to` and forbids `from`;
|
||||
`lost` and `consumed` require `from` and forbid `to`; `transferred` requires
|
||||
both holders. `party` denotes collective possession. A transfer cannot use
|
||||
`party` for either holder and its two normalized holders must differ.
|
||||
|
||||
Only an evidenced discovery or possession change belongs in this artifact.
|
||||
It does not infer quantities or holders, convert currency denominations,
|
||||
calculate balances, or merge nearby events. Campaign references may
|
||||
disambiguate names but are never event evidence. Currency uses the ordinary
|
||||
`name` field and an explicit `quantity` only when the transcript establishes
|
||||
one; each denomination remains a separate event.
|
||||
|
||||
Normalization trims display whitespace, orders and removes exact duplicate
|
||||
source references, then orders events by valid source chronology, name identity
|
||||
and display value, kind, holders, quantity, and reference sequence. It
|
||||
collapses only entries with the same normalized durable fields and complete
|
||||
valid evidence.
|
||||
|
||||
The [JSON output contract](json-output.md) defines publication. See
|
||||
[D&D module internals](../internal/dnd.md) for implementation details and the
|
||||
[NPC-interaction artifact](dnd-npc-interaction-artifacts.md) for a distinct
|
||||
kind of occurrence.
|
||||
69
docs/integrations/dnd-npc-artifacts.md
Normal file
69
docs/integrations/dnd-npc-artifacts.md
Normal file
@@ -0,0 +1,69 @@
|
||||
# D&D NPC Artifact
|
||||
|
||||
This contract defines the durable NPC registry produced by `dnd/npcs`. It is a
|
||||
minimal, source-grounded identity registry for other D&D artifacts, not a
|
||||
character sheet or a relationship summary.
|
||||
|
||||
## Identity and compatibility
|
||||
|
||||
| Property | Value |
|
||||
| --- | --- |
|
||||
| Artifact kind | `dnd/npc-list` |
|
||||
| Schema ID | `notarius.dnd.npcs` |
|
||||
| Schema name | `notarius_dnd_npcs_v1` |
|
||||
| Schema version | `v1` |
|
||||
| Media type | `application/json` |
|
||||
| Identity policy | `dnd.npcs.identity.v1` |
|
||||
|
||||
`v1` accepts one strict JSON object with required `npcs`; the array may be
|
||||
empty. NPC and source-reference objects reject unknown fields. An incompatible
|
||||
artifact shape or identity-policy change uses a new version or policy.
|
||||
|
||||
## Wire shape and identity
|
||||
|
||||
Each NPC has these required fields:
|
||||
|
||||
| Field | Contract |
|
||||
| --- | --- |
|
||||
| `id` | `npc:sha256:` followed by 64 lowercase hexadecimal characters. |
|
||||
| `name` | Non-empty canonical display name. |
|
||||
| `source_refs` | One or more transcript evidence ranges for the identity. |
|
||||
|
||||
A source reference has exactly `source_id`, `start_unit_id`, and `end_unit_id`.
|
||||
The source ID identifies the transcript, unit IDs are positive inclusive unit
|
||||
identifiers, and the start may not follow the end.
|
||||
|
||||
```json
|
||||
{
|
||||
"npcs": [
|
||||
{
|
||||
"id": "npc:sha256:99a16589618a04f535a7d21fdcc71a0b1c05d22f752cd492065b1086d97bc3d7",
|
||||
"name": "Mira Thorn",
|
||||
"source_refs": [
|
||||
{"source_id": "session-7", "start_unit_id": 4, "end_unit_id": 5}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The ID is deterministic: normalize the name to Unicode NFKC, normalize the
|
||||
supported apostrophe forms, collapse whitespace, case-fold it, SHA-256 the
|
||||
result, then prefix the lowercase hexadecimal digest with `npc:sha256:`. Each
|
||||
canonical identity and ID appears at most once. Normalization collapses records
|
||||
with the same canonical identity, retains their earliest position, and merges
|
||||
their canonicalized evidence; it does not add aliases, roles, descriptions, or
|
||||
relationship fields.
|
||||
|
||||
## Scope and consumers
|
||||
|
||||
Only individually identifiable NPC names with transcript evidence belong in
|
||||
this artifact. Groups, generic roles, invented labels, and descriptive
|
||||
enrichment are excluded. Its source references prove registry provenance; they
|
||||
do not become evidence for a spell, interaction, or combat occurrence.
|
||||
|
||||
This registry can ground actor or caster names in the [spell](dnd-spell-artifacts.md)
|
||||
and [combat-turn](dnd-combat-turn-artifacts.md) artifacts. It is required to
|
||||
resolve the canonical `name` in an [NPC interaction](dnd-npc-interaction-artifacts.md).
|
||||
The [JSON output contract](json-output.md) defines publication, and
|
||||
[D&D module internals](../internal/dnd.md) owns pipeline mechanics.
|
||||
78
docs/integrations/dnd-npc-interaction-artifacts.md
Normal file
78
docs/integrations/dnd-npc-interaction-artifacts.md
Normal file
@@ -0,0 +1,78 @@
|
||||
# D&D NPC Interaction Artifact
|
||||
|
||||
This contract defines the durable occurrence list produced by
|
||||
`dnd/npc-interactions`. It records discrete, source-grounded interactions with
|
||||
NPCs already present in a normalized registry; it does not extend that registry
|
||||
or summarize the session.
|
||||
|
||||
## Identity and compatibility
|
||||
|
||||
| Property | Value |
|
||||
| --- | --- |
|
||||
| Artifact kind | `dnd/npc-interaction-list` |
|
||||
| Schema ID | `notarius.dnd.npc_interactions` |
|
||||
| Schema name | `notarius_dnd_npc_interactions_v1` |
|
||||
| Schema version | `v1` |
|
||||
| Media type | `application/json` |
|
||||
|
||||
`v1` is a strict JSON object with required `interactions`; the array may be
|
||||
empty. Interaction and source-reference objects reject unknown fields. An
|
||||
incompatible shape change requires a new schema version.
|
||||
|
||||
## Wire shape
|
||||
|
||||
Each interaction has these required fields:
|
||||
|
||||
| Field | Contract |
|
||||
| --- | --- |
|
||||
| `name` | Non-empty canonical display name from the required NPC registry. |
|
||||
| `kind` | One of the interaction categories below. |
|
||||
| `source_refs` | One or more transcript evidence ranges. |
|
||||
|
||||
Each source reference has exactly `source_id`, `start_unit_id`, and
|
||||
`end_unit_id`. It identifies an inclusive range in the current transcript;
|
||||
unit IDs are positive and the start may not follow the end. Extraction evidence
|
||||
for an interaction is confined to its accepted chunk.
|
||||
|
||||
```json
|
||||
{
|
||||
"interactions": [
|
||||
{
|
||||
"name": "Mira Thorn",
|
||||
"kind": "dialogue",
|
||||
"source_refs": [
|
||||
{"source_id": "session-7", "start_unit_id": 12, "end_unit_id": 13}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Interaction categories
|
||||
|
||||
| Kind | Meaning |
|
||||
| --- | --- |
|
||||
| `mentioned` | The NPC is referred to but is not established as present or communicating. |
|
||||
| `noncombat_presence` | The NPC is present and relevant without meaningful dialogue or combat participation. |
|
||||
| `dialogue` | The NPC speaks, responds, or meaningfully participates in a non-combat exchange. |
|
||||
| `combat_ally` | The NPC actively participates in combat on the party's side. |
|
||||
| `combat_opponent` | The NPC actively participates in combat against the party. |
|
||||
| `other` | A clearly evidenced direct occurrence not covered by another category. |
|
||||
|
||||
The categories do not represent motives, relationships, state, or events that
|
||||
the cited transcript does not establish. An `other` entry is not a substitute
|
||||
for uncertain classification.
|
||||
|
||||
## Identity, evidence, and order
|
||||
|
||||
The required normalized [NPC artifact](dnd-npc-artifacts.md) resolves `name`.
|
||||
Registry references are provenance only and never replace an interaction's own
|
||||
evidence. Normalization canonicalizes recognized registry names, orders and
|
||||
deduplicates exact source references, then orders interactions by valid source
|
||||
chronology, NPC comparison identity, display name, kind, and reference sequence.
|
||||
Only entries with the same canonical name, kind, and complete valid evidence
|
||||
sequence are collapsed; distinct categories or evidence remain separate.
|
||||
|
||||
See the [combat-turn artifact](dnd-combat-turn-artifacts.md) for combat-action
|
||||
occurrences and the [JSON output contract](json-output.md) for publication.
|
||||
Pipeline mechanics are described in [D&D module internals](../internal/dnd.md).
|
||||
69
docs/integrations/dnd-scene-description-artifacts.md
Normal file
69
docs/integrations/dnd-scene-description-artifacts.md
Normal file
@@ -0,0 +1,69 @@
|
||||
# D&D Scene-Description Artifact
|
||||
|
||||
This contract defines the durable output of `dnd/scene-descriptions`. Each
|
||||
record classifies one accepted transcript chunk and gives it a minimal
|
||||
source-grounded title and summary.
|
||||
|
||||
## Identity and compatibility
|
||||
|
||||
| Property | Value |
|
||||
| --- | --- |
|
||||
| Artifact kind | `dnd/scene-description-list` |
|
||||
| Schema ID | `notarius.dnd.scene_descriptions` |
|
||||
| Schema name | `notarius_dnd_scene_descriptions_v1` |
|
||||
| Schema version | `v1` |
|
||||
| Media type | `application/json` |
|
||||
|
||||
`v1` is a strict JSON object with required non-empty `scenes`. Scene and
|
||||
source-reference objects reject unknown fields. An incompatible shape change
|
||||
requires a new schema version.
|
||||
|
||||
## Wire shape
|
||||
|
||||
Each scene has exactly these required fields:
|
||||
|
||||
| Field | Contract |
|
||||
| --- | --- |
|
||||
| `id` | Non-empty accepted chunk ID, assigned by Notarius. |
|
||||
| `source_ref` | The assigned inclusive source range for that chunk. |
|
||||
| `kind` | `combat`, `narrative`, `recap`, or `meta`. |
|
||||
| `title` | Non-empty, trimmed, source-grounded title. |
|
||||
| `summary` | Non-empty, trimmed, source-grounded summary. |
|
||||
|
||||
`source_ref` has exactly `source_id`, `start_unit_id`, and `end_unit_id`.
|
||||
Its source ID identifies the input transcript; its positive unit IDs identify
|
||||
the chunk's inclusive range, with the start no later than the end.
|
||||
|
||||
```json
|
||||
{
|
||||
"scenes": [
|
||||
{
|
||||
"id": "chunk-000001",
|
||||
"source_ref": {"source_id": "session-7", "start_unit_id": 1, "end_unit_id": 3},
|
||||
"kind": "narrative",
|
||||
"title": "Arrival at the watchtower",
|
||||
"summary": "The party reaches the ruined watchtower and begins to investigate it."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Meaning and normalized form
|
||||
|
||||
`combat` identifies a chunk where active combat is the central activity.
|
||||
`narrative` is current in-world play that is not principally combat, recap, or
|
||||
meta discussion. `recap` is primarily a recounting of an earlier session, and
|
||||
`meta` is primarily out-of-character discussion. The artifact does not add
|
||||
participants, confidence, events, or information absent from the chunk.
|
||||
|
||||
Normalization trims title and summary, orders scenes by source position and
|
||||
then ID, and removes exact duplicate records. A reused ID with different
|
||||
durable fields, or the same source range with different kind, title, or
|
||||
summary, is invalid. It does not merge adjacent ranges, alter prose, or infer
|
||||
missing scenes.
|
||||
|
||||
The [combat-turn artifact](dnd-combat-turn-artifacts.md) uses an exact matching
|
||||
`combat` scene only as eligibility control; scene title, summary, and source
|
||||
reference never become combat evidence. Publication is defined by the
|
||||
[JSON output contract](json-output.md); implementation details live in
|
||||
[D&D module internals](../internal/dnd.md).
|
||||
@@ -1,90 +1,73 @@
|
||||
# D&D Spell Artifact
|
||||
|
||||
This document is the durable serialized artifact contract for the production
|
||||
D&D spell extractor. Selectable extractor keys are cataloged in
|
||||
[Configuration](../config.md#implemented-production-modules).
|
||||
This contract defines the durable output of the `dnd/spells` extractor and
|
||||
normalizer. It records source-grounded spell-casting occurrences; it is not a
|
||||
spellbook, a rules lookup result, or a record of hypothetical casts.
|
||||
|
||||
## Identity
|
||||
## Identity and compatibility
|
||||
|
||||
- Artifact kind: `dnd/spell-list`
|
||||
- Prompt ID: `dnd.spells`
|
||||
- Response schema key: `dnd_spells`
|
||||
- Response schema ID: `notarius.dnd.spells`
|
||||
- Response schema name: `notarius_dnd_spells_v1`
|
||||
- Response schema version: `v1`
|
||||
- Media type: `application/json`
|
||||
| Property | Value |
|
||||
| --- | --- |
|
||||
| Artifact kind | `dnd/spell-list` |
|
||||
| Schema ID | `notarius.dnd.spells` |
|
||||
| Schema name | `notarius_dnd_spells_v1` |
|
||||
| Schema version | `v1` |
|
||||
| Media type | `application/json` |
|
||||
|
||||
The durable JSON Schema is owned by the D&D spell artifact codec. The
|
||||
extractor's private LLM response schema is a separate transport contract: its
|
||||
source-reference objects omit `source_id`, which the extractor assigns while
|
||||
mapping the response to the canonical artifact. The LLM DTO and transport
|
||||
schema are not part of this durable contract.
|
||||
`v1` is a single strict JSON object. It requires `spell_casts`; the array may
|
||||
be empty. Each spell-cast object and source-reference object rejects unknown
|
||||
fields. An incompatible shape change requires a new schema version.
|
||||
|
||||
The output contains canonical spell casts derived from transcript evidence.
|
||||
Source IDs are assigned from the input identity; source-unit ranges identify
|
||||
the evidence location.
|
||||
## Wire shape
|
||||
|
||||
## Output Shape
|
||||
Each `spell_casts` entry has these required fields:
|
||||
|
||||
The extractor payload is a JSON object with one required top-level array. Its
|
||||
structure is:
|
||||
| Field | Contract |
|
||||
| --- | --- |
|
||||
| `caster` | Non-empty in-world character or creature name. |
|
||||
| `spell` | Non-empty spell name. |
|
||||
| `source_refs` | One or more transcript evidence ranges. |
|
||||
|
||||
```text
|
||||
{"spell_casts": [<spell-cast object>, ...]}
|
||||
```
|
||||
|
||||
`spell_casts` must be present. It may be empty when no spell casts are found.
|
||||
When multiple chunk results are combined, spell casts remain in chunk order.
|
||||
When the payload is written as durable output, its logical path is derived from
|
||||
the configured artifact lane ID as defined by the
|
||||
[JSON output contract](json-output.md#output-payload-files).
|
||||
|
||||
## Spell-Cast Fields
|
||||
|
||||
Each spell cast contains exactly these required fields:
|
||||
|
||||
- `caster`: in-world character or creature casting the spell;
|
||||
- `spell`: spell name;
|
||||
- `effect`: concise spell effect in the scene;
|
||||
- `narrative_description`: short description of the spell cast in context;
|
||||
- `source_refs`: transcript source references with extractor-assigned source
|
||||
IDs and evidence unit ranges. It must contain at least one entry.
|
||||
|
||||
All four string fields must be non-empty. `caster` is the in-world caster, not
|
||||
the transcript speaker. Unknown fields are rejected.
|
||||
|
||||
## Source References
|
||||
|
||||
Each source reference contains exactly three required fields: `source_id`,
|
||||
`start_unit_id`, and `end_unit_id`. The source ID must match the input identity.
|
||||
The unit IDs must be positive integers present in the input, and the start unit
|
||||
must not appear after the end unit. Unknown fields are rejected.
|
||||
|
||||
Reference slot keys and accepted file types are defined in
|
||||
[Configuration](../config.md#implemented-production-modules). References are
|
||||
supporting disambiguation material, not source evidence, and are not
|
||||
addressable through `source_refs`.
|
||||
|
||||
## Manifest Metadata
|
||||
|
||||
The extractor adds prompt and response-schema provenance under the artifact lane
|
||||
manifest metadata:
|
||||
Every source reference has exactly `source_id`, `start_unit_id`, and
|
||||
`end_unit_id`. The source ID identifies the input transcript; the unit IDs are
|
||||
positive inclusive unit identifiers, and the start may not follow the end in
|
||||
that source. References are evidence for the cast, not campaign-reference or
|
||||
NPC-registry provenance.
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"extractor": {
|
||||
"prompt_id": "dnd.spells",
|
||||
"prompt_version": "v1",
|
||||
"prompt_sha256": "sha256:...",
|
||||
"response_schema_key": "dnd_spells",
|
||||
"response_schema_id": "notarius.dnd.spells",
|
||||
"response_schema_name": "notarius_dnd_spells_v1",
|
||||
"response_schema_version": "v1",
|
||||
"response_schema_sha256": "sha256:..."
|
||||
"spell_casts": [
|
||||
{
|
||||
"caster": "Mira Thorn",
|
||||
"spell": "Fireball",
|
||||
"source_refs": [
|
||||
{"source_id": "session-7", "start_unit_id": 12, "end_unit_id": 13}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Raw prompt and schema content are not included in manifest metadata.
|
||||
## Evidence and normalized form
|
||||
|
||||
An entry represents an actual cast or an unambiguous declared attempt. A spell
|
||||
mention, rules discussion, plan, or catalog match alone is not an occurrence.
|
||||
The configured catalog checks the name; it does not establish evidence.
|
||||
|
||||
When normalization is selected, recognized spell names use the effective
|
||||
catalog's canonical display name. Source references are put in canonical source
|
||||
order and exact duplicate references are removed. A later entry is collapsed
|
||||
only when it has the same canonical spell, the same case- and
|
||||
whitespace-insensitive caster identity, and the same complete valid reference
|
||||
sequence. Remaining entries retain their merged order.
|
||||
|
||||
The optional normalized [NPC artifact](dnd-npc-artifacts.md) can ground a
|
||||
caster name. Its own references remain registry provenance and are never copied
|
||||
into `source_refs`.
|
||||
|
||||
## Related contracts
|
||||
|
||||
The [spell-catalog overlay contract](dnd-spell-catalog-overlays.md) defines
|
||||
the configured catalog additions. The [JSON output contract](json-output.md)
|
||||
defines where this logical artifact is published; [D&D module internals](../internal/dnd.md)
|
||||
describes extraction and validation mechanics.
|
||||
|
||||
72
docs/integrations/dnd-spell-catalog-overlays.md
Normal file
72
docs/integrations/dnd-spell-catalog-overlays.md
Normal file
@@ -0,0 +1,72 @@
|
||||
# D&D Spell-Catalog Overlays
|
||||
|
||||
This document defines the optional JSON overlay consumed by the D&D spell
|
||||
extractor. An overlay contributes campaign spell names and aliases for
|
||||
recognition. It does not define spell rules, effects, levels, classes, or
|
||||
transcript evidence. Bind the optional `spell_catalog` reference as described
|
||||
in [Configuration](../config.md#references-and-ordered-handoffs).
|
||||
|
||||
## Contract Identity
|
||||
|
||||
| Property | Value |
|
||||
| --- | --- |
|
||||
| Consumer | D&D spell extraction and normalization |
|
||||
| Reference slot | `spell_catalog` |
|
||||
| Media type | `application/json` |
|
||||
| Required schema version | `notarius.dnd.spell-catalog-overlay.v1` |
|
||||
| Base catalog | Embedded D&D 5e 2014 SRD catalog |
|
||||
|
||||
At most one overlay document may be bound. The maintained example is
|
||||
[dnd-spell-catalog.json](../../examples/dnd-spell-catalog.json).
|
||||
|
||||
## Wire Shape
|
||||
|
||||
This is a minimal valid overlay:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": "notarius.dnd.spell-catalog-overlay.v1",
|
||||
"catalogs": [
|
||||
{
|
||||
"id": "campaign.example",
|
||||
"ruleset": "dnd-5e-2014",
|
||||
"source": {"title": "Example campaign spells"},
|
||||
"spells": [{"name": "Aegis of Emberfall"}]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Required | Meaning and constraints |
|
||||
| --- | --- | --- |
|
||||
| `schema_version` | Yes | Exactly `notarius.dnd.spell-catalog-overlay.v1`. |
|
||||
| `catalogs` | Yes | Non-empty array of catalog objects with unique IDs. |
|
||||
| `catalogs[].id` | Yes | Non-empty trimmed string. |
|
||||
| `catalogs[].ruleset` | Yes | Exactly `dnd-5e-2014`. |
|
||||
| `catalogs[].source.title` | Yes | Non-empty trimmed string. |
|
||||
| `catalogs[].source.version` | No | String when present. |
|
||||
| `catalogs[].source.url` | No | String when present. |
|
||||
| `catalogs[].source.license` | No | String when present. |
|
||||
| `catalogs[].spells` | Yes | Non-empty array of spell objects. |
|
||||
| `catalogs[].spells[].name` | Yes | Non-empty trimmed string. |
|
||||
| `catalogs[].spells[].aliases` | No | Array of non-empty trimmed strings when present. |
|
||||
|
||||
Unknown fields are rejected at every object level. The document must contain
|
||||
one JSON value; `null` is not accepted for optional strings or aliases.
|
||||
|
||||
## Composition And Compatibility
|
||||
|
||||
Notarius starts with the embedded base catalog, then applies overlay catalogs
|
||||
in ascending catalog-ID order. A new canonical spell name adds a recognition
|
||||
entry. If an overlay names an existing canonical spell, it augments that spell
|
||||
with aliases while retaining the established display spelling.
|
||||
|
||||
Repeated aliases for the same spell are accepted. A canonical-name, canonical-
|
||||
to-alias, or alias-to-alias collision between different spells is rejected,
|
||||
including a collision with the embedded catalog. Matching uses the catalog’s
|
||||
case, whitespace, and apostrophe normalization, so authors should avoid names
|
||||
or aliases that normalize to another spell.
|
||||
|
||||
The overlay is a recognition aid only. The durable spell-artifact schema and
|
||||
source-evidence rules are defined by the
|
||||
[D&D spell artifact contract](dnd-spell-artifacts.md).
|
||||
116
docs/integrations/evidence-context.md
Normal file
116
docs/integrations/evidence-context.md
Normal file
@@ -0,0 +1,116 @@
|
||||
# Published Evidence Context
|
||||
|
||||
This contract defines the optional `source/evidence-context` artifact emitted
|
||||
by the production JSON output. Its configuration is owned by
|
||||
[Configuration](../config.md#module-bindings-and-validators); its logical-file
|
||||
discovery is owned by [Published JSON Output](json-output.md).
|
||||
|
||||
## Identity And Discovery
|
||||
|
||||
When enabled, the JSON bundle contains `evidence-context.json` and an
|
||||
`index.json` `evidence_context` descriptor with the same six fields as other
|
||||
pipeline-wide artifact descriptors.
|
||||
|
||||
| Property | Value |
|
||||
| --- | --- |
|
||||
| Artifact kind | `source/evidence-context` |
|
||||
| Media type | `application/json` |
|
||||
| Schema ID | `notarius.source.evidence_context` |
|
||||
| Schema name | `notarius_source_evidence_context_v1` |
|
||||
| Schema version | `v1` |
|
||||
| Logical file | `evidence-context.json` |
|
||||
|
||||
Consumers must discover the file from the descriptor, verify all six descriptor
|
||||
fields, and decode only a supported schema version. The descriptor is optional:
|
||||
its absence means evidence publication was not enabled for that bundle.
|
||||
|
||||
## Payload
|
||||
|
||||
The v1 payload is a JSON object with required `source_id`, `source_digest`,
|
||||
`window_units`, `selected_lanes`, and `contexts` fields. `selected_lanes` and
|
||||
`contexts` are always arrays; an enabled configuration with no accepted direct
|
||||
evidence publishes `contexts: []`.
|
||||
|
||||
```json
|
||||
{
|
||||
"source_id": "session-alpha",
|
||||
"source_digest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
"window_units": 1,
|
||||
"selected_lanes": ["npcs", "spells"],
|
||||
"contexts": [
|
||||
{
|
||||
"context_ref": {
|
||||
"source_id": "session-alpha",
|
||||
"start_unit_id": 10,
|
||||
"end_unit_id": 20
|
||||
},
|
||||
"evidence_refs": [
|
||||
{
|
||||
"lane_id": "spells",
|
||||
"source_ref": {
|
||||
"source_id": "session-alpha",
|
||||
"start_unit_id": 10,
|
||||
"end_unit_id": 10
|
||||
}
|
||||
}
|
||||
],
|
||||
"units": [
|
||||
{
|
||||
"id": 10,
|
||||
"kind": "transcript_segment",
|
||||
"text": "Aria casts Cure Wounds.",
|
||||
"ref": {
|
||||
"source_id": "session-alpha",
|
||||
"start_unit_id": 10,
|
||||
"end_unit_id": 10
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"kind": "transcript_segment",
|
||||
"text": "The party regroups.",
|
||||
"ref": {
|
||||
"source_id": "session-alpha",
|
||||
"start_unit_id": 20,
|
||||
"end_unit_id": 20
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Each context requires a `context_ref` object and `evidence_refs` and `units`
|
||||
arrays. `context_ref` identifies the first and last included unit. Each
|
||||
evidence entry contains a selected `lane_id` and an original `source_ref`. A
|
||||
unit uses the existing source-unit shape: required `id`, `kind`, `text`, and
|
||||
self `ref`, plus optional JSON-object `metadata`. Fixed payload objects reject
|
||||
unknown fields; unit metadata may contain application-defined JSON values.
|
||||
|
||||
## Citations And Context
|
||||
|
||||
`evidence_refs` are the authoritative citations. They identify the direct
|
||||
references emitted by accepted normalized artifacts. `context_ref` and the
|
||||
units collection include those cited units plus nearby source units selected by
|
||||
the configured window. They are explanatory context, not widened citations.
|
||||
|
||||
Only accepted outputs from the configured lane allowlist contribute. Rejected,
|
||||
failed, absent, and lane-filtered outputs do not contribute. The artifact never
|
||||
contains raw input bytes, prompts, model responses, auxiliary reference
|
||||
content, credentials, or filesystem paths.
|
||||
|
||||
## Ordering And Compatibility
|
||||
|
||||
The selected lane allowlist is lexical. Contexts and units are in source
|
||||
document position order, not numeric unit-ID order. Direct evidence entries
|
||||
are deterministically ordered by lane and source reference. Overlapping or
|
||||
contiguous windows merge, and each source unit appears at most once in the
|
||||
resulting contexts.
|
||||
|
||||
The artifact is additive to the JSON bundle and is not a lane payload,
|
||||
normalized-output count, checkpoint, or generated reference. Consumers that
|
||||
do not need it must tolerate the absent optional descriptor. Consumers that do
|
||||
use it should preserve the artifact and its schema identity with the run
|
||||
provenance, and should treat its source text and metadata as sensitive durable
|
||||
content.
|
||||
@@ -1,189 +1,133 @@
|
||||
# JSON Output
|
||||
# Published JSON Output
|
||||
|
||||
This document is the durable JSON output file-format contract produced by the
|
||||
production JSON encoder and written by the CLI. Selectable output-encoder keys
|
||||
are cataloged in
|
||||
[Configuration](../config.md#implemented-production-modules).
|
||||
This document defines the logical JSON bundle emitted by the production JSON
|
||||
output encoder. The bundle’s physical destination, atomic publication, and
|
||||
retention are operational concerns; see [Operations](../operations.md#output-bundles).
|
||||
Output configuration, including chunk-map and evidence-context publication, belongs in
|
||||
[Configuration](../config.md#module-bindings-and-validators).
|
||||
|
||||
The output module produces the logical bundle described here. The CLI's
|
||||
physical placement and lifecycle for that bundle are defined in
|
||||
[Operations](../operations.md#output-directory).
|
||||
## Bundle Layout
|
||||
|
||||
## Files
|
||||
All paths below are logical, relative, slash-separated bundle paths. The
|
||||
encoder always emits the first four JSON files below and adds lane or
|
||||
pipeline-wide artifact files when their corresponding artifacts are available:
|
||||
|
||||
The encoder writes:
|
||||
A subprocess caller first obtains the physical bundle root from the
|
||||
[run-result receipt](run-result.md), then resolves `index.json` beneath that
|
||||
root for the logical discovery described here.
|
||||
|
||||
- `index.json`
|
||||
- `manifest.json`
|
||||
- `lanes/<lane-id>.json`, one file per normalized serialized artifact
|
||||
- `rejected.json`
|
||||
- `warnings.json`
|
||||
| Path | Purpose |
|
||||
| --- | --- |
|
||||
| `index.json` | Entry point that names the other published files and lane payloads. |
|
||||
| `manifest.json` | Run provenance and result summaries. |
|
||||
| `rejected.json` | Rejected pipeline outputs. |
|
||||
| `warnings.json` | Accepted-output and run warnings. |
|
||||
| `lanes/<safe-lane-id>.json` | One normalized artifact payload for each lane. |
|
||||
| `chunk-map.json` | Optional accepted chunk map, when its export is enabled and available. |
|
||||
| `evidence-context.json` | Optional source-context artifact, when evidence publication is enabled. |
|
||||
|
||||
Files are pretty-printed JSON with a trailing newline when the payload is JSON.
|
||||
Logical file paths are relative, slash-separated, and may not contain `..`.
|
||||
JSON files are pretty-printed with a trailing newline. Lane payloads are
|
||||
accepted only when their media type is `application/json`.
|
||||
|
||||
## `index.json`
|
||||
|
||||
Shape:
|
||||
`index.json` is the bundle’s discovery document. An approved run with no
|
||||
normalized lanes has this valid minimal index:
|
||||
|
||||
```json
|
||||
{
|
||||
"manifest_file": "manifest.json",
|
||||
"output_files": [
|
||||
{
|
||||
"lane_id": "spells",
|
||||
"media_type": "application/json",
|
||||
"file": "lanes/spells.json",
|
||||
"module_key": "noop",
|
||||
"schema_id": "notarius.dnd.spells",
|
||||
"schema_name": "notarius_dnd_spells_v1",
|
||||
"schema_version": "v1"
|
||||
}
|
||||
],
|
||||
"output_files": [],
|
||||
"rejected_file": "rejected.json",
|
||||
"warnings_file": "warnings.json"
|
||||
}
|
||||
```
|
||||
|
||||
`output_files` is sorted by lane ID. Output file names are produced by
|
||||
sanitizing the lane ID:
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `manifest_file` | Yes | Always `manifest.json`. |
|
||||
| `output_files` | Yes | Lane descriptors sorted by `lane_id`. |
|
||||
| `rejected_file` | Yes | Always `rejected.json`. |
|
||||
| `warnings_file` | Yes | Always `warnings.json`. |
|
||||
| `chunk_map` | No | Descriptor for the pipeline-wide `chunk-map.json`; never a lane descriptor. |
|
||||
| `evidence_context` | No | Descriptor for the pipeline-wide `evidence-context.json`; never a lane descriptor. |
|
||||
|
||||
- characters outside `A-Z`, `a-z`, `0-9`, `.`, `_`, and `-` become `_`;
|
||||
- repeated `..` sequences are replaced;
|
||||
- leading and trailing `.`, `_`, and `-` are trimmed;
|
||||
- empty sanitized names are rejected;
|
||||
- two lanes that sanitize to the same output file are rejected.
|
||||
Each lane descriptor has required `lane_id` and `file`. It may also include
|
||||
`media_type`, `module_key`, `schema_id`, `schema_name`, and `schema_version`
|
||||
when supplied by the normalized artifact. Each pipeline-wide artifact
|
||||
descriptor (`chunk_map` or `evidence_context`) contains `artifact_kind`,
|
||||
`file`, `media_type`, `schema_id`, `schema_name`, and `schema_version`. Their
|
||||
payloads are defined by the [Accepted Chunk Map contract](chunk-map.md) and
|
||||
[Published Evidence Context](evidence-context.md), respectively.
|
||||
|
||||
`manifest_file`, `rejected_file`, and `warnings_file` contain the fixed paths
|
||||
shown above. Each `output_files` entry requires `lane_id` and `file`. It also
|
||||
contains the normalized payload `media_type`, normalizer `module_key`, and
|
||||
response `schema_id`, `schema_name`, and `schema_version` when those values are
|
||||
available.
|
||||
The lane path is derived from its lane ID. Characters outside letters, digits,
|
||||
periods, underscores, and hyphens become underscores; `..` sequences are
|
||||
neutralized; leading and trailing periods and underscores are removed. A lane
|
||||
that produces an empty name, or two lanes that produce the same path, makes
|
||||
output encoding fail.
|
||||
|
||||
## Lane Payloads
|
||||
|
||||
Each `lanes/<safe-lane-id>.json` file is the codec-owned normalized JSON for
|
||||
that lane. Consumers should use the index descriptor’s schema identity rather
|
||||
than infer a lane schema from its name. The current D&D payload contracts are
|
||||
[spells](dnd-spell-artifacts.md), [NPCs](dnd-npc-artifacts.md),
|
||||
[NPC interactions](dnd-npc-interaction-artifacts.md),
|
||||
[combat turns](dnd-combat-turn-artifacts.md),
|
||||
[item events](dnd-item-event-artifacts.md), and
|
||||
[scene descriptions](dnd-scene-description-artifacts.md).
|
||||
|
||||
## `manifest.json`
|
||||
|
||||
`manifest.json` contains a run manifest. This abridged example shows its core
|
||||
structure:
|
||||
`manifest.json` is published provenance, not a copy of lane payloads or a
|
||||
checkpoint store. Fields without a value may be omitted. Its top-level fields
|
||||
group into the following externally observable summaries:
|
||||
|
||||
```json
|
||||
{
|
||||
"run_id": "run-123",
|
||||
"pipeline_id": "dnd-session",
|
||||
"artifact_lanes": [
|
||||
{
|
||||
"id": "spells",
|
||||
"extractor": "dnd/spells",
|
||||
"merger": "appendorder",
|
||||
"normalizer": "noop"
|
||||
}
|
||||
],
|
||||
"validation_status": "approved",
|
||||
"started_at": "2026-01-01T00:00:00Z",
|
||||
"completed_at": "2026-01-01T00:00:01Z"
|
||||
}
|
||||
```
|
||||
| Group | Fields |
|
||||
| --- | --- |
|
||||
| Run identity and result | `run_id`, `pipeline_id`, `pipeline_digest`, `schema_version`, `validation_status`, `started_at`, `completed_at` |
|
||||
| Resolved components | `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, `output_encoder`, `artifact_lanes`, `validator_chains`, `module_metadata` |
|
||||
| Source and references | `source_digests`, `references` |
|
||||
| Published result summaries | `normalized_outputs`, `rejected_outputs` |
|
||||
| Execution summaries | `chunk_plan`, `checkpoint_decisions`, `llm_profiles`, `metadata` |
|
||||
|
||||
Fields with empty values may be omitted by JSON encoding.
|
||||
`references` records provenance such as the target, slot, origin, digest,
|
||||
media type, size, and generated-artifact identity. It does not contain
|
||||
reference content. `normalized_outputs` and `rejected_outputs` likewise
|
||||
summarize results without embedding lane payload bytes. A chunk-plan summary is
|
||||
provenance for the plan used by this run; cache records, debug artifacts, and
|
||||
other operational state are not published as bundle files.
|
||||
|
||||
The manifest fields are:
|
||||
Each `llm_profiles` entry identifies effective, non-secret LLM execution
|
||||
provenance:
|
||||
|
||||
- `run_id`, `pipeline_id`, and `pipeline_digest`: run and resolved-pipeline
|
||||
identity;
|
||||
- `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, and
|
||||
`output_encoder`: resolved module keys;
|
||||
- `chunk_plan`: payload-free provenance for the effective chunk plan. `mode`
|
||||
is the effective cache mode; `action` is `reused`, `generated`,
|
||||
`refreshed`, or `bypassed` when a plan was materialized. `requested_module`
|
||||
is the current pipeline chunker, while `producer_input_module`,
|
||||
`producer_module`, `producer_llm_profile`, `producer_references`,
|
||||
`producer_metadata`, `source_digest`, `plan_digest`, `plan_schema_version`,
|
||||
and `created_at` describe the stored or generated producer when available.
|
||||
A cached plan can therefore identify a producer different from the requested
|
||||
module. This object never embeds ranges, units, annotations, prompts,
|
||||
responses, or reference content;
|
||||
- `module_metadata` and `artifact_lanes`: module and per-lane provenance,
|
||||
including prompt and response-schema provenance when provided;
|
||||
- `validator_chains`: resolved validation points and validators;
|
||||
- `source_digests` and `references`: source and reference provenance;
|
||||
- `normalized_outputs` and `rejected_outputs`: payload-free result summaries;
|
||||
- `llm_profiles`: selected profile IDs and provider or model names when
|
||||
available;
|
||||
- `metadata`: the effective prompt `session_id`;
|
||||
- `validation_status`: `approved` or `rejected`;
|
||||
- `started_at` and `completed_at`: UTC run timestamps.
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `id` | Yes | Selected PromptKit profile identifier. |
|
||||
| `provider` | No | Notarius adapter provider identifier. |
|
||||
| `model` | No | Effective provider model identifier. |
|
||||
| `backend_id` | No | Effective PromptKit backend registration identifier. Endpoint-only profiles omit it. |
|
||||
| `reasoning_effort` | No | Effective opaque provider reasoning setting. An empty or explicitly cleared setting is omitted. |
|
||||
|
||||
`source_digests` contains source document digests only. Bound references are
|
||||
recorded separately under `references`, which contains provenance only: target
|
||||
stage, lane ID when present, slot name, origin type and URI, digest, media
|
||||
type, byte size, and binding source. Reference content is not written to
|
||||
durable output.
|
||||
These values describe observed execution; they are not a backend-registration
|
||||
interface. Entries that differ by backend or effective reasoning remain
|
||||
distinct even when their profile, provider, and model are otherwise equal.
|
||||
|
||||
Reference `stage` is `chunk`, `extract`, `merge`, or `normalize`. `lane_id` is
|
||||
omitted for chunk references and present for extract, merge, and normalize
|
||||
references.
|
||||
## Rejections And Warnings
|
||||
|
||||
`validation_status` is `approved` when no outputs were rejected and `rejected`
|
||||
when one or more outputs were rejected.
|
||||
`rejected.json` is always an object with a `rejected` array. Each entry has
|
||||
required `stage` and `message`; `step_id`, `lane_id`, `module_key`, `chunk_id`,
|
||||
`chunk_index`, `validator_name`, `reason_code`, `attempt_count`, and
|
||||
`diagnostic_artifact_path` are present only when applicable.
|
||||
|
||||
Producer warnings and the current run's chunk-validation warnings remain in
|
||||
`warnings.json`. The manifest records only provenance and decision summaries;
|
||||
empty producer-only values are omitted for compatibility with existing readers.
|
||||
`warnings.json` is always an object with a `warnings` array. Each warning has
|
||||
`reason_code` and `message`; `scope` is optional. Both arrays are empty when
|
||||
there is nothing to report.
|
||||
|
||||
`validator_chains` records the resolved validator chain for each validation
|
||||
point. Entries include stage, lane ID when applicable, module key, and validators
|
||||
with key and execution class. Empty chains are recorded with an empty
|
||||
`validators` array, including chains resolved from explicit empty config
|
||||
overrides.
|
||||
## Compatibility
|
||||
|
||||
`normalized_outputs` summarizes each normalized lane output without embedding
|
||||
payload bytes. Entries include lane ID, normalizer module key, source ID, media
|
||||
type, and response schema provenance where available.
|
||||
|
||||
`rejected_outputs` summarizes rejected module outputs without embedding raw
|
||||
payload bytes. Entries include stage, lane, module, chunk, validator or reason,
|
||||
message, attempt count, and optional diagnostic artifact path.
|
||||
|
||||
## Output Payload Files
|
||||
|
||||
Each normalized serialized artifact is written to
|
||||
`lanes/<sanitized-lane-id>.json`. The JSON output encoder is domain-neutral and
|
||||
accepts only artifacts whose codec media type is `application/json`. The file
|
||||
contains the codec-owned JSON bytes pretty-printed.
|
||||
|
||||
The schema of each lane payload is owned by that artifact contract. For the
|
||||
current D&D spell lane, see [D&D Spell Artifact](dnd-spell-artifacts.md).
|
||||
|
||||
## `rejected.json`
|
||||
|
||||
Shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"rejected": []
|
||||
}
|
||||
```
|
||||
|
||||
When output validation rejects an output, each entry contains `stage` and
|
||||
`message`. It includes `lane_id`, `module_key`, `chunk_id`, `chunk_index`,
|
||||
`validator_name`, `reason_code`, `attempt_count`, and
|
||||
`diagnostic_artifact_path` when applicable.
|
||||
|
||||
## `warnings.json`
|
||||
|
||||
Shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"warnings": [
|
||||
{
|
||||
"scope": "extract",
|
||||
"reason_code": "example",
|
||||
"message": "human-readable warning"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`warnings` is an empty array when no warnings are reported.
|
||||
Each warning requires `reason_code` and `message`; `scope` is omitted when it is
|
||||
empty.
|
||||
The index is the authoritative map from a logical lane to its published
|
||||
payload. Consumers must tolerate omitted optional manifest and descriptor
|
||||
fields, and should rely on the linked artifact contract for each lane’s JSON
|
||||
shape. This contract describes the published logical bundle only; it does not
|
||||
promise a filesystem layout or expose internal state formats.
|
||||
|
||||
106
docs/integrations/pkg-promptkit.md
Normal file
106
docs/integrations/pkg-promptkit.md
Normal file
@@ -0,0 +1,106 @@
|
||||
# PromptKit Integration
|
||||
|
||||
Notarius pins
|
||||
[`gitea.maximumdirect.net/eric/promptkit` v0.5.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0)
|
||||
as its in-process prompt engine. The upstream
|
||||
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/consumers/pkg-promptkit.md)
|
||||
owns the public engine API, and the upstream
|
||||
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md)
|
||||
owns prompt, profile, and schema file contracts.
|
||||
|
||||
## Supported Boundary
|
||||
|
||||
Notarius relies on the root `promptkit` package to:
|
||||
|
||||
- construct an `Engine` with filesystem-backed prompt, schema, and optional
|
||||
operator and application-fallback profile sources;
|
||||
- prepare one frozen execution from a `RunRequest` with named inline artifacts,
|
||||
variables, a direct session ID, prompt identity, and profile selection, then
|
||||
record credential-redacted details and run that exact execution;
|
||||
- return rendered debug material, validated structured output, selected
|
||||
profile, backend, effective model metadata, and token usage;
|
||||
- register the optional conventional `local` backend through `BackendLocal`,
|
||||
`LocalBackend`, and `WithBackend`;
|
||||
- distinguish structured-output validation failure from execution failure; and
|
||||
- identify a missing explicit profile through `ErrProfileNotFound` and backend
|
||||
admission exhaustion through `ErrCapacityExceeded`.
|
||||
|
||||
The pinned
|
||||
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/backends.go)
|
||||
owns the registration and backend-capacity contract.
|
||||
|
||||
For one completion, the adapter calls `PrepareExecution`, takes a
|
||||
caller-owned `Details` snapshot, and calls `RunPrepared` for that same opaque
|
||||
prepared execution. It defers `Discard` for every unexecuted handle. Explicit
|
||||
profile preflight uses `Engine.InspectProfile`; it does not prepare a synthetic
|
||||
prompt. PromptKit's prepared handle, inspection result, and capacity-error
|
||||
types stay inside the Notarius LLM adapter.
|
||||
|
||||
When a PromptKit profile and runtime override leave `temperature`, `max_tokens`,
|
||||
or `top_p` unset, Notarius leaves that control unset as well. Compatible
|
||||
providers therefore apply their own defaults; an operator that requires a
|
||||
specific sampling value must select it explicitly in the profile or runtime
|
||||
override.
|
||||
|
||||
Notarius does not use PromptKit's optional `ArtifactReader`. It materializes
|
||||
source and reference content itself and supplies owned inline artifacts at the
|
||||
adapter boundary. It also retains responsibility for pipeline retries,
|
||||
scheduling, debug persistence, redaction, profile provenance, and conversion
|
||||
from private model responses into durable domain artifacts.
|
||||
|
||||
Notarius sends its trimmed run session through PromptKit's direct session
|
||||
field, which is authoritative for provider session behavior. It also retains
|
||||
the same value as the `session_id` prompt variable for maintained prompt
|
||||
compatibility. Session IDs are stable, non-secret correlation identifiers and
|
||||
may be exposed to providers and provider observability.
|
||||
|
||||
Notarius records PromptKit's selected backend ID and effective reasoning
|
||||
setting as optional run-manifest provenance. Endpoint-only profiles have no
|
||||
backend ID. Debug prompt material also retains the selected backend ID and
|
||||
PromptKit's stable lower-case `effective_model_params` JSON, which may include
|
||||
`backend_id`. Notarius production configuration exposes one optional
|
||||
conventional `local` registration. It does not expose a general user-defined
|
||||
PromptKit backend registry. Endpoint-only profiles remain supported unchanged.
|
||||
|
||||
Notarius retains its application-wide scheduled client around the PromptKit
|
||||
adapter. PromptKit may apply a narrower limit for the selected backend;
|
||||
endpoint-only profiles have no such backend limit. The adapter translates
|
||||
PromptKit capacity rejection into the provider-neutral Notarius
|
||||
`ErrLLMCapacityExceeded` contract. It may include the normalized selected
|
||||
backend ID in safe diagnostic context, without exposing PromptKit's capacity
|
||||
error type, and leaves retries to the calling pipeline stage.
|
||||
|
||||
## Profile Sources And Compatibility
|
||||
|
||||
Notarius gives PromptKit the configured operator profile source, registered
|
||||
application fallback profile assets, and optional backend registration through
|
||||
the same construction path for inspection and execution. PromptKit owns the
|
||||
resulting source precedence and strict profile parsing: a matching operator
|
||||
profile is a complete replacement for a fallback or built-in profile, while an
|
||||
invalid matching document fails instead of falling through. The operator
|
||||
configuration and deployment workflow are defined in
|
||||
[Configuration](../config.md#promptkit-profiles) and
|
||||
[Operations](../operations.md#promptkit-profile-deployment).
|
||||
|
||||
Notarius supports this boundary against PromptKit v0.5.0. Its fallback source,
|
||||
prepared-execution, inspection, and typed capacity APIs are used as public
|
||||
upstream contracts; other PromptKit APIs or file-format behavior are not
|
||||
implicitly supported. A dependency upgrade requires reviewing the adapter,
|
||||
profile-source construction, and this compatibility statement against the
|
||||
pinned upstream documentation.
|
||||
|
||||
## Notarius Ownership
|
||||
|
||||
[LLM Runtime Internals](../internal/llm.md) describes how Notarius mounts
|
||||
module assets, maps its transport-neutral completion contract, prepares and
|
||||
executes requests, validates output, records provenance, captures debug
|
||||
material, redacts errors, and preserves timeout ownership.
|
||||
[D&D Module Internals](../internal/dnd.md) owns the embedded
|
||||
`dnd-extraction` fallback profile and the maintained D&D prompt defaults.
|
||||
[Configuration](../config.md#promptkit-profiles) defines how a Notarius
|
||||
configuration selects one PromptKit profile source and optionally registers
|
||||
the conventional local backend.
|
||||
|
||||
PromptKit API or format changes outside this boundary are not implicitly
|
||||
supported. Updating the pinned version requires reviewing the adapter and
|
||||
profile/configuration contracts against the upstream documentation.
|
||||
68
docs/integrations/run-result.md
Normal file
68
docs/integrations/run-result.md
Normal file
@@ -0,0 +1,68 @@
|
||||
# Run Result Receipt
|
||||
|
||||
`notarius run --json` writes this receipt to standard output when a run
|
||||
completes successfully. It lets a subprocess caller discover the physical root
|
||||
of the published output bundle without parsing interactive command output.
|
||||
Command syntax, streams, and exit statuses are defined in the
|
||||
[CLI reference](../cli.md); logical files within the bundle are defined in the
|
||||
[Published JSON Output contract](json-output.md).
|
||||
|
||||
## Schema
|
||||
|
||||
The current schema version is `notarius.run-result.v1`.
|
||||
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `schema_version` | Yes | Exactly `notarius.run-result.v1`. |
|
||||
| `run_id` | Yes | The finalized Notarius run identifier. |
|
||||
| `pipeline_id` | Yes | The effective pipeline identifier. |
|
||||
| `output_directory` | Yes | Absolute path to the published, run-specific output bundle. |
|
||||
| `index_file` | For the production JSON output | Logical path `index.json`; omitted for other output modules. |
|
||||
| `normalized_output_count` | Yes | Number of final normalized outputs. |
|
||||
| `rejected_output_count` | Yes | Number of recorded rejected outputs. |
|
||||
| `warning_count` | Yes | Number of final run warnings. |
|
||||
| `validation_status` | Yes | The final run manifest validation status. |
|
||||
| `debug_directory` | No | Absolute path to the run-specific debug bundle when requested debug capture completed. |
|
||||
|
||||
For the production `json` output module, `index_file` is present only when the
|
||||
completed run returned exactly one logical output file named `index.json`.
|
||||
For another output module, its absence does not indicate a failed run.
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": "notarius.run-result.v1",
|
||||
"run_id": "run-1770000000000000000-0123456789abcdef0123456789abcdef",
|
||||
"pipeline_id": "dnd-session",
|
||||
"output_directory": "/work/results/run-1770000000000000000-0123456789abcdef0123456789abcdef",
|
||||
"index_file": "index.json",
|
||||
"normalized_output_count": 6,
|
||||
"rejected_output_count": 2,
|
||||
"warning_count": 1,
|
||||
"validation_status": "rejected"
|
||||
}
|
||||
```
|
||||
|
||||
## Paths And Bundle Discovery
|
||||
|
||||
`output_directory` and `debug_directory`, when present, are lexical absolute
|
||||
paths. They identify the paths used by Notarius and do not resolve symlinks.
|
||||
`output_directory` is the run-specific bundle, not the configured output root.
|
||||
|
||||
The receipt is a summary and discovery document. It does not contain lane
|
||||
descriptors, payloads, manifest data, rejections, warnings, or file contents.
|
||||
For the production JSON output, resolve `index_file` beneath
|
||||
`output_directory`, reject path escapes, and use the
|
||||
[Published JSON Output contract](json-output.md) to discover logical files and
|
||||
lane payloads.
|
||||
|
||||
## Delivery And Compatibility
|
||||
|
||||
Notarius writes the receipt only after the output bundle has been published and
|
||||
any requested debug terminal reporting has completed. Standard output is not
|
||||
transactional: a result-write failure returns a nonzero status and can leave
|
||||
partial bytes. Consumers must ignore standard output unless the process exits
|
||||
with status 0.
|
||||
|
||||
Future versions may add optional fields to this schema. Consumers must tolerate
|
||||
unknown fields. An incompatible field or semantic change requires a new
|
||||
`schema_version` value.
|
||||
@@ -1,69 +1,73 @@
|
||||
# Seriatim Transcript JSON
|
||||
# Seriatim Transcript Input
|
||||
|
||||
This document is the external input contract consumed by the production
|
||||
Seriatim input adapter. Selectable input-adapter keys are cataloged in
|
||||
[Configuration](../config.md#implemented-production-modules).
|
||||
This document defines the JSON transcript accepted by the production Seriatim
|
||||
input adapter. It is a source input, not a durable lane artifact. Configure the
|
||||
input adapter through [Configuration](../config.md#production-module-keys).
|
||||
|
||||
## Adapter
|
||||
## Contract Identity
|
||||
|
||||
- Source format: `application/vnd.seriatim+json`
|
||||
| Property | Value |
|
||||
| --- | --- |
|
||||
| Consumer | Seriatim input adapter |
|
||||
| Media type | `application/vnd.seriatim+json` |
|
||||
| Source document kind | `transcript` |
|
||||
| Source-unit kind | `transcript_segment` |
|
||||
|
||||
## Accepted Shape
|
||||
|
||||
The input must be one JSON object with top-level `metadata` and `segments`
|
||||
fields. This covers the maintained minimal fixture and Seriatim intermediate
|
||||
output that provides the same required segment fields.
|
||||
The input is one JSON object containing `metadata` and a non-empty `segments`
|
||||
array. This minimal document is valid:
|
||||
|
||||
The maintained example is
|
||||
[examples/seriatim-minimal-transcript.json](../../examples/seriatim-minimal-transcript.json).
|
||||
```json
|
||||
{
|
||||
"metadata": {"id": "session-alpha"},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 0,
|
||||
"end": 4,
|
||||
"speaker": "Aria",
|
||||
"text": "Aria casts Cure Wounds."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Required top-level fields:
|
||||
The maintained two-segment input is
|
||||
[seriatim-minimal-transcript.json](../../examples/seriatim-minimal-transcript.json).
|
||||
|
||||
- `metadata`: an object. Its entries are accepted as source metadata.
|
||||
- `segments`: a non-empty array of segment objects.
|
||||
| Field | Required | Meaning and constraints |
|
||||
| --- | --- | --- |
|
||||
| `metadata` | Yes | JSON object. Its entries become source metadata; no particular metadata key is otherwise required. |
|
||||
| `segments` | Yes | Non-empty array of segment objects, kept in input order. |
|
||||
| `segments[].id` | Yes | Positive canonical decimal integer, supplied as a JSON number or string. IDs must be unique. |
|
||||
| `segments[].start` | Yes | Finite, non-negative numeric value, supplied as a JSON number or string. |
|
||||
| `segments[].end` | Yes | Finite, non-negative numeric value that is not earlier than `start`. |
|
||||
| `segments[].speaker` | Yes | String that is non-empty after trimming. |
|
||||
| `segments[].text` | Yes | String that is non-empty after trimming. Its original text is retained. |
|
||||
|
||||
Required segment fields:
|
||||
Additional top-level and segment fields are ignored. A missing required field,
|
||||
`null` in place of an object or array, malformed JSON, or more than one
|
||||
top-level JSON value is rejected.
|
||||
|
||||
- `id`: a positive integer JSON number or canonical decimal string without
|
||||
leading zeros or surrounding whitespace;
|
||||
- `start`: a finite, non-negative JSON number or numeric string;
|
||||
- `end`: a finite, non-negative JSON number or numeric string that is not less
|
||||
than `start`;
|
||||
- `speaker`: a non-empty string;
|
||||
- `text`: a non-empty string.
|
||||
## Source Identity And References
|
||||
|
||||
Other top-level and segment fields, such as `categories`, are ignored.
|
||||
The adapter chooses the source ID in this order:
|
||||
|
||||
Multiple top-level JSON values are rejected.
|
||||
1. a non-empty source ID supplied by the calling request;
|
||||
2. non-empty string `metadata.id`;
|
||||
3. non-empty string `metadata.source_id`;
|
||||
4. `seriatim:` followed by the first 16 hexadecimal characters of the raw
|
||||
input’s SHA-256 digest.
|
||||
|
||||
## Validation
|
||||
Each accepted segment becomes one source unit whose unit ID is `segments[].id`.
|
||||
Its self-reference uses the derived source ID and the same segment ID for both
|
||||
range endpoints. Artifact contracts use those segment IDs when they cite
|
||||
transcript evidence.
|
||||
|
||||
The adapter rejects empty input, malformed JSON, multiple top-level JSON values,
|
||||
non-object segment values, duplicate segment IDs, and any violation of the
|
||||
shape or field constraints above.
|
||||
## Compatibility
|
||||
|
||||
Segment text is preserved as provided, but it must not be empty after trimming.
|
||||
|
||||
## Derived Identity
|
||||
|
||||
Notarius identifies the parsed source in this order:
|
||||
|
||||
1. `metadata.id`, when it is a non-empty string after trimming;
|
||||
2. `metadata.source_id`, when it is a non-empty string after trimming;
|
||||
3. `seriatim:<first-16-hex-chars-of-raw-sha256>`.
|
||||
|
||||
The exact raw input SHA-256 remains the basis of the fallback source ID. The
|
||||
source digest recorded in output provenance is instead the SHA-256 of the
|
||||
canonical generic source document, excluding the digest field itself. It covers
|
||||
the derived source identity, document kind and format, ordered units and their
|
||||
self-references, and accepted metadata. Segment IDs become the unit IDs used by
|
||||
artifact source references; each produced unit carries a self-reference whose
|
||||
source ID is the derived document ID and whose start and end IDs both equal the
|
||||
segment ID.
|
||||
|
||||
## Compatibility Limit
|
||||
|
||||
This contract covers only Seriatim transcript JSON with the top-level
|
||||
`metadata` object and `segments` array described here. Broader Seriatim output
|
||||
schemas are compatible only when they provide these required fields with the
|
||||
accepted types.
|
||||
This adapter accepts only the shape described here. A broader Seriatim export
|
||||
is usable only when it supplies this object, metadata, and segment shape with
|
||||
the stated types and constraints. Unknown additional fields do not add
|
||||
Notarius behavior.
|
||||
|
||||
163
docs/internal/cli.md
Normal file
163
docs/internal/cli.md
Normal file
@@ -0,0 +1,163 @@
|
||||
# CLI Internals
|
||||
|
||||
This document describes **internal/cli**, Notarius's production composition
|
||||
root. The [CLI reference](../cli.md) owns command syntax and exit statuses;
|
||||
[Configuration](../config.md) owns configuration values; and
|
||||
[Operations](../operations.md) owns filesystem layout, recovery, and operator
|
||||
procedures.
|
||||
|
||||
## Inputs, Outputs, And Boundaries
|
||||
|
||||
The CLI accepts process arguments, standard streams, and injectable options
|
||||
used by tests and embedding code. It writes command results to the supplied
|
||||
streams and returns a process exit status. For a run, it also creates the
|
||||
production catalog and runtime collaborators, hands a prepared pipeline and
|
||||
source bytes to the framework, and places the logical files returned by the
|
||||
runner.
|
||||
|
||||
It is the only boundary allowed to compose concrete registries, LLM clients,
|
||||
cache/checkpoint collaborators, debug recorders, and physical output paths.
|
||||
Pipeline modules receive interfaces and request data rather than CLI streams or
|
||||
filesystem roots. The [Architecture](../policy/architecture.md) defines this
|
||||
composition-root boundary; [Pipeline Internals](pipeline.md) owns resolution,
|
||||
preparation, and runner mechanics after their inputs are supplied.
|
||||
|
||||
## Dispatch And Configuration Handoff
|
||||
|
||||
The root dispatcher handles help, configuration validation, pipeline listing,
|
||||
and a pipeline run. It normalizes injectable options before dispatch so that a
|
||||
missing production dependency fails as a command error rather than reaching
|
||||
execution.
|
||||
|
||||
Commands that need configuration use one shared loader. The CLI discovers the
|
||||
file, parses it through **internal/core/config**, starts from defaults, applies
|
||||
the file and supported environment overrides, and then validates it for the
|
||||
command. The configured discovery and precedence contract is in
|
||||
[Configuration](../config.md), while the loading and resolution mechanics are
|
||||
in [Configuration Internals](configuration.md).
|
||||
|
||||
Configuration validation without a selected pipeline checks structural
|
||||
configuration only. Validation with a selected pipeline also builds the
|
||||
effective catalog, resolves the pipeline, and verifies every explicit effective
|
||||
PromptKit profile. Selected LLM-backed input, chunk, lane, output, and validator
|
||||
profiles are inspected
|
||||
against the configured PromptKit source and backend registrations without
|
||||
loading a prompt or performing generation, so an unknown or invalid profile
|
||||
fails before pipeline preparation. Credential availability remains an
|
||||
execution-time concern. Pipeline listing validates configuration before
|
||||
returning normalized, sorted identifiers.
|
||||
|
||||
## Production Composition
|
||||
|
||||
The production composition helper allocates every framework registry and the
|
||||
prompt-asset registry, then registers the generic, Seriatim, and D&D module
|
||||
families in that order. The resulting registries provide both the module
|
||||
catalog used for resolution and the concrete constructors used for preparation.
|
||||
Tests may provide a catalog or registries instead; production code must not
|
||||
silently merge an injected partial catalog with production registrations.
|
||||
|
||||
The production LLM factory builds one PromptKit-backed client from the resolved
|
||||
**promptkit.profile_dir** or **promptkit.profile_file** source, attaches the
|
||||
profile-provenance recorder, creates one scheduler from the effective global
|
||||
LLM limit, and wraps the client before it reaches modules. Registration and LLM
|
||||
construction errors are returned before a pipeline is prepared. Configuration
|
||||
field definitions remain in [Configuration](../config.md#promptkit-profiles);
|
||||
the D&D registrar's fallback profile assets and the adapter mechanics remain in
|
||||
[LLM Runtime](llm.md).
|
||||
|
||||
The factory also accepts `LLMRuntimeOverrides`, whose reasoning pointer
|
||||
preserves inherit, replace, and clear states across the composition boundary.
|
||||
Run orchestration constructs this value from the mutually exclusive
|
||||
`--reasoning-effort` and `--clear-reasoning-effort` controls. Absence preserves
|
||||
a nil pointer, replacement is trimmed, and clear uses a non-nil empty string.
|
||||
The same override reaches the one shared production client, checkpoint
|
||||
identity, and debug invocation metadata. Persistent reasoning configuration
|
||||
remains owned by PromptKit profiles; Notarius configuration has no reasoning
|
||||
field.
|
||||
|
||||
## Run Orchestration
|
||||
|
||||
After parsing and validating a run invocation, the CLI performs this ordered
|
||||
handoff:
|
||||
|
||||
1. load and validate configuration, then apply command-level operational
|
||||
overrides;
|
||||
2. create and validate a safe run identity, then allocate a debug bundle only
|
||||
when requested;
|
||||
3. build the effective catalog, resolve requested reference changes, resolve
|
||||
the effective pipeline, and inspect its explicit effective PromptKit
|
||||
profiles;
|
||||
4. materialize external or generated references and record redacted invocation
|
||||
and resolution provenance when debug capture is enabled;
|
||||
5. construct registries, the scheduled LLM client, prepared modules, and the
|
||||
requested cache/checkpoint collaborators;
|
||||
6. read the source input and invoke the framework runner; and
|
||||
7. write the runner's logical output files only after a successful run, then
|
||||
complete the command report and user-facing result.
|
||||
|
||||
Preparation happens before source parsing, so module construction and
|
||||
dependency failures cannot begin stage execution. The CLI also preserves the
|
||||
framework's result and warning information when it writes summaries and the
|
||||
final command result. Detailed state lifecycle, resume handling, and physical
|
||||
path confinement are maintained in [Run State Internals](state.md) and
|
||||
[Operations](../operations.md).
|
||||
|
||||
For `run --json`, the CLI constructs and encodes its private run-result receipt
|
||||
after a successful runner result is available, before it publishes logical
|
||||
output files. It writes the prepared receipt to standard output only after
|
||||
output publication and requested debug terminalization succeed. A receipt-write
|
||||
failure exits with runtime status 1 and may leave partial standard-output bytes,
|
||||
but the already-published output bundle remains complete and requested debug
|
||||
reporting remains successfully terminalized. The CLI reports a bounded
|
||||
command-owned error and does not repeat terminal reporting. The receipt remains
|
||||
a CLI reporting concern rather than a framework or output-module responsibility;
|
||||
its public contract is the
|
||||
[run-result receipt](../integrations/run-result.md).
|
||||
|
||||
## Failure Mapping And Terminal Reporting
|
||||
|
||||
Argument, flag, and invocation-combination failures are reported to standard
|
||||
error before runtime composition and use the syntax error class. Once an
|
||||
invocation is syntactically valid, configuration loading and validation,
|
||||
resolution, registration, profile checks, reference materialization, module
|
||||
construction, input reads, runner failures, output publication, and requested
|
||||
debug handling use the runtime failure class. The public status numbers and
|
||||
stream contract are defined in the [CLI reference](../cli.md#output-streams-and-exit-statuses).
|
||||
|
||||
When debug capture has been allocated, one command-state value records the
|
||||
known run result. Guarded terminalization writes a success report once, or
|
||||
attempts a failure report and error record once. A persistence failure is
|
||||
reported in addition to the original failure and never replaces it. If a debug
|
||||
path exists, failure output includes that path so the retained diagnostic data
|
||||
is discoverable.
|
||||
|
||||
## Invariants To Preserve
|
||||
|
||||
- Only the CLI composes production implementations and physical runtime roots.
|
||||
- Configuration and resolved composition failures occur before module
|
||||
preparation or source parsing.
|
||||
- A runner's logical files are published only after a successful run.
|
||||
- Production registries and a caller-supplied catalog or registries are
|
||||
alternative composition sources, not an implicit mixture.
|
||||
- A requested debug bundle has one terminal report attempt; its persistence
|
||||
errors supplement rather than obscure the primary command error.
|
||||
- User-facing flags, paths, exit codes, and configuration fields are defined
|
||||
by their public documentation, not duplicated here.
|
||||
|
||||
## Focused Tests
|
||||
|
||||
- **internal/cli/command_contract_test.go** covers dispatch, help, syntax and
|
||||
runtime error classes, discovery, validation, and listing.
|
||||
- **internal/cli/run_contract_test.go** covers the run handoff, publication,
|
||||
debug reporting, and command-owned state collaborators.
|
||||
- **internal/cli/production_contract_test.go** covers registrar composition,
|
||||
production catalog contents, assets, and representative configuration
|
||||
validation.
|
||||
- **internal/cli/reference_contract_test.go** covers CLI reference overrides,
|
||||
origin separation, and materialization boundaries.
|
||||
- **internal/cli/state_hardening_test.go** covers safe run identity, state
|
||||
roots, and failure ordering.
|
||||
|
||||
Run **go test ./internal/cli** after changing command composition or command
|
||||
behavior. Pair it with **go test ./internal/core/config** when the configuration
|
||||
handoff changes.
|
||||
144
docs/internal/configuration.md
Normal file
144
docs/internal/configuration.md
Normal file
@@ -0,0 +1,144 @@
|
||||
# Configuration Internals
|
||||
|
||||
This document describes the maintainer-facing configuration boundary in
|
||||
**internal/core/config**. The [Configuration](../config.md) reference owns the
|
||||
file format, fields, defaults, precedence contract, and selectable keys. The
|
||||
[CLI reference](../cli.md) owns command syntax; this document does not redefine
|
||||
either interface.
|
||||
|
||||
## Boundary
|
||||
|
||||
The configuration package turns a selected YAML file and supported environment
|
||||
values into a validated, independently owned configuration. It then resolves a
|
||||
requested pipeline against a module catalog before the framework prepares or
|
||||
runs anything.
|
||||
|
||||
| Boundary | Inputs | Outputs | Does not own |
|
||||
| --- | --- | --- | --- |
|
||||
| Loading | Selected file path and environment lookup | Parsed file model and a populated **Config** | Choosing the file path or reporting a command result. |
|
||||
| Validation | **Config** | Structural configuration errors with pipeline, lane, or binding context | Module availability, capabilities, or construction. |
|
||||
| Resolution | Valid **Config**, selected pipeline and lanes, runtime reference changes, LLM override, and module catalog | **EffectiveConfig** with a **ResolvedPipeline** | Materializing reference bytes, preparing modules, execution, or filesystem state. |
|
||||
| Summary | **Config** or **EffectiveConfig** | Detached redacted payload suitable for debug summaries | Redacting arbitrary process state or provider traffic. |
|
||||
|
||||
The CLI discovers a configuration file, invokes this package, and supplies the
|
||||
result to the framework. Configuration never reads an input file, constructs a
|
||||
module, or creates output, cache, or debug paths. Those responsibilities remain
|
||||
at their respective [CLI](cli.md), [pipeline](pipeline.md), and
|
||||
[run-state](state.md) boundaries.
|
||||
|
||||
## Loading And Validation
|
||||
|
||||
The CLI loads configuration in this order:
|
||||
|
||||
1. parse the selected YAML file strictly into the file model;
|
||||
2. start from **Default**;
|
||||
3. apply the file model; and
|
||||
4. apply the supported environment overrides.
|
||||
|
||||
This establishes the public precedence order without giving environment input a
|
||||
second file schema. Loading and application reject malformed YAML, unsupported
|
||||
file versions, unknown fields, invalid values, and identifiers that are empty
|
||||
or collide after whitespace normalization. The file application also makes the
|
||||
effective extraction-worker default follow the effective LLM limit. A present
|
||||
PromptKit local-backend object requires and trims its endpoint, defaults its
|
||||
omitted concurrency limit to zero, and is copied so the parsed file model
|
||||
cannot alias the populated **Config**. A pipeline `llm_profile` is
|
||||
presence-aware: omission remains empty, while a present blank value is
|
||||
rejected and a non-empty file value is trimmed before it reaches **Config**.
|
||||
|
||||
**Config.Validate** checks configuration-only invariants before resolution. It
|
||||
rejects incompatible profile sources, invalid state-surface values, unsupported
|
||||
concurrency settings, malformed bindings and references, invalid retries, and
|
||||
invalid pipeline, step, or lane structure. PromptKit local-backend validation
|
||||
accepts only an absolute HTTP or HTTPS endpoint with a host and no user
|
||||
information, query, or fragment, and rejects a negative local concurrency
|
||||
limit. Its errors retain the closest known pipeline, lane, and binding context.
|
||||
It deliberately does not require modules to be registered: that requires a
|
||||
catalog and belongs to resolution.
|
||||
|
||||
The exact user-selectable values and validation rules are defined in
|
||||
[Configuration](../config.md). Keep additions to the file model, an
|
||||
environment override, its validation, and that reference in the same change.
|
||||
|
||||
## Effective Resolution
|
||||
|
||||
**Config.Resolve** first recomputes derived concurrency defaults and validates
|
||||
the configuration. It normalizes the requested pipeline ID, copies the selected
|
||||
profile, and passes the non-empty command-level LLM profile override, requested
|
||||
lane selection, and reference changes to the framework resolver.
|
||||
|
||||
After module and validator selection, the resolver applies the effective
|
||||
profile policy to LLM-backed bindings only: command override, binding profile,
|
||||
pipeline profile, then the prompt default. Deterministic bindings remain
|
||||
profile-free, and no second inheritance decision occurs during execution. The
|
||||
public field definitions and precedence are owned by
|
||||
[Configuration](../config.md#pipelines).
|
||||
|
||||
The framework resolver supplies defaults, selects lanes, resolves validator
|
||||
chains, checks registered module and artifact compatibility, validates module
|
||||
options, and returns the fixed ordered pipeline shape. The resulting
|
||||
**EffectiveConfig** retains the selected ID, requested selection and reference
|
||||
changes, a clone of the input configuration, and the resolved pipeline.
|
||||
Callers may therefore retain or modify their input slices and maps without
|
||||
changing the resolved result, and later consumers cannot mutate the original
|
||||
configuration through the effective value. This ownership includes the nested
|
||||
PromptKit local-backend value.
|
||||
|
||||
Resolution failures stop before module construction and source parsing. They
|
||||
include an error path for an unconfigured pipeline, missing module, missing
|
||||
capability, incompatible artifact variant, invalid option, invalid reference,
|
||||
or invalid lane selection. CLI code maps these valid-invocation failures to the
|
||||
runtime error class described in the [CLI reference](../cli.md#output-streams-and-exit-statuses).
|
||||
|
||||
## Resolved Identity And Redaction
|
||||
|
||||
The framework assigns the resolved pipeline a deterministic SHA-256 digest
|
||||
after defaults, lane selection, module bindings, reference bindings, validator
|
||||
chains, effective LLM profiles, and artifact schema identity have been
|
||||
resolved. The digest excludes
|
||||
its own stored value. It identifies resolved composition rather than raw YAML
|
||||
bytes, a debug payload, or all runtime state. The CLI records it as invocation
|
||||
provenance before execution; cache and checkpoint identity have additional
|
||||
owners in [Run State Internals](state.md).
|
||||
|
||||
Configuration summaries must use **Redacted**, **RedactedSummaryPayload**, or
|
||||
**RedactedResolvedPipelinePayload**, never a direct configuration marshal.
|
||||
Those methods copy every binding and nested option container, replace values
|
||||
whose key is credential-shaped with **[REDACTED]**, and omit materialized
|
||||
reference content while retaining safe binding and reference provenance. The
|
||||
payload must not alias the source configuration or resolved pipeline.
|
||||
PromptKit's local endpoint and concurrency limit are preserved as non-secret
|
||||
configuration metadata in the independently owned summary; the object contains
|
||||
no credential value. This redaction is deliberately narrow: it protects
|
||||
configuration summaries and does not authorize recording arbitrary environment
|
||||
values or provider requests.
|
||||
|
||||
## Invariants To Preserve
|
||||
|
||||
- Defaults, YAML values, and environment values are applied in one direction;
|
||||
later sources may override only their supported operational settings.
|
||||
- A configuration is structurally valid before it is resolved, and a resolved
|
||||
pipeline is compatible with the supplied catalog before preparation begins.
|
||||
- Whitespace-normalized identifiers are unique wherever they identify a
|
||||
pipeline, step, lane, worker, or reference slot.
|
||||
- Resolution and summary generation return detached data. Redaction must cover
|
||||
every configured and resolved binding, including nested validator bindings.
|
||||
- The resolved digest changes when resolved composition changes and never
|
||||
includes itself.
|
||||
|
||||
## Focused Tests
|
||||
|
||||
- **internal/core/config/file_config_contract_test.go** covers strict file
|
||||
parsing, normalization, file application, and structural rejection.
|
||||
- **internal/core/config/env_contract_test.go** covers supported operational
|
||||
overrides and their precedence.
|
||||
- **internal/core/config/validation_contract_test.go** covers configuration
|
||||
invariants and contextual failures.
|
||||
- **internal/core/config/effective_config_contract_test.go** covers defaults,
|
||||
selections, overrides, resolution context, digest changes, and ownership.
|
||||
- **internal/core/config/redaction_test.go** covers recursive credential
|
||||
redaction, reference-content exclusion, and non-aliasing payloads.
|
||||
|
||||
Run **go test ./internal/core/config** after changing this boundary. Changes to
|
||||
the handoff or resolved-composition semantics also need the focused framework
|
||||
pipeline tests.
|
||||
@@ -1,102 +0,0 @@
|
||||
# Diagnostics Internals
|
||||
|
||||
`internal/core/diagnostics` provides the scoped writer and retention decision
|
||||
used by `internal/cli`. The physical layout, artifact inventory, retention
|
||||
semantics, failure inspection, and cleanup procedures are canonical in
|
||||
[Operations](../operations.md#diagnostics-directory). Configuration fields and
|
||||
defaults are canonical in [Configuration](../config.md#diagnostics).
|
||||
|
||||
## Run Directory
|
||||
|
||||
`NewRunDirectory` normalizes empty constructor inputs, creates the effective
|
||||
diagnostics root when needed, and allocates a unique timestamp-based child
|
||||
directory. It retries a bounded number of collisions before failing. The
|
||||
resulting `RunDirectory` retains its creation time and retention mode for later
|
||||
metadata and cleanup decisions.
|
||||
|
||||
The package does not resolve workspace configuration. `internal/cli` derives
|
||||
effective workspace settings first and passes the diagnostics root into the
|
||||
constructor.
|
||||
|
||||
## Scoped Writers
|
||||
|
||||
Typed methods on `RunDirectory` write invocation metadata, redacted effective
|
||||
configuration, resolved pipeline/reference data, checkpoint events, source data
|
||||
when explicitly requested, manifests, reports, warnings, redacted chunk-plan
|
||||
summaries, and error text. The
|
||||
current filenames and their operator-facing contents are listed in
|
||||
[Operations](../operations.md#diagnostics-directory).
|
||||
|
||||
The chunk-plan summary records the effective mode, source and candidate
|
||||
digests, requested module, lookup decision, materialization action, validation
|
||||
decision, and publication decision. Its closed decision values make failures
|
||||
and recoverable invalid records inspectable without serializing plan ranges,
|
||||
annotations, source content, reference content, prompts, model responses, or
|
||||
raw invalid-file bytes.
|
||||
|
||||
JSON methods indent their payload and append a newline. All artifact writes use
|
||||
a temporary file in the target directory, apply the requested permissions, and
|
||||
rename it into place. Artifact resolution accepts only a single relative base
|
||||
name; absolute paths, separators, and paths escaping the run directory fail
|
||||
before writing.
|
||||
|
||||
## Redacted Configuration
|
||||
|
||||
`WriteRedactedEffectiveConfig` accepts a `RedactedDiagnosticsPayload` provider
|
||||
rather than a raw config value. `internal/core/config` implements that contract
|
||||
by cloning effective config data and removing secret-shaped values before JSON
|
||||
encoding. The diagnostics package therefore never needs configuration-specific
|
||||
field knowledge.
|
||||
|
||||
## Retention Coordination
|
||||
|
||||
`ShouldRetainRunDirectory` is a pure decision over the effective retention mode,
|
||||
run success, and warning presence. `ApplyRetention` uses that result to remove
|
||||
only its own run directory. Unsupported modes retain data as a fail-safe, though
|
||||
normal CLI execution rejects them during config validation.
|
||||
|
||||
The meaning of each supported mode belongs in
|
||||
[Operations](../operations.md#retention); this package implements that contract
|
||||
without loading config or inspecting run artifacts.
|
||||
|
||||
## CLI State Flow
|
||||
|
||||
When diagnostics are enabled, `internal/cli` creates the run directory after
|
||||
configuration loading and before pipeline resolution. It then writes artifacts
|
||||
as state becomes available: invocation data, effective resolution data,
|
||||
pipeline results, and the final report. This ordering permits later failures to
|
||||
retain the context already established.
|
||||
|
||||
Failures before construction have no `RunDirectory`. Later failures write an
|
||||
error log, preserve any available partial manifest and chunk-plan summary, and
|
||||
apply a failed-run
|
||||
retention decision. A diagnostics write failure is itself a command failure so
|
||||
the CLI does not report success after losing requested inspection data.
|
||||
|
||||
When diagnostics are disabled, the CLI carries a nil run directory and the
|
||||
shared `writeDiagnostics` helper turns writes into no-ops. User-facing errors
|
||||
still go to stderr; that invocation behavior is documented in
|
||||
[Operations](../operations.md#failures).
|
||||
|
||||
## Package Guarantees
|
||||
|
||||
- A `RunDirectory` writes and removes only within its allocated directory.
|
||||
- JSON and error artifacts use atomic replacement.
|
||||
- Nil receivers and invalid typed payloads return errors rather than panicking.
|
||||
- Retention never removes a failed run and never targets the diagnostics root.
|
||||
- Diagnostics models contain inspection metadata, not the durable output
|
||||
contract.
|
||||
- Checkpoint and debug serializers remain separate framework components.
|
||||
- Secret-handling follows the invariant in
|
||||
[Architecture](../policy/architecture.md#state-output-and-safety).
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
- `internal/core/diagnostics/run_dir_test.go`: allocation, artifact confinement,
|
||||
atomic writes, retention, and failure behavior.
|
||||
- `internal/core/diagnostics/artifacts_test.go`: stable artifact identifiers.
|
||||
- `internal/core/config/redaction_test.go`: clone-and-redact payload behavior.
|
||||
- `internal/core/workspace/settings_test.go`: effective diagnostics-root and
|
||||
enablement handoff.
|
||||
- `internal/cli/run_test.go`: creation timing, artifact sequencing, disabled
|
||||
diagnostics, overrides, failures, and retention integration.
|
||||
155
docs/internal/dnd.md
Normal file
155
docs/internal/dnd.md
Normal file
@@ -0,0 +1,155 @@
|
||||
# D&D Module Internals
|
||||
|
||||
This guide records the conventions shared by the production D&D module family.
|
||||
It complements [Module Internals](modules.md), which owns generic registration
|
||||
and extension mechanics, and [Configuration](../config.md), which owns the
|
||||
selectable keys, bindings, reference syntax, and default validator chains.
|
||||
|
||||
## Durable Artifact Contracts
|
||||
|
||||
The six lanes have separate durable wire contracts. This guide deliberately
|
||||
does not repeat their JSON shapes or schemas.
|
||||
|
||||
| Lane | Durable contract |
|
||||
| --- | --- |
|
||||
| Spells | [spell artifacts](../integrations/dnd-spell-artifacts.md) |
|
||||
| NPCs | [NPC artifacts](../integrations/dnd-npc-artifacts.md) |
|
||||
| Combat turns | [combat-turn artifacts](../integrations/dnd-combat-turn-artifacts.md) |
|
||||
| Item events | [item-event artifacts](../integrations/dnd-item-event-artifacts.md) |
|
||||
| NPC interactions | [NPC-interaction artifacts](../integrations/dnd-npc-interaction-artifacts.md) |
|
||||
| Scene descriptions | [scene-description artifacts](../integrations/dnd-scene-description-artifacts.md) |
|
||||
|
||||
## Family Composition
|
||||
|
||||
The D&D registrar registers the family’s artifact codecs, extractors, typed
|
||||
append-order mergers, normalizers, validators, prompt assets, fallback LLM
|
||||
profile asset, and default validator chains. Each extractor and normalizer has
|
||||
a stable module spec, explicit execution class, strict option decoding, and a
|
||||
typed builder. Scene chunking, every extractor, and NPC normalization are
|
||||
registered as `llm_backed`; the remaining current D&D mergers and normalizers
|
||||
are `deterministic`. The metadata is available to catalog inspection and
|
||||
resolved-pipeline debug data and determines which selected bindings inherit the
|
||||
pipeline profile. Configuration remains the canonical owner of the exact keys,
|
||||
profile precedence, and validator order.
|
||||
|
||||
Private structured-LLM response schemas are deliberately minimal. They reject
|
||||
invalid JSON structure, missing required fields, incompatible types, and
|
||||
unknown fields, while preserving semantic candidates for deterministic
|
||||
validation. Do not promote a private response envelope into a durable schema;
|
||||
the contracts above define durable data.
|
||||
|
||||
## Prompt Construction
|
||||
|
||||
D&D extractors assemble prompts from an ordered manifest of shared and
|
||||
module-owned assets. Reuse the shared D&D system, evidence, identity,
|
||||
reference, and transcript assets instead of copying their text into individual
|
||||
modules. A manifest’s declared sequence, including cache-control placement, is
|
||||
part of the prompt behavior.
|
||||
|
||||
Every maintained D&D LLM prompt selects `dnd-extraction` as its default
|
||||
profile. The D&D registrar embeds that fallback profile with the maintained
|
||||
OpenRouter model, timeout, and service-tier policy. An operator may provide a
|
||||
complete profile with the same ID through the configured PromptKit source; that
|
||||
definition replaces the fallback rather than merging with it. The fallback
|
||||
leaves reasoning and optional sampling controls unspecified. Deployment profile
|
||||
selection and the maintained operator example are documented in
|
||||
[Configuration](../config.md#promptkit-profiles).
|
||||
|
||||
All extraction prompts share this four-message rendered prefix: the system
|
||||
message without cache control, the identity message without cache control, the
|
||||
campaign-reference message with ephemeral cache control, and the chunk
|
||||
transcript message with ephemeral cache control. This gives equivalent
|
||||
extraction requests the same reusable prefix through their source material.
|
||||
|
||||
Extraction-evidence policy, generated NPC registries, spell catalogs, module
|
||||
tasks, and instructions follow the transcript because they are not universal
|
||||
across all extraction lanes. The final instructions message carries ephemeral
|
||||
cache control; evidence, registry, catalog, and task messages do not. Preserve
|
||||
this division when changing an extractor or its assets so prompt-cache behavior
|
||||
remains stable.
|
||||
|
||||
The other D&D LLM prompts intentionally follow different patterns. Scene
|
||||
chunking has no sibling extraction lane with which to share its full transcript,
|
||||
so it renders campaign references before its task and instructions, then places
|
||||
the cacheable full transcript last. NPC normalization keeps its task and
|
||||
cacheable instructions before the candidate collection, followed by the
|
||||
cacheable transcript windows: candidates must be available before their
|
||||
supporting evidence is evaluated, and those windows are not a cross-lane
|
||||
prefix. Mounted assets and their declared message order determine the prompt
|
||||
fingerprint, so intentional prompt edits continue to invalidate stale
|
||||
checkpoints.
|
||||
|
||||
All extractors use the shared prompt-input preparation rules. The current chunk
|
||||
is copied into transcript material; player, party, glossary, and compatible
|
||||
campaign references are context for disambiguation, not source evidence.
|
||||
Reference prompt material is canonically ordered before it is rendered, which
|
||||
keeps equivalent inputs stable across runs.
|
||||
|
||||
## Evidence, Candidates, And Normalization
|
||||
|
||||
The current transcript is the only durable evidence source. Extractors assign
|
||||
the current source identity, preserve candidate evidence ranges for validators,
|
||||
and canonically order or remove exact duplicate ranges without asking the
|
||||
model to repair semantic errors. Campaign context and generated artifacts may
|
||||
ground names or control routing, but they never establish evidence for a D&D
|
||||
result.
|
||||
|
||||
Default chains keep responsibilities separate: structural validators assess the
|
||||
candidate, source-reference validators resolve cited ranges against the current
|
||||
source, durable-schema validation checks an approved representation, and
|
||||
relatedness validators report advisory evidence concerns. The configured order
|
||||
is documented in
|
||||
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||
|
||||
Normalizers are deterministic for spells, combat turns, item events, NPC
|
||||
interactions, and scene descriptions. They canonicalize display values and
|
||||
evidence, use source-document order for stable output, and issue bounded
|
||||
warnings for changes or collapsed duplicates. The NPC normalizer is the
|
||||
intentional exception: it first produces a deterministic candidate set, then
|
||||
uses a bounded structured-LLM proposal to reconcile identity groups. Invalid
|
||||
or unusable proposals retain the deterministic result and surface retry or
|
||||
fallback diagnostics; the model does not directly replace durable records.
|
||||
|
||||
## Generated References And Grounding
|
||||
|
||||
Normalized D&D artifacts can be handed to a later step through a generated
|
||||
reference binding. The framework verifies artifact compatibility and retains
|
||||
producer provenance; consumers resolve the handed-off artifact into an
|
||||
immutable, validated projection for each operation. External files are checked
|
||||
during preparation, while generated artifacts are resolved at the handoff.
|
||||
|
||||
NPC registries are names-only grounding projections: they may canonicalize
|
||||
actors for spells and combat turns and are required for NPC interactions, but
|
||||
they do not supply evidence. Scene-description registries are eligibility-only
|
||||
projections: they retain the current chunk’s classification data, not scene
|
||||
prose or evidence, and exist to route combat extraction.
|
||||
|
||||
## Lane-Specific Rules
|
||||
|
||||
The following differences are intentional and should remain explicit when a
|
||||
shared helper changes.
|
||||
|
||||
| Lane | Intentional behavior |
|
||||
| --- | --- |
|
||||
| Spells | May use a spell-catalog overlay and optional NPC grounding; the catalog validator supplies domain-specific semantic checks. |
|
||||
| NPCs | Does not consume an NPC registry. Its normalizer is the LLM-assisted reconciliation exception described above. |
|
||||
| Combat turns | Requires a scene-description artifact. It calls the LLM only for an exact `combat` classification; exact non-combat classifications return an accepted empty result, while missing or mismatched classifications return an empty result with a bounded warning. Optional NPC grounding never becomes evidence. |
|
||||
| Item events | Uses campaign context for disambiguation but has no NPC-registry or scene-description dependency. |
|
||||
| NPC interactions | Requires the normalized NPC registry at extraction and normalization, using it for canonical actor grounding only. |
|
||||
| Scene descriptions | Produces the classifications consumed by combat routing; it does not consume an NPC registry or provide evidence for combat artifacts. |
|
||||
|
||||
The combat and scene-description contracts describe their exact handoff and
|
||||
empty-result behavior in more detail:
|
||||
[combat turns](../integrations/dnd-combat-turn-artifacts.md) and
|
||||
[scene descriptions](../integrations/dnd-scene-description-artifacts.md).
|
||||
|
||||
## Focused Verification
|
||||
|
||||
When changing D&D behavior, test the affected codec, extractor, normalizer,
|
||||
validator, prompt-asset manifest, and registry projection. Also test generated
|
||||
handoffs at the integration boundary and run the full D&D module suite:
|
||||
|
||||
~~~sh
|
||||
go test ./internal/modules/dnd/...
|
||||
go test ./internal/modules/integration/...
|
||||
~~~
|
||||
@@ -1,148 +1,251 @@
|
||||
# LLM Runtime Internals
|
||||
|
||||
`internal/framework/llm` implements Notarius's transport boundary for structured
|
||||
completion. It contains the Scriptorium adapter, concurrency scheduler,
|
||||
prompt/schema registries, selected-profile recording, and provider-error
|
||||
redaction.
|
||||
`internal/framework/llm` is Notarius’s provider-independent structured
|
||||
completion boundary. It adapts framework requests to PromptKit, bounds
|
||||
provider calls, assembles registered prompt and schema assets, records selected
|
||||
profiles, and redacts provider errors. The architectural boundary is defined in
|
||||
[Architecture](../policy/architecture.md#llm-boundary); profile sources,
|
||||
credentials, and concurrency settings belong in
|
||||
[Configuration](../config.md#promptkit-profiles) and
|
||||
[Configuration](../config.md#concurrency-output-cache-and-debug).
|
||||
|
||||
Provider-neutral ownership rules are defined in
|
||||
[Architecture](../policy/architecture.md#llm-boundary). Profile sources,
|
||||
credentials, and concurrency settings are defined in
|
||||
[Configuration](../config.md).
|
||||
## Structured Completion Boundary
|
||||
|
||||
## Structured Contract
|
||||
Modules and LLM-backed validators depend only on
|
||||
`contracts.StructuredLLMClient`. A completion request supplies a prompt ID and
|
||||
version, optional profile and session IDs, named input material, variables, and
|
||||
a caller-owned decode target. The successful response returns the validated raw
|
||||
structured bytes together with non-secret provider, model, profile, and token
|
||||
metadata.
|
||||
|
||||
Modules and LLM-backed validators depend on
|
||||
`contracts.StructuredLLMClient.CompleteStructured`. A request identifies a
|
||||
prompt and optional profile/session, supplies named input materials and
|
||||
variables, and provides a caller-owned decoding target. A successful response
|
||||
contains the validated raw structured bytes plus non-secret provider, model,
|
||||
profile, and token metadata.
|
||||
The caller owns the domain behavior: it chooses the prompt, prepares inputs,
|
||||
selects the private response schema, and interprets the decoded result. The
|
||||
adapter does not own source evidence, artifact conversion, normalization, or
|
||||
durable schemas. Those responsibilities remain with the module and its
|
||||
[integration contract](../integrations/).
|
||||
|
||||
The caller owns prompt selection, response-schema selection, and interpretation
|
||||
of the decoded result. `LLMInputMaterial` keeps source and reference bytes with
|
||||
their origin metadata so the adapter can pass named artifacts to Scriptorium
|
||||
without exposing Scriptorium types through stage contracts.
|
||||
`PromptKitClient` validates the request target and prompt identity, maps each
|
||||
named material to a PromptKit inline artifact while preserving its origin URI,
|
||||
maps the trimmed request session to PromptKit's direct per-run session field,
|
||||
retains the same value as the `session_id` prompt variable for maintained
|
||||
prompt compatibility, and forwards profile selection. It then creates one
|
||||
frozen prepared execution, captures its caller-owned credential-redacted
|
||||
details for debug material, and executes that exact snapshot through
|
||||
PromptKit's prepared-execution boundary. The direct field
|
||||
is authoritative for provider session behavior. A session ID is a stable,
|
||||
non-secret correlation identifier and may be exposed to providers and provider
|
||||
observability. The adapter returns PromptKit’s validated raw bytes rather than
|
||||
re-encoding the decoded target. An empty optional material is represented as
|
||||
one space so its named input is retained by PromptKit.
|
||||
|
||||
## Production Construction
|
||||
Client construction may also receive a run-wide reasoning-effort override from
|
||||
the CLI factory boundary. The adapter copies the caller-owned pointer and
|
||||
creates a fresh PromptKit execution override for each request: a nil pointer
|
||||
inherits the selected profile, a non-empty value replaces it, and an empty
|
||||
value clears inherited reasoning. The CLI's mutually exclusive
|
||||
`--reasoning-effort` and `--clear-reasoning-effort` controls select those
|
||||
states. With neither flag, profile behavior remains unchanged. Because
|
||||
production constructs one shared client, the selected state applies uniformly
|
||||
to module calls, retries, and LLM-backed validators for the whole run.
|
||||
|
||||
`internal/cli` constructs the production runtime by:
|
||||
An empty request profile lets the prompt select its configured default. Before a
|
||||
run begins, the CLI asks the adapter to inspect every explicit profile on the
|
||||
resolved selected LLM-backed bindings and validators, including inherited
|
||||
pipeline profiles. Inspection resolves the profile and its selected backend and
|
||||
target without loading a prompt, reading credentials, admitting capacity, or
|
||||
contacting a provider, so a missing or invalid explicit profile fails before
|
||||
stage execution while a valid `api_key_env` may remain unset. Calls record the
|
||||
profile actually selected by PromptKit. The recorder trims and deduplicates
|
||||
non-secret profile identity, provider, model, selected backend ID, and
|
||||
effective reasoning values for manifest use. Entries that differ in backend or
|
||||
reasoning remain distinct and deterministically ordered. Endpoint-only profiles
|
||||
retain an empty backend ID, which the published JSON omits. Successful
|
||||
completion responses and recorded profile manifests identify the adapter
|
||||
provider as `promptkit`.
|
||||
|
||||
1. allocating the asset registry populated by the generic, Seriatim, and D&D
|
||||
package-family registrars;
|
||||
2. creating a `ScriptoriumClient` from the effective profile source;
|
||||
3. attaching an `LLMProfileRecorder`;
|
||||
4. creating a scheduler from the effective concurrency limit;
|
||||
5. returning a `ScheduledClient` wrapper;
|
||||
6. decorating that shared client before preparation when debug recording is
|
||||
enabled; and
|
||||
7. injecting that one shared client into complete pipeline preparation before
|
||||
the source file is read or the runner is invoked.
|
||||
The CLI's profile-inspection engine and the production adapter use the same
|
||||
profile-source construction to apply the configured profile directory or file,
|
||||
the optional registered fallback profile assets, and the optional conventional
|
||||
`local` backend. Preflight therefore resolves the same profile sources and
|
||||
backend membership as runtime without performing generation. Fallback assets
|
||||
are mounted only when at least one source is registered. The production D&D
|
||||
registrar contributes its `dnd-extraction` fallback, and the maintained D&D
|
||||
prompts select that logical ID by default. PromptKit owns source precedence and
|
||||
profile parsing: an operator-provided matching profile takes precedence over a
|
||||
fallback profile without Notarius merging either document.
|
||||
When the registration is absent, a profile selecting `backend: local` fails
|
||||
inspection instead of falling back to a built-in or endpoint-only target.
|
||||
|
||||
The D&D scene chunker and spell extractor retain this injected client and use
|
||||
it for every structured completion. Operation requests do not carry an LLM
|
||||
client.
|
||||
Before execution, the adapter also contributes a non-secret checkpoint
|
||||
fingerprint for the effective PromptKit profile source. It combines the
|
||||
identity of PromptKit's compiled-in profile catalog with a deterministic digest
|
||||
of every YAML profile in the configured profile directory, or of the configured
|
||||
profile file, and a deterministic digest of the flattened fallback profile
|
||||
assets. The fingerprint contains neither profile content nor source paths. It
|
||||
covers inherited pipeline profiles, explicit binding profiles, and
|
||||
prompt-selected defaults, so changing a model or other profile setting cannot
|
||||
reuse checkpoints created under the
|
||||
prior profile source. This cache identity is independent of durable
|
||||
profile provenance: run manifests continue to list only profiles actually
|
||||
observed during LLM calls. When the local backend is registered, a second
|
||||
fingerprint hashes its trimmed endpoint behind a stable marker. Changing that
|
||||
semantic execution target invalidates checkpoint reuse. The raw endpoint is not
|
||||
stored in checkpoint identity, and the local concurrency limit is excluded
|
||||
because it changes scheduling rather than execution semantics.
|
||||
|
||||
The CLI separately gathers explicit profile IDs from resolved LLM-capable stage
|
||||
and validator bindings. It prepares a small internal check prompt for each ID so
|
||||
missing or invalid profiles fail before pipeline execution. The runtime profile
|
||||
override syntax and scope are defined in the
|
||||
[CLI reference](../cli.md#run); binding rules are defined in
|
||||
[Configuration](../config.md#module-bindings).
|
||||
## Shared Provider-Call Limit
|
||||
|
||||
## Scriptorium Adapter
|
||||
Production construction creates one PromptKit client and wraps it in one
|
||||
scheduled client. The scheduler has a fixed, positive permit limit, serves
|
||||
queued calls in FIFO order, and removes a queued call when its context is
|
||||
cancelled. A granted permit is released exactly once on every completion path.
|
||||
|
||||
`ScriptoriumClient` converts a Notarius request into a Scriptorium `RunRequest`.
|
||||
It validates the decoding target and prompt identity, maps named input materials
|
||||
to inline artifacts, forwards explicit profile and session context, delegates
|
||||
rendering/provider execution/structured validation, and unmarshals successful
|
||||
JSON into the caller target.
|
||||
The scheduled wrapper surrounds every `CompleteStructured` call, so concurrent
|
||||
lanes, pipeline retries, and LLM-backed validators share the same provider-call
|
||||
ceiling. This ceiling is independent of pipeline worker concurrency; changing
|
||||
worker counts cannot exceed the configured LLM limit. The configuration field
|
||||
and its effective default are owned by
|
||||
[Configuration](../config.md#concurrency-output-cache-and-debug).
|
||||
|
||||
Empty optional input material is represented by a single space so Scriptorium
|
||||
retains the named input. The client returns Scriptorium's validated structured
|
||||
bytes rather than re-encoding the caller target, allowing modules to preserve
|
||||
the runtime result exactly.
|
||||
|
||||
Selected profile, provider, model, and token metadata are mapped into the
|
||||
Notarius response. The recorder deduplicates profiles by identity and supplies
|
||||
manifest-safe profile summaries after actual calls; manifest population does
|
||||
not guess the selected prompt default in advance.
|
||||
|
||||
Generated-output validation failures and provider failures are wrapped with
|
||||
prompt context. Error strings pass through bearer-token redaction before they
|
||||
cross the runtime boundary.
|
||||
|
||||
## Scheduling
|
||||
|
||||
`Scheduler` uses a bounded permit count and a FIFO waiter queue. Immediate
|
||||
acquisition increments the in-flight count; queued acquisition waits for a
|
||||
permit or context cancellation. Cancellation removes a queued waiter, while a
|
||||
cancelled waiter that has already received a permit releases it.
|
||||
|
||||
`ScheduledClient` acquires a permit around each structured completion and
|
||||
defers release on every result path. The effective limit and default are
|
||||
configuration facts in [Configuration](../config.md#defaults).
|
||||
|
||||
This provider-call ceiling is independent of the pipeline's extract worker
|
||||
limit. Concurrent lanes, retries, and validators all use the same scheduled
|
||||
client, so increasing framework workers cannot exceed `total_llm`. Pipeline
|
||||
dispatch and cancellation mechanics are documented in
|
||||
[Pipeline Internals](pipeline.md#execution-flow).
|
||||
PromptKit applies a second, independent admission limit when the selected
|
||||
profile names a limited backend. It sits beneath the Notarius scheduled client,
|
||||
so it may narrow but cannot expand the application-wide limit. Built-in
|
||||
OpenRouter profiles select PromptKit's reserved backend and its upstream
|
||||
capacity policy. A positive configured local-backend limit bounds active local
|
||||
generations inside PromptKit; zero leaves that backend unlimited there.
|
||||
Endpoint-only profiles do not select a PromptKit backend and remain limited
|
||||
only by the Notarius scheduler.
|
||||
|
||||
## Prompt And Schema Assets
|
||||
|
||||
`AssetRegistry` combines caller-owned prompt filesystems under stable prefixes
|
||||
and rejects invalid or conflicting registrations. Production module packages
|
||||
register their own prompt and schema assets; generic framework code contains no
|
||||
D&D prompt content. `internal/framework/promptfs` provides the domain-neutral
|
||||
filesystem composition helper used to combine module-owned files with shared
|
||||
domain prompt fragments.
|
||||
An `AssetRegistry` collects prompt, schema, and optional fallback-profile
|
||||
filesystems from production module families. It flattens registered roots into
|
||||
the corresponding PromptKit filesystems and rejects invalid roots, unreadable
|
||||
assets, duplicate paths, and missing prompt or schema files during preparation.
|
||||
Fallback assets receive a safe content digest for checkpoint identity; raw
|
||||
paths and bytes are never included. The framework’s `promptfs` helper combines
|
||||
module-owned prompt files with reusable domain fragments without making the
|
||||
framework depend on D&D content.
|
||||
|
||||
Schema helpers load embedded JSON Schema with identity and digest metadata,
|
||||
return defensive copies, and expose a diagnostics map that omits schema bytes.
|
||||
The small framework registry contains only generic test schemas; production
|
||||
schemas remain package-owned.
|
||||
Each LLM-backed module owns its prompt declaration, package-specific assets,
|
||||
and private response schema. Shared D&D wording is owned by the D&D shared
|
||||
asset package; the detailed D&D conventions are in
|
||||
[D&D Module Internals](dnd.md). The mounted prompt assets used by a module also
|
||||
determine its prompt fingerprint. Schema loaders validate JSON, attach identity
|
||||
and digest metadata, make defensive copies, and expose diagnostics without raw
|
||||
schema bytes.
|
||||
|
||||
## Debug And Redaction Boundaries
|
||||
Private response schemas validate a model transport envelope. They are not the
|
||||
durable artifact schema and should not be documented as an external wire
|
||||
contract. Durable formats and compatibility rules remain in the
|
||||
[integration contracts](../integrations/).
|
||||
|
||||
The pipeline may wrap the client with a debug recorder that captures prepared
|
||||
prompt/response material for an explicitly enabled debug run. Default
|
||||
diagnostics and manifests receive identities, hashes, usage, and selected
|
||||
profile summaries rather than prompt, source, reference, schema, or response
|
||||
content.
|
||||
## Prompt Maintenance And Backend Caching
|
||||
|
||||
The Scriptorium error wrapper removes bearer credential values from surfaced
|
||||
provider errors; `RedactSecrets` and `ErrorWithSecretsRedacted` support known
|
||||
secret values elsewhere in the runtime. Config diagnostics use a separate
|
||||
clone-and-redact path in `internal/core/config`. These mechanisms implement the
|
||||
security invariant in
|
||||
[Architecture](../policy/architecture.md#state-output-and-safety); operator
|
||||
handling of debug data is defined in [Operations](../operations.md#debug).
|
||||
Prompt message order and shared asset bytes are runtime behavior. Backend cache
|
||||
reuse depends on identical preceding roles, rendered bytes, and cache-control
|
||||
metadata—not merely equivalent meaning. Keep reusable shared assets
|
||||
byte-identical and preserve each prompt’s declared ordering and cache controls
|
||||
when editing it.
|
||||
|
||||
## Failure Behavior
|
||||
For sibling prompts that can reuse the same source material, order universal
|
||||
shared context first, request source material next, and module-specific
|
||||
suffixes last. Put a cache boundary at a reusable prefix that is useful to the
|
||||
backend. Redundant intermediate cache boundaries do not extend that reusable
|
||||
prefix and add no value.
|
||||
|
||||
- Invalid targets, missing prompt IDs, malformed structured output, and
|
||||
Scriptorium failures return contextual errors to the calling module.
|
||||
- Scheduler construction rejects non-positive limits; acquisition respects
|
||||
context cancellation.
|
||||
- Asset registration rejects invalid roots, missing content, and path conflicts.
|
||||
- Schema loading distinguishes missing assets, invalid JSON, and invalid
|
||||
metadata.
|
||||
- Profile validation errors occur during CLI preparation when an explicit
|
||||
selected ID cannot be prepared.
|
||||
Prompt-family owners may choose a different sequence when their inputs and
|
||||
reuse pattern differ. The D&D family’s extraction, scene-chunking, and NPC
|
||||
normalization policies are maintained in [D&D Module Internals](dnd.md#prompt-construction).
|
||||
Do not add tests that enforce prompt prose; prompt tests should verify the
|
||||
meaningful input placement and cache controls of the prompt being changed.
|
||||
|
||||
## Tests To Inspect
|
||||
## Validation, Repair, And Retries
|
||||
|
||||
- `internal/framework/llm/scriptorium_client_test.go` and
|
||||
`scriptorium_api_test.go`: adapter mapping and local HTTP integration.
|
||||
- `internal/framework/llm/scheduler_test.go` and
|
||||
`scheduled_client_test.go`: permits, FIFO behavior, cancellation, and wrapper
|
||||
release.
|
||||
- `internal/framework/llm/asset_registry_test.go` and
|
||||
`schema_registry_test.go`: asset composition, validation, and defensive
|
||||
copies.
|
||||
- `internal/framework/llm/secrets_test.go`: provider-error redaction.
|
||||
- `internal/cli/run_test.go`: profile validation, production client wiring,
|
||||
manifest recording, and debug integration.
|
||||
- Module-local `scriptorium_assets_test.go` files: prompt inputs and package
|
||||
asset registration.
|
||||
PromptKit performs prompt rendering, provider execution, and the prompt’s
|
||||
structured-output validation. The adapter reports an empty result, validation
|
||||
failure, empty structured body, or decode failure as
|
||||
`ErrInvalidStructuredOutput`, while retaining the returned raw bytes and debug
|
||||
material when they exist. Provider failures remain operational errors rather
|
||||
than output-validation failures.
|
||||
|
||||
When PromptKit rejects backend admission before generation, the adapter maps
|
||||
`promptkit.ErrCapacityExceeded` to
|
||||
`contracts.ErrLLMCapacityExceeded`, retaining prompt context and a redacted
|
||||
upstream diagnostic without exposing the PromptKit sentinel or capacity-error
|
||||
type as a framework contract. When supplied, the normalized selected backend
|
||||
ID appears only in that safe application-owned diagnostic context. A canceled
|
||||
caller context takes precedence. The adapter does not retry capacity failures;
|
||||
the pipeline's existing binding attempt policy sees the operational error and
|
||||
decides whether to rerun the complete operation.
|
||||
|
||||
Prompt-declared repair is executed within PromptKit’s structured-output flow.
|
||||
The current production D&D prompt manifests set repair attempts to zero. That
|
||||
setting does not replace pipeline retry behavior: a binding’s configured retry
|
||||
count reruns its stage attempt after an error or rejection, and an exhausted
|
||||
rejection is a recorded output rather than a provider error. The pipeline owns
|
||||
attempt lifecycle, validation chains, and retry diagnostics; see
|
||||
[Pipeline Internals](pipeline.md#validation-retries-and-output) and the
|
||||
[binding reference](../config.md#module-bindings-and-validators).
|
||||
|
||||
## Timeout Ownership
|
||||
|
||||
The caller context remains the outer cancellation authority. PromptKit applies
|
||||
a positive effective generation timeout as an inner request deadline; an
|
||||
explicit zero disables only that generation deadline. The HTTP client timeout
|
||||
is a separate transport-wide cap. Notarius forwards the caller context and
|
||||
does not install another timeout wrapper around PromptKit.
|
||||
|
||||
The selected PromptKit profile owns generation settings. Notarius binding
|
||||
retries remain outside the adapter and repeat the complete module operation
|
||||
and validation chain. PromptKit does not add a provider retry loop.
|
||||
Operator-facing behavior is summarized in
|
||||
[Operations](../operations.md#operational-limits), and the pinned upstream
|
||||
contract is identified in
|
||||
[PromptKit Integration](../integrations/pkg-promptkit.md).
|
||||
|
||||
## Observability And Redaction
|
||||
|
||||
When debug recording is enabled, the pipeline decorates the shared client. The
|
||||
wrapper records prepared prompt and response material, timing, selected profile
|
||||
and backend, effective model parameters, and call identifiers in the run’s
|
||||
debug bundle, including material available from a failed structured completion.
|
||||
Effective parameters use PromptKit's stable lower-case JSON field names and may
|
||||
include `backend_id`. For a successful completion, a debug-write failure is
|
||||
surfaced; when the completion already failed, its call error remains the
|
||||
result. Debug-bundle location, retention, and handling are operational concerns
|
||||
documented in [Operations](../operations.md#debug-bundles).
|
||||
|
||||
Run manifests receive selected profile summaries, including optional effective
|
||||
backend and reasoning provenance, and component identities—not prompt, schema,
|
||||
source, reference, or response content. The published field semantics belong
|
||||
to the [JSON output contract](../integrations/json-output.md#manifestjson).
|
||||
Provider error text is wrapped with prompt context and bearer credentials are
|
||||
redacted before it crosses the runtime boundary. Known-secret redaction is
|
||||
available to other runtime collaborators; it does not make prompt or response
|
||||
contents safe for general logging.
|
||||
|
||||
## Failure Boundaries
|
||||
|
||||
- Construction fails for missing asset registries, mutually exclusive profile
|
||||
sources, invalid asset registration, or a non-positive scheduler limit.
|
||||
- Preparation failures, unavailable explicit profiles, provider failures, and
|
||||
context cancellation propagate to the calling stage with context.
|
||||
- Backend admission exhaustion is a provider-neutral operational error and is
|
||||
not classified as invalid structured output or validator rejection.
|
||||
- Malformed or schema-invalid provider output is classified separately as
|
||||
invalid structured output so the module or pipeline can apply its own retry
|
||||
and rejection policy.
|
||||
- Domain semantic checks, evidence decisions, and deterministic normalization
|
||||
run outside the provider adapter.
|
||||
|
||||
## Focused Verification
|
||||
|
||||
Read the LLM adapter, scheduler, asset registry, schema loader, and redaction
|
||||
tests when changing this boundary. Prompt changes also require the owning
|
||||
module’s asset tests, and retry or debug changes require focused pipeline or
|
||||
CLI coverage. The focused runtime and D&D checks are:
|
||||
|
||||
~~~sh
|
||||
go test ./internal/framework/llm/... ./internal/modules/dnd/...
|
||||
~~~
|
||||
|
||||
@@ -1,247 +1,113 @@
|
||||
# Module And Validator Internals
|
||||
# Module Internals
|
||||
|
||||
Production module and validator implementations live under their domain-first
|
||||
trees in `internal/modules`.
|
||||
The selectable keys, configuration options, reference slots, and default
|
||||
validator chain are canonical in the
|
||||
[module](../config.md#implemented-production-modules) and
|
||||
[validator](../config.md#implemented-production-validators) catalogs in
|
||||
Configuration.
|
||||
This guide owns the mechanics for implementing and registering production
|
||||
modules. [Configuration](../config.md) owns selectable keys, binding syntax,
|
||||
reference configuration, and default validator chains. Durable input and output
|
||||
shapes belong in [integration contracts](../integrations/).
|
||||
|
||||
## Extension Pattern
|
||||
The D&D family has additional shared conventions and domain-specific
|
||||
exceptions. See [D&D Module Internals](dnd.md) rather than adding them here.
|
||||
|
||||
A stage module package provides a stable key, constructor, contract
|
||||
implementation, `ModuleSpec`, `Register`, and focused behavior and registration
|
||||
tests. A validator package follows the same pattern with `ValidatorSpec` and the
|
||||
validator registry. Package-family registrars compose those leaf registrations
|
||||
into the production catalog and own family-level policy such as default
|
||||
validator chains and prompt asset collection.
|
||||
## Module Boundary
|
||||
|
||||
Production input, chunk, output, and D&D spell-extract packages register strict
|
||||
option decoders and run-local builders. Preparation decodes their options into
|
||||
implementation-owned values and injects dependencies. The spell extractor is
|
||||
typed over the canonical D&D model. D&D validators, merge, and normalize use
|
||||
typed variants; JSON representation validators use serialized requests; and
|
||||
unconditional validators expose separate chunk and typed variants. The D&D
|
||||
production registrar registers only the canonical typed spell implementations.
|
||||
A module is a typed implementation registered for one pipeline stage. Its
|
||||
`ModuleSpec` is the public-to-the-framework declaration of its stable key,
|
||||
stage, execution class, required and provided capabilities, artifact kind, and
|
||||
accepted reference slots. The execution class states whether a module is
|
||||
`deterministic` or `llm_backed`; registries retain it for catalog inspection and
|
||||
resolved-pipeline debug data without constructing the module. The framework
|
||||
uses the declaration to resolve a configured binding before it builds the
|
||||
implementation. After selection, the resolver applies profile inheritance only
|
||||
to bindings whose declared execution class is `llm_backed` and rejects a
|
||||
binding-specific profile on a deterministic module. The user-facing precedence
|
||||
contract belongs in [Configuration](../config.md#pipelines).
|
||||
|
||||
Prepared extractors, extract validators, and codecs may be reused concurrently
|
||||
by the run-wide extract pool. Production implementations are immutable after
|
||||
construction: they retain only typed options, immutable assets, or the shared
|
||||
concurrency-safe LLM client. Implementations that introduce mutable state must
|
||||
synchronize that state without creating a separate provider scheduler.
|
||||
Implementations that accept options must provide both an option validator and
|
||||
a builder. The validator is used while resolving configuration; the builder
|
||||
decodes the same options and constructs the implementation from the prepared
|
||||
`BuildRequest`. Reject unknown options in both paths. A builder receives only
|
||||
the dependencies and materialized references that the framework prepared for
|
||||
that operation, so it must not re-read configuration or files.
|
||||
|
||||
Specs expose capability and execution metadata without constructing an
|
||||
implementation. Registry entries separately expose option validation and
|
||||
run-local construction. Chunk, extract, merge, and normalize modules that accept
|
||||
auxiliary material declare identical reference slots from both
|
||||
`ReferenceSlots()` and `ModuleSpec().ReferenceSlots`; registration tests enforce
|
||||
that agreement. Runtime delivery uses the corresponding stage request's
|
||||
`References` field.
|
||||
Registry helpers register the typed builder for a stage-specific registry.
|
||||
They are preferable to hand-written untyped registration because they retain
|
||||
the artifact type at the framework boundary. Registrars validate the registries
|
||||
they need, register each leaf implementation, and add any family-owned assets
|
||||
or default validator chains. They return contextual errors so production
|
||||
composition fails at startup rather than at the first run.
|
||||
|
||||
LLM-backed extensions own their prompt definitions and response schemas under
|
||||
package-local embedded assets. Shared filesystem composition belongs in
|
||||
`internal/framework/promptfs`; reusable D&D prompt fragments, reference
|
||||
declarations, prompt-input assembly, and source-unit helpers belong in
|
||||
`internal/modules/dnd/shared`. Stage contracts expose only Notarius structured-
|
||||
completion types, not Scriptorium public types.
|
||||
An artifact family can register an optional typed evidence projector alongside
|
||||
its codec. The projector returns defensive copies of the artifact's direct
|
||||
generic source references and must use the codec's exact Go type. It does not
|
||||
interpret surrounding context or publish files; the pipeline validates the
|
||||
capability during preparation and the output boundary owns publication. See
|
||||
the [Published Evidence Context contract](../integrations/evidence-context.md)
|
||||
for the durable result.
|
||||
|
||||
Reference material may inform a module or prompt but must not become source
|
||||
evidence. The resolver and materializer behavior is described in
|
||||
[Pipeline Internals](pipeline.md#reference-materialization).
|
||||
## Production Composition
|
||||
|
||||
## Input Adapter
|
||||
Production composition is intentionally split by family:
|
||||
|
||||
### `internal/modules/seriatim/input/transcript`
|
||||
- The generic registrar provides the unit chunker, generic JSON validators,
|
||||
and JSON output encoder.
|
||||
- The Seriatim registrar provides the transcript input adapter. Its external
|
||||
input behavior is defined by the [Seriatim contract](../integrations/seriatim.md).
|
||||
- The D&D registrar provides its codecs, extractors, mergers, normalizers,
|
||||
validators, prompt assets, fallback profile asset, and default chains. Its behavioral conventions
|
||||
are documented in [D&D Module Internals](dnd.md).
|
||||
|
||||
The adapter decodes the supported transcript JSON, selects the source identity,
|
||||
computes canonical source provenance, validates segments, and maps each segment
|
||||
into a generic source unit with a self-reference plus speaker and timestamp
|
||||
metadata. It accepts no module options. Its spec advertises the transcript
|
||||
capabilities consumed by D&D modules.
|
||||
The CLI owns the composition that invokes these registrars. A module package
|
||||
may register its own family but must not assemble the CLI or make framework
|
||||
packages depend on production extensions.
|
||||
|
||||
Parsing is strict about required values and duplicate unit IDs but deliberately
|
||||
ignores unrelated Seriatim fields. The external format and derived-identity
|
||||
rules are defined in the
|
||||
[Seriatim contract](../integrations/seriatim.md).
|
||||
## Adding Or Changing A Module
|
||||
|
||||
## Chunkers
|
||||
1. Choose the pipeline stage and the typed artifact boundary. Put external
|
||||
input or durable artifact formats in the relevant integration contract,
|
||||
not in this guide or in a private LLM response type.
|
||||
2. Define a stable `ModuleSpec` with an explicit execution class, the exact
|
||||
capabilities, and reference slots needed for the operation. Model a
|
||||
producer/consumer handoff as an artifact-compatible slot; configuration
|
||||
then chooses an external file or a generated binding.
|
||||
3. Implement strict option decoding, construction, and the typed stage
|
||||
interface. Preserve caller ownership: do not retain mutable request data
|
||||
and return defensive copies where an implementation exposes stored data.
|
||||
4. Register the module through its typed registry helper and add it to the
|
||||
owning family registrar. Add a default validator chain only when that
|
||||
family owns the behavior; otherwise require an explicit compatible chain.
|
||||
5. Update the selectable-key and chain reference in
|
||||
[Configuration](../config.md#production-module-keys), the applicable
|
||||
integration contract, and focused tests. Keep the configuration document
|
||||
as the sole list of production keys and validator order.
|
||||
|
||||
Chunkers implement `contracts.Chunker.Plan`. A plan identifies ordered source
|
||||
unit ranges and may carry optional namespaced JSON annotations; it does not
|
||||
contain materialized chunk content. The framework canonicalizes annotations,
|
||||
validates ranges against the current source, and materializes chunk IDs,
|
||||
indexes, references, content, units, and generic metadata. Annotation
|
||||
namespaces remain optional data: generic framework code and downstream modules
|
||||
must not require D&D scene annotations or import `dnd/scenes`.
|
||||
## Validation And References
|
||||
|
||||
### `internal/modules/generic/chunk/units`
|
||||
Validators operate on the value produced at their configured stage. A default
|
||||
chain is ordered behavior, not a set: JSON parsing, structural checks,
|
||||
domain-specific checks, durable-schema checks, and advisory checks may have
|
||||
different responsibilities and failure handling. The active default chains and
|
||||
override rules are maintained in
|
||||
[Configuration](../config.md#production-validator-keys-and-default-chains).
|
||||
|
||||
The generic chunker validates the source document and returns ranges over units
|
||||
in configured windows. Overlap changes the next window start but never reorders
|
||||
units. Framework materialization derives the resulting chunk identity and
|
||||
generic metadata from those ranges.
|
||||
Reference slots are part of the module specification. They describe the
|
||||
accepted artifact kind, media type, size, and whether a binding is required;
|
||||
the framework validates those constraints before construction. An external
|
||||
reference is materialized during preparation. A generated reference is a
|
||||
compatible normalized artifact handed from an earlier pipeline step at
|
||||
operation time. The configuration reference rules, including precedence and
|
||||
ordered-handoff requirements, are maintained in
|
||||
[Configuration](../config.md#references-and-ordered-handoffs).
|
||||
|
||||
The accepted options and defaults are defined in
|
||||
[Configuration](../config.md#implemented-production-modules). Generic
|
||||
framework validation canonicalizes the returned unit slices before extraction.
|
||||
The chunker decodes its options during construction and retains only the typed
|
||||
window settings used by `Plan`.
|
||||
## Focused Verification
|
||||
|
||||
### `internal/modules/dnd/chunk/scenes`
|
||||
Exercise the leaf implementation and its registration path when changing a
|
||||
module. Registry and registrar tests cover duplicate keys, required registries,
|
||||
and typed construction; pipeline resolution tests cover capabilities, options,
|
||||
and reference compatibility. Domain packages should additionally test their
|
||||
codecs, validators, normalizers, and any integration handoffs they own.
|
||||
|
||||
The scene chunker prepares a structured Scriptorium request from the full
|
||||
transcript, session, and optional D&D reference inputs. It validates the model's
|
||||
scene boundaries against source-unit IDs and converts them into deterministic
|
||||
plan ranges with optional scene annotations. Preparation injects the shared
|
||||
structured LLM client into the chunker; `Plan`
|
||||
supplies only the run-specific profile, session, source, references, and
|
||||
metadata.
|
||||
Run the affected package tests while iterating. The complete module suite is:
|
||||
|
||||
Scene validation requires sequential, contiguous, non-overlapping coverage from
|
||||
the first source unit through the last. Scene descriptions, boundaries,
|
||||
confidence, and participants are module-owned annotations. Boundary caveats
|
||||
become warnings. Malformed
|
||||
structured output is returned as an error; there is no fallback chunker.
|
||||
|
||||
The package embeds its prompt and response schema and reports their non-secret
|
||||
identity and hashes through singleton module metadata. Shared D&D assets supply
|
||||
reference declarations and prompt inputs; their user-facing keys and accepted
|
||||
file types remain canonical in [Configuration](../config.md).
|
||||
|
||||
## Extractor
|
||||
|
||||
### `internal/modules/dnd/extract/spells`
|
||||
|
||||
The spell extractor prepares a structured request from one chunk, the
|
||||
chunk-scoped source input, the session, and optional D&D reference inputs. It
|
||||
decodes the model response, assigns the generic source identity to every source
|
||||
reference, canonicalizes duplicate references, orders spell casts by their
|
||||
earliest cited unit, and returns `dnd.SpellList`.
|
||||
|
||||
The extractor owns its private model-response DTO, embedded prompt, LLM response
|
||||
schema, strict option decoder, injected shared LLM client, and prompt/schema
|
||||
manifest metadata. The separate `internal/modules/dnd/codec/spells` package
|
||||
owns the durable schema and stable JSON representation for artifact kind
|
||||
`dnd/spell-list`. The runner keeps the result typed through validators and later
|
||||
stages, using the codec only for checkpoint, debug, and output boundaries.
|
||||
Shared D&D helpers keep prompt input
|
||||
names and source-unit reference conversion consistent with the scene chunker.
|
||||
|
||||
The durable payload and manifest metadata shapes are defined in the
|
||||
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
|
||||
|
||||
## Merger And Normalizer
|
||||
|
||||
### `internal/modules/generic/merge/appendorder`
|
||||
|
||||
The merger passes typed values to an injected combine function in framework
|
||||
source-chunk order. The D&D registrar specializes it with a spell-list append
|
||||
function.
|
||||
|
||||
### `internal/modules/generic/normalize/noop`
|
||||
|
||||
The normalizer returns the merged domain value unchanged and is reusable for
|
||||
any registered artifact type.
|
||||
|
||||
## Output Encoder
|
||||
|
||||
### `internal/modules/generic/output/json`
|
||||
|
||||
The JSON encoder sorts normalized results by lane, derives collision-checked
|
||||
safe logical names, pretty-prints JSON payloads, and assembles the logical index,
|
||||
manifest, rejected-result, warning, and lane files. Invalid JSON, unsupported
|
||||
media types, unsafe names, and sanitized-name collisions are errors.
|
||||
|
||||
The encoder returns logical files only. The CLI places them on disk, and the
|
||||
[JSON output contract](../integrations/json-output.md) defines their external
|
||||
paths and schemas.
|
||||
|
||||
## Generic Validators
|
||||
|
||||
The generic validator implementations live under
|
||||
`internal/modules/generic/validate`.
|
||||
|
||||
The unconditional accept and reject validators provide explicit chunk and
|
||||
typed-artifact variants used primarily for controlled composition and tests.
|
||||
|
||||
The serialized JSON syntax validator uses `encoding/json` to reject malformed
|
||||
representation bytes. The serialized JSON Schema validator requires schema
|
||||
bytes, parses the instance and schema with `jsonschema`, and distinguishes
|
||||
payload rejection from schema loading or compilation errors. The framework
|
||||
serialized-validation request carries either canonical chunk bytes or artifact
|
||||
codec bytes according to its target context. Neither validator calls the LLM.
|
||||
|
||||
## D&D Spell Validators
|
||||
|
||||
All three validators receive `dnd.SpellList` directly. The shape validator
|
||||
rejects missing or empty spell fields and empty reference lists. The
|
||||
source-reference validator applies generic source-reference validation to every
|
||||
cited range. The relatedness validator warns when a case-insensitive spell name
|
||||
is absent from all cited source text.
|
||||
|
||||
These validators are deterministic. Their selectable keys and production order
|
||||
are defined in
|
||||
[Configuration](../config.md#implemented-production-validators); their durable
|
||||
payload rules are defined in the
|
||||
[artifact contract](../integrations/dnd-spell-artifacts.md).
|
||||
|
||||
## Production Registration
|
||||
|
||||
Production composition occurs through family registrars. The CLI allocates one
|
||||
complete framework registry set and one LLM asset registry. It invokes
|
||||
`internal/modules/generic/register`,
|
||||
`internal/modules/seriatim/register`, and `internal/modules/dnd/register` in
|
||||
that order, then exposes the matching catalog for resolution. The generic and
|
||||
Seriatim registrars own their production leaf registrations. The D&D registrar
|
||||
owns D&D leaf registrations, the spell default-validator chain, and D&D
|
||||
prompt/schema asset collection.
|
||||
|
||||
Concrete implementation packages do not import generic implementation
|
||||
packages directly. A concrete family's `register` package is its composition
|
||||
point for specializing reusable generic implementations, while the generic
|
||||
registrar composes only generic children.
|
||||
|
||||
Core and framework production packages do not import production extensions.
|
||||
CLI production code is the sole application composition root for extensions
|
||||
and imports only exact family registrar packages. Other production packages,
|
||||
including commands and newly introduced package trees, do not import module
|
||||
packages directly. Compatibility tests in the CLI, core, and framework trees
|
||||
may import roots and implementation leaves directly. Other non-module tests do
|
||||
not receive that exemption. White-box tests within module families retain the
|
||||
production family boundaries. `internal/modules/integration` is test
|
||||
infrastructure: its black-box tests may compose multiple families, but it is
|
||||
not a production module family or production dependency target.
|
||||
|
||||
## Adding An Extension
|
||||
|
||||
When adding a production module or validator:
|
||||
|
||||
1. implement the stage or validator contract and package-local key;
|
||||
2. expose and test its spec, constructor, and registration function;
|
||||
3. keep format or domain parsing inside the concrete package;
|
||||
4. add package-owned prompt/schema assets when the extension is LLM-backed;
|
||||
5. register it through its package-family registrar and add a default chain
|
||||
there only when production policy requires one;
|
||||
6. add resolution and composition coverage for capabilities, options,
|
||||
references, and validation behavior;
|
||||
7. update the selectable-key catalog in [Configuration](../config.md), the
|
||||
relevant external contract, this inventory, and maintained examples when
|
||||
user-visible behavior changes.
|
||||
|
||||
Do not add the extension to `docs/development.md`; that file routes by task and
|
||||
does not inventory implementations.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
- Package-local `*_test.go` files under the module or validator being changed.
|
||||
- `internal/framework/pipeline/typed_resolution_test.go`: typed registry, spec,
|
||||
and heterogeneous artifact composition.
|
||||
- `internal/framework/pipeline/default_modules_test.go`: framework binding
|
||||
defaults.
|
||||
- `internal/cli/run_test.go`: production catalog, config resolution, and
|
||||
end-to-end CLI composition.
|
||||
- `internal/framework/promptfs/*_test.go` and
|
||||
`internal/modules/dnd/shared/*_test.go`: shared prompt and reference assembly.
|
||||
- `internal/modules/integration/*_test.go`: black-box composition across
|
||||
production extension domains.
|
||||
~~~sh
|
||||
go test ./internal/modules/...
|
||||
~~~
|
||||
|
||||
@@ -1,143 +1,58 @@
|
||||
# Internal Overview
|
||||
|
||||
This document inventories the implemented Notarius components. Normative
|
||||
This document is the implemented component map for Notarius. Normative
|
||||
boundaries and dependency direction belong in
|
||||
[Architecture](../policy/architecture.md); external behavior belongs in the
|
||||
[CLI](../cli.md), [Configuration](../config.md),
|
||||
[Architecture](../policy/architecture.md). User and operator contracts belong
|
||||
in the [CLI](../cli.md), [Configuration](../config.md),
|
||||
[Operations](../operations.md), and [integration contracts](../integrations/).
|
||||
|
||||
## Execution Path
|
||||
|
||||
`cmd/notarius` delegates to `internal/cli`, the production composition root.
|
||||
The CLI loads configuration, builds the production catalogs and runtime
|
||||
collaborators, invokes `internal/framework/pipeline`, and places the logical
|
||||
output files returned by the runner. Diagnostics, checkpoints, and debug
|
||||
recorders are optional side-channel collaborators supplied at this boundary.
|
||||
~~~
|
||||
cmd/notarius -> internal/cli -> configuration and production composition
|
||||
-> internal/framework/pipeline -> logical output files
|
||||
-> internal/cli -> durable output and optional state/debug data
|
||||
~~~
|
||||
|
||||
Resolution produces a fixed ordered workflow and a sorted set of artifact
|
||||
lanes. Preparation constructs the complete module and validator set before the
|
||||
runner receives source bytes. Source parsing and chunking are serial; extraction
|
||||
uses a bounded run-wide worker pool, followed by serial per-lane merge and
|
||||
normalize continuations that may overlap across lanes.
|
||||
The CLI is the application boundary: it discovers configuration, composes
|
||||
production registries and runtime collaborators, invokes the framework, and
|
||||
places returned files. The framework resolves and prepares a fixed extraction
|
||||
pipeline, then returns logical results without owning process behavior or
|
||||
physical state roots.
|
||||
|
||||
## Application Boundary
|
||||
## Components
|
||||
|
||||
| Package | Implemented responsibility |
|
||||
| --- | --- |
|
||||
| `cmd/notarius` | Executable entry point and process exit delegation. |
|
||||
| `internal/cli` | Command parsing, config discovery, package-family registrar invocation, LLM client construction, reference materialization, workspace collaborator setup, durable writes, and user-facing results. |
|
||||
|
||||
## Core Packages
|
||||
|
||||
| Package | Implemented responsibility |
|
||||
| --- | --- |
|
||||
| `internal/core/artifacts` | Run-manifest and provenance models. |
|
||||
| `internal/core/config` | Defaults, YAML parsing, environment overrides, validation, redaction, and effective pipeline resolution. |
|
||||
| `internal/core/diagnostics` | Scoped run directories, diagnostics writers, atomic writes, and retention decisions. |
|
||||
| `internal/core/source` | Generic source documents, units, chunks, canonical references, lookup, validation, and deterministic source digests. |
|
||||
| `internal/core/workspace` | Effective workspace settings, confined paths and writes, and checkpoint identity and manifest models. |
|
||||
|
||||
## Framework Packages
|
||||
|
||||
| Package | Implemented responsibility |
|
||||
| --- | --- |
|
||||
| `internal/framework/contracts` | Source-stage contracts plus artifact identity, schema, serialized representation, codec, validator, reference, output, and structured-completion interfaces and data types. |
|
||||
| `internal/framework/pipeline` | Module and artifact-codec registries, option validation, profile resolution, capability checks, reference materialization, complete pipeline preparation, retries, orchestration, warnings, and manifest population. |
|
||||
| `internal/framework/validate` | Shared validator decision and cardinality helpers. |
|
||||
| `internal/framework/llm` | Scriptorium-backed structured completions, prompt/schema registration, scheduling, profile recording, and secret redaction. |
|
||||
| `internal/framework/promptfs` | Builds module prompt filesystems from module-owned and caller-provided shared prompt assets. |
|
||||
| `internal/framework/checkpoint` | Workspace-backed checkpoint loading, recording, and payload serialization. |
|
||||
| `internal/framework/chunkplan` | Source-addressed chunk-plan filesystem storage, envelope validation, and atomic publication. |
|
||||
| `internal/framework/debug` | Workspace-backed framework and LLM debug recording. |
|
||||
|
||||
Framework contracts provide typed artifact, provenance-wrapper, chunk-validator,
|
||||
serialized-validator, and
|
||||
typed-validator interfaces. The runner owns handoff provenance, validation
|
||||
sequencing, rejection handling, checkpoint and debug boundaries, and final
|
||||
manifest assembly.
|
||||
|
||||
Artifact registries support heterogeneous typed extraction entries and
|
||||
kind-specific merger, normalizer, and validator variants. Resolution derives a
|
||||
lane's kind from its extractor, requires the matching codec, verifies exact Go
|
||||
type equality across the lane, and records schema identity in the resolved lane
|
||||
and pipeline digest. Registry entries carry separate option-validation and
|
||||
run-local construction closures. Preparation injects shared dependencies and
|
||||
constructs input, chunk, validators, ordered lanes, and output before source
|
||||
parsing. Production modules use strict construction-time option decoding, and
|
||||
LLM-backed modules retain the injected shared client. The D&D family registers
|
||||
the canonical `dnd/spell-list` codec, typed spell extractor and validators, and
|
||||
kind-specific generic merge and normalize strategies; generic JSON validators
|
||||
use the serialized-validation contract. The runner executes lanes through
|
||||
private exact-type-checked closures, coordinates extract results independently
|
||||
of completion timing, and serializes artifacts only through their codec at
|
||||
checkpoint, debug, and output boundaries.
|
||||
|
||||
## Production Extensions
|
||||
|
||||
The canonical catalogs of user-selectable
|
||||
[module](../config.md#implemented-production-modules) and
|
||||
[validator](../config.md#implemented-production-validators) keys are in
|
||||
Configuration. The implemented module packages are:
|
||||
|
||||
| Package | Implemented responsibility |
|
||||
| --- | --- |
|
||||
| `internal/modules/seriatim/input/transcript` | Parses the supported Seriatim transcript format into the generic source model. |
|
||||
| `internal/modules/generic/chunk/units` | Splits ordered source units by unit count and overlap. |
|
||||
| `internal/modules/dnd/chunk/scenes` | Produces contiguous D&D scene chunks from structured model output. |
|
||||
| `internal/modules/dnd` | Owns the canonical D&D spell-list and spell-cast artifact types. |
|
||||
| `internal/modules/dnd/codec/spells` | Strictly decodes and stably encodes the durable D&D spell-list representation. |
|
||||
| `internal/modules/dnd/extract/spells` | Maps private structured model output to canonical source-grounded D&D spell lists. |
|
||||
| `internal/modules/generic/merge/appendorder` | Combines accepted extraction results in chunk order. |
|
||||
| `internal/modules/generic/normalize/noop` | Preserves accepted merged output. |
|
||||
| `internal/modules/generic/output/json` | Encodes manifests, lane payloads, warnings, and rejections as logical JSON files. |
|
||||
|
||||
`internal/modules/dnd/shared` owns reusable D&D prompt fragments,
|
||||
reference declarations, prompt input assembly, and source-unit reference
|
||||
helpers. Domain-neutral prompt filesystem composition lives in
|
||||
`internal/framework/promptfs`.
|
||||
|
||||
Generic validators under `internal/modules/generic/validate` provide
|
||||
unconditional test decisions, JSON syntax validation, and JSON Schema
|
||||
validation. D&D spell validators under `internal/modules/dnd/validate/spells`
|
||||
consume the canonical spell-list type directly to provide shape,
|
||||
source-reference, and source-relatedness decisions.
|
||||
|
||||
Production composition is grouped behind package-family registrars, and every
|
||||
implemented production extension uses its domain-first tree:
|
||||
|
||||
| Package | Implemented responsibility |
|
||||
| --- | --- |
|
||||
| `internal/modules/generic/register` | Registers domain-neutral chunk, merge, normalize, output, and validator implementations. |
|
||||
| `internal/modules/seriatim/register` | Registers the Seriatim input adapter. |
|
||||
| `internal/modules/dnd/register` | Registers D&D modules, validators, default validator policy, and prompt/schema assets. |
|
||||
|
||||
The CLI allocates the framework registries and asset registry, then invokes
|
||||
these registrars in generic, Seriatim, and D&D order.
|
||||
|
||||
Implementation details for all production extensions are in
|
||||
[Module Internals](modules.md).
|
||||
|
||||
## Run-State Components
|
||||
|
||||
| Surface | Implemented owners | Internal purpose |
|
||||
| Area | Implemented owners | Responsibility |
|
||||
| --- | --- | --- |
|
||||
| Durable output | Output module, pipeline runner, and CLI writer | Return logical consumer files and place them for a run. |
|
||||
| Diagnostics | `internal/core/diagnostics` and `internal/cli` | Record redacted invocation, resolution, result, and failure inspection data. |
|
||||
| Checkpoints | `internal/framework/checkpoint` and `internal/core/workspace` | Validate and serialize reusable extract, merge, and normalize outcomes. |
|
||||
| Chunk-plan cache | `internal/framework/chunkplan` and `internal/cli` | Persist and select source-addressed plans before framework materialization. |
|
||||
| Debug artifacts | `internal/framework/debug` and pipeline instrumentation | Capture sensitive framework-boundary and LLM-call material. |
|
||||
| Executable and command boundary | **cmd/notarius**, **internal/cli** | Process entry, command dispatch, configuration discovery, production composition, runtime collaborator setup, durable file placement, and user-facing reporting. |
|
||||
| Configuration | **internal/core/config** | Defaults, strict YAML parsing, environment overrides, structural validation, effective resolution, redaction, and resolved-composition summaries. |
|
||||
| Generic models | **internal/core/source**, **internal/core/artifacts**, **internal/framework/contracts** | Source documents and chunks, manifests and provenance, plus typed artifact, reference, validation, output, and structured-completion contracts. |
|
||||
| Pipeline framework | **internal/framework/pipeline** | Registries, profile and reference resolution, typed preparation, validation, retry coordination, ordered execution, handoff, and result assembly. |
|
||||
| LLM and prompt runtime | **internal/framework/llm**, **internal/framework/promptfs** | Provider-neutral structured completions, scheduling, profile recording, prompt assets, schema registration, and credential-shaped-value redaction. |
|
||||
| Runtime state | **internal/core/fileio**, **internal/core/debugbundle**, **internal/framework/checkpoint**, **internal/framework/chunkplan**, **internal/framework/chunkmap**, **internal/framework/debug** | Confined atomic files, debug bundles, checkpoint and chunk-plan state, accepted chunk maps, and pipeline-facing debug recording. |
|
||||
| Production extensions | **internal/modules/generic**, **internal/modules/seriatim**, **internal/modules/dnd** | Domain-neutral extensions, Seriatim input support, and D&D extraction families registered into the production catalog. |
|
||||
|
||||
Physical layout, retention, recovery, and sensitive-data handling are defined
|
||||
in [Operations](../operations.md). Concrete stage modules receive recorder
|
||||
interfaces and request data, not workspace paths.
|
||||
Generic core and framework packages do not depend on production extensions.
|
||||
Concrete extensions depend inward on their contracts and are registered only at
|
||||
the CLI composition boundary.
|
||||
|
||||
## Focused Documentation
|
||||
|
||||
- [Pipeline Internals](pipeline.md): resolution, execution, validation, retries,
|
||||
checkpoint/debug hooks, and result assembly.
|
||||
- [Module Internals](modules.md): production modules, validators, assets,
|
||||
registration, and the contributor recipe for adding an extension.
|
||||
- [LLM Runtime](llm.md): structured completion contracts, Scriptorium adapter,
|
||||
assets, scheduling, profile recording, and redaction.
|
||||
- [Diagnostics Internals](diagnostics.md): scoped writers, retention
|
||||
coordination, CLI failure flow, and path safety.
|
||||
- [Configuration Internals](configuration.md): loading, validation, effective
|
||||
resolution, redaction, and resolved-composition identity.
|
||||
- [CLI Internals](cli.md): command dispatch, production composition, run
|
||||
orchestration, and terminal reporting.
|
||||
- [Pipeline Internals](pipeline.md): resolution, preparation, execution,
|
||||
validation, typed handoff, and framework state hooks.
|
||||
- [Run State Internals](state.md): output, cache, debug collaborator
|
||||
composition, and path safety.
|
||||
- [LLM Runtime](llm.md): structured completion, scheduling, prompt assets,
|
||||
profiles, and secret handling.
|
||||
- [Module Internals](modules.md): generic extension registration, module
|
||||
construction, validation, and reference mechanics.
|
||||
- [D&D Module Internals](dnd.md): shared D&D extractor conventions, generated
|
||||
reference projections, and lane-specific exceptions. Durable D&D and
|
||||
Seriatim data shapes remain in the [integration contracts](../integrations/).
|
||||
|
||||
Use this map to find an owner, then read the focused document and its tests
|
||||
before changing behavior.
|
||||
|
||||
@@ -1,327 +1,185 @@
|
||||
# Pipeline Internals
|
||||
|
||||
The implemented resolver and runner live in `internal/framework/pipeline`.
|
||||
Their fixed workflow and ownership boundaries are defined by
|
||||
[Architecture](../policy/architecture.md#system-shape). Configuration fields,
|
||||
defaults, and selectable keys are defined in
|
||||
[Configuration](../config.md#pipelines).
|
||||
This document describes the framework-owned pipeline mechanics in
|
||||
**internal/framework/pipeline**. [Configuration](../config.md) owns selectable
|
||||
profiles, bindings, and retry settings; [Operations](../operations.md) owns
|
||||
state lifecycle and recovery; and the [integration contracts](../integrations/)
|
||||
own durable output shapes. Concrete production extensions are covered by
|
||||
[Module Internals](modules.md).
|
||||
|
||||
Resolution fixes the selected lanes and all stage bindings; preparation
|
||||
constructs every selected implementation before the runner begins source work.
|
||||
After serial input parsing and plan selection or generation, the runner
|
||||
materializes chunks and dispatches extract work to
|
||||
one bounded run-wide worker pool in chunk-first, lane-second order. Each lane's
|
||||
merge and normalize operations remain serial and may overlap other lanes once
|
||||
all extracts for that lane are terminal.
|
||||
## Boundary
|
||||
|
||||
## Resolution
|
||||
The pipeline framework accepts a resolved composition, registries, shared
|
||||
dependencies, input bytes, and state/debug collaborators. It returns logical
|
||||
output files, normalized artifacts, recorded rejections and warnings, manifest
|
||||
provenance, and checkpoint decisions. The CLI owns process arguments,
|
||||
configuration discovery, physical roots, and placement of returned output
|
||||
files.
|
||||
|
||||
`internal/core/config.Config.Resolve` validates the loaded configuration,
|
||||
selects the named profile, applies the runtime inputs supplied by the CLI, and
|
||||
calls `pipeline.ResolvePipeline`.
|
||||
The framework has one fixed shape:
|
||||
|
||||
`ResolvePipeline`:
|
||||
~~~
|
||||
input -> chunk -> extract -> merge -> normalize -> output
|
||||
~~~
|
||||
|
||||
1. selects and sorts artifact lanes;
|
||||
2. completes omitted bindings using the documented configuration defaults;
|
||||
3. looks up each module and validator spec without constructing it;
|
||||
4. for a typed extractor, derives its artifact kind, requires the codec, and
|
||||
selects exact-type merger, normalizer, and validator variants;
|
||||
5. checks required and provided capabilities in workflow order;
|
||||
6. resolves target-aware reference bindings and validator chains;
|
||||
7. validates each selected module and validator option set through its registry
|
||||
entry; and
|
||||
8. calculates a digest over the resolved structure, including typed artifact
|
||||
kind and schema identity and the effective validator policy in its resolved
|
||||
execution order.
|
||||
Input and chunking are pipeline-wide. A selected artifact lane owns extract,
|
||||
merge, and normalize; output aggregates the terminal lane outcomes. A pipeline
|
||||
is an ordered list of steps, not an arbitrary workflow graph.
|
||||
|
||||
Resolution returns a `ResolvedPipeline` containing ordered lanes, concrete
|
||||
bindings, validator chains, reference targets, and the digest. It does not read
|
||||
reference bytes or construct runtime modules. CLI lane and reference selector
|
||||
syntax is defined in the [CLI reference](../cli.md#run).
|
||||
## Resolve, Materialize, Prepare
|
||||
|
||||
The digest includes each resolved validator chain's stage, lane, owning module,
|
||||
ordered validator bindings, execution classes, targets, and artifact kinds.
|
||||
Changing a default chain or an explicit override therefore changes pipeline
|
||||
identity whenever it changes the effective validator policy.
|
||||
Resolution turns a configured pipeline profile into a **ResolvedPipeline**.
|
||||
It normalizes the pipeline and lane identities, applies stage defaults, selects
|
||||
requested lanes where that is supported, resolves validator chains, checks
|
||||
module capabilities and typed artifact compatibility, validates options, and
|
||||
assigns a deterministic resolved-composition digest. The resolved pipeline
|
||||
contains bindings and declared reference targets, not external reference bytes.
|
||||
After selection, the resolver applies command, binding, and pipeline profile
|
||||
precedence to LLM-backed bindings and validators only; prompt defaults remain
|
||||
an empty resolved binding profile. Deterministic bindings remain profile-free.
|
||||
These effective values are part of the digest, so execution and checkpoint
|
||||
consumers do not repeat profile inheritance.
|
||||
Configuration resolution supplies the selected profile and catalog; see
|
||||
[Configuration Internals](configuration.md).
|
||||
|
||||
## Reference Materialization
|
||||
External reference materialization happens before preparation. The materializer
|
||||
checks that each slot is declared by the selected module, resolves a file path
|
||||
relative to the correct configuration or working-directory origin, reads
|
||||
UTF-8 text, verifies media type and size limits, and retains bounded
|
||||
provenance. A generated-artifact selector remains declared but has no bytes
|
||||
until its producing step completes.
|
||||
|
||||
The CLI calls `MaterializeReferences` after resolution and before constructing
|
||||
the LLM client or running the pipeline. The materializer checks each binding
|
||||
against its resolved target declaration, reads and validates the file, and
|
||||
builds both a `contracts.ReferenceSet` and provenance-only metadata on the
|
||||
corresponding `ResolvedReferenceTarget`.
|
||||
Preparation is the construction boundary. It validates the resolved shape and
|
||||
registry set, clones the resolved data, then constructs the input adapter,
|
||||
chunker, stage-local validators, every typed lane, and output encoder with
|
||||
cloned options, references, and shared dependencies. It also collects stable
|
||||
checkpoint fingerprints. Missing registrations, incompatible typed entries,
|
||||
nil implementations, and constructor failures are reported before source
|
||||
parsing or any stage operation begins.
|
||||
|
||||
The runner clones the resulting set into the chunk, extract, merge, or normalize
|
||||
request that owns the target. LLM-backed extensions may convert those items into
|
||||
named prompt inputs. Reference content remains separate from source evidence and
|
||||
source digests.
|
||||
An output encoder can opt into source-evidence publication through its output
|
||||
policy. Preparation keeps the configured lane allowlist and active lanes
|
||||
separate, then verifies an exact typed evidence projector and registered codec
|
||||
for each active lane. The resulting private plan is immutable; lanes excluded
|
||||
by invocation filtering remain configured but do not acquire a projector for
|
||||
that run.
|
||||
|
||||
Binding precedence, path resolution, accepted content, and media-type behavior
|
||||
are configuration contracts; see [Configuration](../config.md#pipelines).
|
||||
Durable provenance is defined in the
|
||||
[JSON output contract](../integrations/json-output.md#manifestjson), while
|
||||
runtime sensitive-data handling belongs in [Operations](../operations.md).
|
||||
## Typed Lanes And References
|
||||
|
||||
## Registries And Specs
|
||||
Each resolved lane has one artifact kind, codec, and exact Go type. The
|
||||
framework uses private type erasure only around those typed operations; every
|
||||
handoff checks exact type and codec identity and reports incompatibility as an
|
||||
error rather than panicking. Encoding through the registered codec is the
|
||||
boundary for output, checkpoints, debug records, and generated references.
|
||||
|
||||
`pipeline.Registries` holds option validators and run-local builders used during
|
||||
resolution and preparation.
|
||||
`pipeline.ModuleCatalog` exposes their specs during configuration validation and
|
||||
resolution. Separate registries exist for every stage and for validators;
|
||||
`ValidatorChainRegistry` stores production default-chain mappings. Both
|
||||
containers also carry an `ArtifactCodecRegistry`. Generic registration records
|
||||
one codec per stable artifact kind, validates its schema metadata and JSON
|
||||
Schema, retains the exact schema digest and Go type, and safely encodes or
|
||||
decodes framework-erased values with typed errors on incompatibility.
|
||||
Reference targets are stage- and lane-specific. External reference bytes are
|
||||
cloned into the operation request. Generated references are built at the next
|
||||
step boundary from exactly one accepted normalized producer output. The
|
||||
framework decodes and re-encodes that output with the registered producer
|
||||
codec, checks its complete schema and media identity, and records a content
|
||||
digest plus bounded producer provenance. A missing, ambiguous, invalid, or
|
||||
incompatible producer prevents the consumer step from starting.
|
||||
|
||||
Typed extractor entries are keyed by module key and declare one artifact kind.
|
||||
Merger, normalizer, and typed-validator variants are keyed by module or
|
||||
validator key plus artifact kind. Chunk and serialized validators occupy
|
||||
separate target namespaces; serialized registrations declare whether they
|
||||
support chunks, artifacts, or both. Duplicate variants and exact Go-type
|
||||
mismatches are rejected deterministically.
|
||||
## Execution And Ordering
|
||||
|
||||
Lane-sensitive merger and normalizer spec discovery always supplies the
|
||||
extractor's artifact kind, so variants under one reusable key may declare
|
||||
different capabilities and reference slots. Kind-neutral registry inspection
|
||||
selects the first registered artifact kind in sorted order.
|
||||
The runner validates its input, installs no-op state collaborators when none
|
||||
were supplied, and serially performs source parsing and chunk-plan selection.
|
||||
An accepted plan is materialized into source-addressed chunks and passes the
|
||||
configured chunk validators before any lane runs. A chunk rejection is a
|
||||
recorded pipeline outcome: lanes do not start, but the output stage can encode
|
||||
the terminal result.
|
||||
|
||||
Production composition registers the D&D spell-list codec and typed extractor,
|
||||
matching typed merge, normalize, and semantic-validator variants, and
|
||||
serialized JSON validators. Every artifact lane resolves through the typed
|
||||
registries and a matching codec.
|
||||
For each ordered step, the runner first builds generated reference sets from
|
||||
the accepted normalized outputs of earlier steps. It then executes the step's
|
||||
lanes. Later steps do not begin until the current step is terminal and its
|
||||
generated handoffs have succeeded.
|
||||
|
||||
A `ModuleSpec` declares its stage plus required and provided capabilities.
|
||||
Chunk, extract, merge, and normalize specs may also declare reference slots.
|
||||
Registry implementations defensively copy spec metadata, reject duplicate keys,
|
||||
and verify that a constructed implementation reports the registered key.
|
||||
Builder registrations accept `ModuleDependencies` and cloned configuration
|
||||
options through one `BuildRequest`. Builders decode those options and retain
|
||||
typed values or injected dependencies in the constructed implementation.
|
||||
Extractors declare their artifact kind, and merger, normalizer, and validator
|
||||
resolution selects the matching typed variant.
|
||||
Within a step, the lane engine dispatches extraction jobs in deterministic
|
||||
chunk-first, lane-second order to a bounded worker group. When all extraction
|
||||
jobs for one lane are terminal, a bounded continuation group can run that
|
||||
lane's merge and normalize work while extraction for other lanes continues.
|
||||
The framework does not create an unbounded goroutine per chunk or lane.
|
||||
|
||||
A `ValidatorSpec` declares a validator key and execution class. Resolution uses
|
||||
the execution class to reject incompatible profile bindings before execution.
|
||||
The current production catalog and default chain are listed only in
|
||||
[Configuration](../config.md#implemented-production-validators).
|
||||
Completion timing does not determine public results. The coordinator restores
|
||||
lane and chunk order before merging results, and selects a framework error by
|
||||
stable stage, lane, and chunk position. A validator rejection records a lane
|
||||
outcome without cancelling unrelated work. A framework error or parent
|
||||
cancellation cancels derived work, prevents queued work from starting, waits
|
||||
for started workers, and prevents output encoding.
|
||||
|
||||
## Preparation And Runner Boundary
|
||||
## Validation, Retries, And Output
|
||||
|
||||
`pipeline.Prepare` receives a resolved pipeline, the registries, and shared
|
||||
module dependencies. It constructs input; chunk and its validators; each lane's
|
||||
extract, merge, and normalize modules and validator chains in resolved order;
|
||||
then output. It stops at the first error with pipeline, stage, lane, module, and
|
||||
validator context as applicable. It never invokes an operation method.
|
||||
Every chunk, extract, merge, and normalize candidate passes its resolved
|
||||
validator chain. Validators receive immutable canonical input appropriate to
|
||||
their target: chunks, typed values, or serialized codec bytes. They may
|
||||
approve, approve with warnings, reject, or fail. A rejection is an ordinary
|
||||
pipeline result; a validator error is a framework error.
|
||||
|
||||
`PreparedPipeline` keeps private constructed executors and exposes cloned
|
||||
resolved input, chunk, lane, and output identities. `pipeline.RunInput` carries
|
||||
that prepared pipeline, raw source input, run identity and timing, optional
|
||||
session and profile metadata, a chunk-plan store and mode, and checkpoint/debug
|
||||
collaborators. The runner
|
||||
parses source bytes through the already constructed input adapter. Later stage
|
||||
requests receive the generic source model; extract requests receive
|
||||
chunk-scoped input material, while chunk, merge, and normalize requests retain
|
||||
access to the original source material. Input, chunk, and output operation
|
||||
requests do not carry raw module options. The chunk request also does not carry
|
||||
an LLM client; an LLM-backed chunker receives the shared client during
|
||||
preparation. Their operation requests retain run-specific source, reference,
|
||||
profile, session, and metadata context as applicable.
|
||||
The runner applies the binding's retry policy around a stage operation and its
|
||||
complete validation chain. It preserves warnings only from the final accepted
|
||||
or rejected attempt. Cancellation stops retries. Normalizer-specific retry
|
||||
directives consume this same budget and validate any final safe fallback through
|
||||
the normalizer chain.
|
||||
|
||||
Prepared lanes retain exact-type-checked erased operation closures. The runner
|
||||
uses those closures to keep each value typed through extraction, validation,
|
||||
merge, and normalization.
|
||||
|
||||
Source validation requires every unit to carry a canonical self-reference to
|
||||
its containing document and its own unit ID. Explicit clone, checkpoint, and
|
||||
debug boundaries retain that reference, and the canonical source digest covers
|
||||
it deterministically. Chunks use the same source model and carry one canonical
|
||||
reference spanning the first selected unit through the last.
|
||||
|
||||
`pipeline.RunOutput` carries the run manifest, accepted normalized serialized
|
||||
artifacts with lane and normalizer provenance,
|
||||
rejected results, warnings, checkpoint events, and logical files returned by the
|
||||
output encoder. The CLI owns diagnostics and durable filesystem writes after the
|
||||
runner returns.
|
||||
|
||||
## Execution Flow
|
||||
|
||||
The runner:
|
||||
|
||||
1. validates its prepared input;
|
||||
2. parses the raw input with the prepared adapter and validates the generic
|
||||
source document;
|
||||
3. selects a stored plan or executes the configured chunker's `Plan` operation;
|
||||
4. canonicalizes and materializes the plan, then validates the resulting
|
||||
chunks;
|
||||
5. dispatches extract jobs in source-chunk then resolved-lane order, starting a
|
||||
bounded lane continuation when all extracts for that lane are terminal;
|
||||
6. invokes the prepared output encoder and validates its logical file results;
|
||||
7. returns the assembled manifest, outcomes, warnings, and files.
|
||||
|
||||
Within each artifact lane, it reuses the prepared extractor, merger, normalizer,
|
||||
and validators while performing these transitions:
|
||||
|
||||
1. extract once per accepted chunk and add runner-owned lane, source, and chunk
|
||||
provenance;
|
||||
2. validate each extract result and omit rejected results from merge input;
|
||||
3. skip the rest of the lane when no extract result is accepted;
|
||||
4. merge accepted extract results in their existing order;
|
||||
5. validate the merge result and skip normalization on rejection;
|
||||
6. normalize the accepted merge result;
|
||||
7. validate and append the accepted normalized result.
|
||||
|
||||
Module-provided warnings and payload warnings are promoted only from attempts
|
||||
whose results are accepted and used.
|
||||
|
||||
## Chunk Plans And Reuse
|
||||
|
||||
`Chunker.Plan` returns a `source.ChunkPlan`: the canonical source digest,
|
||||
ordered unit-ID ranges, and optional plan or range annotations. The framework
|
||||
owns plan canonicalization and materialization. It creates the generic chunks
|
||||
and therefore owns their IDs, indexes, source references, JSON content, units,
|
||||
media type, and generic metadata. Plan and range annotations are independently
|
||||
owned raw JSON and become `Chunk.PlanAnnotations` and `Chunk.Annotations`.
|
||||
|
||||
In `auto`, the runner looks up the source digest before invoking the chunker. A
|
||||
valid hit is materialized and sent through the current run's configured chunk
|
||||
validators; it does not invoke the chunk module, consume its retry budget, or
|
||||
make a chunk-stage LLM call. A missing, invalid, or unmaterializable record
|
||||
generates a candidate. `refresh` generates without lookup; `bypass` generates
|
||||
without cache access. Generated plans are published only after the full chunk
|
||||
validator chain approves them. A validator rejection is a regular rejected
|
||||
pipeline outcome and never replaces a cached plan.
|
||||
|
||||
The store is source-addressed, not pipeline-addressed. Changes to pipeline
|
||||
configuration, requested chunker, options, references, lanes, validators, or
|
||||
LLM profile do not prevent a source-digest hit. The manifest records both the
|
||||
currently requested chunker and the effective plan producer. Cache state and
|
||||
paths are configured and operated outside the runner; see
|
||||
[Configuration](../config.md#workspace) and [Operations](../operations.md).
|
||||
|
||||
The extract job channel has the same capacity as the effective extract worker
|
||||
count, so dispatch applies backpressure. A fixed continuation executor prevents
|
||||
ready or checkpoint-reused lanes from creating one goroutine each. Workers and
|
||||
continuations publish lane-local results; the coordinator is the only writer of
|
||||
aggregate output and merges those results in resolved lane and source-chunk
|
||||
order.
|
||||
|
||||
## Plan Canonicalization And Chunk Materialization
|
||||
|
||||
Plan canonicalization requires canonical JSON annotations, a matching source
|
||||
digest, at least one range, existing ordered boundaries, and increasing range
|
||||
starts. Ranges may overlap or leave gaps; a chunker may impose stricter policy.
|
||||
Materialization deterministically reconstructs each range from the current
|
||||
source document and copies annotations without interpreting their namespaces.
|
||||
|
||||
Before lane execution, generic chunk validation checks the materialized chunks'
|
||||
identities, order, source references, content, media type, units, and metadata.
|
||||
No chunk checkpoint participates in plan selection: plan storage is the only
|
||||
chunk-reuse mechanism. Extract, merge, and normalize checkpoints continue to
|
||||
use materialized chunk digests as their dependencies.
|
||||
|
||||
## Validation And Retries
|
||||
|
||||
Chunk, extract, merge, and normalize results pass through the resolved validator
|
||||
chain for their stage and module. Chunk validators receive canonical chunks;
|
||||
typed validators receive the domain value; and serialized validators receive
|
||||
canonical chunk JSON or artifact codec bytes. Validators execute in resolved
|
||||
order and stop at the first error or rejection. An empty chain approves the
|
||||
result.
|
||||
|
||||
`runWithRetry` applies the effective retry policy around module execution and
|
||||
its complete validation chain. A module or validator error becomes a framework
|
||||
error when attempts are exhausted. A rejection becomes a recorded
|
||||
`RejectedOutput` when attempts are exhausted. Cancellation stops retry
|
||||
processing immediately.
|
||||
|
||||
Rejected output is a non-fatal pipeline outcome and does not advance. Warnings
|
||||
from discarded attempts are not promoted. Configuration owns retry counts and
|
||||
validator overrides; see [Module Bindings](../config.md#module-bindings).
|
||||
After terminal lane work, the runner assembles manifest provenance, normalized
|
||||
artifacts, rejections, warnings, and an optional accepted chunk map. When an
|
||||
output policy selected evidence lanes, it decodes accepted serialized normalize
|
||||
outputs through their registered codecs and invokes the prepared typed
|
||||
projectors. Rejected or absent lanes contribute nothing. This reconstruction is
|
||||
also used after normalized-checkpoint reuse, so no second typed output channel
|
||||
is retained. The runner passes the resulting owned artifact to the output
|
||||
encoder, which returns logical files and does not choose a physical directory.
|
||||
The CLI publishes those files only after the runner returns without a framework
|
||||
error. Logical file names and schemas are defined by the [output integration
|
||||
contracts](../integrations/).
|
||||
|
||||
## Checkpoint And Debug Hooks
|
||||
|
||||
The runner depends on recorder and loader interfaces, using no-op
|
||||
implementations when collaborators are absent. Each checkpointed workflow
|
||||
boundary records a running, succeeded, or failed transition. Reuse decisions
|
||||
are consulted in workflow order and accepted payloads are cloned before
|
||||
entering the normal handoff path. Typed extract, merge, and normalize
|
||||
checkpoints store codec bytes with artifact kind, schema ID and version, exact
|
||||
schema digest, and media type. Reuse compares that identity with the prepared
|
||||
codec and decodes through the codec; missing identity, mismatches, corrupt
|
||||
bytes, and decode failures become explicit reuse misses and execute the step
|
||||
normally. Dependency fingerprints and debug content digests use the same stable
|
||||
codec bytes that cross those boundaries.
|
||||
The runner receives checkpoint and debug interfaces rather than roots. It
|
||||
records workflow transitions and reuse decisions through the supplied
|
||||
collaborators, and clones reusable artifacts before they re-enter normal typed
|
||||
handoff. Generated-reference dependencies participate in checkpoint decisions.
|
||||
Selective recomputation can require a canonical accepted normalized predecessor
|
||||
before a dependent lane starts.
|
||||
|
||||
Debug instrumentation wraps run, stage, attempt, validator, and structured LLM
|
||||
boundaries. Every executed chunk, extract, merge, and normalize attempt writes
|
||||
one terminal envelope for acceptance, validator rejection, module or validator
|
||||
error, or applicable candidate or final serialization error. The envelope
|
||||
contains its attempt-local warnings, any available candidate and rejection,
|
||||
and terminal error text; failures before a candidate exists omit that payload.
|
||||
Only LLM calls made by the module operation belong to the module attempt.
|
||||
Validator calls retain independent scopes under `validate/` and are not
|
||||
duplicated into the module envelope. A failed terminal-envelope write is a
|
||||
non-retryable framework error and is joined with any primary attempt error.
|
||||
Debug data is never used as a checkpoint source. Typed artifact debug envelopes
|
||||
are domain-neutral, redact sensitive metadata and bytes through the common
|
||||
debug policy, and record codec identity plus schema and content digests.
|
||||
Debug recording is attempt-scoped and application-owned. A failure to persist
|
||||
required debug data is a framework error. State roots, persistence, reason-code
|
||||
meanings, resume, and cleanup are intentionally owned by
|
||||
[Run State Internals](state.md) and [Operations](../operations.md).
|
||||
|
||||
Merge and normalize attempts serialize their in-memory candidate with the
|
||||
codec's required candidate encoder before typed validation. Serialized
|
||||
validators and attempt debug use that candidate representation, which carries
|
||||
the codec media type and schema identity but is never checkpointed or passed
|
||||
downstream. Only a validator-approved value is encoded through the strict final
|
||||
codec and made eligible for a checkpoint or stage output.
|
||||
## Invariants To Preserve
|
||||
|
||||
Checkpoint identity, physical layout, reuse behavior, and debug artifact
|
||||
handling are operator contracts in [Operations](../operations.md). Serialization
|
||||
and recorder implementation are inventoried in
|
||||
[Internal Overview](overview.md#run-state-components).
|
||||
- The six fixed stages remain explicit; a pipeline is not a general DAG.
|
||||
- Resolution and preparation reject statically discoverable incompatibility
|
||||
before parsing or execution.
|
||||
- Every typed lane uses one compatible artifact kind, codec, and exact Go type.
|
||||
- Generated references come only from one earlier accepted normalized producer
|
||||
and carry canonical identity rather than an unverified value.
|
||||
- Rejections are recorded outcomes; framework errors cancel derived work and
|
||||
prevent output encoding.
|
||||
- Public ordering and selected errors are independent of goroutine completion
|
||||
order.
|
||||
- Pipeline modules receive collaborators and data, never CLI streams or
|
||||
physical output, cache, or debug roots.
|
||||
|
||||
## Results And Failures
|
||||
## Focused Tests
|
||||
|
||||
The runner owns manifest assembly and handoff summaries but not the durable JSON
|
||||
schema. It records resolved module and lane provenance, validator chains,
|
||||
source/reference identities, selected LLM profiles, normalized and rejected
|
||||
summaries, status, and timing. Serialized artifact content remains outside the manifest.
|
||||
Module metadata providers may add non-secret singleton or lane-scoped metadata.
|
||||
- **internal/framework/pipeline/profile_test.go** and
|
||||
**typed_resolution_test.go** cover resolution, defaults, ordered steps,
|
||||
compatibility, validators, references, and resolved identity.
|
||||
- **internal/framework/pipeline/preparation_test.go** covers complete
|
||||
construction before execution and contextual construction failures.
|
||||
- **internal/framework/pipeline/references_test.go** and **handoff_test.go**
|
||||
cover external materialization, generated references, provenance, and typed
|
||||
producer checks.
|
||||
- **internal/framework/pipeline/runner_concurrency_test.go** covers bounded
|
||||
execution, ordered steps, stable error selection, rejections, and
|
||||
cancellation.
|
||||
- **internal/framework/pipeline/runner_chunk_plan_test.go**,
|
||||
**runner_typed_checkpoint_test.go**, and
|
||||
**runner_accepted_checkpoint_test.go** cover state hooks and reuse behavior.
|
||||
- **internal/framework/pipeline/runner_attempt_debug_test.go** and
|
||||
**runner_terminal_debug_test.go** cover attempt and terminal debug behavior.
|
||||
|
||||
Execution errors include stage, module, lane, or validator context. Once a
|
||||
manifest exists, a failing run returns it with failed status and completion
|
||||
time. Successful status reflects whether any result was rejected. The
|
||||
durable manifest and logical file schemas are defined in the
|
||||
[JSON output contract](../integrations/json-output.md).
|
||||
|
||||
On a framework failure, the runner cancels its derived context, stops submitting
|
||||
new extract work, drains started tasks, and skips the output encoder. Parent
|
||||
cancellation takes precedence. Otherwise context-cancellation fallout is
|
||||
discarded when a substantive error exists, and the primary error is selected by
|
||||
stage, resolved lane, and source chunk rather than completion time.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
- `internal/core/config/effective_config_test.go`: config-to-resolution boundary.
|
||||
- `internal/framework/pipeline/profile_test.go`: selection, defaults,
|
||||
capabilities, validator chains, and digest behavior.
|
||||
- `internal/framework/pipeline/artifact_codec_registry_test.go`: typed codec
|
||||
metadata, registration, erasure safety, strict decoding, and cloning.
|
||||
- `internal/framework/pipeline/typed_resolution_test.go`: heterogeneous typed
|
||||
lane resolution and preparation, target-specific validators,
|
||||
incompatibilities, ordering, and schema-sensitive pipeline identity.
|
||||
- `internal/framework/pipeline/runner_concurrency_test.go`: bounded dispatch and
|
||||
continuations, reverse completion, stable errors, rejection, cancellation,
|
||||
retries, and independent provider-call limits.
|
||||
- `internal/framework/pipeline/preparation_test.go`: option validation,
|
||||
construction order, dependency failures, and the before-source-work boundary.
|
||||
- `internal/framework/pipeline/references_test.go`: target resolution and
|
||||
materialization.
|
||||
- `internal/cli/run_test.go`: production stage transitions, retries, rejections,
|
||||
warnings, debug hooks, manifests, and end-to-end composition.
|
||||
- `internal/modules/integration/*_test.go` and
|
||||
`internal/modules/seriatim/input/transcript/runner_test.go`: typed runner
|
||||
composition across concrete module families.
|
||||
- `internal/framework/checkpoint/*_test.go`: checkpoint serialization and reuse
|
||||
collaborators.
|
||||
Run **go test ./internal/framework/pipeline ./internal/cli** after changing a
|
||||
pipeline boundary. Use the more focused tests above while iterating.
|
||||
|
||||
147
docs/internal/state.md
Normal file
147
docs/internal/state.md
Normal file
@@ -0,0 +1,147 @@
|
||||
# Run State Internals
|
||||
|
||||
This document describes the implementation collaborators behind output, cache,
|
||||
and debug state. User-visible fields belong in [Configuration](../config.md),
|
||||
and physical layout, retention, recovery, reason codes, and cleanup belong in
|
||||
[Operations](../operations.md).
|
||||
|
||||
## Composition
|
||||
|
||||
`internal/cli` is the only physical-path composition root. It resolves the
|
||||
effective configuration, selects exact roots, allocates requested debug bundles,
|
||||
constructs cache collaborators, writes logical output files, and reports paths.
|
||||
Pipeline modules receive interfaces and request data, never output, cache, or
|
||||
debug roots.
|
||||
|
||||
The CLI creates no chunk-plan store in bypass mode. It creates a checkpoint
|
||||
recorder only when recording is enabled and a checkpoint loader only for a
|
||||
resume invocation. It allocates debug state only after a safe run identity has
|
||||
been generated and only when debug capture was requested. These choices keep
|
||||
the three state families independently composable.
|
||||
|
||||
## Output And Cache
|
||||
|
||||
The pipeline runner returns logical output files. After validating every
|
||||
logical name, the CLI exclusively creates the run directory beneath the
|
||||
selected output root and performs confined, atomic file writes within it.
|
||||
The runner supplies an accepted chunk map as an optional, defensively owned
|
||||
output-request artifact. The JSON encoder alone decides whether its explicit
|
||||
option writes the map and optional index descriptor; neither the map payload
|
||||
nor its annotations are copied into the run manifest. The durable fields are
|
||||
owned by the [Accepted Chunk Map contract](../integrations/chunk-map.md).
|
||||
|
||||
`internal/framework/chunkplan` owns source-addressed plan storage, validation,
|
||||
and atomic publication. Its store is constructed only when the selected mode is
|
||||
not `bypass`.
|
||||
|
||||
`internal/framework/checkpoint` owns checkpoint identity, manifests, payload
|
||||
codecs, loader, and recorder. The CLI constructs a recorder whenever checkpoint
|
||||
recording is enabled and constructs a loader only for a `--resume` invocation.
|
||||
Identity incorporates explicit stable semantic fingerprints collected from
|
||||
prepared modules and validators in addition to configuration, input,
|
||||
references, runtime overrides, observed LLM profiles, and the LLM runtime's
|
||||
non-secret effective profile-source identity. A profile source change therefore
|
||||
causes a cold miss even when the configured profile ID remains unchanged.
|
||||
The serialized
|
||||
`workspace_schema_version` identifiers are frozen wire-compatibility fields;
|
||||
they do not describe a current public state surface.
|
||||
|
||||
Ordered-step lane checkpoints include the step identity in their storage scope.
|
||||
When a later lane consumes a generated artifact, its dependency fingerprints
|
||||
include the producer's artifact kind, complete schema identity, media type,
|
||||
canonical content digest, and size. Ordinary resume compares those fingerprints
|
||||
when progressively loading consumer stage checkpoints, so changed producer
|
||||
content produces `dependency_invalidated` rather than stale downstream reuse.
|
||||
Selective recomputation instead requires each unselected producer's accepted
|
||||
normalized artifact; invalid accepted state records its specific bounded reason
|
||||
and stops before the dependent. The selected step and its transitive dependents
|
||||
record `forced_recompute`.
|
||||
|
||||
Ordinary resume loads extract, merge, and normalize checkpoints progressively
|
||||
and may execute later lane stages after an earlier cache miss. Selective
|
||||
recomputation instead asks the loader for the required producer's accepted
|
||||
normalize artifact. That lookup reuses the existing normalize files, requires
|
||||
workspace schema v3 plus an exact non-empty invocation identity, and deliberately
|
||||
does not require extract or merge checkpoint files or dependency fingerprints.
|
||||
The runner performs canonical codec and producer-provenance validation before
|
||||
cloning the artifact into normal step output. Success restores only stored
|
||||
normalize warnings and emits one normalize decision; failure retains the files,
|
||||
records the decision, and stops without executing the producer or consumer.
|
||||
|
||||
The loader assigns a typed category and reason code at each validation site;
|
||||
diagnostic prose is not classified after the fact. The runner then applies
|
||||
forced-execution policy, validates reusable artifact bytes through the prepared
|
||||
codec once, returns the canonical hydrated value to the stage, and records the
|
||||
final decision before enforcing a required-predecessor failure. That failure
|
||||
names only the step, lane, and stable reason code. Decision detail is selected
|
||||
from code-owned descriptions by reason code and then UTF-8 normalized and
|
||||
bounded; callers cannot supply arbitrary diagnostic prose. Typed categories and
|
||||
codes remain intact through pipeline events and become strings only in manifest
|
||||
and debug-summary JSON.
|
||||
[Operations](../operations.md#checkpoint-recording-resume-and-recompute) is the
|
||||
canonical operator-facing reason-code reference.
|
||||
|
||||
`internal/core/fileio` provides confined atomic file writes used by state
|
||||
collaborators. The chunk-plan store retains its stronger entry validation.
|
||||
|
||||
The CLI constructs selective-recomputation policy from resolved generated
|
||||
artifact dependencies. It forces the selected step and transitive consumers,
|
||||
while marking unforced producers as required reusable inputs. The runner owns
|
||||
the actual hydration and rejection decisions; the [Operations guide](../operations.md#checkpoint-recording-resume-and-recompute)
|
||||
owns the operator workflow and stable reason-code meanings.
|
||||
|
||||
## Debug Bundles
|
||||
|
||||
`internal/core/debugbundle` allocates an explicitly requested per-run bundle
|
||||
with `summary/` and `trace/` roots. `SummaryWriter` persists redacted command,
|
||||
resolution, run, warning, and failure artifacts. `internal/framework/debug`
|
||||
implements the pipeline-facing trace recorder under the trace root.
|
||||
|
||||
The CLI allocates a bundle before pipeline resolution and treats requested
|
||||
summary or trace persistence failures as command failures. The pipeline's debug
|
||||
boundaries redact sensitive metadata and credential-shaped bytes while allowing
|
||||
application-owned trace material. Debug data is never a checkpoint source or
|
||||
cache input.
|
||||
|
||||
Generated reference bytes exist only in cloned operation requests and are not
|
||||
written as paths into checkpoints, manifests, or debug summaries. Those state
|
||||
surfaces retain canonical identities and bounded producer provenance so that a
|
||||
resume decision can be explained without copying generated campaign content.
|
||||
|
||||
After allocation, one CLI-owned state value accumulates the known report paths,
|
||||
pipeline outcome counts, and validation status. A single guarded terminalization
|
||||
operation writes the success report, or makes one attempt each to write the
|
||||
failure report and error log. Terminal persistence failures are reported
|
||||
separately and never replace the command's primary error.
|
||||
|
||||
## Invariants To Preserve
|
||||
|
||||
- Modules receive state collaborators and request data, never physical roots.
|
||||
- Output logical paths are validated before a run directory is allocated, and
|
||||
files are atomically written within that directory.
|
||||
- Chunk-plan publication occurs only for accepted plans; bypass does not
|
||||
construct or touch a plan store.
|
||||
- Checkpoint recording and checkpoint loading remain separate collaborators.
|
||||
- Debug state is opt-in, is not cache input, and terminal reporting does not
|
||||
obscure the command's primary failure.
|
||||
|
||||
## Tests To Inspect
|
||||
|
||||
- `internal/cli/run_contract_test.go`: command-owned state allocation,
|
||||
terminalization, and output/report boundaries.
|
||||
- `internal/cli/cache_contract_test.go`: cache-mode precedence, root selection,
|
||||
and resume collaborator construction.
|
||||
- `internal/cli/state_hardening_test.go`: independent roots, reuse, failures,
|
||||
permissions, cleanup, and redaction.
|
||||
- `internal/cli/recompute_policy_test.go`: forced dependents and required
|
||||
reusable predecessors for selective recomputation.
|
||||
- `internal/cli/recompute_execution_contract_test.go`: selective recomputation,
|
||||
filesystem recovery, deterministic decisions, and failed predecessor state.
|
||||
- `internal/cli/production_contract_test.go`: production composition and
|
||||
configuration validation at the CLI boundary.
|
||||
- `internal/cli/example_contract_test.go`: maintained example ownership.
|
||||
- `internal/core/debugbundle/*_test.go`: bundle allocation and summary writes.
|
||||
- `internal/framework/checkpoint/*_test.go`: checkpoint serialization and
|
||||
reuse.
|
||||
- `internal/framework/chunkplan/store_test.go`: plan envelope, confinement,
|
||||
publication, and permissions.
|
||||
@@ -1,298 +1,291 @@
|
||||
# Operations
|
||||
|
||||
This is the canonical reference for operating implemented Notarius runs.
|
||||
This is the canonical guide for operating Notarius runtime state. The
|
||||
[CLI reference](cli.md) owns command syntax and exit statuses, while
|
||||
[Configuration](config.md) owns fields, defaults, and precedence. Maintainers
|
||||
who need implementation mechanics should read [Run State Internals](internal/state.md).
|
||||
|
||||
## Normal Run
|
||||
## State Surfaces
|
||||
|
||||
A run reads one source file, resolves one configured pipeline, executes its
|
||||
modules, writes durable output, and writes diagnostics when enabled. Start with
|
||||
the [README quickstart](../README.md), then use the [CLI reference](cli.md) for
|
||||
invocation options.
|
||||
Each run can use independent roots with different retention and access-control
|
||||
needs.
|
||||
|
||||
For production, configure an application-owned workspace such as
|
||||
`/var/lib/notarius` and ensure the Notarius process can create files below it.
|
||||
For local development, prefer an ignored project-local workspace such as
|
||||
`./.notarius/workspace`. See [Configuration](config.md#workspace) for workspace
|
||||
fields.
|
||||
| Surface | Purpose | Created when | Retention |
|
||||
| --- | --- | --- | --- |
|
||||
| Output | Durable user-facing result bundle | A pipeline completes and returns logical output files | Keep until consumers no longer need it. |
|
||||
| Chunk-plan cache | Reconstructible source-addressed plan | The configured cache mode permits cache I/O | Keep while reuse is useful. |
|
||||
| Checkpoint cache | Reconstructible execution and recovery state | Checkpoint recording is enabled | Keep only while recovery or reuse is useful. |
|
||||
| Debug bundle | Explicit diagnostic record | A run requests debug collection | Keep only under an intentional sensitive-data retention policy. |
|
||||
|
||||
## Output Directory
|
||||
Output, cache, and debug roots are never merged or cleaned automatically. Use
|
||||
separate locations and permissions for operators or services that must not
|
||||
share application data.
|
||||
|
||||
Durable output is written to:
|
||||
## Roots And Permissions
|
||||
|
||||
```text
|
||||
The configured output and debug directories are exact roots. An empty cache
|
||||
directory selects a per-user root:
|
||||
|
||||
~~~
|
||||
<os.UserCacheDir>/notarius/chunk-plans
|
||||
<os.UserCacheDir>/notarius/checkpoints
|
||||
~~~
|
||||
|
||||
The field definitions and configuration examples are in [Configuration](config.md).
|
||||
On supported Unix systems, output directories and files are created with
|
||||
requested modes **0755** and **0644**. Chunk-plan, checkpoint, and debug
|
||||
directories and files use **0700** and **0600**. The operating system's umask
|
||||
may impose stricter output modes. Cache and debug roots may contain sensitive
|
||||
source-derived data, so provision them for one trusted account or service. An
|
||||
output bundle can also contain source content when its JSON output enables
|
||||
evidence publication. Apply an appropriate umask and output-root access policy
|
||||
before enabling that option; the requested output modes alone may not be
|
||||
suitable for transcript-bearing bundles.
|
||||
|
||||
## PromptKit Profile Deployment
|
||||
|
||||
Profile deployment has four distinct layers:
|
||||
|
||||
| Layer | Owner | Operational role |
|
||||
| --- | --- | --- |
|
||||
| Prompts and schemas | Notarius module families | Embedded request and structured-output definitions. They are not deployment profile files. |
|
||||
| Fallback profiles | Notarius module families | Embedded application defaults, including D&D's `dnd-extraction` profile. |
|
||||
| Built-in profiles | PromptKit | Upstream catalog entries available when no higher-precedence source defines an ID. |
|
||||
| Operator profiles | Deployment filesystem | Complete environment-specific definitions selected by `promptkit.profile_file` or `promptkit.profile_dir`. |
|
||||
|
||||
The maintained D&D pipeline uses the workload ID `dnd-extraction`. The
|
||||
embedded fallback makes that ID usable without an operator file. Production,
|
||||
development, and local deployments can each install a different complete
|
||||
definition for the same ID, retaining the pipeline while choosing their own
|
||||
model, backend, timeout, or reasoning policy. An operator definition wins over
|
||||
the fallback; it is not merged with it. The configuration field and full
|
||||
precedence rules are owned by [Configuration](config.md#promptkit-profiles).
|
||||
|
||||
Use a profile source owned by the service account, keep it readable only by
|
||||
the intended operator, and supply provider credentials through the service
|
||||
environment—not in the Notarius configuration or profile YAML. The maintained
|
||||
[operator profile](../examples/profiles/dnd-extraction.yml) is secret-free and
|
||||
can be copied as a format starting point. Validate a deployment without a
|
||||
provider call or credentials:
|
||||
|
||||
~~~sh
|
||||
notarius config validate --config /etc/notarius/config.yml --pipeline dnd-session
|
||||
~~~
|
||||
|
||||
Profile paths are currently resolved from the process working directory, not
|
||||
from the configuration file. The complete example's
|
||||
`./examples/profiles/dnd-extraction.yml` path is valid for a repository-root
|
||||
invocation only. Use absolute paths such as
|
||||
`/etc/notarius/profiles/dnd-extraction.yml` for services and containers.
|
||||
|
||||
## Run Lifecycle
|
||||
|
||||
Use the [run command](cli.md#run) to start a pipeline. A valid invocation loads
|
||||
and resolves configuration before module preparation and source parsing. It
|
||||
then performs any permitted cache lookup, executes the pipeline, and publishes
|
||||
logical output files only after a successful runner result.
|
||||
|
||||
On success, the command reports the output bundle path. A warning-bearing run
|
||||
still succeeds and reports its warning count on standard error. Errors and
|
||||
their exit classes are defined in the [CLI reference](cli.md#output-streams-and-exit-statuses).
|
||||
|
||||
## Output Bundles
|
||||
|
||||
Each successful run receives a generated safe run identifier and writes beneath:
|
||||
|
||||
~~~
|
||||
<output-root>/<run-id>/
|
||||
```
|
||||
~~~
|
||||
|
||||
The output root and its invocation-specific override are defined in the
|
||||
[CLI reference](cli.md#run). Output writes are atomic per file. The
|
||||
[JSON output contract](integrations/json-output.md) defines the logical files,
|
||||
paths, schemas, and media types inside each run directory.
|
||||
The [JSON output contract](integrations/json-output.md) owns the logical files
|
||||
and their schemas. Before creating the run directory, Notarius validates every
|
||||
logical output path. It refuses an existing run directory without changing it.
|
||||
Files are written atomically; if a later write fails, the newly created partial
|
||||
run directory remains for inspection and is never removed automatically.
|
||||
|
||||
## Diagnostics Directory
|
||||
|
||||
Diagnostics are written under:
|
||||
|
||||
```text
|
||||
<diagnostics-work-dir>/<run-id>/
|
||||
```
|
||||
|
||||
When a workspace directory is configured, diagnostics are written under
|
||||
`<workspace.directory>/diagnostics/<run-id>/`. An invocation-specific override
|
||||
changes only the diagnostics root, not the workspace root. Configuration and
|
||||
environment controls are defined in [Configuration](config.md); the override
|
||||
flag is defined in the [CLI reference](cli.md#run).
|
||||
|
||||
Diagnostics can be disabled through configuration. When disabled, Notarius
|
||||
does not create a diagnostics run directory or write diagnostics artifacts;
|
||||
concise failures are still printed to stderr.
|
||||
|
||||
Implemented diagnostics artifacts:
|
||||
|
||||
- `invocation.json`: command metadata such as operation, config path, input
|
||||
path, selected lanes, run ID, and pipeline digest when available.
|
||||
- `effective-config.json`: resolved config without raw API keys.
|
||||
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
|
||||
- `resolved-references.json`: resolved reference provenance, including target
|
||||
stage, lane ID when present, origin, digest, media type, byte size, and
|
||||
binding source, without reference content.
|
||||
- `checkpoint-events.json`: checkpoint steps that were reused or executed
|
||||
during an explicit resume invocation.
|
||||
- `chunk-plan.json`: redacted plan-cache lookup, validation, and publication
|
||||
summary. It contains identifiers and decisions, never source units, plan
|
||||
annotations, reference content, prompts, model responses, or invalid-file
|
||||
bytes.
|
||||
- `run-manifest.json`: the same run manifest written to durable output when it
|
||||
is available, including top-level module metadata when present.
|
||||
- `warnings.json`: warning list.
|
||||
- `run-report.json`: counts, status, output path, diagnostics path, and run ID.
|
||||
- `error.log`: failure message, written after diagnostics directory creation
|
||||
when a run fails.
|
||||
Treat an output bundle as durable user data. Do not use cache-cleanup policy to
|
||||
remove it. An optional accepted chunk map is also durable output and can carry
|
||||
source- or model-derived annotations; its content and compatibility contract
|
||||
are defined in [Accepted Chunk Map](integrations/chunk-map.md). An optional
|
||||
[evidence context](integrations/evidence-context.md) contains source-unit text
|
||||
and metadata. It is not a cache or debug artifact: retain it with the output
|
||||
bundle only for as long as consumers need it, and apply source-content access
|
||||
controls to the entire bundle. Selected lanes may collectively cite most of a
|
||||
transcript, so a broad allowlist can make the evidence artifact nearly as
|
||||
sensitive and large as the source itself.
|
||||
|
||||
## Chunk-Plan Cache
|
||||
|
||||
The chunk-plan cache is independent of the workspace and checkpoints. Its
|
||||
configuration and selection precedence are defined in
|
||||
[Configuration](config.md#workspace); the invocation override is documented in
|
||||
the [CLI reference](cli.md#run).
|
||||
Chunk plans live beneath the selected chunk-plan root:
|
||||
|
||||
When no root is configured, a normal Linux user uses
|
||||
`$XDG_CACHE_HOME/notarius/chunk-plans` when `XDG_CACHE_HOME` is a valid absolute
|
||||
path, or `$HOME/.cache/notarius/chunk-plans` when it is unset. A relative
|
||||
`XDG_CACHE_HOME` is a configuration error. A configured
|
||||
`workspace.chunk_cache.directory` is the root itself, not a parent to which
|
||||
Notarius adds a suffix.
|
||||
|
||||
Each source digest has one file:
|
||||
|
||||
```text
|
||||
~~~
|
||||
<chunk-plan-root>/<source-sha256-hex>/plan.json
|
||||
```
|
||||
~~~
|
||||
|
||||
Directories are created with `0700` permissions and plan files with `0600`.
|
||||
`auto` reuses a complete valid plan or regenerates an absent or invalid one;
|
||||
`refresh` deliberately regenerates; `bypass` performs no cache I/O. A stored
|
||||
plan is still validated and materialized against the current source before use,
|
||||
and the current run's chunk validators always run. Invalid state is recoverable:
|
||||
an `auto` run regenerates and atomically replaces it only after validation
|
||||
succeeds. Delete an exact cache root or digest directory only when regeneration
|
||||
cost is acceptable.
|
||||
One validated canonical plan is active for each source digest. The plan stores
|
||||
boundaries and provenance, not a second copy of the entire source. This
|
||||
source-addressed policy is recorded in [ADR-0005](adr/0005-cache-canonical-chunk-plans-by-source.md).
|
||||
|
||||
Publication uses atomic replacement. Concurrent readers observe a complete old
|
||||
or new plan, and concurrent writers leave one complete valid winner; there is
|
||||
no history, lock protocol, or rollback facility. Do not share a cache root
|
||||
between mutually untrusted users because plans can contain source-derived
|
||||
structure and annotations.
|
||||
The configured cache mode controls one invocation:
|
||||
|
||||
For a system-wide Linux deployment under a dedicated service account, configure
|
||||
and provision a separate restrictive root such as:
|
||||
- **auto** looks for a valid active plan. Missing or invalid state causes a new
|
||||
plan to be generated; an accepted new plan is atomically published.
|
||||
- **refresh** skips lookup, generates a plan with the configured chunker, and
|
||||
atomically replaces the active plan after it is accepted.
|
||||
- **bypass** performs no chunk-plan cache I/O. It does not resolve or create a
|
||||
chunk-plan root.
|
||||
|
||||
```yaml
|
||||
workspace:
|
||||
chunk_cache:
|
||||
directory: /var/cache/notarius/chunk-plans
|
||||
```
|
||||
A reused plan is still materialized and validated against the current source.
|
||||
If a prior plan no longer gives acceptable results, use a refresh run rather
|
||||
than editing cache files. Deleting a plan is recoverable but can repeat costly
|
||||
chunking work.
|
||||
|
||||
`/var/cache/notarius/chunk-plans` is a recommended configured service root, not
|
||||
the unprivileged default. The operator or package installer must create it with
|
||||
restrictive service-account ownership and permissions before use.
|
||||
## Checkpoint Recording, Resume, And Recompute
|
||||
|
||||
## Checkpoints
|
||||
Checkpoint recording is an explicit configuration choice and is disabled by
|
||||
default. When enabled, each run records stage transitions and the state needed
|
||||
for compatible recovery. A run records checkpoints even when it does not ask
|
||||
to reuse them. Checkpoint payloads can contain source-derived and intermediate
|
||||
application data, so treat the entire root as sensitive.
|
||||
|
||||
When checkpoint writing is enabled for a configured workspace, runs write
|
||||
checkpoints under:
|
||||
Checkpoint loading is separate: [**--resume**](cli.md#run) asks a run to reuse
|
||||
compatible recorded work. A resume request fails when checkpoint recording is
|
||||
disabled. Without **--resume**, a recording-enabled run executes normally and
|
||||
does not load checkpoint state. Compatibility includes the resolved pipeline,
|
||||
input, selected lanes, runtime overrides, reference provenance, LLM-profile
|
||||
provenance, the effective PromptKit profile-source fingerprint, and
|
||||
prepared-component fingerprints. When a local PromptKit backend is configured,
|
||||
compatibility also includes a non-secret fingerprint of its endpoint. Changing
|
||||
profile content or the local endpoint causes a cold miss; changing only the
|
||||
local concurrency limit does not. A changed identity produces a cold miss;
|
||||
Notarius does not migrate, rewrite, or delete older checkpoint directories.
|
||||
Reasoning-effort inheritance, replacement, and explicit clearing are distinct
|
||||
runtime identities, so checkpoints created under one state are not reused by
|
||||
either of the others.
|
||||
|
||||
```text
|
||||
<workspace.directory>/checkpoints/<pipeline-id>/<input-key>-<source-or-input-digest>/<pipeline-digest>/<identity-digest>/
|
||||
```
|
||||
Checkpoint state is confined below an identity-specific path:
|
||||
|
||||
Each workflow step owns its own manifest and payload files. There is no
|
||||
root-level checkpoint summary. Ordinary invocations execute the pipeline
|
||||
normally and refresh checkpoints. An explicit resume invocation reuses valid
|
||||
checkpoints and executes any missing, invalid, or incompatible step normally.
|
||||
Configuration controls checkpoint writing, while the explicit resume option is
|
||||
defined in the [Configuration](config.md#workspace) and
|
||||
[CLI](cli.md#run) references.
|
||||
~~~
|
||||
<checkpoint-root>/<pipeline-id>/<input-key>-<source-or-input-digest-prefix>/<pipeline-digest-prefix>/<identity-digest-prefix>/
|
||||
~~~
|
||||
|
||||
Checkpoints do not include raw prompts, raw reference contents, raw LLM request
|
||||
payloads, or debug traces. They can still contain source text, intermediate
|
||||
extracted content, rejected outputs, metadata, warnings, and content digests.
|
||||
Treat checkpoint directories as sensitive local state.
|
||||
### Selective Recompute
|
||||
|
||||
A checkpoint is reused only when its stored status, dependencies, payloads, and
|
||||
digests match the current invocation. Changes to input bytes, the resolved
|
||||
pipeline, selected lanes, the runtime LLM profile override, or bound reference
|
||||
content invalidate reuse. The resolved pipeline identity includes effective
|
||||
default and explicitly overridden validator chains, so adding, removing,
|
||||
reordering, or reconfiguring a validator invalidates checkpoints even when the
|
||||
pipeline profile itself is unchanged.
|
||||
[**--recompute-step**](cli.md#run) requires both **--resume** and enabled
|
||||
checkpoint recording. It forces the selected ordered step and every lane that
|
||||
depends on it through generated artifact references. Unrelated lanes remain
|
||||
eligible for reuse.
|
||||
|
||||
Typed artifact checkpoints additionally record codec-owned bytes, artifact
|
||||
kind, schema ID and version, exact schema digest, and media type. A missing or
|
||||
mismatched codec identity, or bytes the current codec cannot decode, is reported
|
||||
as a checkpoint reuse miss. The affected operation executes normally and, when
|
||||
checkpoint writing is enabled, replaces the incompatible checkpoint.
|
||||
For an earlier producer required by a forced consumer, Notarius requires a
|
||||
compatible accepted normalized artifact. It validates that artifact before
|
||||
hydrating it and does not silently rerun the producer. If that state is
|
||||
missing, rejected, corrupt, non-canonical, or incompatible, the run stops
|
||||
before its dependent starts. Rerun the required producer deliberately instead
|
||||
of copying or editing checkpoint files.
|
||||
|
||||
Current checkpoint manifests use workspace schema `notarius.workspace.v2`.
|
||||
Manifests written with `notarius.workspace.v1` are incompatible because their
|
||||
chunk provenance has an older shape. On the first explicit resume after an
|
||||
upgrade, each affected checkpoint is treated as a reuse miss and its workflow
|
||||
step executes normally. The compatibility check does not migrate or delete the
|
||||
v1 files; when checkpoint writing is enabled, normal execution refreshes the
|
||||
affected checkpoint files in the current schema.
|
||||
## Checkpoint Decisions And Recovery
|
||||
|
||||
Runs do not reuse checkpoints unless explicitly requested. Without reuse, the
|
||||
workflow executes normally and refreshes checkpoint files when checkpointing is
|
||||
enabled.
|
||||
Checkpoint events classify work as **executed**, **reused**,
|
||||
**forced_recompute**, or **dependency_invalidated**. Their stable reason codes
|
||||
are written to run diagnostics and provenance. Use the code, not a copied
|
||||
error message, to decide what to repair.
|
||||
|
||||
## Debug
|
||||
| Reason code | Recovery meaning |
|
||||
| --- | --- |
|
||||
| **loading_disabled** | This invocation did not permit checkpoint loading. |
|
||||
| **checkpoint_missing**, **checkpoint_path_invalid**, **checkpoint_read_failed**, **checkpoint_decode_failed** | The stored checkpoint could not be located or read safely; normal resume work can execute again. |
|
||||
| **workspace_schema_incompatible**, **identity_mismatch**, **stage_mismatch**, **step_mismatch**, **lane_mismatch**, **module_mismatch** | Stored state belongs to a different compatible scope or identity; allow a fresh run to create new state. |
|
||||
| **status_not_reusable** | The recorded operation did not end in reusable state. |
|
||||
| **dependency_mismatch** | A dependency changed; dependent work is invalidated rather than reused. |
|
||||
| **artifact_payload_invalid**, **artifact_digest_mismatch**, **artifact_codec_incompatible**, **artifact_not_canonical** | A stored artifact cannot safely be hydrated; rerun the producer instead of modifying the cache. |
|
||||
| **checkpoint_reused** | A normal checkpoint passed compatibility checks. |
|
||||
| **accepted_artifact_reused** | A required predecessor's accepted normalized artifact was safely hydrated. |
|
||||
| **recompute_step** | Selective recomputation deliberately forced this work. |
|
||||
|
||||
When debug recording is enabled for a configured workspace, runs write debug
|
||||
artifacts under:
|
||||
Reason detail is bounded code-owned text. It is diagnostic information, not a
|
||||
path-discovery or data-recovery mechanism, and does not contain checkpoint,
|
||||
source, reference, credential, or environment content.
|
||||
|
||||
```text
|
||||
<workspace.directory>/debug/<run-id>/
|
||||
```
|
||||
## Debug Bundles
|
||||
|
||||
Debug output is per invocation. It is independent of checkpointing and is not
|
||||
used for resume. Enabling debug does not write checkpoints, and enabling resume
|
||||
checkpointing does not write debug output.
|
||||
Only a [debug-enabled run](cli.md#run) creates a bundle:
|
||||
|
||||
Debug artifacts include inputs and outputs for source, chunk, extract, merge,
|
||||
normalize, and output work, structured LLM request and response data, validator
|
||||
requests and results, timing, and retry attempt metadata. LLM calls made inside
|
||||
a module retry write `prompt-000N.json`, `response-000N.json`, and
|
||||
`response-content-000N.*` files under that attempt directory and are linked
|
||||
from its `llm_calls` array. Validator calls use separate attempt scopes under
|
||||
`validate/` and are not duplicated into the module attempt. Prompt content is
|
||||
written inline in the prompt artifact. The response metadata and body use the
|
||||
paired files described above; the body is pretty-printed JSON when possible
|
||||
and raw text otherwise. Retrying stages use these stable module-attempt paths:
|
||||
~~~
|
||||
<debug-root>/<run-id>/
|
||||
summary/
|
||||
trace/
|
||||
~~~
|
||||
|
||||
```text
|
||||
chunk/attempt-<NN>.json
|
||||
extract/<lane-id>/chunk-<NNNNNN>/attempt-<NN>.json
|
||||
merge/<lane-id>/attempt-<NN>.json
|
||||
merge/<lane-id>/attempt-<NN>/prompt-<NNNN>.json
|
||||
merge/<lane-id>/attempt-<NN>/response-<NNNN>.json
|
||||
merge/<lane-id>/attempt-<NN>/response-content-<NNNN>.<ext>
|
||||
The summary contains redacted invocation and resolution information plus run,
|
||||
warning, checkpoint, chunk-plan, and terminal reporting artifacts. The trace
|
||||
contains allowlisted application diagnostic records and can include source or
|
||||
derived application data. Neither surface is a cache input. Do not treat a
|
||||
debug bundle as safe to share merely because its configuration summary is
|
||||
redacted. Invocation metadata omits reasoning effort when it is inherited,
|
||||
records the replacement value when one is supplied, and records an empty value
|
||||
when inherited reasoning was explicitly cleared.
|
||||
|
||||
normalize/<lane-id>/attempt-<NN>.json
|
||||
normalize/<lane-id>/attempt-<NN>/prompt-<NNNN>.json
|
||||
normalize/<lane-id>/attempt-<NN>/response-<NNNN>.json
|
||||
normalize/<lane-id>/attempt-<NN>/response-content-<NNNN>.<ext>
|
||||
```
|
||||
|
||||
Every executed chunk, extract, merge, and normalize attempt has one terminal
|
||||
envelope recording acceptance, validator rejection, or a module, validator,
|
||||
candidate-serialization, or final-serialization error as applicable. It
|
||||
includes attempt-local warnings and any available candidate or rejection. A
|
||||
failure before a candidate exists has no candidate payload. If the envelope
|
||||
cannot be persisted, the run does not retry that module attempt and reports the
|
||||
debug failure together with any primary attempt error.
|
||||
|
||||
Chunk-plan candidates, materialized chunks, annotations, and chunk-attempt
|
||||
details appear only in these opt-in debug artifacts. They are intentionally not
|
||||
included in normal manifests or the `chunk-plan.json` diagnostics summary.
|
||||
|
||||
Checkpoint-reused extract, merge, and normalize work retains the stage-level
|
||||
input and output artifacts but has no retry-attempt artifacts
|
||||
because no module attempt executed. Debug artifacts may contain source
|
||||
material, reference material, prompt inputs, model outputs, and other sensitive
|
||||
data. Typed artifact
|
||||
envelopes include domain-neutral codec identity, redacted metadata and content,
|
||||
and digests of the stable codec bytes. API keys are not written, and obvious
|
||||
credential-shaped values and sensitive map keys are redacted, but debug
|
||||
directories should still be protected as sensitive local state.
|
||||
|
||||
## Retention
|
||||
|
||||
Diagnostics retention uses the effective mode selected through configuration;
|
||||
see [Configuration](config.md#diagnostics) for the fields, environment
|
||||
overrides, precedence, and default.
|
||||
|
||||
- `auto`: keep failed runs and successful runs with warnings; remove successful
|
||||
warning-free runs.
|
||||
- `always`: keep every diagnostics run directory.
|
||||
- `never`: remove successful run directories; failed runs are still retained.
|
||||
|
||||
## Failures
|
||||
|
||||
Failures before diagnostics directory creation, such as a missing config file or
|
||||
an unusable diagnostics work directory, are printed to stderr and may not have a
|
||||
diagnostics run directory.
|
||||
|
||||
Failures after diagnostics directory creation are printed to stderr and written
|
||||
to `error.log`. Depending on where the failure occurred, the directory may also
|
||||
contain artifacts written before the failure.
|
||||
|
||||
If durable output writing fails after the pipeline completes, diagnostics are
|
||||
retained for inspection.
|
||||
|
||||
## Warnings
|
||||
|
||||
A successful run with warnings exits with code `0`, prints a warning count to
|
||||
stderr, and writes warnings to durable output and diagnostics when retained.
|
||||
|
||||
The [JSON output contract](integrations/json-output.md) defines durable warning
|
||||
and validation-status fields.
|
||||
Notarius never creates debug state without an explicit request and never
|
||||
automatically deletes a requested bundle. If allocation succeeds, the command
|
||||
reports its path on both success and later failure. A summary, trace, or
|
||||
terminal-report persistence failure fails the command while preserving any
|
||||
already-written diagnostic data for inspection.
|
||||
|
||||
## Cleanup
|
||||
|
||||
It is safe to remove specific old run directories after their output and
|
||||
diagnostics are no longer needed:
|
||||
Cleanup is manual and destructive. First inspect the exact leaf directory,
|
||||
then remove only that leaf; do not use a glob or a parent root as the target.
|
||||
|
||||
```sh
|
||||
rm -rf /tmp/notarius/run-1234567890
|
||||
rm -rf ./notarius-output/run-1234567890
|
||||
```
|
||||
~~~
|
||||
rm -rf -- /srv/notarius/output/run-1721300000000000000-0123456789abcdef0123456789abcdef
|
||||
rm -rf -- /srv/notarius/chunk-plans/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
|
||||
rm -rf -- /srv/notarius/checkpoints/example/seriatim-0123456789abcdef/0123456789abcdef/0123456789abcdef
|
||||
rm -rf -- /srv/notarius/debug/run-1721300000000000000-0123456789abcdef0123456789abcdef
|
||||
~~~
|
||||
|
||||
Workspace checkpoint and debug directories can also be removed when no longer
|
||||
needed. Remove exact identity or run directories, for example:
|
||||
|
||||
```sh
|
||||
rm -rf /var/lib/notarius/checkpoints/dnd-session/seriatim-abcdef123456/7890abcd1234/identityabcd1234
|
||||
rm -rf /var/lib/notarius/debug/run-1234567890
|
||||
```
|
||||
|
||||
Chunk-plan cache entries can likewise be removed by exact digest directory or
|
||||
configured root. Removal is recoverable, but the next non-bypass run may need
|
||||
to regenerate plans and repeat any chunk-stage LLM work.
|
||||
|
||||
Use exact run-directory paths. Avoid broad cleanup commands against parent
|
||||
directories unless they are part of your own operational policy.
|
||||
Deleting output permanently removes user data. Deleting chunk plans or
|
||||
checkpoints is recoverable but may repeat expensive provider or pipeline work.
|
||||
Deleting a debug bundle removes troubleshooting evidence and a retained copy of
|
||||
application data. Notarius has no cache garbage collector, rollback operation,
|
||||
or automatic cleanup command.
|
||||
|
||||
## Operational Limits
|
||||
|
||||
Provider retries and timeouts are handled by Scriptorium according to the
|
||||
selected execution profile. Pipeline module retry settings are defined in
|
||||
[Configuration](config.md#module-bindings). There is no separate CLI retry
|
||||
command.
|
||||
Provider execution settings and the generation timeout come from the selected
|
||||
PromptKit profile. The invocation-only **--reasoning-effort** and
|
||||
**--clear-reasoning-effort** controls may replace or clear that profile setting
|
||||
for all LLM-backed calls in one run without changing the profile. PromptKit
|
||||
v0.5.0 does not add a provider retry loop. Notarius binding retries rerun the
|
||||
complete module operation and validation chain as defined by
|
||||
[module bindings](config.md#module-bindings-and-validators).
|
||||
|
||||
Extract worker concurrency and actual provider-call concurrency are separate
|
||||
limits. Their configuration, defaults, and validation are defined in
|
||||
[Configuration](config.md#concurrency). Cancellation stops undispatched extract
|
||||
work; already started work is allowed to finish or observe cancellation before
|
||||
the run reports failure.
|
||||
Timeouts are layered. Caller cancellation is the outer authority. A positive
|
||||
effective generation timeout adds an inner request deadline, while zero
|
||||
disables only that generation deadline. The HTTP client timeout remains a
|
||||
transport-wide cap. Notarius does not add another timeout around PromptKit.
|
||||
The pinned upstream boundary and profile-format links are in
|
||||
[PromptKit Integration](integrations/pkg-promptkit.md).
|
||||
|
||||
Notarius writes local files only. Remote storage and archive management are not
|
||||
part of the implemented CLI.
|
||||
Concurrency has two independent layers. Notarius **total_llm** is the
|
||||
application-wide provider-call limit shared by all backends, modules, retries,
|
||||
and validators. PromptKit may impose a narrower admission limit for the
|
||||
selected backend. The effective active-generation bound is the intersection of
|
||||
both limits and can therefore be lower than **total_llm**. Built-in OpenRouter
|
||||
profiles use PromptKit's upstream backend limit; endpoint-only profiles have no
|
||||
PromptKit backend limit and remain bounded by Notarius. For the configured
|
||||
local backend, a zero **concurrency_limit** leaves only the Notarius scheduler
|
||||
as a call limit. A positive value makes the effective active local-generation
|
||||
bound the smaller of **total_llm** and that local limit.
|
||||
|
||||
For a positive local limit, PromptKit owns its default waiting capacity and
|
||||
admission behavior. When a PromptKit backend has admitted all active and queued
|
||||
work, a new call fails as capacity exhaustion before generation. The adapter
|
||||
maps that failure to Notarius's existing provider-neutral capacity error and
|
||||
does not retry it. The calling stage's configured retry policy applies
|
||||
normally, and the run fails if those attempts are exhausted. Caller
|
||||
cancellation remains authoritative. Configuration contracts are documented
|
||||
under [PromptKit profiles](config.md#promptkit-profiles) and
|
||||
[concurrency](config.md#concurrency-output-cache-and-debug). Extract-worker
|
||||
limits and actual provider-call limits are independent. Notarius writes local
|
||||
filesystem state only; remote storage, archival, and retention automation are
|
||||
outside the implemented CLI.
|
||||
|
||||
@@ -79,6 +79,14 @@ Pipeline resolution requires a compatible codec and matching kind-specific
|
||||
variants before a typed lane can be accepted. Framework-owned erasure remains
|
||||
private and must report type incompatibility as an error rather than a panic.
|
||||
|
||||
An artifact kind may additionally provide a typed evidence projection that
|
||||
copies its direct generic source references. Preparation proves that projection
|
||||
matches the artifact codec's exact Go type before retaining it for an output
|
||||
policy. The runner reconstructs evidence only from accepted serialized
|
||||
normalized artifacts, and the output boundary owns any resulting publication.
|
||||
Generic framework code never infers evidence by inspecting domain JSON or
|
||||
depends on domain artifact types.
|
||||
|
||||
Auxiliary references provide context or disambiguation. They are not source
|
||||
evidence and must not be converted into source references.
|
||||
|
||||
@@ -114,8 +122,8 @@ plan, while the current run still applies its configured chunk validators to
|
||||
the materialized chunks.
|
||||
|
||||
The framework owns orchestration and handoff provenance. Modules return logical
|
||||
results and warnings; they do not own CLI reporting, workspace paths, durable
|
||||
file placement, checkpoints, or diagnostics.
|
||||
results and warnings; they do not own CLI reporting, physical output, cache, or
|
||||
debug roots, durable file placement, or checkpoint and debug lifecycle.
|
||||
|
||||
After pipeline-wide chunking, extraction uses bounded framework concurrency.
|
||||
One run-wide worker pool receives chunk-scoped lane jobs in deterministic
|
||||
@@ -169,7 +177,8 @@ individual modules.
|
||||
The application-wide LLM scheduler bounds actual provider calls independently
|
||||
of framework worker limits. Every LLM-backed module, retry, and validator uses
|
||||
the single injected scheduled client, including work performed by overlapping
|
||||
lanes.
|
||||
lanes. Provider runtime adapters may enforce a narrower backend-specific limit
|
||||
beneath this mandatory application-wide scheduler.
|
||||
|
||||
## Configuration And Provenance
|
||||
|
||||
@@ -188,31 +197,33 @@ record identities and summaries rather than secret or large payload content.
|
||||
|
||||
## State, Output, And Safety
|
||||
|
||||
Durable output, diagnostics, checkpoints, and debug artifacts are separate
|
||||
surfaces with separate ownership:
|
||||
Notarius exposes three filesystem surfaces with independent roots and
|
||||
lifecycle:
|
||||
|
||||
- output modules define logical durable output; the application boundary owns
|
||||
filesystem placement;
|
||||
- diagnostics provide redacted run inspection and are not the durable output
|
||||
contract;
|
||||
- checkpoints support validated stage reuse and are not diagnostics;
|
||||
- debug artifacts are opt-in inspection data and may contain sensitive source,
|
||||
prompt, reference, and model-output content.
|
||||
- output is durable user data; output modules define logical files and the CLI
|
||||
owns their placement;
|
||||
- cache is reconstructible state, with separate chunk-plan and checkpoint
|
||||
families; and
|
||||
- debug is explicitly requested inspection data, combining a redacted summary
|
||||
with a detailed trace.
|
||||
|
||||
Chunk-plan cache state is an additional independent surface. It is keyed only
|
||||
by canonical source digest, is not rooted under `workspace.directory`, and is
|
||||
not a checkpoint or a diagnostic. A cache record is atomically replaced as one
|
||||
complete plan envelope; it has no history, locking, or rollback interface.
|
||||
Invalid records are recoverable cache misses rather than pipeline state.
|
||||
Chunk plans are keyed only by canonical source digest. Configured checkpoint
|
||||
recording is independent of checkpoint reuse; checkpoints are loaded only for
|
||||
an invocation that explicitly requests resume. Debug is never a cache input and
|
||||
is never created without an explicit request. Pipeline modules receive
|
||||
collaborator interfaces and never physical roots.
|
||||
|
||||
Writes of durable state are atomic where practical. Paths for writes, moves,
|
||||
overwrites, and deletion must be narrow and explicit. Cleanup that can lose data
|
||||
is opt-in.
|
||||
Writes are atomic where practical. Paths for writes, moves, overwrites, and
|
||||
deletion must be narrow and explicit. Notarius never automatically deletes
|
||||
output or requested debug bundles; cache cleanup is explicit and recoverable.
|
||||
|
||||
Secrets must not appear in errors, logs, diagnostics, manifests,
|
||||
documentation, examples, or redacted configuration. Default logs and
|
||||
diagnostics must not include large source, prompt, reference, or artifact
|
||||
payloads.
|
||||
Secrets must not appear in errors, logs, output, cache, debug summaries,
|
||||
traces, manifests, documentation, examples, or redacted configuration. Debug
|
||||
collection is allowlisted to application-owned payloads and must not capture
|
||||
unrelated process environment values or filesystem content. Trace data may
|
||||
contain application data and therefore inherits its sensitivity; operators own
|
||||
access controls and retention. Physical layout and operation are defined in
|
||||
[Operations](../operations.md).
|
||||
|
||||
## Architectural Non-Goals
|
||||
|
||||
|
||||
@@ -61,9 +61,10 @@ secret values.
|
||||
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
|
||||
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, diagnostics use, retention, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||
|
||||
296
docs/policy/testing.md
Normal file
296
docs/policy/testing.md
Normal file
@@ -0,0 +1,296 @@
|
||||
# Testing Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||
|
||||
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||
|
||||
## Every test has a cost
|
||||
|
||||
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||
|
||||
A test must be:
|
||||
|
||||
- written and reviewed;
|
||||
- understood by future maintainers and coding agents;
|
||||
- executed in local and CI workflows;
|
||||
- diagnosed when it fails;
|
||||
- updated when legitimate behavior changes;
|
||||
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||
|
||||
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||
|
||||
A test is warranted only when the confidence it provides justifies these costs.
|
||||
|
||||
Apply this cost-benefit analysis at two levels:
|
||||
|
||||
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||
|
||||
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||
|
||||
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||
|
||||
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||
|
||||
## Default testing style
|
||||
|
||||
Use a **classical/Detroit-style** approach:
|
||||
|
||||
- Test observable behavior, resulting state, contracts, and invariants.
|
||||
- Use real internal collaborators when they are fast and deterministic.
|
||||
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
|
||||
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
|
||||
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
|
||||
|
||||
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||
|
||||
## Test execution requirements
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
|
||||
|
||||
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
|
||||
|
||||
## What deserves tests
|
||||
|
||||
Prioritize tests for:
|
||||
|
||||
1. Public and package-level contracts.
|
||||
2. Domain rules and important invariants.
|
||||
3. Boundary conditions and malformed input.
|
||||
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||
5. Serialization, schemas, compatibility, and round trips.
|
||||
6. Previously observed or plausible regressions.
|
||||
7. Representative integration and end-to-end workflows.
|
||||
|
||||
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||
|
||||
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||
|
||||
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||
|
||||
## Choose the right test boundary
|
||||
|
||||
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||
|
||||
This is often the package API, but it may instead be:
|
||||
|
||||
- a smaller pure function when dense domain logic is most clearly isolated there;
|
||||
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||
|
||||
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
|
||||
|
||||
## Test behavior, not implementation
|
||||
|
||||
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
|
||||
|
||||
Before adding or retaining a test, ask:
|
||||
|
||||
> What realistic defect would this test catch?
|
||||
|
||||
A test is suspect when its main purpose is to detect that someone:
|
||||
|
||||
- changed an internal constant;
|
||||
- renamed or split a private helper;
|
||||
- reordered equivalent internal operations;
|
||||
- changed incidental formatting;
|
||||
- replaced one correct algorithm with another; or
|
||||
- refactored internal object structure without changing behavior.
|
||||
|
||||
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||
|
||||
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
|
||||
|
||||
## Expected effects of different changes
|
||||
|
||||
Use the following expectations when evaluating test failures and test maintenance:
|
||||
|
||||
| Change | Expected effect on tests |
|
||||
|---|---|
|
||||
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
|
||||
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
|
||||
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
|
||||
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
|
||||
|
||||
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
|
||||
|
||||
## Separate mechanism from policy
|
||||
|
||||
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||
|
||||
For example, do not encode an internal concurrency limit indirectly:
|
||||
|
||||
```go
|
||||
// Production policy:
|
||||
const maxConcurrency = 4
|
||||
|
||||
// Brittle test:
|
||||
err := startProcesses(5)
|
||||
require.Error(t, err)
|
||||
```
|
||||
|
||||
Instead, test the mechanism relationally:
|
||||
|
||||
```go
|
||||
const limit = 2
|
||||
runner := NewRunner(limit)
|
||||
|
||||
require.NoError(t, runner.Start(limit))
|
||||
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||
```
|
||||
|
||||
The test should prove:
|
||||
|
||||
- the configured limit is accepted; and
|
||||
- one beyond the configured limit is rejected.
|
||||
|
||||
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||
|
||||
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||
|
||||
For concurrency limits, test both kinds of behavior when relevant:
|
||||
|
||||
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||
|
||||
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||
|
||||
## Avoid semantic duplication across layers
|
||||
|
||||
Each behavior should have a clear test owner.
|
||||
|
||||
- Parser tests own parsing cases.
|
||||
- Validator tests own validation rules.
|
||||
- Domain tests own transformations and invariants.
|
||||
- Adapter tests own external integration behavior.
|
||||
- Orchestrator tests own coordination and failure propagation.
|
||||
- CLI tests own argument and configuration mapping.
|
||||
- End-to-end tests prove that representative assembled workflows work.
|
||||
|
||||
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||
|
||||
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||
|
||||
## Use test doubles deliberately
|
||||
|
||||
Choose the least elaborate test double that provides the required control or observation.
|
||||
|
||||
As a default:
|
||||
|
||||
1. Prefer real collaborators when they are fast and deterministic.
|
||||
2. Use small in-memory fakes when realistic stateful behavior is helpful.
|
||||
3. Use stubs when a dependency only needs to provide controlled responses.
|
||||
4. Use mocks when the interaction itself is contractual.
|
||||
|
||||
Mocks are appropriate when the contract includes facts such as:
|
||||
|
||||
- a notification is sent exactly once;
|
||||
- a transaction is committed only after successful writes;
|
||||
- cancellation reaches a subprocess;
|
||||
- an expensive API is called no more than once; or
|
||||
- a security audit event is emitted.
|
||||
|
||||
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||
|
||||
## Go-specific guidance
|
||||
|
||||
Use:
|
||||
|
||||
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||
- `t.TempDir()` for real filesystem behavior;
|
||||
- `httptest.Server` for realistic HTTP interactions;
|
||||
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||
- golden files only when the complete output is intentionally stable;
|
||||
- integration tests where correctness depends on component interaction; and
|
||||
- a small number of representative end-to-end tests.
|
||||
|
||||
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||
|
||||
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
|
||||
|
||||
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||
|
||||
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||
|
||||
## Coverage
|
||||
|
||||
Coverage is a diagnostic, not a target.
|
||||
|
||||
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||
|
||||
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||
|
||||
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||
|
||||
## Regression tests
|
||||
|
||||
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||
|
||||
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||
|
||||
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||
|
||||
## Deleting or rewriting tests
|
||||
|
||||
Tests are maintained code, not permanent historical artifacts.
|
||||
|
||||
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||
|
||||
Strong candidates include tests that:
|
||||
|
||||
- require updates after harmless internal changes;
|
||||
- directly assert private constants without protecting a real contract;
|
||||
- duplicate the same policy across several layers;
|
||||
- verify mock choreography rather than outcomes;
|
||||
- snapshot large amounts of incidental output;
|
||||
- test trivial private helpers already exercised through stable package behavior;
|
||||
- protect risks already covered more effectively elsewhere;
|
||||
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||
- no longer correspond to a plausible failure mode.
|
||||
|
||||
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||
|
||||
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||
|
||||
## Reviewing a proposed test
|
||||
|
||||
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||
|
||||
1. What realistic defect would it catch?
|
||||
2. How likely is that defect?
|
||||
3. How consequential would it be?
|
||||
4. Is the behavior already protected elsewhere?
|
||||
5. At which layer should this behavior be owned?
|
||||
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||
8. What should cause this test to fail?
|
||||
9. What legitimate changes should not cause this test to fail?
|
||||
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||
11. Is there a smaller or more direct test that protects the same risk?
|
||||
|
||||
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||
|
||||
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||
|
||||
## Definition of sufficient
|
||||
|
||||
A test suite is sufficient when:
|
||||
|
||||
- important contracts and invariants are protected;
|
||||
- meaningful boundaries and failure modes are exercised;
|
||||
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||
- important external boundaries have realistic integration coverage;
|
||||
- representative complete workflows are tested;
|
||||
- failures provide useful signal rather than redundant noise;
|
||||
- legitimate internal changes usually do not require test edits; and
|
||||
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||
|
||||
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
|
||||
|
||||
The governing rule is:
|
||||
|
||||
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||
@@ -1,166 +0,0 @@
|
||||
# ADR-0005 Feature Roadmap
|
||||
|
||||
This roadmap defines the intended end state for
|
||||
[ADR-0005](../adr/0005-cache-canonical-chunk-plans-by-source.md). The feature
|
||||
is not implemented. The ordered coding work belongs in the
|
||||
[implementation plan](implementation.md).
|
||||
|
||||
## Intent
|
||||
|
||||
Notarius may run multiple extraction passes over the same source. Chunking,
|
||||
especially LLM-backed chunking over a large transcript, can be substantially
|
||||
more expensive than later per-chunk operations. Stable chunk material also
|
||||
improves the opportunity for provider-side prompt-cache reads across passes.
|
||||
|
||||
Notarius therefore prioritizes stable, aggressively reused source partitions
|
||||
over automatically applying later chunk-module, reference, prompt, model, or
|
||||
pipeline changes. An operator must explicitly request repartitioning.
|
||||
|
||||
## Target State
|
||||
|
||||
### Canonical source plan
|
||||
|
||||
Each validated generic source document has at most one stored chunk plan. The
|
||||
source document's canonical digest selects that plan. Pipeline identity,
|
||||
selected lanes, configured chunk module and options, references, validators,
|
||||
prompts, schemas, and LLM settings do not participate in lookup.
|
||||
|
||||
In the default mode, the first accepted plan for a source is reused by later
|
||||
pipelines and invocations. A configured chunk module runs only when no valid
|
||||
stored plan exists, when reuse is bypassed for one invocation, or when the
|
||||
operator requests refresh.
|
||||
|
||||
### Plan and annotation contract
|
||||
|
||||
A plan contains an ordered, non-empty collection of inclusive source-unit
|
||||
ranges. The framework validates these boundaries and deterministically
|
||||
materializes runtime chunks from the current source document.
|
||||
|
||||
Chunk modules may attach optional domain annotations at plan or range scope.
|
||||
Annotations are namespaced, validated JSON values. They are preserved through
|
||||
storage and materialization but are never a hard cross-domain capability.
|
||||
Downstream modules may rely on the generic chunk contract only.
|
||||
|
||||
The store contains boundaries, annotations, warnings, and provenance. It does
|
||||
not contain fully materialized chunks or duplicated source-unit content.
|
||||
|
||||
### Persistence
|
||||
|
||||
Chunk plans use a dedicated cache root that is independent of
|
||||
`workspace.directory` and the locations of checkpoints, diagnostics, debug
|
||||
artifacts, and durable output. The default is the platform-appropriate per-user
|
||||
cache directory: on Linux, `$XDG_CACHE_HOME/notarius/chunk-plans` when
|
||||
`XDG_CACHE_HOME` is set to a valid absolute path, otherwise
|
||||
`$HOME/.cache/notarius/chunk-plans` when it is unset, through the platform
|
||||
cache-directory resolver. An invalid relative `XDG_CACHE_HOME` is an error, not
|
||||
a fallback.
|
||||
|
||||
Operators may set `workspace.chunk_cache.directory` to replace that root. The
|
||||
recommended system-wide setting for a dedicated Linux service account is
|
||||
`/var/cache/notarius/chunk-plans`. This is an operational recommendation, not
|
||||
the application default; the service account must own the directory and it must
|
||||
not be shared across mutually untrusted users.
|
||||
|
||||
One mutable, versioned plan file is stored beneath the chunk-plan root under the
|
||||
full canonical source digest. Normal publication and refresh replace that file
|
||||
atomically; history, rollback, multiple variants, and content deduplication are
|
||||
outside this feature.
|
||||
|
||||
Plan state is potentially sensitive. Paths are confined, directories and files
|
||||
use restrictive permissions, default diagnostics omit annotation and source
|
||||
payloads, and debug output remains explicitly opt-in.
|
||||
|
||||
### Runtime policy
|
||||
|
||||
The effective chunk-cache mode is one of:
|
||||
|
||||
- `auto`: load a valid stored plan; otherwise generate, validate, and publish
|
||||
one;
|
||||
- `bypass`: do not read or write plan state for this invocation; or
|
||||
- `refresh`: generate and validate a plan, then atomically replace stored state.
|
||||
|
||||
`auto` is the default. Invalid or incompatible stored state is a reported miss
|
||||
in `auto`; it is replaced only after a newly generated plan is accepted. A
|
||||
failed generation never overwrites prior state. Concurrent writes must never
|
||||
expose partial data and use last-successful-atomic-write semantics.
|
||||
|
||||
The public CLI override is
|
||||
`--chunk_cache <auto|bypass|refresh>`. Configuration and environment values use
|
||||
the same three modes, with CLI taking highest precedence.
|
||||
|
||||
### Validation and execution
|
||||
|
||||
Framework plan validation and deterministic materialization run for generated
|
||||
and reused plans. The current pipeline's configured chunk-validator chain then
|
||||
validates the materialized chunks.
|
||||
|
||||
On a cache hit, the configured chunk module is constructed during normal
|
||||
pipeline preparation but its operation is not invoked and it makes no LLM call.
|
||||
A validator rejection of a structurally valid reused plan is a run outcome; it
|
||||
does not implicitly authorize rechunking.
|
||||
|
||||
Only the generic `chunks` capability is hard. Domain annotations such as D&D
|
||||
scene information are opportunistic and cannot be required solely because the
|
||||
current pipeline selected the module that normally produces them.
|
||||
|
||||
Canonical plans are the sole owner of chunk reuse. Invocation-scoped
|
||||
checkpoints no longer load or record chunk outputs. Downstream checkpoint
|
||||
identity continues to depend on the digest of the effective materialized
|
||||
chunks, so refresh invalidates affected downstream work.
|
||||
|
||||
### Provenance
|
||||
|
||||
Every run distinguishes the chunk module requested by the resolved pipeline
|
||||
from the producer of the effective stored plan. Durable provenance records the
|
||||
source and plan digests, plan schema version, effective cache mode and action,
|
||||
producer module, producer references and LLM profile where applicable, and
|
||||
non-sensitive producer metadata.
|
||||
|
||||
Default diagnostics record lookup, validation, generation, and publication
|
||||
decisions without source or annotation payloads. Opt-in debug artifacts may
|
||||
contain complete plan and annotation material and are treated as sensitive.
|
||||
|
||||
### Deployment documentation
|
||||
|
||||
The implemented configuration reference identifies the dedicated directory
|
||||
field, its environment override, and the per-user default. The operations guide
|
||||
documents the resulting filesystem layout and permissions, and recommends
|
||||
`/var/cache/notarius/chunk-plans` for a system-wide Linux deployment running as
|
||||
a dedicated service account. It also explains that cache deletion is
|
||||
recoverable but may repeat expensive chunk generation. Neither document
|
||||
presents the system-wide path as the default for an ordinary unprivileged
|
||||
invocation.
|
||||
|
||||
## Compatibility Policy
|
||||
|
||||
Existing fully materialized chunk checkpoint files are not migrated or promoted
|
||||
to canonical plans. They are ignored for chunk reuse after this feature lands.
|
||||
The first `auto` run generates the source's plan through the configured chunk
|
||||
module.
|
||||
|
||||
The existing source digest is the canonical lookup identity. Any source change
|
||||
that alters that digest creates a separate plan. Earlier plan schema versions,
|
||||
malformed files, and digest mismatches are incompatible and follow the invalid
|
||||
`auto`-miss behavior.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
This feature does not provide:
|
||||
|
||||
- plan editing, comparison, history, rollback, or garbage collection;
|
||||
- remote or shared plan storage;
|
||||
- multiple active or automatically selected plan variants for one source;
|
||||
- mandatory domain annotation contracts;
|
||||
- automatic rechunking because configuration or model inputs changed; or
|
||||
- a guarantee that an LLM provider will report cache hits.
|
||||
|
||||
## Completion Outcomes
|
||||
|
||||
The feature is complete when independent runs over the same canonical source
|
||||
reuse byte-stable materialized chunks across pipeline, lane, reference,
|
||||
chunk-module, and LLM configuration changes; bypass and refresh obey their
|
||||
documented state semantics; provenance identifies the effective producer;
|
||||
legacy chunk checkpoints cannot compete with plan reuse; and all focused,
|
||||
integration, compatibility, CLI, and repository-wide validation passes. The
|
||||
configuration and operations references must also document both the per-user
|
||||
default and the recommended system-wide Linux setting.
|
||||
@@ -2,49 +2,222 @@
|
||||
|
||||
Current Notarius behavior is documented in the canonical README, CLI,
|
||||
configuration, operations, internal, and integration docs. This roadmap records
|
||||
future work only.
|
||||
future work only. Items are ordered roughly by current value and specificity,
|
||||
not as committed release dates.
|
||||
|
||||
## Candidate Product Work
|
||||
## Near-Term D&D Pipeline
|
||||
|
||||
### Combat Enemy Ledger
|
||||
|
||||
- Add a D&D artifact that identifies enemies faced during combat and supports
|
||||
an end-of-session encounter ledger.
|
||||
- Track each enemy's observed state using a small controlled vocabulary such as
|
||||
`active`, `killed`, `fled`, `captured`, or `incapacitated`, while preserving
|
||||
an explicit unresolved state when the transcript does not establish an
|
||||
outcome.
|
||||
- Preserve the evidence for enemy participation and state changes rather than
|
||||
inferring a terminal outcome from combat ending or an enemy disappearing
|
||||
from the conversation.
|
||||
- Define how repeated mentions, groups of unnamed enemies, summoned or allied
|
||||
creatures, and the same enemy appearing in multiple combats affect identity
|
||||
and ledger entries.
|
||||
- Evaluate whether the ledger should be extracted directly, derived from
|
||||
combat-turn artifacts, or use a sequential pipeline that consumes combat
|
||||
turns and the normalized NPC registry as grounding references.
|
||||
|
||||
### Location Extraction
|
||||
|
||||
- Add a D&D artifact for locations visited by the party or otherwise mentioned
|
||||
in the transcript.
|
||||
- Distinguish observed visits from references, plans, recalled places, and
|
||||
uncertain or inferred locations so a mention alone is not reported as a
|
||||
visit.
|
||||
- Preserve transcript evidence for each visit or mention and reconcile aliases,
|
||||
nested places, and repeated appearances without collapsing distinct
|
||||
locations that share a generic name.
|
||||
- Define how the location artifact should ground later narrative reports and
|
||||
whether future event artifacts should retain canonical location identities.
|
||||
|
||||
### Evaluate Spell Extraction And Normalization
|
||||
|
||||
- Evaluate ordinary extraction retries and the completed normalization path
|
||||
against a human-reviewed transcript set before adding repair-aware retries or
|
||||
an LLM-backed semantic validator.
|
||||
- Maintain a small set of human-reviewed transcripts and outputs for prompt,
|
||||
validator, and normalizer development. Treat model-quality review as an
|
||||
iterative human evaluation aid, not a deterministic correctness gate.
|
||||
|
||||
### Evaluate The Shared D&D Scene Plan
|
||||
|
||||
- Reassess whether one shared scene plan provides enough context for NPC,
|
||||
spell, combat, interaction, and scene-description lanes after real-world use.
|
||||
Add more complex chunking only in response to demonstrated failures.
|
||||
|
||||
## Cross-Cutting LLM Runtime
|
||||
|
||||
### Deterministic Prompt Session Identity
|
||||
|
||||
- Replace the source-document-ID default for prompt sessions with one
|
||||
predictable, procedurally generated session ID for the complete
|
||||
source-processing workload.
|
||||
- Preserve an explicit non-empty `--session-id` as the highest-precedence
|
||||
override. Otherwise, derive the default only from the effective input module
|
||||
identity and the exact raw input bytes.
|
||||
- Use a versioned, bounded representation such as
|
||||
`notarius:v1:<sha256(input-module + NUL + raw-input)>`. The exact encoding
|
||||
must fit PromptKit's session length contract and must not embed source
|
||||
content.
|
||||
- Keep the derived session stable across runs, pipelines, selected lanes,
|
||||
ordered steps, retries, resume, recomputation, LLM profiles, reasoning
|
||||
overrides, and output, debug, or cache settings.
|
||||
- Do not include file-backed references, generated references, reference
|
||||
contents, or the composition of a reference bundle in session derivation.
|
||||
References may change between prompt calls within one pipeline without
|
||||
changing routing affinity.
|
||||
- Resolve the authoritative session before checkpoint construction and use the
|
||||
same value for checkpoint runtime identity, every prompt-facing module,
|
||||
PromptKit's direct session field, the compatibility `session_id` prompt
|
||||
variable, run-manifest metadata, and debug metadata.
|
||||
- Keep routing identity separate from cache and checkpoint content identity.
|
||||
Exact prompt prefixes, reference contents, model settings, and other
|
||||
generation-affecting inputs must continue to participate in their existing
|
||||
hashes and checkpoint fingerprints even though they do not change the
|
||||
session.
|
||||
- Treat the generated value as a provider-visible, stable pseudonymous
|
||||
correlation identifier. Do not introduce an installation-specific HMAC or
|
||||
secret unless a concrete multi-tenant or privacy requirement justifies
|
||||
sacrificing deterministic identity across installations.
|
||||
|
||||
### Raise The Default Application-Wide LLM Limit
|
||||
|
||||
- Raise the default `concurrency.total_llm` value from 1 to 16 so ordinary
|
||||
single-backend runs can use PromptKit's expected OpenRouter capacity and
|
||||
lower-capacity local backends without an unnecessarily narrower Notarius
|
||||
limit.
|
||||
- Keep the Notarius application-wide scheduler mandatory and require
|
||||
`total_llm` to remain a positive integer. Do not make the default unlimited:
|
||||
endpoint-only profiles, an unrestricted local backend, injected clients, and
|
||||
aggregate work across several backends may have no narrower PromptKit limit.
|
||||
- Continue defaulting `concurrency.stage_workers.extract` to the effective
|
||||
`total_llm`, making its default 16 as part of the same change. Preserve an
|
||||
explicit lower extract-worker setting when an operator wants less queued or
|
||||
concurrent extraction work.
|
||||
- Define effective provider concurrency as the intersection of the Notarius
|
||||
application-wide limit, the selected PromptKit backend limit when present,
|
||||
and the work made available by stage execution. A Notarius limit of 16 does
|
||||
not narrow a backend already limited to 16, while a local backend limited to
|
||||
4 remains bounded at 4.
|
||||
- Treat the default as an application-wide safety ceiling across profiles,
|
||||
backends, modules, retries, and validators. A run that intentionally needs
|
||||
the combined capacity of several backends may configure a higher
|
||||
`total_llm` and an appropriate extract-worker count explicitly.
|
||||
- Retain the existing configuration and environment override surfaces. Update
|
||||
canonical configuration, operations, and internal documentation together
|
||||
when the default changes.
|
||||
- Reconsider decoupling the extract-worker default from `total_llm` only after
|
||||
mixed-backend workloads demonstrate a need for a high global emergency
|
||||
ceiling with a lower default work-production rate.
|
||||
|
||||
## Shared Normalization And Quality Work
|
||||
|
||||
### Generic LLM-Assisted Deduplication
|
||||
|
||||
- Add a reusable normalizer that asks an LLM to identify duplicate sets in a
|
||||
list and propose one replacement element for each set.
|
||||
- Define the minimum domain-neutral input contract, initially an ordered list
|
||||
whose elements have stable unique IDs. Artifact-kind registrations or
|
||||
adapters may expose that structure without moving domain rules into the
|
||||
generic package.
|
||||
- Keep mutation deterministic: parse and validate the model's duplicate groups,
|
||||
require every referenced ID to exist, reject overlapping or malformed groups,
|
||||
prevent unrelated insertion or deletion, and apply only approved replacement
|
||||
operations in code.
|
||||
- Preserve provenance needed for audit and downstream validation, and emit
|
||||
warnings describing every collapsed group.
|
||||
- Evaluate batching and context-window limits before applying the normalizer to
|
||||
large artifact collections.
|
||||
|
||||
The model may use its own domain knowledge to judge semantic duplication; the
|
||||
generic implementation is responsible only for the common proposal contract,
|
||||
safety checks, and deterministic application of accepted changes.
|
||||
|
||||
### Validation And Review
|
||||
|
||||
- Add domain validators and production default chains alongside each new D&D
|
||||
artifact.
|
||||
- Add production LLM-backed validators only when a concrete review policy
|
||||
benefits from model judgment and deterministic checks are insufficient.
|
||||
- Add validator diagnostics and timing summaries if operators need more detail
|
||||
than the current [durable output bundle](../integrations/json-output.md)
|
||||
provides.
|
||||
- Add validator compatibility metadata if deployments need config-time proof
|
||||
that a validator is suitable for a particular stage, module, or artifact
|
||||
kind.
|
||||
- Add media-type validators when non-JSON artifact representations are
|
||||
introduced.
|
||||
|
||||
## Further Reference Evolution
|
||||
|
||||
- Make prior-run artifacts easier to bind as references without changing the
|
||||
existing module-facing reference-item contract.
|
||||
- Add structured or parsed references, such as typed NPC registries, rosters,
|
||||
or spell catalogs, when opaque UTF-8 prompt material is no longer sufficient.
|
||||
- Add per-slot or per-chunk inclusion policies so large references are not
|
||||
repeated in every prompt unnecessarily.
|
||||
- Add token budgeting and model context-window management for reference
|
||||
content.
|
||||
- Add reference caching, preprocessing, summarization, embedding, or retrieval
|
||||
only when reference size and observed model behavior justify them.
|
||||
- Extend generated references to prior-run artifacts or derived summaries only
|
||||
after same-run ordered handoffs establish the required provenance and
|
||||
lifecycle semantics.
|
||||
|
||||
## Design Considerations To Revisit
|
||||
|
||||
These concerns are relevant to ordered artifact dependencies but are not
|
||||
committed near-term features.
|
||||
|
||||
### Cross-artifact identity links
|
||||
|
||||
Evaluate whether downstream D&D artifacts should retain canonical NPC IDs from
|
||||
the generated NPC reference in addition to normalized display names. Any such
|
||||
contract must define player-character, unknown-actor, missing-NPC, and
|
||||
superseded-identity behavior before implementation. Deterministic validation
|
||||
may confirm that a linked ID exists in the consumed NPC artifact, but the link
|
||||
must never substitute for transcript evidence that the downstream event
|
||||
occurred.
|
||||
|
||||
### Artifact contract evolution
|
||||
|
||||
Define compatibility and migration policy before generated-reference chains
|
||||
must span multiple schema versions or long-lived historical artifacts. The
|
||||
policy should address stable identifier semantics, which schema changes permit
|
||||
checkpoint reuse, when an older artifact may be decoded or adapted, and when a
|
||||
producer or all dependents must be recomputed. Do not add a general migration
|
||||
framework until an actual contract change requires one.
|
||||
|
||||
## Blue-Sky Platform And Operations
|
||||
|
||||
These ideas are intentionally less specified. Promote one into an earlier
|
||||
section only after a concrete workflow, contract, and priority emerge.
|
||||
|
||||
### Platform Extensions
|
||||
|
||||
- Additional input adapters, such as Markdown or note-export formats.
|
||||
- Additional D&D extractors beyond spell casts.
|
||||
- Add non-file reference producers, such as prior-run artifacts, derived
|
||||
summaries, or entity registries, without changing module-facing reference
|
||||
item contracts.
|
||||
- Add token budgeting and model context-window management for reference content.
|
||||
- Add per-slot or per-chunk inclusion policies so modules can avoid repeating
|
||||
large reference content in every prompt when that becomes important.
|
||||
- Add structured or parsed references, such as typed roster schemas, when a
|
||||
module has a clear need for more than opaque UTF-8 text.
|
||||
- Add reference caching, preprocessing, summarization, embedding, or retrieval
|
||||
if references become large enough to require preprocessing.
|
||||
- Cross-lane entity normalization.
|
||||
- Cross-chunk semantic deduplication.
|
||||
- Additional validator packages and production default chains for future
|
||||
modules.
|
||||
- Production LLM-backed validators when there is a concrete review policy that
|
||||
benefits from model judgment.
|
||||
- Validator diagnostics and timing summaries if operators need more detail than
|
||||
the current [durable output bundle](../integrations/json-output.md) provides.
|
||||
- Media-type validators for non-JSON module outputs when such modules are
|
||||
introduced.
|
||||
- Validator compatibility metadata if real deployments need config-time
|
||||
enforcement that a validator is suitable for a specific stage or module.
|
||||
- Batching or context-window controls for LLM-backed validators if validator
|
||||
inputs become large enough to require them.
|
||||
- Additional output encoders.
|
||||
- Concurrent cross-lane entity normalization or broader workflow composition.
|
||||
- Batching or specialized context-window controls for LLM-backed validators.
|
||||
|
||||
## Candidate Operational Work
|
||||
### Distribution And Operations
|
||||
|
||||
- Packaged release artifacts for alpha distribution.
|
||||
- A documented versioning and release process.
|
||||
- Optional generated example output fixtures with a regeneration procedure.
|
||||
- Additional diagnostics or reporting views if operator workflows need them.
|
||||
- Optional generated example-output fixtures with a regeneration procedure.
|
||||
- Additional diagnostics or reporting views.
|
||||
|
||||
## Candidate Workspace Work
|
||||
### Workspace And Storage
|
||||
|
||||
- Default-idempotent run behavior with an explicit force override.
|
||||
- Remote workspace storage.
|
||||
- Workspace garbage collection.
|
||||
- Workspace archival policy.
|
||||
- Workspace garbage collection and archival policies.
|
||||
- Cross-machine checkpoint reuse.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
318
docs/roadmap/promptkit.md
Normal file
318
docs/roadmap/promptkit.md
Normal file
@@ -0,0 +1,318 @@
|
||||
# PromptKit v0.5 Integration And LLM Profile Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
This roadmap defines the target state for upgrading Notarius from PromptKit
|
||||
v0.3.0 to v0.5.0 and adopting the upstream runtime and profile facilities that
|
||||
directly improve Notarius. It also defines the application policy for stable,
|
||||
domain-oriented LLM profile names, operator overrides, pipeline inheritance,
|
||||
profile validation, provider defaults, checkpoint identity, and documentation.
|
||||
|
||||
The ordered work needed to reach this state belongs in
|
||||
[the implementation plan](implementation.md). Current behavior remains defined
|
||||
by the canonical documentation outside `docs/roadmap/` until the corresponding
|
||||
work is implemented.
|
||||
|
||||
## Background
|
||||
|
||||
Notarius currently pins PromptKit v0.3.0. Its adapter prepares a request once
|
||||
for debug material and then independently runs the original request, causing
|
||||
PromptKit to prepare the same logical call a second time. The CLI validates an
|
||||
explicit profile by preparing a synthetic prompt. PromptKit profile selection
|
||||
can be repeated on individual module bindings or replaced for one invocation
|
||||
with `--llm-profile`, but a configured pipeline cannot yet declare one inherited
|
||||
profile policy.
|
||||
|
||||
PromptKit v0.4.0 and v0.5.0 add the upstream boundaries needed to improve these
|
||||
areas:
|
||||
|
||||
- [v0.4.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/releases/v0.4.0.md)
|
||||
adds opaque prepared executions, exact profile and prompt inspection, and a
|
||||
typed backend-capacity error;
|
||||
- [v0.5.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/releases/v0.5.0.md)
|
||||
adds application fallback profile filesystems and stops sending unset
|
||||
optional sampling controls as framework-selected provider values; and
|
||||
- the [v0.5.0 format contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md)
|
||||
defines the resulting profile-source and execution-setting precedence.
|
||||
|
||||
A source-compatibility test of the current Notarius repository against
|
||||
PromptKit v0.5.0 completed successfully. The work is therefore primarily an
|
||||
intentional runtime and configuration migration rather than a repair for a
|
||||
breaking Go API change.
|
||||
|
||||
## Goals
|
||||
|
||||
- Pin and document PromptKit v0.5.0 as Notarius's supported upstream contract.
|
||||
- Execute the exact prepared request snapshot whose safe details are recorded
|
||||
in Notarius debug material.
|
||||
- Validate configured PromptKit profiles through the upstream inspection API
|
||||
without synthetic prompts, provider calls, or credential-value access.
|
||||
- Give Notarius an application-owned, operator-overridable
|
||||
`dnd-extraction` profile fallback.
|
||||
- Let a pipeline choose one default LLM profile without repeating that ID on
|
||||
every LLM-backed binding.
|
||||
- Apply profile inheritance and run-wide overrides only where the resolved
|
||||
module or validator can use an LLM.
|
||||
- Preserve accurate checkpoint invalidation, effective profile provenance,
|
||||
redaction, cancellation, concurrency, and provider-neutral module contracts.
|
||||
- Provide operators with one clear deployment pattern for production,
|
||||
development, and local profile definitions.
|
||||
|
||||
## Target End State
|
||||
|
||||
### PromptKit Runtime Boundary
|
||||
|
||||
Notarius depends on PromptKit v0.5.0 and uses its public APIs rather than
|
||||
reimplementing source or execution resolution.
|
||||
|
||||
For each structured completion, the adapter:
|
||||
|
||||
1. builds one PromptKit run request from the provider-neutral Notarius request;
|
||||
2. calls `PrepareExecution` once;
|
||||
3. immediately arranges an idempotent `Discard` for every unexecuted handle;
|
||||
4. obtains credential-redacted `Details` for debug and response metadata; and
|
||||
5. calls `RunPrepared` so generation uses that exact frozen snapshot.
|
||||
|
||||
The debug prompt and successful result therefore describe the same selected
|
||||
profile, rendered messages, input bytes, session, output contract, and effective
|
||||
settings even when a filesystem-backed source changes concurrently. PromptKit
|
||||
handle types remain private to `internal/framework/llm`.
|
||||
|
||||
PromptKit admission failures continue to match Notarius's provider-neutral
|
||||
`ErrLLMCapacityExceeded` contract. When PromptKit supplies a `CapacityError`,
|
||||
the adapter obtains the normalized backend ID through `errors.As` and may add it
|
||||
to safe application-owned diagnostics without parsing upstream error wording.
|
||||
The backend ID does not become a provider-specific module contract.
|
||||
|
||||
### Optional Provider Controls
|
||||
|
||||
Notarius accepts PromptKit v0.5.0's new behavior for `temperature`,
|
||||
`max_tokens`, and `top_p`: an unset setting is omitted from compatible provider
|
||||
requests and the provider chooses its own default. Notarius does not restore
|
||||
PromptKit's former implicit `top_p: 1` value globally.
|
||||
|
||||
An operator who requires a particular value specifies it in the selected
|
||||
PromptKit profile. The application fallback described below intentionally
|
||||
leaves these controls unset. A human-reviewed D&D extraction comparison should
|
||||
be performed after the upgrade, but paid or nondeterministic model output is
|
||||
not part of the default automated test suite.
|
||||
|
||||
### Profile Inspection
|
||||
|
||||
Pipeline-aware configuration validation uses `Engine.InspectProfile` for every
|
||||
effective explicit profile ID. It verifies that the profile exists, parses and
|
||||
validates, resolves its backend and target, and is compatible with the engine's
|
||||
registered backends. It does not create a synthetic prompt, load prompt inputs,
|
||||
contact a provider, or require credential values to exist in the validation
|
||||
process environment.
|
||||
|
||||
Credential availability is execution-time state. PromptKit preparation still
|
||||
enforces the selected profile's credential contract before generation. This
|
||||
keeps `notarius config validate` useful in build and deployment validation
|
||||
environments where secrets are deliberately absent.
|
||||
|
||||
PromptKit construction for inspection and execution uses one shared internal
|
||||
profile-source and backend-option path. The CLI does not expose PromptKit public
|
||||
types across the Notarius LLM boundary merely to perform inspection.
|
||||
|
||||
`InspectPrompt` is not adopted merely because it exists. It remains available
|
||||
for a later, separately defined module-to-prompt interface preflight if a
|
||||
concrete validation requirement justifies that additional contract.
|
||||
|
||||
### Application And Operator Profile Sources
|
||||
|
||||
Notarius embeds one ordinary PromptKit YAML profile with the stable ID
|
||||
`dnd-extraction`. It is an application fallback registered through
|
||||
`WithFallbackProfileFS`, is owned by the D&D module family, and initially
|
||||
preserves the current effective D&D baseline:
|
||||
|
||||
- backend: PromptKit's built-in `openrouter` backend;
|
||||
- model: `openai/gpt-5.6-luna`;
|
||||
- reasoning effort: unset, allowing OpenAI's backend to apply its default of
|
||||
`medium`;
|
||||
- generation timeout: 240 seconds;
|
||||
- service tier: `flex`; and
|
||||
- no application-selected `temperature`, `max_tokens`, or `top_p`.
|
||||
|
||||
All maintained D&D LLM prompt definitions use `dnd-extraction` as their
|
||||
`default_profile`. The ID communicates workload intent rather than a provider,
|
||||
model, or environment. Changing the embedded fallback is an intentional
|
||||
Notarius execution-policy change and participates in checkpoint identity.
|
||||
|
||||
Effective profile definitions resolve in PromptKit's order:
|
||||
|
||||
1. programmatic in-memory profiles used by tests or explicit consumers;
|
||||
2. the operator source configured by `promptkit.profile_file` or
|
||||
`promptkit.profile_dir`;
|
||||
3. the Notarius application fallback source; and
|
||||
4. PromptKit's embedded built-in catalog.
|
||||
|
||||
Only an absent ID falls through to the next source. A matching profile is a
|
||||
complete definition: fields are not merged with a lower-precedence definition,
|
||||
and a malformed matching operator profile fails rather than silently selecting
|
||||
the application fallback.
|
||||
|
||||
Production, development, and local deployments should normally provide
|
||||
different complete definitions for the same `dnd-extraction` ID. An operator
|
||||
source is optional because the application fallback keeps the maintained D&D
|
||||
workflow usable, but a deployment that needs an intentional model or backend
|
||||
policy should configure its own definition.
|
||||
|
||||
### Domain Ownership And Asset Assembly
|
||||
|
||||
The D&D fallback profile remains under `internal/modules/dnd` and is registered
|
||||
by the D&D registrar, consistent with ADR-0004. Generic LLM plumbing knows how
|
||||
to collect and flatten application fallback profile filesystems but contains no
|
||||
D&D model or policy knowledge.
|
||||
|
||||
The shared asset registry detects invalid roots, unreadable sources, and
|
||||
duplicate flattened paths. PromptKit remains responsible for strict profile
|
||||
YAML parsing, duplicate profile-ID detection, source precedence, and effective
|
||||
target resolution. The same assembled fallback source is supplied to runtime
|
||||
execution and CLI profile inspection.
|
||||
|
||||
### Explicit Module Execution Metadata
|
||||
|
||||
Every registered input, chunk, extract, merge, normalize, and output module
|
||||
declares one required execution class: `deterministic` or `llm_backed`.
|
||||
Validator registrations continue to declare the same distinction through their
|
||||
validator specifications.
|
||||
|
||||
The registered specification is authoritative for configuration resolution.
|
||||
Current production classifications are:
|
||||
|
||||
- the D&D scene chunker, all D&D extractors, and the D&D NPC normalizer are
|
||||
LLM-backed;
|
||||
- the Seriatim input adapter, generic chunker, all current mergers, all other
|
||||
current normalizers, and the JSON output encoder are deterministic; and
|
||||
- current validators retain their declared classifications.
|
||||
|
||||
Missing or unsupported execution metadata is a registration error. Explicitly
|
||||
assigning `llm_profile` to a deterministic module or validator is a pipeline
|
||||
resolution error. The framework does not infer execution class by inspecting
|
||||
domain package names or concrete implementation types at runtime.
|
||||
|
||||
The module specification replaces the chunk runner's special runtime
|
||||
execution-class probe. Effective resolved bindings already express the result:
|
||||
only LLM-backed bindings may retain a non-empty profile.
|
||||
|
||||
### Pipeline-Level Profile Default
|
||||
|
||||
Configuration version 4 gains one optional non-empty pipeline field:
|
||||
|
||||
```yaml
|
||||
pipelines:
|
||||
dnd-session:
|
||||
llm_profile: dnd-extraction
|
||||
```
|
||||
|
||||
No configuration-version increment is required because the field is additive
|
||||
and existing files remain valid. An explicitly present blank value is invalid.
|
||||
|
||||
For every selected LLM-backed module and validator, the effective profile uses
|
||||
this precedence:
|
||||
|
||||
1. non-empty run-wide `--llm-profile` override;
|
||||
2. binding-specific `llm_profile`;
|
||||
3. pipeline-level `llm_profile`; and
|
||||
4. the prompt definition's `default_profile`, represented by an empty effective
|
||||
Notarius binding profile.
|
||||
|
||||
The run-wide override and inherited pipeline default never attach to a
|
||||
deterministic binding. Binding-specific exceptions remain available when one
|
||||
operation needs a different cost, latency, quality, backend, or reasoning
|
||||
policy.
|
||||
|
||||
Inheritance is resolved after module and validator selection, including
|
||||
`--only` lane filtering, but before effective-pipeline validation, digest
|
||||
construction, explicit-profile inspection, checkpoint construction,
|
||||
preparation, execution, or provenance capture. Only profiles used by selected
|
||||
LLM-backed bindings are inspected. An unused pipeline default in a pipeline
|
||||
with no selected LLM-backed work does not require an otherwise unused profile
|
||||
to exist.
|
||||
|
||||
The resolved pipeline contains effective binding profiles rather than a second
|
||||
runtime inheritance mechanism. Two pipelines that differ only by spelling the
|
||||
same effective policy once as a pipeline default and once on every LLM-backed
|
||||
binding have the same semantic resolved digest. Changing an effective profile
|
||||
changes the digest and applicable checkpoint identity.
|
||||
|
||||
### Provenance And Checkpoints
|
||||
|
||||
The PromptKit profile-source checkpoint fingerprint covers:
|
||||
|
||||
- the PromptKit v0.5.0 built-in profile catalog identity;
|
||||
- exact application fallback profile asset content; and
|
||||
- exact configured operator profile YAML content, when present.
|
||||
|
||||
The existing local-backend target fingerprint remains separate and continues
|
||||
to exclude scheduling-only concurrency limits. Fingerprints contain hashes and
|
||||
stable markers, not profile contents, filesystem paths, endpoints, credentials,
|
||||
or other secrets.
|
||||
|
||||
Changing the PromptKit version, application fallback, operator profile, or
|
||||
effective pipeline profile makes incompatible LLM checkpoints ineligible for
|
||||
reuse. The dependency upgrade is expected to invalidate checkpoints produced
|
||||
under v0.3.0.
|
||||
|
||||
Successful run manifests continue to record only profiles actually selected by
|
||||
PromptKit, including their effective model, backend, and reasoning metadata.
|
||||
Debug output reports the same effective execution snapshot used for generation.
|
||||
|
||||
### Operator Documentation And Examples
|
||||
|
||||
Canonical documentation clearly distinguishes:
|
||||
|
||||
- Notarius prompt and schema assets embedded in the application;
|
||||
- Notarius application fallback profiles embedded in the application;
|
||||
- PromptKit's own embedded built-in profiles; and
|
||||
- operator profile files on the deployment filesystem.
|
||||
|
||||
The configuration reference owns the pipeline field, profile-source fields,
|
||||
validation rules, and precedence. Operations owns deployment layout, working
|
||||
directory behavior, credentials, and environment-specific profile management.
|
||||
The PromptKit integration document owns the pinned upstream contract and
|
||||
source-precedence boundary. Internal documents describe asset registration,
|
||||
resolution, inspection, prepared execution, fingerprinting, and tests without
|
||||
duplicating user-facing field definitions.
|
||||
|
||||
The maintained examples continue to include only the minimal and complete D&D
|
||||
configurations. They use the stable `dnd-extraction` policy, and one maintained
|
||||
PromptKit profile file under `examples/` demonstrates an operator override.
|
||||
Examples remain secret-free and are validated without live provider calls.
|
||||
|
||||
## Out Of Scope
|
||||
|
||||
- Implementing the separate deterministic prompt-session identity roadmap
|
||||
item.
|
||||
- Changing the default `concurrency.total_llm` value; PromptKit's retained
|
||||
OpenRouter capacity of 16 remains relevant to that separate item.
|
||||
- Adding model evaluation as a deterministic or CI correctness gate.
|
||||
- Automatically selecting production, development, or local environments.
|
||||
Deployment configuration chooses the operator profile source.
|
||||
- Profile inheritance, partial profile merging, or cross-profile aliases.
|
||||
- Exposing PromptKit types to modules, validators, durable output contracts, or
|
||||
public configuration structures.
|
||||
- Adopting `InspectPrompt` without a separately justified prompt-interface
|
||||
validation contract.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- Notarius builds and its offline test suite passes with PromptKit v0.5.0.
|
||||
- Every structured completion executes the exact snapshot used for safe debug
|
||||
prompt details.
|
||||
- Profile preflight uses profile inspection and no synthetic prompt.
|
||||
- The embedded `dnd-extraction` fallback resolves without an operator source,
|
||||
and a matching valid operator profile replaces it completely.
|
||||
- Every production module has explicit, correct execution metadata.
|
||||
- Pipeline, binding, CLI, and prompt-default precedence behaves as defined for
|
||||
modules and validators, while deterministic bindings remain profile-free.
|
||||
- Effective profiles participate in pipeline digests, profile inspection,
|
||||
checkpoint identity, debug records, and run provenance at the appropriate
|
||||
boundaries.
|
||||
- The dependency and application fallback changes invalidate incompatible old
|
||||
checkpoints without exposing profile or credential content.
|
||||
- Canonical documentation and maintained examples accurately describe and
|
||||
exercise the implemented operator workflow.
|
||||
- Default tests remain deterministic, offline, credential-free, and focused on
|
||||
Notarius-owned behavior rather than duplicating PromptKit's upstream suite.
|
||||
85
examples/dnd-complete-transcript.json
Normal file
85
examples/dnd-complete-transcript.json
Normal file
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"metadata": {
|
||||
"id": "session-ravenfall",
|
||||
"title": "The Ravenfall Watchtower"
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"start": 0,
|
||||
"end": 14,
|
||||
"speaker": "DM",
|
||||
"text": "Recap: last session, the party learned that Elder Rowan vanished near the Ravenfall watchtower."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"start": 14,
|
||||
"end": 25,
|
||||
"speaker": "Player",
|
||||
"text": "Out of character, we agree to investigate the watchtower before the next game."
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"start": 25,
|
||||
"end": 39,
|
||||
"speaker": "DM",
|
||||
"text": "Aria and Borin arrive at the ruined Ravenfall watchtower as dusk settles over the road."
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"start": 39,
|
||||
"end": 55,
|
||||
"speaker": "Mira Thorn",
|
||||
"text": "Mira Thorn steps from the doorway and says, \"Elder Rowan warned me that Kesh would return for the relic.\""
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"start": 55,
|
||||
"end": 70,
|
||||
"speaker": "DM",
|
||||
"text": "Mira leads the party to a hidden cache. The party discovers a moonblade and acquires 20 silver pieces."
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"start": 70,
|
||||
"end": 83,
|
||||
"speaker": "Aria",
|
||||
"text": "Aria hands her healing potion to Borin so he can carry it into the tower."
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"start": 83,
|
||||
"end": 96,
|
||||
"speaker": "DM",
|
||||
"text": "Kesh, the goblin captain, orders the raiders to attack. Roll initiative."
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"start": 96,
|
||||
"end": 110,
|
||||
"speaker": "DM",
|
||||
"text": "On Kesh's turn, he strikes Borin with his scimitar. Borin drinks the healing potion on his turn."
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"start": 110,
|
||||
"end": 124,
|
||||
"speaker": "Aria",
|
||||
"text": "Aria casts Cure Wounds on Borin, then invokes Aegis of Emberfall as Kesh closes in."
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"start": 124,
|
||||
"end": 137,
|
||||
"speaker": "DM",
|
||||
"text": "Kesh casts Shield as a reaction against Borin's counterattack, but the party drives the raiders away."
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"start": 137,
|
||||
"end": 150,
|
||||
"speaker": "DM",
|
||||
"text": "After the battle, Aria pays 5 silver pieces to repair the watchtower gate."
|
||||
}
|
||||
]
|
||||
}
|
||||
103
examples/dnd-complete.config.yml
Normal file
103
examples/dnd-complete.config.yml
Normal file
@@ -0,0 +1,103 @@
|
||||
version: 4
|
||||
promptkit:
|
||||
profile_file: ./examples/profiles/dnd-extraction.yml
|
||||
concurrency:
|
||||
total_llm: 2
|
||||
stage_workers:
|
||||
extract: 2
|
||||
output:
|
||||
directory: ./notarius-output
|
||||
cache:
|
||||
chunk_plans:
|
||||
mode: auto
|
||||
directory: ./notarius-cache/chunk-plans
|
||||
checkpoints:
|
||||
enabled: true
|
||||
directory: ./notarius-cache/checkpoints
|
||||
debug:
|
||||
directory: ./notarius-debug
|
||||
pipelines:
|
||||
dnd-session:
|
||||
llm_profile: dnd-extraction
|
||||
input: seriatim
|
||||
# Stable campaign context is shared by every module that accepts these slots.
|
||||
references:
|
||||
party: ./dnd-party.txt
|
||||
glossary: ./dnd-glossary.txt
|
||||
chunk:
|
||||
module: dnd/scenes
|
||||
retries: 2
|
||||
output:
|
||||
module: json
|
||||
options:
|
||||
include_chunk_map: true
|
||||
evidence_context:
|
||||
enabled: true
|
||||
window_units: 3
|
||||
lanes:
|
||||
- item-events
|
||||
- npcs
|
||||
- spells
|
||||
- combat-turns
|
||||
- npc-interactions
|
||||
steps:
|
||||
# Establish session-wide reference artifacts alongside independent item events.
|
||||
- id: describe-session
|
||||
artifacts:
|
||||
item-events:
|
||||
extract:
|
||||
module: dnd/item-events
|
||||
retries: 2
|
||||
merge: appendorder
|
||||
normalize: dnd/item-events
|
||||
npcs:
|
||||
extract:
|
||||
module: dnd/npcs
|
||||
retries: 2
|
||||
merge: appendorder
|
||||
normalize:
|
||||
module: dnd/npcs
|
||||
retries: 2
|
||||
scene-descriptions:
|
||||
extract:
|
||||
module: dnd/scene-descriptions
|
||||
retries: 2
|
||||
merge: appendorder
|
||||
normalize: dnd/scene-descriptions
|
||||
- id: extract-events
|
||||
# Accepted NPC grounding and scene-description eligibility artifacts are
|
||||
# supplied in memory to their compatible consumers in this step.
|
||||
references:
|
||||
npcs:
|
||||
artifact:
|
||||
step: describe-session
|
||||
lane: npcs
|
||||
scene_descriptions:
|
||||
artifact:
|
||||
step: describe-session
|
||||
lane: scene-descriptions
|
||||
artifacts:
|
||||
spells:
|
||||
extract:
|
||||
module: dnd/spells
|
||||
retries: 2
|
||||
references:
|
||||
spell_catalog: ./dnd-spell-catalog.json
|
||||
merge: appendorder
|
||||
# Stage-local file references are intentionally bound at each stage.
|
||||
normalize:
|
||||
module: dnd/spells
|
||||
references:
|
||||
spell_catalog: ./dnd-spell-catalog.json
|
||||
combat-turns:
|
||||
extract:
|
||||
module: dnd/combat-turns
|
||||
retries: 2
|
||||
merge: appendorder
|
||||
normalize: dnd/combat-turns
|
||||
npc-interactions:
|
||||
extract:
|
||||
module: dnd/npc-interactions
|
||||
retries: 2
|
||||
merge: appendorder
|
||||
normalize: dnd/npc-interactions
|
||||
8
examples/dnd-glossary.txt
Normal file
8
examples/dnd-glossary.txt
Normal file
@@ -0,0 +1,8 @@
|
||||
Ravenfall watchtower: a ruined watchtower near the party's current route.
|
||||
Mira Thorn: the watchtower's keeper.
|
||||
Elder Rowan: a missing local scholar.
|
||||
Kesh: a goblin captain leading raiders.
|
||||
Moonblade: a blade found in the watchtower's hidden cache.
|
||||
Cure Wounds: a healing spell.
|
||||
Shield: a defensive reaction spell.
|
||||
Aegis of Emberfall: a campaign spell recorded in the supplied catalog overlay.
|
||||
@@ -1,10 +1,9 @@
|
||||
version: 2
|
||||
workspace:
|
||||
chunk_cache:
|
||||
mode: bypass
|
||||
version: 4
|
||||
pipelines:
|
||||
dnd-session:
|
||||
llm_profile: dnd-extraction
|
||||
input: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
normalize: dnd/spells
|
||||
@@ -1,3 +1,2 @@
|
||||
Aria: party cleric and recurring healer.
|
||||
Borin: fighter ally.
|
||||
Bandit mage: hostile spellcaster.
|
||||
21
examples/dnd-spell-catalog.json
Normal file
21
examples/dnd-spell-catalog.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"schema_version": "notarius.dnd.spell-catalog-overlay.v1",
|
||||
"catalogs": [
|
||||
{
|
||||
"id": "notarius.example-campaign",
|
||||
"ruleset": "dnd-5e-2014",
|
||||
"source": {
|
||||
"title": "Notarius example campaign spell names",
|
||||
"version": "1",
|
||||
"url": "",
|
||||
"license": ""
|
||||
},
|
||||
"spells": [
|
||||
{
|
||||
"name": "Aegis of Emberfall",
|
||||
"aliases": ["Emberfall Aegis"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
Cure Wounds: healing spell cast by touch.
|
||||
Shield: defensive reaction spell.
|
||||
@@ -1,29 +0,0 @@
|
||||
version: 2
|
||||
concurrency:
|
||||
total_llm: 1
|
||||
stage_workers:
|
||||
extract: 1
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: auto
|
||||
resume:
|
||||
enabled: false
|
||||
debug:
|
||||
enabled: false
|
||||
chunk_cache:
|
||||
directory: /var/cache/notarius/chunk-plans
|
||||
pipelines:
|
||||
dnd-session:
|
||||
input: seriatim
|
||||
references:
|
||||
party: ./dnd-spells-roster.txt
|
||||
glossary: ./dnd-spells-glossary.txt
|
||||
chunk:
|
||||
module: generic
|
||||
options:
|
||||
max_units: 50
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
5
examples/profiles/dnd-extraction.yml
Normal file
5
examples/profiles/dnd-extraction.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
id: dnd-extraction
|
||||
backend: openrouter
|
||||
model: openai/gpt-5.6-luna
|
||||
timeout_seconds: 240
|
||||
service_tier: flex
|
||||
4
go.mod
4
go.mod
@@ -3,9 +3,9 @@ module gitea.maximumdirect.net/eric/notarius
|
||||
go 1.25.5
|
||||
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/scriptorium v0.11.0
|
||||
gitea.maximumdirect.net/eric/promptkit v0.5.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require golang.org/x/text v0.14.0 // indirect
|
||||
require golang.org/x/text v0.40.0
|
||||
|
||||
8
go.sum
8
go.sum
@@ -1,11 +1,11 @@
|
||||
gitea.maximumdirect.net/eric/scriptorium v0.11.0 h1:rjvbt9FTaWHxYlHq7QlUzmMVUt3QdbTmeCkmH81N//o=
|
||||
gitea.maximumdirect.net/eric/scriptorium v0.11.0/go.mod h1:FQ5lEuNxmrQyNgIomkpZdxvfTC0jWjbXYuq3tbJWF64=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.5.0 h1:jnpazLyyNhWrB2xzwwtUkNUfktkTdkENTwuSPnKiYrc=
|
||||
gitea.maximumdirect.net/eric/promptkit v0.5.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
315
internal/cli/assembled_spell_pipeline_contract_test.go
Normal file
315
internal/cli/assembled_spell_pipeline_contract_test.go
Normal file
@@ -0,0 +1,315 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||
)
|
||||
|
||||
const assembledSpellExtractorKey = "test/dnd/spell-casts"
|
||||
|
||||
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
||||
registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{})
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
chunkIndexes := extractor.chunkIndexesSnapshot()
|
||||
sort.Ints(chunkIndexes)
|
||||
if !reflect.DeepEqual(chunkIndexes, []int{0, 1}) {
|
||||
t.Fatalf("extractor chunk indexes = %#v, want two chunk-boundary calls", chunkIndexes)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("run output = %#v, want approved normalized output without rejections", output)
|
||||
}
|
||||
if output.NormalizeOutputs[0].NormalizerKey != spellnormalize.Key {
|
||||
t.Fatalf("normalized output module = %q, want %q", output.NormalizeOutputs[0].NormalizerKey, spellnormalize.Key)
|
||||
}
|
||||
|
||||
var normalized dnd.SpellList
|
||||
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil {
|
||||
t.Fatalf("decode normalized output: %v", err)
|
||||
}
|
||||
if len(normalized.SpellCasts) != 2 {
|
||||
t.Fatalf("normalized casts = %#v, want collapsed duplicate plus distinct evidence", normalized.SpellCasts)
|
||||
}
|
||||
first, distinct := normalized.SpellCasts[0], normalized.SpellCasts[1]
|
||||
if first.Spell != "Cure Wounds" || first.Caster != " Aria \t" {
|
||||
t.Fatalf("retained cast = %#v, want canonical spell with first occurrence caster", first)
|
||||
}
|
||||
if !reflect.DeepEqual(first.SourceRefs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}) {
|
||||
t.Fatalf("retained refs = %#v, want sorted complete evidence", first.SourceRefs)
|
||||
}
|
||||
if distinct.Spell != "Cure Wounds" || distinct.Caster != "aria" || !reflect.DeepEqual(distinct.SourceRefs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}) {
|
||||
t.Fatalf("distinct cast = %#v, want separate evidence event", distinct)
|
||||
}
|
||||
|
||||
wantWarningReasons := []string{
|
||||
spellnormalize.ReasonCodeSpellNameCanonicalized,
|
||||
spellnormalize.ReasonCodeSourceReferencesNormalized,
|
||||
spellnormalize.ReasonCodeDuplicateSpellCastCollapsed,
|
||||
"spell_not_near_source",
|
||||
}
|
||||
gotWarningReasons := make([]string, len(output.Warnings))
|
||||
for index, warning := range output.Warnings {
|
||||
gotWarningReasons[index] = warning.ReasonCode
|
||||
}
|
||||
if !reflect.DeepEqual(gotWarningReasons, wantWarningReasons) {
|
||||
t.Fatalf("warnings = %#v, want deterministic normalize and validation warnings", output.Warnings)
|
||||
}
|
||||
if output.Warnings[2].Scope != "spell_casts[0]" || !strings.Contains(output.Warnings[2].Message, "retained input index 0") || !strings.Contains(output.Warnings[2].Message, "removed input indices [1]") {
|
||||
t.Fatalf("duplicate warning = %#v, want retained and removed merged indices", output.Warnings[2])
|
||||
}
|
||||
|
||||
warningsFile := decodeAssembledOutput[struct {
|
||||
Warnings []contracts.Warning `json:"warnings"`
|
||||
}](t, output.OutputFiles, "warnings.json")
|
||||
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) {
|
||||
t.Fatalf("warnings file = %#v, run warnings = %#v, want manifest output path to preserve warnings", warningsFile.Warnings, output.Warnings)
|
||||
}
|
||||
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
|
||||
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key {
|
||||
t.Fatalf("manifest lanes = %#v, want assembled spell normalizer", manifest.ArtifactLanes)
|
||||
}
|
||||
normalizerMetadata, ok := manifest.ArtifactLanes[0].Metadata["normalizer"].(map[string]any)
|
||||
_, hasOverlayIDs := normalizerMetadata["catalog_overlay_ids"]
|
||||
if !ok || normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !hasOverlayIDs {
|
||||
t.Fatalf("normalizer manifest metadata = %#v, want base ID, digest, and overlay IDs", manifest.ArtifactLanes[0].Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true})
|
||||
var normalizeChain *pipeline.ResolvedValidatorChain
|
||||
for index := range resolved.ValidatorChains {
|
||||
chain := &resolved.ValidatorChains[index]
|
||||
if chain.Stage == pipeline.StageNormalize && chain.ModuleKey == spellnormalize.Key && chain.LaneID == "spells" {
|
||||
normalizeChain = chain
|
||||
break
|
||||
}
|
||||
}
|
||||
if normalizeChain == nil || len(normalizeChain.Validators) != 1 || normalizeChain.Validators[0].Binding.Module != "generic/always_accept" {
|
||||
t.Fatalf("normalize validator chain = %#v, want explicit always-accept override", normalizeChain)
|
||||
}
|
||||
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
})
|
||||
if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output)
|
||||
}
|
||||
for _, warning := range output.Warnings {
|
||||
if warning.ReasonCode == "spell_not_near_source" {
|
||||
t.Fatalf("warnings = %#v, want explicit validator override to replace default relatedness chain", output.Warnings)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledSpellPipelineRejectsUnknownSpellWithoutPromotingAttemptWarning(t *testing.T) {
|
||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "rejected" || len(output.NormalizeOutputs) != 0 || len(output.Rejected) != 1 {
|
||||
t.Fatalf("run output = %#v, want one rejected normalize candidate and no normalized output", output)
|
||||
}
|
||||
rejection := output.Rejected[0]
|
||||
if rejection.Stage != string(pipeline.StageNormalize) || rejection.LaneID != "spells" || rejection.ModuleKey != spellnormalize.Key || rejection.ValidatorName != "extract/dnd/spells/catalog" || rejection.ReasonCode != "unknown_spell" {
|
||||
t.Fatalf("rejection = %#v, want durable normalize catalog rejection", rejection)
|
||||
}
|
||||
rejectedFile := decodeAssembledOutput[struct {
|
||||
Rejected []contracts.RejectedOutput `json:"rejected"`
|
||||
}](t, output.OutputFiles, "rejected.json")
|
||||
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
|
||||
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
|
||||
}
|
||||
for _, warning := range output.Warnings {
|
||||
if warning.ReasonCode == spellnormalize.ReasonCodeSpellNameUnresolved {
|
||||
t.Fatalf("warnings = %#v, want rejected-attempt warning to remain non-durable", output.Warnings)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledSpellPipelinePromotesUnknownSpellWarningWhenOverrideAccepts(t *testing.T) {
|
||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true, unknownSpell: true})
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("run output = %#v, want accepted unknown spell with explicit validator override", output)
|
||||
}
|
||||
var normalized dnd.SpellList
|
||||
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil {
|
||||
t.Fatalf("decode normalized output: %v", err)
|
||||
}
|
||||
if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" {
|
||||
t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts)
|
||||
}
|
||||
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Warnings[0].Scope != "spell_casts[0]" {
|
||||
t.Fatalf("warnings = %#v, want promoted scoped unresolved-name warning", output.Warnings)
|
||||
}
|
||||
warningsFile := decodeAssembledOutput[struct {
|
||||
Warnings []contracts.Warning `json:"warnings"`
|
||||
}](t, output.OutputFiles, "warnings.json")
|
||||
if !reflect.DeepEqual(warningsFile.Warnings, output.Warnings) {
|
||||
t.Fatalf("warnings file = %#v, run warnings = %#v, want durable unresolved-name warning", warningsFile.Warnings, output.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
type assembledSpellPipelineOptions struct {
|
||||
normalizeValidatorOverride bool
|
||||
unknownSpell bool
|
||||
}
|
||||
|
||||
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
|
||||
t.Helper()
|
||||
components := productionTestComponents(t)
|
||||
extractor := &assembledSpellExtractor{unknownSpell: options.unknownSpell}
|
||||
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
|
||||
Key: assembledSpellExtractorKey,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||
Requires: []string{"chunks", "source.transcript"},
|
||||
Provides: []string{"dnd.spell_casts"},
|
||||
ArtifactKind: dnd.SpellListKind,
|
||||
}, func() (contracts.Extractor[dnd.SpellList], error) {
|
||||
return extractor, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register assembled extractor: %v", err)
|
||||
}
|
||||
|
||||
normalize := pipeline.Binding(spellnormalize.Key)
|
||||
if options.normalizeValidatorOverride {
|
||||
normalize.Validators = pipeline.ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []pipeline.ModuleBinding{pipeline.Binding("generic/always_accept")},
|
||||
}
|
||||
}
|
||||
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
|
||||
ID: "assembled-dnd-spells",
|
||||
Input: pipeline.Binding("seriatim"),
|
||||
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"spells": {Extract: pipeline.Binding(assembledSpellExtractorKey), Normalize: normalize},
|
||||
},
|
||||
Output: pipeline.Binding("json"),
|
||||
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
return components.registries, resolved, extractor
|
||||
}
|
||||
|
||||
type assembledSpellExtractor struct {
|
||||
mu sync.Mutex
|
||||
chunkIndexes []int
|
||||
unknownSpell bool
|
||||
}
|
||||
|
||||
func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey }
|
||||
|
||||
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (e *assembledSpellExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||
}
|
||||
if req.Chunk == nil || req.Source == nil {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("assembled extractor requires source and chunk")
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.chunkIndexes = append(e.chunkIndexes, req.Chunk.Index)
|
||||
e.mu.Unlock()
|
||||
refOne := source.SourceRef{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}
|
||||
refTwo := source.SourceRef{SourceID: req.Source.ID, StartUnitID: 2, EndUnitID: 2}
|
||||
if e.unknownSpell {
|
||||
if req.Chunk.Index == 0 {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{{
|
||||
Caster: "Aria", Spell: "Mysterious Burst", SourceRefs: []source.SourceRef{refOne},
|
||||
}}}}, nil
|
||||
}
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{}}}, nil
|
||||
}
|
||||
switch req.Chunk.Index {
|
||||
case 0:
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{{
|
||||
Caster: " Aria \t", Spell: " cure wounds ", SourceRefs: []source.SourceRef{refTwo, refOne},
|
||||
}}}}, nil
|
||||
case 1:
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{
|
||||
{Caster: "aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{refOne, refTwo}},
|
||||
{Caster: "aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{refTwo}},
|
||||
}}}, nil
|
||||
default:
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("unexpected assembled chunk index %d", req.Chunk.Index)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return append([]int(nil), e.chunkIndexes...)
|
||||
}
|
||||
|
||||
func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T {
|
||||
t.Helper()
|
||||
for _, file := range files {
|
||||
if file.Name != name {
|
||||
continue
|
||||
}
|
||||
var value T
|
||||
if err := json.Unmarshal(file.Bytes, &value); err != nil {
|
||||
t.Fatalf("decode %s: %v", name, err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
t.Fatalf("output files = %#v, want %q", files, name)
|
||||
return *new(T)
|
||||
}
|
||||
371
internal/cli/cache_contract_test.go
Normal file
371
internal/cli/cache_contract_test.go
Normal file
@@ -0,0 +1,371 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRunChunkPlanModePrecedenceAndValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fileMode string
|
||||
envMode string
|
||||
cliMode string
|
||||
wantStores int
|
||||
}{
|
||||
{name: "default", wantStores: 1},
|
||||
{name: "file", fileMode: "bypass"},
|
||||
{name: "environment", envMode: "bypass"},
|
||||
{name: "cli", envMode: "refresh", cliMode: "bypass"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
if tt.name == "default" {
|
||||
removeStateTestConfigLine(t, roots.config, " mode: auto\n")
|
||||
} else if tt.fileMode != "" {
|
||||
replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: "+tt.fileMode+"\n")
|
||||
}
|
||||
|
||||
var stores []string
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LookupEnv = func(name string) (string, bool) {
|
||||
if name == "NOTARIUS_CACHE_CHUNK_PLANS_MODE" && tt.envMode != "" {
|
||||
return tt.envMode, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
||||
stores = append(stores, root)
|
||||
return chunkplan.NewFilesystemStore(root)
|
||||
}
|
||||
|
||||
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input}
|
||||
if tt.cliMode != "" {
|
||||
args = append(args, "--chunk_cache", tt.cliMode)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
if len(stores) != tt.wantStores {
|
||||
t.Fatalf("chunk plan store roots = %v, want %d stores", stores, tt.wantStores)
|
||||
}
|
||||
if tt.wantStores == 1 && stores[0] != roots.plans {
|
||||
t.Fatalf("chunk plan store root = %q, want %q", stores[0], roots.plans)
|
||||
}
|
||||
if tt.wantStores == 0 {
|
||||
assertAbsent(t, roots.plans)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("invalid cli syntax is a usage error", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "invalid"}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
fileConfig bool
|
||||
}{
|
||||
{name: "invalid environment mode"},
|
||||
{name: "invalid file mode", fileConfig: true},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
if tt.fileConfig {
|
||||
replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: invalid\n")
|
||||
} else {
|
||||
opts.LookupEnv = func(name string) (string, bool) {
|
||||
if name == "NOTARIUS_CACHE_CHUNK_PLANS_MODE" {
|
||||
return "invalid", true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunChunkPlanRootSelectionAndFailures(t *testing.T) {
|
||||
t.Run("empty configured root uses the per-user cache root", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
||||
userCache := filepath.Join(t.TempDir(), "user-cache")
|
||||
var stores []string
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return userCache, nil }
|
||||
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
||||
stores = append(stores, root)
|
||||
return chunkplan.NewFilesystemStore(root)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
wantRoot := filepath.Join(userCache, "notarius", "chunk-plans")
|
||||
if len(stores) != 1 || stores[0] != wantRoot {
|
||||
t.Fatalf("chunk plan store roots = %v, want [%q]", stores, wantRoot)
|
||||
}
|
||||
assertFile(t, filepath.Join(wantRoot, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
|
||||
assertAbsent(t, roots.plans)
|
||||
})
|
||||
|
||||
t.Run("bypass avoids default cache dependencies", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
||||
userCacheCalls := 0
|
||||
storeCalls := 0
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) {
|
||||
userCacheCalls++
|
||||
return "", errors.New("user cache must not be resolved")
|
||||
}
|
||||
opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) {
|
||||
storeCalls++
|
||||
return nil, errors.New("chunk plan store must not be constructed")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if userCacheCalls != 0 || storeCalls != 0 {
|
||||
t.Fatalf("user cache calls=%d store calls=%d, want none", userCacheCalls, storeCalls)
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
assertAbsent(t, roots.plans)
|
||||
})
|
||||
|
||||
t.Run("user cache resolution failure has context and no output", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("cache home unavailable") }
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "resolve chunk plan root") || !strings.Contains(stderr.String(), "cache home unavailable") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
|
||||
t.Run("store construction failure has context and no output", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
||||
return nil, fmt.Errorf("store unavailable")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
||||
want := fmt.Sprintf("create chunk plan store at %q", roots.plans)
|
||||
if code != 1 || !strings.Contains(stderr.String(), want) || !strings.Contains(stderr.String(), "store unavailable") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
|
||||
t.Run("checkpoint root resolution failure has context and no output", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache unavailable") }
|
||||
result := runStateTest(t, roots, opts, false, true, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "resolve checkpoint root") || !strings.Contains(result.stderr, "checkpoint cache unavailable") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunAutoReusesPlanWhenRunInputsChange(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configText := replaceRequiredOnce(t, string(data), " chunk: test/chunk\n", ` chunk:
|
||||
module: test/chunk
|
||||
options:
|
||||
strategy: first
|
||||
`)
|
||||
configText = replaceRequiredOnce(t, configText, " output: test/output\n", ` other:
|
||||
extract: test/extract
|
||||
merge: test/merge
|
||||
normalize: test/normalize
|
||||
output: test/output
|
||||
`)
|
||||
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
referencePath := filepath.Join(filepath.Dir(roots.input), "reference.txt")
|
||||
if err := os.WriteFile(referencePath, []byte("reference content"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
harness := newStateTestHarness()
|
||||
var firstStdout, firstStderr bytes.Buffer
|
||||
first := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &firstStdout, &firstStderr, harness.options())
|
||||
if first != 0 {
|
||||
t.Fatalf("first run code=%d stdout=%q stderr=%q", first, firstStdout.String(), firstStderr.String())
|
||||
}
|
||||
configData, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configText = replaceRequiredOnce(t, string(configData), "strategy: first", "strategy: second")
|
||||
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
second := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--only", "items", "--reference", "chunk.cache-reference=" + referencePath,
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if second != 0 {
|
||||
t.Fatalf("second run code=%d stdout=%q stderr=%q", second, stdout.String(), stderr.String())
|
||||
}
|
||||
harness.mu.Lock()
|
||||
chunkCalls := harness.chunkCalls
|
||||
harness.mu.Unlock()
|
||||
if chunkCalls != 1 {
|
||||
t.Fatalf("chunk calls across changed run inputs = %d, want 1", chunkCalls)
|
||||
}
|
||||
assertFile(t, filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
|
||||
assertAnyFile(t, roots.output)
|
||||
}
|
||||
|
||||
func TestRunResumeSelectsConfiguredOrPerUserCheckpointRoot(t *testing.T) {
|
||||
for _, configured := range []bool{true, false} {
|
||||
name := "per-user root"
|
||||
if configured {
|
||||
name = "configured root"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
if !configured {
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
||||
}
|
||||
userCache := filepath.Join(t.TempDir(), "user-cache")
|
||||
userCacheCalls := 0
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) {
|
||||
userCacheCalls++
|
||||
return userCache, nil
|
||||
}
|
||||
result := runStateTest(t, roots, opts, false, true, "bypass")
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
wantRoot := roots.checkpoints
|
||||
wantCalls := 0
|
||||
if !configured {
|
||||
wantRoot = filepath.Join(userCache, "notarius", "checkpoints")
|
||||
wantCalls = 1
|
||||
}
|
||||
if userCacheCalls != wantCalls {
|
||||
t.Fatalf("user cache calls = %d, want %d", userCacheCalls, wantCalls)
|
||||
}
|
||||
assertAnyFile(t, wantRoot)
|
||||
if !configured {
|
||||
assertAbsent(t, roots.checkpoints)
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("disabled avoids checkpoint root resolution", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
|
||||
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache must not be resolved") }
|
||||
result := runStateTest(t, roots, opts, false, false, "bypass")
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
assertAbsent(t, roots.checkpoints)
|
||||
})
|
||||
|
||||
t.Run("resume requires enabled checkpoint recording", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
|
||||
result := runStateTest(t, roots, newStateTestHarness().options(), true, true, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "--resume requires cache.checkpoints.enabled: true") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
|
||||
func TestConfigCommandsDoNotResolveRunState(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"config", "validate", "--config"},
|
||||
{"pipelines", "list", "--config"},
|
||||
} {
|
||||
name := strings.Join(args[:2], "-")
|
||||
t.Run(name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("state root must not be resolved") }
|
||||
opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) {
|
||||
return nil, errors.New("chunk plan store must not be constructed")
|
||||
}
|
||||
command := append([]string(nil), args...)
|
||||
command = append(command, roots.config)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(command, &stdout, &stderr, opts)
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func replaceStateTestConfigLine(t *testing.T, path, old, new string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := replaceRequiredOnce(t, string(data), old, new)
|
||||
if err := os.WriteFile(path, []byte(text), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func removeStateTestConfigLine(t *testing.T, path, line string) {
|
||||
replaceStateTestConfigLine(t, path, line, "")
|
||||
}
|
||||
|
||||
func assertNoRunState(t *testing.T, roots stateTestRoots) {
|
||||
t.Helper()
|
||||
assertAbsent(t, roots.output)
|
||||
assertAbsent(t, roots.plans)
|
||||
assertAbsent(t, roots.checkpoints)
|
||||
assertAbsent(t, roots.debug)
|
||||
}
|
||||
@@ -21,15 +21,16 @@ type productionComponents struct {
|
||||
|
||||
func newProductionComponents() (productionComponents, error) {
|
||||
registries := pipeline.Registries{
|
||||
Inputs: pipeline.NewInputAdapterRegistry(),
|
||||
Chunkers: pipeline.NewChunkerRegistry(),
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
|
||||
Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(),
|
||||
Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(),
|
||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
Inputs: pipeline.NewInputAdapterRegistry(),
|
||||
Chunkers: pipeline.NewChunkerRegistry(),
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
|
||||
ArtifactEvidence: pipeline.NewArtifactEvidenceRegistry(),
|
||||
Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(),
|
||||
Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(),
|
||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
}
|
||||
assets := llm.NewAssetRegistry()
|
||||
registrars := []struct {
|
||||
@@ -88,29 +89,31 @@ func effectiveRegistries(opts Options) (pipeline.Registries, error) {
|
||||
|
||||
func catalogFromRegistries(registries pipeline.Registries) pipeline.ModuleCatalog {
|
||||
return pipeline.ModuleCatalog{
|
||||
Inputs: registries.Inputs,
|
||||
Chunkers: registries.Chunkers,
|
||||
ArtifactCodecs: registries.ArtifactCodecs,
|
||||
Extractors: registries.Extractors,
|
||||
Mergers: registries.Mergers,
|
||||
Normalizers: registries.Normalizers,
|
||||
Validators: registries.Validators,
|
||||
ValidatorChains: registries.ValidatorChains,
|
||||
Outputs: registries.Outputs,
|
||||
Inputs: registries.Inputs,
|
||||
Chunkers: registries.Chunkers,
|
||||
ArtifactCodecs: registries.ArtifactCodecs,
|
||||
ArtifactEvidence: registries.ArtifactEvidence,
|
||||
Extractors: registries.Extractors,
|
||||
Mergers: registries.Mergers,
|
||||
Normalizers: registries.Normalizers,
|
||||
Validators: registries.Validators,
|
||||
ValidatorChains: registries.ValidatorChains,
|
||||
Outputs: registries.Outputs,
|
||||
}
|
||||
}
|
||||
|
||||
func registriesFromCatalog(catalog pipeline.ModuleCatalog) pipeline.Registries {
|
||||
return pipeline.Registries{
|
||||
Inputs: catalog.Inputs,
|
||||
Chunkers: catalog.Chunkers,
|
||||
ArtifactCodecs: catalog.ArtifactCodecs,
|
||||
Extractors: catalog.Extractors,
|
||||
Mergers: catalog.Mergers,
|
||||
Normalizers: catalog.Normalizers,
|
||||
Validators: catalog.Validators,
|
||||
ValidatorChains: catalog.ValidatorChains,
|
||||
Outputs: catalog.Outputs,
|
||||
Inputs: catalog.Inputs,
|
||||
Chunkers: catalog.Chunkers,
|
||||
ArtifactCodecs: catalog.ArtifactCodecs,
|
||||
ArtifactEvidence: catalog.ArtifactEvidence,
|
||||
Extractors: catalog.Extractors,
|
||||
Mergers: catalog.Mergers,
|
||||
Normalizers: catalog.Normalizers,
|
||||
Validators: catalog.Validators,
|
||||
ValidatorChains: catalog.ValidatorChains,
|
||||
Outputs: catalog.Outputs,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,6 +121,7 @@ func isEmptyCatalog(catalog pipeline.ModuleCatalog) bool {
|
||||
return catalog.Inputs == nil &&
|
||||
catalog.Chunkers == nil &&
|
||||
catalog.ArtifactCodecs == nil &&
|
||||
catalog.ArtifactEvidence == nil &&
|
||||
catalog.Extractors == nil &&
|
||||
catalog.Mergers == nil &&
|
||||
catalog.Normalizers == nil &&
|
||||
@@ -130,6 +134,7 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
|
||||
return registries.Inputs == nil &&
|
||||
registries.Chunkers == nil &&
|
||||
registries.ArtifactCodecs == nil &&
|
||||
registries.ArtifactEvidence == nil &&
|
||||
registries.Extractors == nil &&
|
||||
registries.Mergers == nil &&
|
||||
registries.Normalizers == nil &&
|
||||
@@ -138,24 +143,13 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
|
||||
registries.Outputs == nil
|
||||
}
|
||||
|
||||
func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
assets, err := productionPromptAssets()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return buildProductionLLMClient(ctx, cfg, profileID, assets)
|
||||
}
|
||||
|
||||
func productionLLMClientFactoryWithAssets(assets *llm.AssetRegistry) LLMClientFactory {
|
||||
return func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
return buildProductionLLMClient(ctx, cfg, profileID, assets)
|
||||
return func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
return buildProductionLLMClient(ctx, cfg, profileID, overrides, assets)
|
||||
}
|
||||
}
|
||||
|
||||
func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -163,14 +157,16 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID
|
||||
return nil, nil, fmt.Errorf("production asset registry must not be nil")
|
||||
}
|
||||
recorder := llm.NewLLMProfileRecorder()
|
||||
client, err := llm.NewScriptoriumClient(llm.ScriptoriumClientConfig{
|
||||
ProfileDir: cfg.Scriptorium.ProfileDir,
|
||||
ProfileFile: cfg.Scriptorium.ProfileFile,
|
||||
Assets: assets,
|
||||
Recorder: recorder,
|
||||
client, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{
|
||||
ProfileDir: cfg.PromptKit.ProfileDir,
|
||||
ProfileFile: cfg.PromptKit.ProfileFile,
|
||||
LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend),
|
||||
Assets: assets,
|
||||
Recorder: recorder,
|
||||
ReasoningEffort: overrides.ReasoningEffort,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create Scriptorium-backed LLM client: %w", err)
|
||||
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)
|
||||
}
|
||||
scheduler, err := llm.NewScheduler(cfg.Concurrency.TotalLLM)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,480 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
type recordingChunkPlanStore struct {
|
||||
record pipeline.ChunkPlanRecord
|
||||
decision pipeline.ChunkPlanDecision
|
||||
loadErr error
|
||||
saveErr error
|
||||
loads int
|
||||
saves int
|
||||
}
|
||||
|
||||
func (s *recordingChunkPlanStore) Load(string) (pipeline.ChunkPlanRecord, pipeline.ChunkPlanDecision, error) {
|
||||
s.loads++
|
||||
return s.record, s.decision, s.loadErr
|
||||
}
|
||||
|
||||
func (s *recordingChunkPlanStore) Save(record pipeline.ChunkPlanRecord) error {
|
||||
s.saves++
|
||||
s.record = record
|
||||
return s.saveErr
|
||||
}
|
||||
|
||||
type recordingChunkPlanFactory struct {
|
||||
roots []string
|
||||
store *recordingChunkPlanStore
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *recordingChunkPlanFactory) build(root string) (pipeline.ChunkPlanStore, error) {
|
||||
f.roots = append(f.roots, root)
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
if f.store == nil {
|
||||
f.store = &recordingChunkPlanStore{decision: pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanMissing}}
|
||||
}
|
||||
return f.store, nil
|
||||
}
|
||||
|
||||
func TestRunChunkCachePrecedence(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fileMode string
|
||||
envMode string
|
||||
flagMode string
|
||||
wantMode pipeline.ChunkCacheMode
|
||||
wantBuild bool
|
||||
}{
|
||||
{name: "file refresh", fileMode: "refresh", wantMode: pipeline.ChunkCacheRefresh, wantBuild: true},
|
||||
{name: "environment over file", fileMode: "bypass", envMode: "refresh", wantMode: pipeline.ChunkCacheRefresh, wantBuild: true},
|
||||
{name: "flag over environment", fileMode: "refresh", envMode: "auto", flagMode: "bypass", wantMode: pipeline.ChunkCacheBypass},
|
||||
{name: "explicit auto", fileMode: "bypass", envMode: "refresh", flagMode: "auto", wantMode: pipeline.ChunkCacheAuto, wantBuild: true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "plans")
|
||||
configPath := writeTestConfig(t, cacheTestConfig(tc.fileMode, root, ""))
|
||||
inputPath := writeFile(t, "input.txt", "input")
|
||||
values := map[string]string{}
|
||||
if tc.envMode != "" {
|
||||
values["NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE"] = tc.envMode
|
||||
}
|
||||
factory := &recordingChunkPlanFactory{}
|
||||
args := []string{"run", "example", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir()}
|
||||
if tc.flagMode != "" {
|
||||
args = append(args, "--chunk_cache", tc.flagMode)
|
||||
}
|
||||
code, stderr := runCacheCommand(t, args, cacheTestOptions(t, values, factory))
|
||||
if code != 0 {
|
||||
t.Fatalf("code = %d stderr = %q", code, stderr)
|
||||
}
|
||||
if got := len(factory.roots) > 0; got != tc.wantBuild {
|
||||
t.Fatalf("store built = %t roots = %#v, want %t", got, factory.roots, tc.wantBuild)
|
||||
}
|
||||
if !tc.wantBuild {
|
||||
return
|
||||
}
|
||||
if tc.wantMode == pipeline.ChunkCacheAuto && (factory.store.loads != 1 || factory.store.saves != 1) {
|
||||
t.Fatalf("auto calls = load %d save %d", factory.store.loads, factory.store.saves)
|
||||
}
|
||||
if tc.wantMode == pipeline.ChunkCacheRefresh && (factory.store.loads != 0 || factory.store.saves != 1) {
|
||||
t.Fatalf("refresh calls = load %d save %d", factory.store.loads, factory.store.saves)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunChunkCacheInvalidValuesHaveEstablishedExitCodes(t *testing.T) {
|
||||
validConfig := writeTestConfig(t, cacheTestConfig("bypass", "", ""))
|
||||
invalidFile := writeTestConfig(t, cacheTestConfig("sometimes", "", ""))
|
||||
inputPath := writeFile(t, "input.txt", "input")
|
||||
tests := []struct {
|
||||
name string
|
||||
config string
|
||||
env map[string]string
|
||||
flag string
|
||||
want int
|
||||
}{
|
||||
{name: "flag", config: validConfig, flag: "sometimes", want: 2},
|
||||
{name: "environment", config: validConfig, env: map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "sometimes"}, want: 1},
|
||||
{name: "file", config: invalidFile, want: 1},
|
||||
{name: "flag does not mask invalid file", config: invalidFile, flag: "bypass", want: 1},
|
||||
{name: "flag does not mask invalid environment", config: validConfig, env: map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "sometimes"}, flag: "bypass", want: 1},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
args := []string{"run", "example", "--config", tc.config, "--input", inputPath, "--output-dir", t.TempDir()}
|
||||
if tc.flag != "" {
|
||||
args = append(args, "--chunk_cache", tc.flag)
|
||||
}
|
||||
code, _ := runCacheCommand(t, args, cacheTestOptions(t, tc.env, &recordingChunkPlanFactory{}))
|
||||
if code != tc.want {
|
||||
t.Fatalf("code = %d, want %d", code, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunChunkPlanRootResolution(t *testing.T) {
|
||||
t.Run("explicit file root", func(t *testing.T) {
|
||||
factory := &recordingChunkPlanFactory{}
|
||||
resolverCalls := 0
|
||||
opts := cacheTestOptions(t, nil, factory)
|
||||
opts.UserCacheDir = func() (string, error) { resolverCalls++; return "", errors.New("must not be called") }
|
||||
configPath := writeTestConfig(t, cacheTestConfig("auto", "/var/cache/notarius/chunk-plans", ""))
|
||||
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts)
|
||||
if code != 0 || stderr != "" || resolverCalls != 0 || len(factory.roots) != 1 || factory.roots[0] != "/var/cache/notarius/chunk-plans" {
|
||||
t.Fatalf("code=%d stderr=%q resolver=%d roots=%#v", code, stderr, resolverCalls, factory.roots)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("environment root", func(t *testing.T) {
|
||||
factory := &recordingChunkPlanFactory{}
|
||||
environmentRoot := filepath.Join(t.TempDir(), "environment-plans")
|
||||
configPath := writeTestConfig(t, cacheTestConfig("auto", filepath.Join(t.TempDir(), "file-plans"), ""))
|
||||
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), cacheTestOptions(t, map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR": environmentRoot}, factory))
|
||||
if code != 0 || stderr != "" || len(factory.roots) != 1 || factory.roots[0] != environmentRoot {
|
||||
t.Fatalf("code=%d stderr=%q roots=%#v", code, stderr, factory.roots)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("per-user default", func(t *testing.T) {
|
||||
factory := &recordingChunkPlanFactory{}
|
||||
cacheDir := filepath.Join(t.TempDir(), "user-cache")
|
||||
opts := cacheTestOptions(t, nil, factory)
|
||||
opts.UserCacheDir = func() (string, error) { return cacheDir, nil }
|
||||
configPath := writeTestConfig(t, cacheTestConfig("auto", "", ""))
|
||||
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts)
|
||||
want := filepath.Join(cacheDir, "notarius", "chunk-plans")
|
||||
if code != 0 || stderr != "" || len(factory.roots) != 1 || factory.roots[0] != want {
|
||||
t.Fatalf("code=%d stderr=%q roots=%#v want=%q", code, stderr, factory.roots, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunBypassSkipsRootAndStore(t *testing.T) {
|
||||
factory := &recordingChunkPlanFactory{err: errors.New("must not build")}
|
||||
resolverCalls := 0
|
||||
opts := cacheTestOptions(t, nil, factory)
|
||||
opts.UserCacheDir = func() (string, error) { resolverCalls++; return "", errors.New("must not resolve") }
|
||||
configPath := writeTestConfig(t, cacheTestConfig("bypass", "", ""))
|
||||
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts)
|
||||
if code != 0 || stderr != "" || resolverCalls != 0 || len(factory.roots) != 0 {
|
||||
t.Fatalf("code=%d stderr=%q resolver=%d roots=%#v", code, stderr, resolverCalls, factory.roots)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunChunkPlanSetupFailures(t *testing.T) {
|
||||
for _, mode := range []string{"auto", "refresh"} {
|
||||
t.Run(mode+" resolver", func(t *testing.T) {
|
||||
factory := &recordingChunkPlanFactory{}
|
||||
opts := cacheTestOptions(t, nil, factory)
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("cache unavailable") }
|
||||
configPath := writeTestConfig(t, cacheTestConfig(mode, "", ""))
|
||||
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts)
|
||||
if code != 1 || !strings.Contains(stderr, "cache unavailable") || len(factory.roots) != 0 {
|
||||
t.Fatalf("code=%d stderr=%q roots=%#v", code, stderr, factory.roots)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("store", func(t *testing.T) {
|
||||
factory := &recordingChunkPlanFactory{err: errors.New("unwritable store")}
|
||||
configPath := writeTestConfig(t, cacheTestConfig("auto", t.TempDir(), ""))
|
||||
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), cacheTestOptions(t, nil, factory))
|
||||
if code != 1 || !strings.Contains(stderr, "unwritable store") {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unwritable filesystem store", func(t *testing.T) {
|
||||
root := writeFile(t, "not-a-directory", "occupied")
|
||||
configPath := writeTestConfig(t, `version: 2
|
||||
workspace:
|
||||
chunk_cache:
|
||||
mode: auto
|
||||
directory: `+root+`
|
||||
pipelines:
|
||||
dnd-session:
|
||||
input: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`)
|
||||
code, stderr := runCacheCommand(t, []string{"run", "dnd-session", "--config", configPath, "--input", writeSeriatimInput(t), "--output-dir", t.TempDir()}, Options{LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), LookupEnv: mapLookup(nil)})
|
||||
if code != 1 || (!strings.Contains(stderr, "load chunk plan") && !strings.Contains(stderr, "save chunk plan")) {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestConfigCommandsDoNotResolveOrCreateChunkPlanState(t *testing.T) {
|
||||
configPath := writeTestConfig(t, cacheTestConfig("auto", "", ""))
|
||||
for _, args := range [][]string{
|
||||
{"config", "validate", "--config", configPath},
|
||||
{"pipelines", "list", "--config", configPath},
|
||||
} {
|
||||
factory := &recordingChunkPlanFactory{err: errors.New("must not build")}
|
||||
resolverCalls := 0
|
||||
opts := cacheTestOptions(t, nil, factory)
|
||||
opts.UserCacheDir = func() (string, error) { resolverCalls++; return "", errors.New("must not resolve") }
|
||||
code, stderr := runCacheCommand(t, args, opts)
|
||||
if code != 0 || stderr != "" || resolverCalls != 0 || len(factory.roots) != 0 {
|
||||
t.Fatalf("args=%v code=%d stderr=%q resolver=%d roots=%#v", args, code, stderr, resolverCalls, factory.roots)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordsExplicitChunkCacheOverrideAndEffectiveMode(t *testing.T) {
|
||||
diagnosticsDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, cacheTestConfig("bypass", "", diagnosticsDir))
|
||||
factory := &recordingChunkPlanFactory{}
|
||||
args := append(cacheRunArgs(t, configPath), "--chunk_cache", "refresh")
|
||||
code, stderr := runCacheCommand(t, args, cacheTestOptions(t, nil, factory))
|
||||
if code != 0 || stderr != "" {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
runDir := onlyChildDir(t, diagnosticsDir)
|
||||
invocation := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactInvocationMetadata)))
|
||||
effective := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactEffectiveConfig)))
|
||||
if !strings.Contains(invocation, `"chunk_cache_override": "refresh"`) || !strings.Contains(effective, `"mode": "refresh"`) {
|
||||
t.Fatalf("invocation=%s effective=%s", invocation, effective)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultAutoReusesPlanAcrossIndependentInvocations(t *testing.T) {
|
||||
cacheBase := filepath.Join(t.TempDir(), "cache")
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
configPath := writeTestConfig(t, `version: 2
|
||||
workspace:
|
||||
directory: `+workspaceDir+`
|
||||
debug:
|
||||
enabled: true
|
||||
pipelines:
|
||||
dnd-session:
|
||||
input: seriatim
|
||||
chunk: dnd/scenes
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`)
|
||||
inputPath := writeFile(t, "source.json", `{
|
||||
"metadata": {"id": "session-alpha"},
|
||||
"segments": [
|
||||
{"id": 1, "start": 0, "end": 1, "speaker": "Aria", "text": "Aria casts Cure Wounds."},
|
||||
{"id": 2, "start": 1, "end": 2, "speaker": "Borin", "text": "Borin recovers."}
|
||||
]
|
||||
}`)
|
||||
client := newFakeRunLLMClient(false)
|
||||
opts := Options{
|
||||
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||
LookupEnv: mapLookup(nil),
|
||||
UserCacheDir: func() (string, error) { return cacheBase, nil },
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
code, stderr := runCacheCommand(t, []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir()}, opts)
|
||||
if code != 0 {
|
||||
t.Fatalf("run %d code=%d stderr=%q", i+1, code, stderr)
|
||||
}
|
||||
}
|
||||
if client.calls != 3 {
|
||||
t.Fatalf("LLM calls = %d, want chunk+extract then extract-only reuse", client.calls)
|
||||
}
|
||||
planRoot := filepath.Join(cacheBase, "notarius", "chunk-plans")
|
||||
entries, err := os.ReadDir(planRoot)
|
||||
if err != nil || len(entries) != 1 {
|
||||
t.Fatalf("plan root entries = %v error=%v", entries, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(planRoot, entries[0].Name(), "plan.json")); err != nil {
|
||||
t.Fatalf("plan file: %v", err)
|
||||
}
|
||||
debugRuns := childDirs(t, filepath.Join(workspaceDir, "debug"))
|
||||
if len(debugRuns) != 2 {
|
||||
t.Fatalf("debug runs = %#v", debugRuns)
|
||||
}
|
||||
attemptCounts := 0
|
||||
for _, runDir := range debugRuns {
|
||||
if _, err := os.Stat(filepath.Join(runDir, "chunk", "attempt-01.json")); err == nil {
|
||||
attemptCounts++
|
||||
} else if !os.IsNotExist(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if attemptCounts != 1 {
|
||||
t.Fatalf("chunk attempt files across runs = %d, want only generating run", attemptCounts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkPlanReuseDependsOnlyOnSourceDigest(t *testing.T) {
|
||||
cacheRoot := filepath.Join(t.TempDir(), "plans")
|
||||
inputPath := writeSeriatimInput(t)
|
||||
referencePath := writeFile(t, "players.txt", "Alyx")
|
||||
profilePath := writeScriptoriumProfileFile(t, "chunk-profile", "http://127.0.0.1:8080/v1", "test-model")
|
||||
seedConfig := writeTestConfig(t, `version: 2
|
||||
workspace:
|
||||
chunk_cache:
|
||||
mode: auto
|
||||
directory: `+cacheRoot+`
|
||||
pipelines:
|
||||
seed:
|
||||
input: seriatim
|
||||
chunk:
|
||||
module: generic
|
||||
options:
|
||||
max_units: 1
|
||||
overlap_units: 0
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`)
|
||||
changedConfig := writeTestConfig(t, `version: 2
|
||||
scriptorium:
|
||||
profile_file: `+profilePath+`
|
||||
workspace:
|
||||
chunk_cache:
|
||||
mode: auto
|
||||
directory: `+cacheRoot+`
|
||||
pipelines:
|
||||
changed:
|
||||
input: seriatim
|
||||
chunk:
|
||||
module: dnd/scenes
|
||||
llm_profile: chunk-profile
|
||||
references:
|
||||
players: `+referencePath+`
|
||||
validators:
|
||||
- generic/always_accept
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`)
|
||||
client := newFakeRunLLMClient(false)
|
||||
opts := Options{LLMClientFactory: fakeLLMFactory(client, nil), LookupEnv: mapLookup(nil)}
|
||||
for _, run := range []struct {
|
||||
pipeline string
|
||||
config string
|
||||
}{{pipeline: "seed", config: seedConfig}, {pipeline: "changed", config: changedConfig}} {
|
||||
code, stderr := runCacheCommand(t, []string{"run", run.pipeline, "--config", run.config, "--input", inputPath, "--output-dir", t.TempDir()}, opts)
|
||||
if code != 0 {
|
||||
t.Fatalf("pipeline %q code=%d stderr=%q", run.pipeline, code, stderr)
|
||||
}
|
||||
}
|
||||
if client.calls != 2 {
|
||||
t.Fatalf("LLM calls = %d, want one extractor call per run and no changed chunker call", client.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumeAndChunkCacheModesRemainIndependent(t *testing.T) {
|
||||
for _, mode := range []pipeline.ChunkCacheMode{pipeline.ChunkCacheAuto, pipeline.ChunkCacheBypass, pipeline.ChunkCacheRefresh} {
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
cacheRoot := filepath.Join(t.TempDir(), "plans")
|
||||
configPath := writeTestConfig(t, cacheTestConfigWithWorkspace(string(mode), cacheRoot, workspaceDir))
|
||||
factory := &recordingChunkPlanFactory{}
|
||||
args := append(cacheRunArgs(t, configPath), "--resume")
|
||||
code, stderr := runCacheCommand(t, args, cacheTestOptions(t, nil, factory))
|
||||
if code != 0 || stderr != "" {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
if mode == pipeline.ChunkCacheBypass {
|
||||
if len(factory.roots) != 0 {
|
||||
t.Fatalf("bypass roots = %#v", factory.roots)
|
||||
}
|
||||
} else if len(factory.roots) != 1 || factory.roots[0] != cacheRoot {
|
||||
t.Fatalf("roots = %#v, want %q", factory.roots, cacheRoot)
|
||||
}
|
||||
if entries := childDirs(t, filepath.Join(workspaceDir, "checkpoints")); len(entries) != 1 {
|
||||
t.Fatalf("checkpoint roots = %#v", entries)
|
||||
}
|
||||
if strings.HasPrefix(cacheRoot, workspaceDir+string(filepath.Separator)) || strings.HasPrefix(workspaceDir, cacheRoot+string(filepath.Separator)) {
|
||||
t.Fatalf("cache root %q and workspace root %q overlap", cacheRoot, workspaceDir)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceDirectoryDoesNotSelectChunkPlanRoot(t *testing.T) {
|
||||
cacheBase := filepath.Join(t.TempDir(), "user-cache")
|
||||
factory := &recordingChunkPlanFactory{}
|
||||
for _, workspaceDir := range []string{filepath.Join(t.TempDir(), "workspace-one"), filepath.Join(t.TempDir(), "workspace-two")} {
|
||||
configPath := writeTestConfig(t, cacheTestConfigWithWorkspace("auto", "", workspaceDir))
|
||||
opts := cacheTestOptions(t, nil, factory)
|
||||
opts.UserCacheDir = func() (string, error) { return cacheBase, nil }
|
||||
code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts)
|
||||
if code != 0 || stderr != "" {
|
||||
t.Fatalf("workspace=%q code=%d stderr=%q", workspaceDir, code, stderr)
|
||||
}
|
||||
}
|
||||
want := filepath.Join(cacheBase, "notarius", "chunk-plans")
|
||||
if len(factory.roots) != 2 || factory.roots[0] != want || factory.roots[1] != want {
|
||||
t.Fatalf("roots = %#v, want %q twice", factory.roots, want)
|
||||
}
|
||||
}
|
||||
|
||||
func cacheTestOptions(t *testing.T, env map[string]string, factory *recordingChunkPlanFactory) Options {
|
||||
t.Helper()
|
||||
registries := fakeExecutionRegistries(t)
|
||||
return Options{
|
||||
Catalog: catalogFromRegistries(registries),
|
||||
Registries: registries,
|
||||
LLMClientFactory: fakeLLMFactory(nil, nil),
|
||||
LookupEnv: mapLookup(env),
|
||||
UserCacheDir: func() (string, error) { return filepath.Join(t.TempDir(), "cache"), nil },
|
||||
ChunkPlanStoreFactory: factory.build,
|
||||
}
|
||||
}
|
||||
|
||||
func cacheRunArgs(t *testing.T, configPath string) []string {
|
||||
t.Helper()
|
||||
return []string{"run", "example", "--config", configPath, "--input", writeFile(t, "input.txt", "input"), "--output-dir", t.TempDir()}
|
||||
}
|
||||
|
||||
func runCacheCommand(t *testing.T, args []string, opts Options) (int, string) {
|
||||
t.Helper()
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := RunWithOptions(args, &stdout, &stderr, opts)
|
||||
return code, stderr.String()
|
||||
}
|
||||
|
||||
func cacheTestConfig(mode, directory, diagnosticsDir string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("version: 2\n")
|
||||
if mode != "" || directory != "" {
|
||||
b.WriteString("workspace:\n chunk_cache:\n")
|
||||
if mode != "" {
|
||||
b.WriteString(" mode: " + mode + "\n")
|
||||
}
|
||||
if directory != "" {
|
||||
b.WriteString(" directory: " + directory + "\n")
|
||||
}
|
||||
}
|
||||
if diagnosticsDir != "" {
|
||||
b.WriteString("diagnostics:\n work_dir: " + diagnosticsDir + "\n retention: always\n")
|
||||
}
|
||||
b.WriteString("pipelines:\n example:\n input: fake/input\n artifacts:\n spells:\n extract: fake/extract\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func cacheTestConfigWithWorkspace(mode, cacheRoot, workspaceDir string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("version: 2\nworkspace:\n directory: " + workspaceDir + "\n resume:\n enabled: true\n chunk_cache:\n mode: " + mode + "\n")
|
||||
if cacheRoot != "" {
|
||||
b.WriteString(" directory: " + cacheRoot + "\n")
|
||||
}
|
||||
b.WriteString("pipelines:\n example:\n input: fake/input\n artifacts:\n spells:\n extract: fake/extract\n")
|
||||
return b.String()
|
||||
}
|
||||
257
internal/cli/command_contract_test.go
Normal file
257
internal/cli/command_contract_test.go
Normal file
@@ -0,0 +1,257 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
|
||||
tests := [][]string{nil, {"help"}, {"--help"}, {"-h"}}
|
||||
for _, args := range tests {
|
||||
name := "no arguments"
|
||||
if len(args) > 0 {
|
||||
name = args[0]
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t))
|
||||
if code != 0 || !strings.Contains(stdout.String(), "Usage:") || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{name: "unknown command", args: []string{"unknown"}, want: "unknown command"},
|
||||
{name: "missing config subcommand", args: []string{"config"}, want: "config requires a subcommand"},
|
||||
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
|
||||
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
|
||||
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(tt.args, &stdout, &stderr, commandContractOptions(t))
|
||||
if code != 2 || !strings.Contains(stderr.String(), tt.want) || stdout.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
|
||||
explicit := writeCommandConfig(t, "explicit", "alpha")
|
||||
environment := writeCommandConfig(t, "environment", "beta")
|
||||
lookup := func(name string) (string, bool) {
|
||||
if name == "NOTARIUS_CONFIG" {
|
||||
return environment, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"pipelines", "list", "--config", explicit}, &stdout, &stderr, commandContractOptionsWithLookup(t, lookup))
|
||||
if code != 0 || stdout.String() != "alpha\nexplicit\n" || stderr.Len() != 0 {
|
||||
t.Fatalf("explicit config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"pipelines", "list"}, &stdout, &stderr, commandContractOptionsWithLookup(t, lookup))
|
||||
if code != 0 || stdout.String() != "beta\nenvironment\n" || stderr.Len() != 0 {
|
||||
t.Fatalf("environment config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigDiscoveryUsesCompiledDefaultOnlyWhenAvailable(t *testing.T) {
|
||||
info, statErr := os.Stat(defaultConfigPath)
|
||||
if statErr != nil && !os.IsNotExist(statErr) {
|
||||
t.Fatalf("stat compiled default config: %v", statErr)
|
||||
}
|
||||
if statErr == nil && !info.Mode().IsRegular() {
|
||||
t.Skipf("compiled default config has unexpected host state: %s", info.Mode())
|
||||
}
|
||||
|
||||
path, err := discoverConfigPath("", commandContractOptions(t))
|
||||
if statErr == nil {
|
||||
if err != nil || path != defaultConfigPath {
|
||||
t.Fatalf("discoverConfigPath() = %q, %v; want compiled default", path, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "config file not found") {
|
||||
t.Fatalf("discoverConfigPath() error = %v, want documented not-found context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigLoadingFailuresReturnOneWithPathContext(t *testing.T) {
|
||||
missing := filepath.Join(t.TempDir(), "missing.yml")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"config", "validate", "--config", missing}, &stdout, &stderr, commandContractOptions(t))
|
||||
if code != 1 || !strings.Contains(stderr.String(), missing) || stdout.Len() != 0 {
|
||||
t.Fatalf("missing config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
malformed := filepath.Join(t.TempDir(), "malformed.yml")
|
||||
if err := os.WriteFile(malformed, []byte("version: [\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"config", "validate", "--config", malformed}, &stdout, &stderr, commandContractOptions(t))
|
||||
if code != 1 || !strings.Contains(stderr.String(), malformed) || !strings.Contains(stderr.String(), "parse config file") || stdout.Len() != 0 {
|
||||
t.Fatalf("malformed config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidateResolvesPipelineAndChecksSelection(t *testing.T) {
|
||||
configPath := writeResolvableCommandConfig(t)
|
||||
options := commandContractOptions(t)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "spells"}, &stdout, &stderr, options)
|
||||
if code != 0 || !strings.Contains(stdout.String(), "valid for pipeline \"demo\"") || stderr.Len() != 0 {
|
||||
t.Fatalf("valid resolution: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "missing"}, &stdout, &stderr, options)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "pipeline \"missing\"") {
|
||||
t.Fatalf("unknown pipeline: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "missing"}, &stdout, &stderr, options)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "lane \"missing\"") {
|
||||
t.Fatalf("unknown lane: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--only", "spells"}, &stdout, &stderr, options)
|
||||
if code != 2 || !strings.Contains(stderr.String(), "--only requires --pipeline") {
|
||||
t.Fatalf("missing pipeline for only: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "spells,,other"}, &stdout, &stderr, options)
|
||||
if code != 2 || !strings.Contains(stderr.String(), "--only must contain") {
|
||||
t.Fatalf("malformed only: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidatePipelineDefaultProfileIsOffline(t *testing.T) {
|
||||
configPath := writeCommandConfigContent(t, `version: 4
|
||||
pipelines:
|
||||
demo:
|
||||
llm_profile: dnd-extraction
|
||||
input: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo"}, &stdout, &stderr, Options{})
|
||||
if code != 0 || !strings.Contains(stdout.String(), "valid for pipeline \"demo\"") || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelinesListSortsNormalizedIDsInTextAndJSON(t *testing.T) {
|
||||
configPath := writeCommandConfig(t, " zeta ", "alpha")
|
||||
options := commandContractOptions(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"pipelines", "list", "--config", configPath}, &stdout, &stderr, options)
|
||||
if code != 0 || stdout.String() != "alpha\nzeta\n" || stderr.Len() != 0 {
|
||||
t.Fatalf("text list: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"pipelines", "list", "--config", configPath, "--json"}, &stdout, &stderr, options)
|
||||
var payload struct {
|
||||
Pipelines []string `json:"pipelines"`
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("JSON list = %q: %v", stdout.String(), err)
|
||||
}
|
||||
if code != 0 || len(payload.Pipelines) != 2 || payload.Pipelines[0] != "alpha" || payload.Pipelines[1] != "zeta" || stderr.Len() != 0 {
|
||||
t.Fatalf("JSON list: code=%d payload=%#v stderr=%q", code, payload, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemovedStructuralFlagsAndRuntimeFailuresKeepExitClasses(t *testing.T) {
|
||||
configPath := writeResolvableCommandConfig(t)
|
||||
options := commandContractOptions(t)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "demo", "--input", "missing-input", "--config", configPath, "--diagnostics-dir", t.TempDir()}, &stdout, &stderr, options)
|
||||
if code != 2 || !strings.Contains(stderr.String(), "flag provided but not defined") {
|
||||
t.Fatalf("removed flag: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"run", "missing", "--input", "missing-input", "--config", configPath, "--chunk_cache", "bypass"}, &stdout, &stderr, options)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "pipeline \"missing\"") || stdout.Len() != 0 {
|
||||
t.Fatalf("valid-runtime failure: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func commandContractOptions(t *testing.T) Options {
|
||||
return commandContractOptionsWithLookup(t, emptyLookup)
|
||||
}
|
||||
|
||||
func commandContractOptionsWithLookup(t *testing.T, lookup func(string) (string, bool)) Options {
|
||||
t.Helper()
|
||||
components, err := newProductionComponents()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Options{
|
||||
Catalog: catalogFromRegistries(components.registries),
|
||||
Registries: components.registries,
|
||||
LookupEnv: lookup,
|
||||
}
|
||||
}
|
||||
|
||||
func writeCommandConfig(t *testing.T, firstID, secondID string) string {
|
||||
t.Helper()
|
||||
content := fmt.Sprintf("version: 4\npipelines:\n %q:\n input: seriatim\n %q:\n input: seriatim\n", firstID, secondID)
|
||||
return writeCommandConfigContent(t, content)
|
||||
}
|
||||
|
||||
func writeResolvableCommandConfig(t *testing.T) string {
|
||||
t.Helper()
|
||||
return writeCommandConfigContent(t, `version: 4
|
||||
pipelines:
|
||||
demo:
|
||||
input: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`)
|
||||
}
|
||||
|
||||
func writeCommandConfigContent(t *testing.T, content string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -1,600 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
|
||||
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
|
||||
spellrelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness"
|
||||
validjson "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/valid_json"
|
||||
validjsonschema "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/valid_json_schema"
|
||||
)
|
||||
|
||||
func TestProductionCompatibilitySnapshot(t *testing.T) {
|
||||
normalizedOptions, err := normalizeOptions(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("normalizeOptions() error = %v, want nil", err)
|
||||
}
|
||||
if normalizedOptions.Catalog.Inputs != normalizedOptions.Registries.Inputs ||
|
||||
normalizedOptions.Catalog.Chunkers != normalizedOptions.Registries.Chunkers ||
|
||||
normalizedOptions.Catalog.Extractors != normalizedOptions.Registries.Extractors ||
|
||||
normalizedOptions.Catalog.Mergers != normalizedOptions.Registries.Mergers ||
|
||||
normalizedOptions.Catalog.Normalizers != normalizedOptions.Registries.Normalizers ||
|
||||
normalizedOptions.Catalog.Validators != normalizedOptions.Registries.Validators ||
|
||||
normalizedOptions.Catalog.ValidatorChains != normalizedOptions.Registries.ValidatorChains ||
|
||||
normalizedOptions.Catalog.Outputs != normalizedOptions.Registries.Outputs {
|
||||
t.Fatal("production catalog and execution registries do not share one composition")
|
||||
}
|
||||
if normalizedOptions.LLMClientFactory == nil {
|
||||
t.Fatal("production LLM client factory is nil")
|
||||
}
|
||||
|
||||
registries, err := productionRegistries()
|
||||
if err != nil {
|
||||
t.Fatalf("productionRegistries() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
keySnapshots := []struct {
|
||||
name string
|
||||
got []string
|
||||
want []string
|
||||
}{
|
||||
{name: "inputs", got: registries.Inputs.RegisteredKeys(), want: []string{"seriatim"}},
|
||||
{name: "chunkers", got: registries.Chunkers.RegisteredKeys(), want: []string{"dnd/scenes", "generic"}},
|
||||
{name: "extractors", got: registries.Extractors.RegisteredKeys(), want: []string{"dnd/spells"}},
|
||||
{name: "mergers", got: registries.Mergers.RegisteredKeys(), want: []string{"appendorder"}},
|
||||
{name: "normalizers", got: registries.Normalizers.RegisteredKeys(), want: []string{"noop"}},
|
||||
{name: "validators", got: registries.Validators.RegisteredKeys(), want: []string{
|
||||
"extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
|
||||
"generic/always_accept", "generic/always_reject", "generic/valid_json", "generic/valid_json_schema",
|
||||
}},
|
||||
{name: "outputs", got: registries.Outputs.RegisteredKeys(), want: []string{"json"}},
|
||||
}
|
||||
for _, snapshot := range keySnapshots {
|
||||
t.Run(snapshot.name, func(t *testing.T) {
|
||||
if !reflect.DeepEqual(snapshot.got, snapshot.want) {
|
||||
t.Fatalf("registered keys = %#v, want compatibility snapshot %#v", snapshot.got, snapshot.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
wantChain := []pipeline.ModuleBinding{
|
||||
pipeline.Binding(validjson.Key),
|
||||
pipeline.Binding(validjsonschema.Key),
|
||||
pipeline.Binding(spellshape.Key),
|
||||
pipeline.Binding(spellsourcerefs.Key),
|
||||
pipeline.Binding(spellrelatedness.Key),
|
||||
}
|
||||
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
||||
t.Fatalf("spell validator chain = %#v, want compatibility snapshot %#v", got, wantChain)
|
||||
}
|
||||
|
||||
assets, err := productionPromptAssets()
|
||||
if err != nil {
|
||||
t.Fatalf("productionPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
assertAssetNames(t, assets.PromptFS, []string{
|
||||
"dnd.scenes/dnd.scenes.yaml",
|
||||
"dnd.scenes/instructions.md",
|
||||
"dnd.scenes/sharedassets/common-dnd-references.md",
|
||||
"dnd.scenes/sharedassets/common-dnd-system.md",
|
||||
"dnd.scenes/sharedassets/common-dnd-transcript.md",
|
||||
"dnd.scenes/task.md",
|
||||
"dnd.spells/dnd.spells.yaml",
|
||||
"dnd.spells/instructions.md",
|
||||
"dnd.spells/sharedassets/common-dnd-references.md",
|
||||
"dnd.spells/sharedassets/common-dnd-system.md",
|
||||
"dnd.spells/sharedassets/common-dnd-transcript.md",
|
||||
"dnd.spells/task.md",
|
||||
})
|
||||
assertAssetNames(t, assets.SchemaFS, []string{
|
||||
"dnd_scenes.v1.json",
|
||||
"dnd_spells_llm.v1.json",
|
||||
})
|
||||
|
||||
identitySnapshot := map[string]map[string]any{
|
||||
"scenes": sceneManifestMetadata(t),
|
||||
"spells": spellManifestMetadata(t),
|
||||
}
|
||||
for name, metadata := range identitySnapshot {
|
||||
for _, key := range []string{"prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_id", "response_schema_name", "response_schema_version", "response_schema_sha256"} {
|
||||
if value, ok := metadata[key].(string); !ok || value == "" {
|
||||
t.Fatalf("%s metadata[%q] = %#v, want non-empty identity", name, key, metadata[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
if got := []any{
|
||||
identitySnapshot["scenes"]["prompt_id"], identitySnapshot["scenes"]["prompt_version"],
|
||||
identitySnapshot["scenes"]["response_schema_key"], identitySnapshot["scenes"]["response_schema_id"], identitySnapshot["scenes"]["response_schema_name"], identitySnapshot["scenes"]["response_schema_version"],
|
||||
}; !reflect.DeepEqual(got, []any{"dnd.scenes", "v1", "dnd_scenes", "notarius.dnd.scenes", "notarius_dnd_scenes_v1", "v1"}) {
|
||||
t.Fatalf("scene identities = %#v, want compatibility snapshot", got)
|
||||
}
|
||||
if got := []any{
|
||||
identitySnapshot["spells"]["prompt_id"], identitySnapshot["spells"]["prompt_version"],
|
||||
identitySnapshot["spells"]["response_schema_key"], identitySnapshot["spells"]["response_schema_id"], identitySnapshot["spells"]["response_schema_name"], identitySnapshot["spells"]["response_schema_version"],
|
||||
}; !reflect.DeepEqual(got, []any{"dnd.spells", "v1", "dnd_spells", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"}) {
|
||||
t.Fatalf("spell identities = %#v, want compatibility snapshot", got)
|
||||
}
|
||||
|
||||
fileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells.config.yml"))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFileConfig() error = %v, want nil", err)
|
||||
}
|
||||
cfg := config.Default()
|
||||
if err := cfg.ApplyFileConfig(fileConfig); err != nil {
|
||||
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
|
||||
}
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
resolved := effective.ResolvedPipeline
|
||||
if resolved.Input.Module != "seriatim" || resolved.Chunk.Module != "generic" || resolved.Output.Module != "json" || len(resolved.ArtifactLanes) != 1 {
|
||||
t.Fatalf("resolved example = %#v, want maintained production topology", resolved)
|
||||
}
|
||||
lane := resolved.ArtifactLanes[0]
|
||||
if lane.ID != "spells" || lane.Extract.Module != "dnd/spells" || lane.Merge.Module != "appendorder" || lane.Normalize.Module != "noop" {
|
||||
t.Fatalf("resolved lane = %#v, want maintained spell lane", lane)
|
||||
}
|
||||
if got := resolvedValidatorKeys(resolved.ValidatorChains, pipeline.StageExtract, "spells", spells.Key); !reflect.DeepEqual(got, []string{
|
||||
"generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
|
||||
}) {
|
||||
t.Fatalf("resolved validator keys = %#v, want compatibility snapshot", got)
|
||||
}
|
||||
|
||||
productionFileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells-production.config.yml"))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFileConfig(production example) error = %v, want nil", err)
|
||||
}
|
||||
productionConfig := config.Default()
|
||||
if err := productionConfig.ApplyFileConfig(productionFileConfig); err != nil {
|
||||
t.Fatalf("ApplyFileConfig(production example) error = %v, want nil", err)
|
||||
}
|
||||
productionEffective, err := productionConfig.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve(production example) error = %v, want nil", err)
|
||||
}
|
||||
productionResolved := productionEffective.ResolvedPipeline
|
||||
if productionConfig.Concurrency.TotalLLM != 1 || productionConfig.Concurrency.StageWorkers["extract"] != 1 || !reflect.DeepEqual(productionResolved.Chunk.Options, map[string]any{"max_units": 50}) {
|
||||
t.Fatalf("production example concurrency/options = %#v/%#v, want compatibility snapshot", productionConfig.Concurrency, productionResolved.Chunk.Options)
|
||||
}
|
||||
bindings := productionResolved.ArtifactLanes[0].ExtractReferences.Bindings
|
||||
if len(bindings) != 2 || bindings[0].SlotName != "glossary" || bindings[0].Source != "./dnd-spells-glossary.txt" || bindings[1].SlotName != "party" || bindings[1].Source != "./dnd-spells-roster.txt" {
|
||||
t.Fatalf("production example reference bindings = %#v, want maintained glossary and party bindings", bindings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintainedSeriatimToDNDCompatibilityBundle(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
client contracts.StructuredLLMClient
|
||||
wantStatus string
|
||||
wantLaneFile bool
|
||||
wantRejectedCount int
|
||||
}{
|
||||
{name: "approved", client: newFakeRunLLMClient(false), wantStatus: "approved", wantLaneFile: true},
|
||||
{name: "validator rejection is nonfatal", client: newFakeRunLLMClient(true), wantStatus: "rejected", wantRejectedCount: 1},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
outputDir := t.TempDir()
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session",
|
||||
"--config", fixturePath(t, "examples/dnd-spells.config.yml"),
|
||||
"--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"),
|
||||
"--output-dir", outputDir,
|
||||
"--diagnostics-dir", t.TempDir(),
|
||||
}, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(test.client, nil)})
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
|
||||
runDir := onlyChildDir(t, outputDir)
|
||||
wantFiles := []string{"index.json", "manifest.json", "rejected.json", "warnings.json"}
|
||||
if test.wantLaneFile {
|
||||
wantFiles = append(wantFiles, "lanes/spells.json")
|
||||
}
|
||||
sort.Strings(wantFiles)
|
||||
if got := relativeFileNames(t, runDir); !reflect.DeepEqual(got, wantFiles) {
|
||||
t.Fatalf("durable files = %#v, want compatibility snapshot %#v", got, wantFiles)
|
||||
}
|
||||
|
||||
var manifest artifacts.RunManifest
|
||||
readJSONFile(t, filepath.Join(runDir, "manifest.json"), &manifest)
|
||||
if manifest.PipelineID != "dnd-session" || manifest.InputModule != "seriatim" || manifest.Chunker != "generic" || manifest.OutputEncoder != "json" {
|
||||
t.Fatalf("manifest module provenance = %#v, want maintained production modules", manifest)
|
||||
}
|
||||
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].ID != "spells" {
|
||||
t.Fatalf("artifact lanes = %#v, want one spells lane", manifest.ArtifactLanes)
|
||||
}
|
||||
laneManifest := manifest.ArtifactLanes[0]
|
||||
if laneManifest.Extractor != "dnd/spells" || laneManifest.Merger != "appendorder" || laneManifest.Normalizer != "noop" {
|
||||
t.Fatalf("manifest lane module provenance = %#v, want maintained production modules", laneManifest)
|
||||
}
|
||||
if len(manifest.Extractors) != 0 || manifest.Merger != "" || manifest.Normalizer != "" {
|
||||
t.Fatalf("legacy top-level lane summaries = %#v/%q/%q, want empty compatibility snapshot", manifest.Extractors, manifest.Merger, manifest.Normalizer)
|
||||
}
|
||||
if manifest.ValidationStatus != test.wantStatus || len(manifest.RejectedOutputs) != test.wantRejectedCount {
|
||||
t.Fatalf("manifest outcome = status %q rejected %#v, want %q/%d", manifest.ValidationStatus, manifest.RejectedOutputs, test.wantStatus, test.wantRejectedCount)
|
||||
}
|
||||
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:1c98d94ae632fb10a2b56f684cd4fb1019cedb1a629e57dc0977cf4a54135be0"}) {
|
||||
t.Fatalf("source digests = %#v, want maintained fixture provenance", manifest.SourceDigests)
|
||||
}
|
||||
if got := manifestValidatorKeys(manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)); !reflect.DeepEqual(got, []string{
|
||||
"generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
|
||||
}) {
|
||||
t.Fatalf("manifest validator chain = %#v, want compatibility snapshot", got)
|
||||
}
|
||||
|
||||
var index struct {
|
||||
ManifestFile string `json:"manifest_file"`
|
||||
OutputFiles []struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
MediaType string `json:"media_type"`
|
||||
File string `json:"file"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
SchemaID string `json:"schema_id"`
|
||||
SchemaName string `json:"schema_name"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
} `json:"output_files"`
|
||||
RejectedFile string `json:"rejected_file"`
|
||||
WarningsFile string `json:"warnings_file"`
|
||||
}
|
||||
readJSONFile(t, filepath.Join(runDir, "index.json"), &index)
|
||||
if index.ManifestFile != "manifest.json" || index.RejectedFile != "rejected.json" || index.WarningsFile != "warnings.json" {
|
||||
t.Fatalf("output index fixed files = %#v, want compatibility snapshot", index)
|
||||
}
|
||||
if test.wantLaneFile {
|
||||
if len(index.OutputFiles) != 1 {
|
||||
t.Fatalf("output index entries = %#v, want one", index.OutputFiles)
|
||||
}
|
||||
wantOutput := struct {
|
||||
LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string
|
||||
}{"spells", "application/json", "lanes/spells.json", "noop", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"}
|
||||
gotOutput := index.OutputFiles[0]
|
||||
got := struct {
|
||||
LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string
|
||||
}{gotOutput.LaneID, gotOutput.MediaType, gotOutput.File, gotOutput.ModuleKey, gotOutput.SchemaID, gotOutput.SchemaName, gotOutput.SchemaVersion}
|
||||
if got != wantOutput {
|
||||
t.Fatalf("output index entries = %#v, want compatibility snapshot %#v", index.OutputFiles, wantOutput)
|
||||
}
|
||||
assertJSONEqual(t, readFile(t, filepath.Join(runDir, "lanes/spells.json")), []byte(`{
|
||||
"spell_casts": [{
|
||||
"caster": "Aria",
|
||||
"spell": "Cure Wounds",
|
||||
"effect": "Heals a wounded ally.",
|
||||
"narrative_description": "Aria casts Cure Wounds.",
|
||||
"source_refs": [{"source_id": "session-alpha", "start_unit_id": 1, "end_unit_id": 1}]
|
||||
}]
|
||||
}`))
|
||||
} else if len(index.OutputFiles) != 0 {
|
||||
t.Fatalf("output index entries = %#v, want none for rejected lane", index.OutputFiles)
|
||||
}
|
||||
|
||||
var warnings struct {
|
||||
Warnings []contracts.Warning `json:"warnings"`
|
||||
}
|
||||
readJSONFile(t, filepath.Join(runDir, "warnings.json"), &warnings)
|
||||
if len(warnings.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want empty compatibility snapshot", warnings.Warnings)
|
||||
}
|
||||
var rejected struct {
|
||||
Rejected []contracts.RejectedOutput `json:"rejected"`
|
||||
}
|
||||
readJSONFile(t, filepath.Join(runDir, "rejected.json"), &rejected)
|
||||
if len(rejected.Rejected) != test.wantRejectedCount {
|
||||
t.Fatalf("rejected outputs = %#v, want %d", rejected.Rejected, test.wantRejectedCount)
|
||||
}
|
||||
if test.wantRejectedCount == 1 {
|
||||
got := rejected.Rejected[0]
|
||||
if got.Stage != "extract" || got.LaneID != "spells" || got.ModuleKey != "dnd/spells" || got.ChunkID != "chunk-000001" || got.ChunkIndex != 0 || got.ValidatorName != "extract/dnd/spells/source_refs" || got.ReasonCode != "invalid_source_refs" || got.AttemptCount != 1 {
|
||||
t.Fatalf("rejection = %#v, want maintained nonfatal validator outcome", got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionLLMCallersShareScheduledClient(t *testing.T) {
|
||||
underlying := newBlockingProductionLLMClient()
|
||||
scheduler, err := frameworkllm.NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler() error = %v, want nil", err)
|
||||
}
|
||||
client := frameworkllm.NewScheduledClient(underlying, scheduler)
|
||||
doc := &source.SourceDocument{
|
||||
ID: "session-alpha", Kind: "transcript", Format: "application/json", Digest: "sha256:source",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: 1, Kind: "segment", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}},
|
||||
{ID: 2, Kind: "segment", Text: "The spell takes effect.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}},
|
||||
},
|
||||
}
|
||||
chunk := source.Chunk{
|
||||
ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 2},
|
||||
Content: []byte(`{"scene":"Aria casts Cure Wounds."}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
}
|
||||
|
||||
var started sync.WaitGroup
|
||||
started.Add(2)
|
||||
errs := make(chan error, 2)
|
||||
go func() {
|
||||
started.Done()
|
||||
chunker, err := scenes.New(client, scenes.Options{})
|
||||
if err == nil {
|
||||
_, err = chunker.Plan(context.Background(), contracts.ChunkRequest{Source: doc})
|
||||
}
|
||||
errs <- err
|
||||
}()
|
||||
go func() {
|
||||
started.Done()
|
||||
extractor, err := spells.New(client, spells.Options{})
|
||||
if err == nil {
|
||||
_, err = extractor.Extract(context.Background(), contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk})
|
||||
}
|
||||
errs <- err
|
||||
}()
|
||||
started.Wait()
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
<-underlying.entered
|
||||
underlying.release <- struct{}{}
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatalf("production LLM caller error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
if underlying.maxActive != 1 {
|
||||
t.Fatalf("maximum concurrent provider calls = %d, want total_llm limit 1", underlying.maxActive)
|
||||
}
|
||||
sort.Strings(underlying.stageNames)
|
||||
if !reflect.DeepEqual(underlying.stageNames, []string{"dnd/scenes", "dnd/spells"}) {
|
||||
t.Fatalf("scheduled stage names = %#v, want both production LLM callers", underlying.stageNames)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionBundlePreservesLaneAndChunkOrder(t *testing.T) {
|
||||
configPath := writeTestConfig(t, `version: 2
|
||||
pipelines:
|
||||
dnd-session:
|
||||
input: seriatim
|
||||
chunk:
|
||||
module: generic
|
||||
options:
|
||||
max_units: 1
|
||||
artifacts:
|
||||
zeta:
|
||||
extract: dnd/spells
|
||||
alpha:
|
||||
extract: dnd/spells
|
||||
`)
|
||||
outputDir := t.TempDir()
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session",
|
||||
"--config", configPath,
|
||||
"--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"),
|
||||
"--output-dir", outputDir,
|
||||
"--diagnostics-dir", t.TempDir(),
|
||||
}, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(orderingProductionLLMClient{}, nil)})
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
|
||||
runDir := onlyChildDir(t, outputDir)
|
||||
var index struct {
|
||||
OutputFiles []struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
} `json:"output_files"`
|
||||
}
|
||||
readJSONFile(t, filepath.Join(runDir, "index.json"), &index)
|
||||
if len(index.OutputFiles) != 2 {
|
||||
t.Fatalf("output index entries = %#v, want two lanes", index.OutputFiles)
|
||||
}
|
||||
if got := []string{index.OutputFiles[0].LaneID, index.OutputFiles[1].LaneID}; !reflect.DeepEqual(got, []string{"alpha", "zeta"}) {
|
||||
t.Fatalf("output lane order = %#v, want resolved lane order", got)
|
||||
}
|
||||
for _, laneID := range []string{"alpha", "zeta"} {
|
||||
var payload struct {
|
||||
SpellCasts []struct {
|
||||
Spell string `json:"spell"`
|
||||
SourceRefs []source.SourceRef `json:"source_refs"`
|
||||
} `json:"spell_casts"`
|
||||
}
|
||||
readJSONFile(t, filepath.Join(runDir, "lanes", laneID+".json"), &payload)
|
||||
if len(payload.SpellCasts) != 2 {
|
||||
t.Fatalf("lane %q spell casts = %#v, want one per source chunk", laneID, payload.SpellCasts)
|
||||
}
|
||||
got := []any{
|
||||
payload.SpellCasts[0].Spell, payload.SpellCasts[0].SourceRefs[0].StartUnitID,
|
||||
payload.SpellCasts[1].Spell, payload.SpellCasts[1].SourceRefs[0].StartUnitID,
|
||||
}
|
||||
if !reflect.DeepEqual(got, []any{"Cure Wounds", 1, "Shield", 2}) {
|
||||
t.Fatalf("lane %q chunk handoff order = %#v, want source chunk order", laneID, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type blockingProductionLLMClient struct {
|
||||
mu sync.Mutex
|
||||
active int
|
||||
maxActive int
|
||||
stageNames []string
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
type orderingProductionLLMClient struct{}
|
||||
|
||||
func (orderingProductionLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
material := req.Inputs["transcript"]
|
||||
unitID := 1
|
||||
spellName := "Cure Wounds"
|
||||
if strings.Contains(string(material.Content), "Shield") {
|
||||
unitID = 2
|
||||
spellName = "Shield"
|
||||
}
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"spell_casts": []map[string]any{{
|
||||
"caster": "Aria",
|
||||
"spell": spellName,
|
||||
"effect": "Fixture effect.",
|
||||
"narrative_description": "Fixture spell cast.",
|
||||
"source_refs": []map[string]any{{
|
||||
"start_unit_id": unitID,
|
||||
"end_unit_id": unitID,
|
||||
}},
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
if err := json.Unmarshal(payload, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: payload}, nil
|
||||
}
|
||||
|
||||
func newBlockingProductionLLMClient() *blockingProductionLLMClient {
|
||||
return &blockingProductionLLMClient{entered: make(chan struct{}, 2), release: make(chan struct{}, 2)}
|
||||
}
|
||||
|
||||
func (client *blockingProductionLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.mu.Lock()
|
||||
client.active++
|
||||
if client.active > client.maxActive {
|
||||
client.maxActive = client.active
|
||||
}
|
||||
client.stageNames = append(client.stageNames, req.StageName)
|
||||
client.mu.Unlock()
|
||||
client.entered <- struct{}{}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
case <-client.release:
|
||||
}
|
||||
|
||||
client.mu.Lock()
|
||||
client.active--
|
||||
client.mu.Unlock()
|
||||
|
||||
var payload []byte
|
||||
switch req.StageName {
|
||||
case scenes.Key:
|
||||
payload = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Spell","primary_mode":"Narrative","main_participants":["Aria"],"summary":"Aria casts a spell.","boundary_note":"Complete source.","boundary_confidence":"High"}],"boundary_caveats":[]}`)
|
||||
case spells.Key:
|
||||
payload = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Healing","narrative_description":"Aria casts Cure Wounds.","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`)
|
||||
}
|
||||
if err := json.Unmarshal(payload, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: payload}, nil
|
||||
}
|
||||
|
||||
func sceneManifestMetadata(t *testing.T) map[string]any {
|
||||
t.Helper()
|
||||
chunker, err := scenes.New(orderingProductionLLMClient{}, scenes.Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("construct scene chunker: %v", err)
|
||||
}
|
||||
return chunker.ManifestMetadata()
|
||||
}
|
||||
|
||||
func spellManifestMetadata(t *testing.T) map[string]any {
|
||||
t.Helper()
|
||||
extractor, err := spells.New(orderingProductionLLMClient{}, spells.Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("construct spell extractor: %v", err)
|
||||
}
|
||||
return extractor.ManifestMetadata()
|
||||
}
|
||||
|
||||
func assertAssetNames(t *testing.T, getFS func() (fs.FS, error), want []string) {
|
||||
t.Helper()
|
||||
fSys, err := getFS()
|
||||
if err != nil {
|
||||
t.Fatalf("asset filesystem error = %v, want nil", err)
|
||||
}
|
||||
var got []string
|
||||
if err := fs.WalkDir(fSys, ".", func(path string, entry fs.DirEntry, err error) error {
|
||||
if err == nil && !entry.IsDir() {
|
||||
got = append(got, path)
|
||||
}
|
||||
return err
|
||||
}); err != nil {
|
||||
t.Fatalf("walk assets: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("asset names = %#v, want compatibility snapshot %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func resolvedValidatorKeys(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID, moduleKey string) []string {
|
||||
for _, chain := range chains {
|
||||
if chain.Stage == stage && chain.LaneID == laneID && chain.ModuleKey == moduleKey {
|
||||
keys := make([]string, 0, len(chain.Validators))
|
||||
for _, validator := range chain.Validators {
|
||||
keys = append(keys, validator.Binding.Module)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func relativeFileNames(t *testing.T, root string) []string {
|
||||
t.Helper()
|
||||
var names []string
|
||||
if err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil || entry.IsDir() {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names = append(names, filepath.ToSlash(rel))
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("walk durable output: %v", err)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
func assertJSONEqual(t *testing.T, got, want []byte) {
|
||||
t.Helper()
|
||||
var gotValue any
|
||||
var wantValue any
|
||||
if err := json.Unmarshal(got, &gotValue); err != nil {
|
||||
t.Fatalf("unmarshal actual JSON: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(want, &wantValue); err != nil {
|
||||
t.Fatalf("unmarshal expected JSON: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(gotValue, wantValue) {
|
||||
t.Fatalf("JSON = %#v, want compatibility snapshot %#v", gotValue, wantValue)
|
||||
}
|
||||
}
|
||||
16
internal/cli/contract_test_helpers_test.go
Normal file
16
internal/cli/contract_test_helpers_test.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func replaceRequiredOnce(t *testing.T, input, old, replacement string) string {
|
||||
t.Helper()
|
||||
if count := strings.Count(input, old); count != 1 {
|
||||
t.Fatalf("replacement marker %q occurs %d times, want exactly once", old, count)
|
||||
}
|
||||
return strings.Replace(input, old, replacement, 1)
|
||||
}
|
||||
|
||||
func emptyLookup(string) (string, bool) { return "", false }
|
||||
208
internal/cli/dnd_combat_contract_test.go
Normal file
208
internal/cli/dnd_combat_contract_test.go
Normal file
@@ -0,0 +1,208 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
|
||||
combatnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/combatturns"
|
||||
)
|
||||
|
||||
func TestProductionCombatConfigurationResolvesTypedLane(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
cfg := productionCombatContractConfig()
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-combat", Catalog: catalogFromRegistries(components.registries)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
if effective.ResolvedPipeline.Chunk.Module != pipeline.DefaultChunkModule {
|
||||
t.Fatalf("chunk module = %q, want %q", effective.ResolvedPipeline.Chunk.Module, pipeline.DefaultChunkModule)
|
||||
}
|
||||
if len(effective.ResolvedPipeline.Steps[0].ArtifactLanes) != 1 {
|
||||
t.Fatalf("artifact lanes = %#v, want one combat lane", effective.ResolvedPipeline.Steps[0].ArtifactLanes)
|
||||
}
|
||||
lane := effective.ResolvedPipeline.Steps[0].ArtifactLanes[0]
|
||||
if lane.ID != "combat" || lane.ArtifactKind != dnd.CombatTurnListKind || lane.Extract.Module != combatextract.Key || lane.Extract.Retries != 2 || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != combatnormalize.Key {
|
||||
t.Fatalf("resolved combat lane = %#v, want typed production composition", lane)
|
||||
}
|
||||
|
||||
catalog := catalogFromRegistries(components.registries)
|
||||
extractSpec, ok := catalog.Extractors.Spec(combatextract.Key)
|
||||
if !ok || !reflect.DeepEqual(extractSpec.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(extractSpec.Provides, []string{"dnd.combat_turns"}) {
|
||||
t.Fatalf("combat extractor spec = %#v, want source and artifact capabilities", extractSpec)
|
||||
}
|
||||
normalizeSpec, ok := catalog.Normalizers.SpecForArtifact(combatnormalize.Key, dnd.CombatTurnListKind)
|
||||
if !ok || !reflect.DeepEqual(normalizeSpec.Requires, []string{"merged"}) || !reflect.DeepEqual(normalizeSpec.Provides, []string{"normalized"}) {
|
||||
t.Fatalf("combat normalizer spec = %#v, want merged/normalized capabilities", normalizeSpec)
|
||||
}
|
||||
mergeSpec, ok := catalog.Mergers.SpecForArtifact(pipeline.DefaultMergeModule, dnd.CombatTurnListKind)
|
||||
if !ok || !reflect.DeepEqual(mergeSpec.Provides, []string{"merged"}) {
|
||||
t.Fatalf("combat merger spec = %#v, want merged capability", mergeSpec)
|
||||
}
|
||||
codecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.CombatTurnListKind)
|
||||
if !ok || codecSpec.Schema.ID != "notarius.dnd.combat_turns" || codecSpec.Schema.Version != "v1" {
|
||||
t.Fatalf("combat codec spec = %#v, want compatible durable schema", codecSpec)
|
||||
}
|
||||
if !hasReferenceSlot(extractSpec.ReferenceSlots, "npcs") || !hasReferenceSlot(extractSpec.ReferenceSlots, "scene_descriptions") || !hasReferenceSlot(normalizeSpec.ReferenceSlots, "npcs") {
|
||||
t.Fatalf("combat reference slots = %#v / %#v, want extraction scene and NPC slots plus normalization NPC slot", extractSpec.ReferenceSlots, normalizeSpec.ReferenceSlots)
|
||||
}
|
||||
sceneSlot := referenceSlot(extractSpec.ReferenceSlots, "scene_descriptions")
|
||||
if !sceneSlot.Required || !reflect.DeepEqual(sceneSlot.AcceptedMediaTypes, []string{"application/json"}) || !reflect.DeepEqual(sceneSlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.SceneDescriptionListKind}) || sceneSlot.MaxBytes != 1048576 {
|
||||
t.Fatalf("scene description slot = %#v, want required approved scene artifact", sceneSlot)
|
||||
}
|
||||
|
||||
wantExtractChain := []pipeline.ModuleBinding{
|
||||
pipeline.Binding("generic/valid_json"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/shape"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/source_refs"),
|
||||
pipeline.Binding("generic/valid_json_schema"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/source_relatedness"),
|
||||
}
|
||||
wantNormalizeChain := []pipeline.ModuleBinding{
|
||||
pipeline.Binding("generic/valid_json"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/shape"),
|
||||
pipeline.Binding("normalize/dnd/combat-turns/invariants"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/source_refs"),
|
||||
pipeline.Binding("generic/valid_json_schema"),
|
||||
pipeline.Binding("extract/dnd/combat-turns/source_relatedness"),
|
||||
}
|
||||
if got := validatorChain(effective.ResolvedPipeline, pipeline.StageExtract, combatextract.Key); !reflect.DeepEqual(got, wantExtractChain) {
|
||||
t.Fatalf("combat extract chain = %#v, want %#v", got, wantExtractChain)
|
||||
}
|
||||
if got := validatorChain(effective.ResolvedPipeline, pipeline.StageNormalize, combatnormalize.Key); !reflect.DeepEqual(got, wantNormalizeChain) {
|
||||
t.Fatalf("combat normalize chain = %#v, want %#v", got, wantNormalizeChain)
|
||||
}
|
||||
if got := validatorChain(effective.ResolvedPipeline, pipeline.StageMerge, pipeline.DefaultMergeModule); len(got) != 0 {
|
||||
t.Fatalf("combat merge chain = %#v, want empty", got)
|
||||
}
|
||||
|
||||
bound, err := cfg.Resolve(config.ResolveInput{
|
||||
PipelineID: "dnd-combat",
|
||||
Catalog: catalog,
|
||||
ReferenceOverrides: []pipeline.ReferenceBinding{
|
||||
{Stage: pipeline.StageExtract, LaneID: "combat", SlotName: "npcs", Source: "npc-run/lanes/npcs.json", BindingSource: contracts.ReferenceBindingSourceCLI},
|
||||
{Stage: pipeline.StageNormalize, LaneID: "combat", SlotName: "npcs", Source: "npc-run/lanes/npcs.json", BindingSource: contracts.ReferenceBindingSourceCLI},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve(bound references) error = %v, want nil", err)
|
||||
}
|
||||
boundLane := bound.ResolvedPipeline.Steps[0].ArtifactLanes[0]
|
||||
if len(boundLane.ExtractReferences.Bindings) != 2 || len(boundLane.NormalizeReferences.Bindings) != 1 || !hasReferenceBinding(boundLane.ExtractReferences.Bindings, "npcs") || !hasReferenceBinding(boundLane.ExtractReferences.Bindings, "scene_descriptions") || !hasReferenceBinding(boundLane.NormalizeReferences.Bindings, "npcs") {
|
||||
t.Fatalf("bound combat references = %#v / %#v, want extraction scene and NPC bindings plus normalization NPC binding", boundLane.ExtractReferences, boundLane.NormalizeReferences)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionCombatConfigurationRequiresSceneDescriptions(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
cfg := productionCombatContractConfig()
|
||||
profile := cfg.Pipelines["dnd-combat"]
|
||||
profile.References = nil
|
||||
cfg.Pipelines["dnd-combat"] = profile
|
||||
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-combat", Catalog: catalogFromRegistries(components.registries)}); err == nil || !strings.Contains(err.Error(), "scene_descriptions") || !strings.Contains(err.Error(), "required") {
|
||||
t.Fatalf("Resolve() error = %v, want required scene reference failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionCombatConfigurationRejectsLooseOptionsAndLaneValidators(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
resolve := func(mutate func(*pipeline.PipelineProfile)) error {
|
||||
cfg := productionCombatContractConfig()
|
||||
profile := cfg.Pipelines["dnd-combat"]
|
||||
mutate(&profile)
|
||||
cfg.Pipelines["dnd-combat"] = profile
|
||||
_, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-combat", Catalog: catalogFromRegistries(components.registries)})
|
||||
return err
|
||||
}
|
||||
if err := resolve(func(profile *pipeline.PipelineProfile) {
|
||||
lane := profile.Artifacts["combat"]
|
||||
lane.Extract.Options = map[string]any{"unexpected": true}
|
||||
profile.Artifacts["combat"] = lane
|
||||
}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("unknown extractor option error = %v, want strict option rejection", err)
|
||||
}
|
||||
if err := resolve(func(profile *pipeline.PipelineProfile) {
|
||||
lane := profile.Artifacts["combat"]
|
||||
lane.Normalize.Options = map[string]any{"unexpected": true}
|
||||
profile.Artifacts["combat"] = lane
|
||||
}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("unknown normalizer option error = %v, want strict option rejection", err)
|
||||
}
|
||||
if err := resolve(func(profile *pipeline.PipelineProfile) {
|
||||
lane := profile.Artifacts["combat"]
|
||||
lane.Validators = []pipeline.ModuleBinding{pipeline.Binding("generic/always_accept")}
|
||||
profile.Artifacts["combat"] = lane
|
||||
}); err == nil || !strings.Contains(err.Error(), "artifact lane level") {
|
||||
t.Fatalf("lane-level validator error = %v, want invalid placement rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionCombatConfigurationResolvesTypedUnconditionalValidators(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
cfg := productionCombatContractConfig()
|
||||
profile := cfg.Pipelines["dnd-combat"]
|
||||
lane := profile.Artifacts["combat"]
|
||||
lane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{pipeline.Binding("generic/always_accept")}}
|
||||
lane.Normalize.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{pipeline.Binding("generic/always_reject")}}
|
||||
profile.Artifacts["combat"] = lane
|
||||
cfg.Pipelines["dnd-combat"] = profile
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-combat", Catalog: catalogFromRegistries(components.registries)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want typed unconditional validators to resolve", err)
|
||||
}
|
||||
if got := validatorChain(effective.ResolvedPipeline, pipeline.StageExtract, combatextract.Key); !reflect.DeepEqual(got, []pipeline.ModuleBinding{pipeline.Binding("generic/always_accept")}) {
|
||||
t.Fatalf("extract override chain = %#v, want typed always-accept", got)
|
||||
}
|
||||
if got := validatorChain(effective.ResolvedPipeline, pipeline.StageNormalize, combatnormalize.Key); !reflect.DeepEqual(got, []pipeline.ModuleBinding{pipeline.Binding("generic/always_reject")}) {
|
||||
t.Fatalf("normalize override chain = %#v, want typed always-reject", got)
|
||||
}
|
||||
}
|
||||
|
||||
func productionCombatContractConfig() config.Config {
|
||||
cfg := config.Default()
|
||||
cfg.Pipelines["dnd-combat"] = pipeline.PipelineProfile{
|
||||
ID: "dnd-combat",
|
||||
Input: pipeline.Binding("seriatim"),
|
||||
Chunk: pipeline.Binding(pipeline.DefaultChunkModule),
|
||||
References: map[string]pipeline.ReferenceSource{"scene_descriptions": pipeline.ExternalReference("scenes.json")},
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"combat": {
|
||||
Extract: pipeline.ModuleBinding{Module: combatextract.Key, Retries: 2},
|
||||
Normalize: pipeline.Binding(combatnormalize.Key),
|
||||
},
|
||||
},
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func hasReferenceSlot(slots []contracts.ReferenceSlot, name string) bool {
|
||||
for _, slot := range slots {
|
||||
if slot.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasReferenceBinding(bindings []pipeline.ReferenceBinding, name string) bool {
|
||||
for _, binding := range bindings {
|
||||
if binding.SlotName == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func referenceSlot(slots []contracts.ReferenceSlot, name string) contracts.ReferenceSlot {
|
||||
for _, slot := range slots {
|
||||
if slot.Name == name {
|
||||
return slot
|
||||
}
|
||||
}
|
||||
return contracts.ReferenceSlot{}
|
||||
}
|
||||
135
internal/cli/dnd_interactions_contract_test.go
Normal file
135
internal/cli/dnd_interactions_contract_test.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
interactioncodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcinteractions"
|
||||
interactionextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcinteractions"
|
||||
npcextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs"
|
||||
interactionnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcinteractions"
|
||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcs"
|
||||
)
|
||||
|
||||
func TestProductionNPCInteractionPipelineResolvesAndPrepares(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
resolved, err := pipeline.ResolvePipeline(npcInteractionProfile(pipeline.GeneratedReference("npcs", "npcs")), pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v", err)
|
||||
}
|
||||
if len(resolved.Steps) != 2 || len(resolved.Steps[1].ArtifactLanes) != 1 {
|
||||
t.Fatalf("resolved pipeline = %#v", resolved)
|
||||
}
|
||||
lane := resolved.Steps[1].ArtifactLanes[0]
|
||||
if lane.ArtifactKind != dnd.NPCInteractionListKind || lane.Extract.Module != interactionextract.Key || lane.Normalize.Module != interactionnormalize.Key {
|
||||
t.Fatalf("interaction lane = %#v", lane)
|
||||
}
|
||||
for _, bindings := range [][]pipeline.ReferenceBinding{lane.ExtractReferences.Bindings, lane.NormalizeReferences.Bindings} {
|
||||
if len(bindings) != 1 || bindings[0].SlotName != "npcs" || bindings[0].Artifact == nil || bindings[0].Artifact.Step != "npcs" || bindings[0].Artifact.Lane != "npcs" {
|
||||
t.Fatalf("generated bindings = %#v", bindings)
|
||||
}
|
||||
}
|
||||
if _, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
|
||||
catalog := catalogFromRegistries(components.registries)
|
||||
codecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.NPCInteractionListKind)
|
||||
if !ok || codecSpec.Schema.ID != interactioncodec.SchemaID || codecSpec.Schema.Version != interactioncodec.SchemaVersion {
|
||||
t.Fatalf("NPC interaction codec spec = %#v", codecSpec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionNPCInteractionReferencesRequireEarlierCompatibleProducer(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
catalog := catalogFromRegistries(components.registries)
|
||||
laterProfile := npcInteractionProfile(pipeline.GeneratedReference("npcs", "npcs"))
|
||||
laterProfile.Steps[0].ID = "seed"
|
||||
laterProfile.Steps[0].Artifacts["seed"] = laterProfile.Steps[0].Artifacts["npcs"]
|
||||
delete(laterProfile.Steps[0].Artifacts, "npcs")
|
||||
laterProfile.Steps = append(laterProfile.Steps, pipeline.PipelineStepProfile{ID: "future", Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"npcs": {Extract: pipeline.Binding(npcextract.Key), Normalize: pipeline.Binding(npcnormalize.Key)},
|
||||
}})
|
||||
laterProfile.Steps[1].References["npcs"] = pipeline.GeneratedReference("future", "npcs")
|
||||
tests := []struct {
|
||||
name string
|
||||
profile pipeline.PipelineProfile
|
||||
want string
|
||||
}{
|
||||
{name: "missing", profile: npcInteractionProfile(pipeline.ReferenceSource{}), want: "source must not be empty"},
|
||||
{name: "same step", profile: npcInteractionProfile(pipeline.GeneratedReference("interactions", "interactions")), want: "earlier step"},
|
||||
{name: "later step", profile: laterProfile, want: "earlier step"},
|
||||
{name: "wrong artifact kind", profile: npcInteractionProfile(pipeline.GeneratedReference("npcs", "npcs")), want: "does not accept artifact kind"},
|
||||
}
|
||||
tests[3].profile.Steps[0].Artifacts["npcs"] = pipeline.ArtifactLaneProfile{Extract: pipeline.Binding("dnd/spells")}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := pipeline.ResolvePipeline(test.profile, pipeline.ResolveOptions{}, catalog)
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionNPCInteractionReferencesRejectIncompatibleExternalRegistries(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
catalog := catalogFromRegistries(components.registries)
|
||||
root := t.TempDir()
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
file string
|
||||
content string
|
||||
prepare bool
|
||||
want string
|
||||
}{
|
||||
{name: "media type", file: "registry.txt", content: "not JSON", want: "media type"},
|
||||
{name: "artifact schema", file: "registry.json", content: `{"npcs":[{"name":"missing required fields"}]}`, prepare: true, want: "NPC registry"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
path := filepath.Join(root, test.file)
|
||||
if err := os.WriteFile(path, []byte(test.content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resolved, err := pipeline.ResolvePipeline(npcInteractionProfile(pipeline.ExternalReference(path)), pipeline.ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v", err)
|
||||
}
|
||||
materialized, _, err := pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
|
||||
if !test.prepare {
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("MaterializeReferences() error = %v, want %q", err, test.want)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v", err)
|
||||
}
|
||||
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Prepare() error = %v, want %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func npcInteractionProfile(reference pipeline.ReferenceSource) pipeline.PipelineProfile {
|
||||
profile := pipeline.PipelineProfile{
|
||||
ID: "dnd-npc-interactions",
|
||||
Input: pipeline.Binding("seriatim"),
|
||||
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
|
||||
Output: pipeline.Binding("json"),
|
||||
Steps: []pipeline.PipelineStepProfile{
|
||||
{ID: "npcs", Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"npcs": {Extract: pipeline.Binding(npcextract.Key), Normalize: pipeline.Binding(npcnormalize.Key)},
|
||||
}},
|
||||
{ID: "interactions", References: map[string]pipeline.ReferenceSource{"npcs": reference}, Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"interactions": {Extract: pipeline.Binding(interactionextract.Key), Normalize: pipeline.Binding(interactionnormalize.Key)},
|
||||
}},
|
||||
},
|
||||
}
|
||||
return profile
|
||||
}
|
||||
151
internal/cli/dnd_npc_contract_test.go
Normal file
151
internal/cli/dnd_npc_contract_test.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs"
|
||||
npcextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs"
|
||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcs"
|
||||
)
|
||||
|
||||
func TestProductionNPCConfigurationResolvesTypedLane(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
catalog := catalogFromRegistries(components.registries)
|
||||
cfg := productionNPCContractConfig()
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
if effective.ResolvedPipeline.Chunk.Module != pipeline.DefaultChunkModule {
|
||||
t.Fatalf("chunk module = %q, want %q", effective.ResolvedPipeline.Chunk.Module, pipeline.DefaultChunkModule)
|
||||
}
|
||||
if len(effective.ResolvedPipeline.Steps[0].ArtifactLanes) != 1 {
|
||||
t.Fatalf("artifact lanes = %#v, want one NPC lane", effective.ResolvedPipeline.Steps[0].ArtifactLanes)
|
||||
}
|
||||
lane := effective.ResolvedPipeline.Steps[0].ArtifactLanes[0]
|
||||
if lane.ID != "npcs" || lane.ArtifactKind != dnd.NPCListKind || lane.Extract.Module != npcextract.Key || lane.Extract.Retries != 2 || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != npcnormalize.Key {
|
||||
t.Fatalf("resolved NPC lane = %#v, want typed production composition", lane)
|
||||
}
|
||||
if len(lane.ExtractReferences.Bindings) != 0 || len(lane.NormalizeReferences.Bindings) != 0 {
|
||||
t.Fatalf("unbound NPC references = %#v / %#v, want none", lane.ExtractReferences, lane.NormalizeReferences)
|
||||
}
|
||||
|
||||
extractSpec, ok := catalog.Extractors.Spec(npcextract.Key)
|
||||
if !ok || !reflect.DeepEqual(extractSpec.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(extractSpec.Provides, []string{"dnd.npcs"}) {
|
||||
t.Fatalf("NPC extractor spec = %#v, want source and artifact capabilities", extractSpec)
|
||||
}
|
||||
mergeSpec, ok := catalog.Mergers.SpecForArtifact(pipeline.DefaultMergeModule, dnd.NPCListKind)
|
||||
if !ok || !reflect.DeepEqual(mergeSpec.Provides, []string{"merged"}) {
|
||||
t.Fatalf("NPC merger spec = %#v, want merged capability", mergeSpec)
|
||||
}
|
||||
normalizeSpec, ok := catalog.Normalizers.SpecForArtifact(npcnormalize.Key, dnd.NPCListKind)
|
||||
if !ok || !reflect.DeepEqual(normalizeSpec.Requires, []string{"merged"}) || !reflect.DeepEqual(normalizeSpec.Provides, []string{"normalized"}) {
|
||||
t.Fatalf("NPC normalizer spec = %#v, want merged/normalized capabilities", normalizeSpec)
|
||||
}
|
||||
codecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.NPCListKind)
|
||||
if !ok || codecSpec.Kind != dnd.NPCListKind || codecSpec.Schema.ID != npccodec.SchemaID || codecSpec.Schema.Version != npccodec.SchemaVersion {
|
||||
t.Fatalf("NPC codec spec = %#v, want typed v1 durable schema", codecSpec)
|
||||
}
|
||||
|
||||
wantExtractChain := []pipeline.ModuleBinding{
|
||||
pipeline.Binding("generic/valid_json"),
|
||||
pipeline.Binding("extract/dnd/npcs/shape"),
|
||||
pipeline.Binding("extract/dnd/npcs/source_refs"),
|
||||
pipeline.Binding("generic/valid_json_schema"),
|
||||
pipeline.Binding("extract/dnd/npcs/source_relatedness"),
|
||||
}
|
||||
wantNormalizeChain := []pipeline.ModuleBinding{
|
||||
pipeline.Binding("generic/valid_json"),
|
||||
pipeline.Binding("extract/dnd/npcs/shape"),
|
||||
pipeline.Binding("normalize/dnd/npcs/identity"),
|
||||
pipeline.Binding("extract/dnd/npcs/source_refs"),
|
||||
pipeline.Binding("generic/valid_json_schema"),
|
||||
pipeline.Binding("extract/dnd/npcs/source_relatedness"),
|
||||
}
|
||||
if got := validatorChain(effective.ResolvedPipeline, pipeline.StageExtract, npcextract.Key); !reflect.DeepEqual(got, wantExtractChain) {
|
||||
t.Fatalf("NPC extract chain = %#v, want %#v", got, wantExtractChain)
|
||||
}
|
||||
if got := validatorChain(effective.ResolvedPipeline, pipeline.StageNormalize, npcnormalize.Key); !reflect.DeepEqual(got, wantNormalizeChain) {
|
||||
t.Fatalf("NPC normalize chain = %#v, want %#v", got, wantNormalizeChain)
|
||||
}
|
||||
if got := validatorChain(effective.ResolvedPipeline, pipeline.StageMerge, pipeline.DefaultMergeModule); len(got) != 0 {
|
||||
t.Fatalf("NPC merge chain = %#v, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionNPCConfigurationValidatesOptionsReferencesAndPlacement(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
resolve := func(mutate func(*pipeline.PipelineProfile)) error {
|
||||
cfg := productionNPCContractConfig()
|
||||
profile := cfg.Pipelines["dnd-session"]
|
||||
mutate(&profile)
|
||||
cfg.Pipelines["dnd-session"] = profile
|
||||
_, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
|
||||
return err
|
||||
}
|
||||
|
||||
if err := resolve(func(profile *pipeline.PipelineProfile) {
|
||||
lane := profile.Artifacts["npcs"]
|
||||
lane.Extract.Options = map[string]any{"unexpected": true}
|
||||
profile.Artifacts["npcs"] = lane
|
||||
}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("unknown extractor option error = %v, want strict option rejection", err)
|
||||
}
|
||||
if err := resolve(func(profile *pipeline.PipelineProfile) {
|
||||
lane := profile.Artifacts["npcs"]
|
||||
lane.Normalize.Options = map[string]any{"unexpected": true}
|
||||
profile.Artifacts["npcs"] = lane
|
||||
}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
t.Fatalf("unknown normalizer option error = %v, want strict option rejection", err)
|
||||
}
|
||||
if err := resolve(func(profile *pipeline.PipelineProfile) {
|
||||
profile.References = pipeline.ExternalReferenceMap(map[string]string{
|
||||
"players": "players.txt",
|
||||
"party": "party.txt",
|
||||
"glossary": "glossary.txt",
|
||||
})
|
||||
}); err != nil {
|
||||
t.Fatalf("optional NPC references error = %v, want resolution success", err)
|
||||
}
|
||||
if err := resolve(func(profile *pipeline.PipelineProfile) {
|
||||
lane := profile.Artifacts["npcs"]
|
||||
lane.Validators = []pipeline.ModuleBinding{pipeline.Binding("normalize/dnd/npcs/identity")}
|
||||
profile.Artifacts["npcs"] = lane
|
||||
}); err == nil || !strings.Contains(err.Error(), "artifact lane level") {
|
||||
t.Fatalf("lane-level validator error = %v, want invalid placement rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func productionNPCContractConfig() config.Config {
|
||||
cfg := config.Default()
|
||||
cfg.Pipelines["dnd-session"] = pipeline.PipelineProfile{
|
||||
ID: "dnd-session",
|
||||
Input: pipeline.Binding("seriatim"),
|
||||
Chunk: pipeline.Binding(pipeline.DefaultChunkModule),
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"npcs": {
|
||||
Extract: pipeline.ModuleBinding{Module: npcextract.Key, Retries: 2},
|
||||
Normalize: pipeline.Binding(npcnormalize.Key),
|
||||
},
|
||||
},
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func validatorChain(resolved pipeline.ResolvedPipeline, stage pipeline.ModuleStage, module string) []pipeline.ModuleBinding {
|
||||
for _, chain := range resolved.ValidatorChains {
|
||||
if chain.Stage == stage && chain.ModuleKey == module {
|
||||
bindings := make([]pipeline.ModuleBinding, len(chain.Validators))
|
||||
for index, validator := range chain.Validators {
|
||||
bindings[index] = validator.Binding
|
||||
}
|
||||
return bindings
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
147
internal/cli/dnd_scene_descriptions_contract_test.go
Normal file
147
internal/cli/dnd_scene_descriptions_contract_test.go
Normal file
@@ -0,0 +1,147 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
||||
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
||||
scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
||||
)
|
||||
|
||||
func TestProductionSceneDescriptionWorkflow(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
cfg := config.Default()
|
||||
cfg.Pipelines["scene-descriptions"] = pipeline.PipelineProfile{
|
||||
ID: "scene-descriptions",
|
||||
Input: pipeline.Binding("seriatim"),
|
||||
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
|
||||
Output: pipeline.Binding("json"),
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"scene-descriptions": {
|
||||
Extract: pipeline.ModuleBinding{Module: sceneextract.Key, LLMProfile: "scene-description-profile"},
|
||||
Normalize: pipeline.Binding(scenenormalize.Key),
|
||||
},
|
||||
},
|
||||
}
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "scene-descriptions", Catalog: catalogFromRegistries(components.registries)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
lane := effective.ResolvedPipeline.Steps[0].ArtifactLanes[0]
|
||||
if lane.ArtifactKind != dnd.SceneDescriptionListKind || lane.Extract.Module != sceneextract.Key || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != scenenormalize.Key {
|
||||
t.Fatalf("resolved lane = %#v, want production scene-description composition", lane)
|
||||
}
|
||||
if len(lane.ExtractReferences.Bindings) != 0 || len(lane.NormalizeReferences.Bindings) != 0 {
|
||||
t.Fatalf("resolved references = %#v / %#v, want no generated or required references", lane.ExtractReferences, lane.NormalizeReferences)
|
||||
}
|
||||
|
||||
llmClient := &sceneDescriptionLLM{}
|
||||
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("run output = %#v, want one approved normalized artifact", output)
|
||||
}
|
||||
wantProfiles := []artifacts.LLMProfileManifest{{
|
||||
ID: "scene-description-profile",
|
||||
Provider: "promptkit",
|
||||
Model: "deterministic",
|
||||
}}
|
||||
if !reflect.DeepEqual(output.Manifest.LLMProfiles, wantProfiles) {
|
||||
t.Fatalf("manifest LLM profiles = %#v, want %#v", output.Manifest.LLMProfiles, wantProfiles)
|
||||
}
|
||||
normalizedOutput := output.NormalizeOutputs[0]
|
||||
if normalizedOutput.NormalizerKey != scenenormalize.Key || normalizedOutput.Artifact.Kind != dnd.SceneDescriptionListKind || normalizedOutput.Artifact.Schema.ID != scenecodec.SchemaID || normalizedOutput.Artifact.Schema.Name != scenecodec.SchemaName || normalizedOutput.Artifact.Schema.Version != scenecodec.SchemaVersion {
|
||||
t.Fatalf("normalized output = %#v, want registered durable scene-description schema", normalizedOutput)
|
||||
}
|
||||
|
||||
var value dnd.SceneDescriptionList
|
||||
if err := json.Unmarshal(normalizedOutput.Artifact.Content, &value); err != nil {
|
||||
t.Fatalf("decode normalized artifact: %v", err)
|
||||
}
|
||||
want := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{
|
||||
{ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, Kind: dnd.SceneKindNarrative, Title: "Aria casts Cure Wounds", Summary: "Aria casts Cure Wounds."},
|
||||
{ID: "chunk-000002", SourceRef: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}, Kind: dnd.SceneKindCombat, Title: "Bandit mage casts Shield", Summary: "The bandit mage casts Shield."},
|
||||
}}
|
||||
if !reflect.DeepEqual(value, want) {
|
||||
t.Fatalf("normalized scene descriptions = %#v, want %#v", value, want)
|
||||
}
|
||||
durable := decodeAssembledOutput[dnd.SceneDescriptionList](t, output.OutputFiles, "lanes/scene-descriptions.json")
|
||||
if !reflect.DeepEqual(durable, want) {
|
||||
t.Fatalf("durable output payload = %#v, want %#v", durable, want)
|
||||
}
|
||||
if len(output.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want grounded descriptions without warnings", output.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
type sceneDescriptionLLM struct {
|
||||
mu sync.Mutex
|
||||
profile *artifacts.LLMProfileManifest
|
||||
}
|
||||
|
||||
func (client *sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
if req.PromptID != sceneextract.PromptID {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
|
||||
}
|
||||
transcript := string(req.Inputs["transcript"].Content)
|
||||
var content string
|
||||
switch {
|
||||
case strings.Contains(transcript, "Cure Wounds"):
|
||||
content = `{"kind":"narrative","title":" Aria casts Cure Wounds ","summary":" Aria casts Cure Wounds. "}`
|
||||
case strings.Contains(transcript, "Shield"):
|
||||
content = `{"kind":"combat","title":"Bandit mage casts Shield","summary":"The bandit mage casts Shield."}`
|
||||
default:
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected transcript material %q", transcript)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(content), out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured response: %w", err)
|
||||
}
|
||||
profile := artifacts.LLMProfileManifest{
|
||||
ID: req.ProfileID,
|
||||
Provider: "promptkit",
|
||||
Model: "deterministic",
|
||||
}
|
||||
client.mu.Lock()
|
||||
client.profile = &profile
|
||||
client.mu.Unlock()
|
||||
return contracts.StructuredCompletionResponse{
|
||||
Content: []byte(content),
|
||||
Provider: profile.Provider,
|
||||
Model: profile.Model,
|
||||
ProfileID: profile.ID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (client *sceneDescriptionLLM) LLMProfileManifests() []artifacts.LLMProfileManifest {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
if client.profile == nil {
|
||||
return nil
|
||||
}
|
||||
return []artifacts.LLMProfileManifest{*client.profile}
|
||||
}
|
||||
266
internal/cli/example_contract_test.go
Normal file
266
internal/cli/example_contract_test.go
Normal file
@@ -0,0 +1,266 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
)
|
||||
|
||||
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
for _, example := range maintainedExampleFiles(t) {
|
||||
t.Run(example.name, func(t *testing.T) {
|
||||
cfg := loadMaintainedExample(t, example.path)
|
||||
raw, err := os.ReadFile(example.transcriptPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read maintained transcript %q: %v", example.transcriptPath, err)
|
||||
}
|
||||
document, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{
|
||||
Path: example.transcriptPath,
|
||||
Raw: raw,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("parse maintained transcript %q: %v", example.transcriptPath, err)
|
||||
}
|
||||
if len(document.Units) == 0 {
|
||||
t.Fatalf("maintained transcript %q has no parsed units", example.transcriptPath)
|
||||
}
|
||||
for _, pipelineID := range example.pipelineIDs {
|
||||
effective, err := cfg.Resolve(resolveInputForMaintainedExample(components, pipelineID))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve maintained example %q: %v", pipelineID, err)
|
||||
}
|
||||
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
|
||||
ConfigPath: example.path,
|
||||
WorkingDir: filepath.Dir(example.path),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err)
|
||||
}
|
||||
if example.name == "complete" {
|
||||
if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-events,npcs,scene-descriptions|extract-events:combat-turns,npc-interactions,spells" {
|
||||
t.Fatalf("complete example steps and lanes = %v, want every D&D extractor in the documented two-step composition", got)
|
||||
}
|
||||
spellLane := referenceContractLane(t, materialized, "spells")
|
||||
if len(spellLane.ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 ||
|
||||
len(spellLane.NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 {
|
||||
t.Fatalf("complete example spell catalog reference was not materialized: %#v", spellLane)
|
||||
}
|
||||
itemEventLane := referenceContractLane(t, materialized, "item-events")
|
||||
for _, references := range []pipeline.ResolvedReferenceTarget{itemEventLane.ExtractReferences, itemEventLane.NormalizeReferences} {
|
||||
if _, found := references.ReferenceSet.Slots["npcs"]; found {
|
||||
t.Fatalf("item event lane unexpectedly depends on generated NPCs: %#v", itemEventLane)
|
||||
}
|
||||
if _, found := references.ReferenceSet.Slots["scene_descriptions"]; found {
|
||||
t.Fatalf("item event lane unexpectedly depends on generated scene descriptions: %#v", itemEventLane)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{"pipelines", "list", "--config", example.path}, &stdout, &stderr, productionOptionsFromComponents(components))
|
||||
if code != 0 || stdout.String() != strings.Join(example.pipelineIDs, "\n")+"\n" || stderr.Len() != 0 {
|
||||
t.Fatalf("pipelines list: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintainedConfigurationExampleSet(t *testing.T) {
|
||||
entries, err := os.ReadDir(repositoryPath("examples"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".config.yml") {
|
||||
names = append(names, entry.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
if got := strings.Join(names, ","); got != "dnd-complete.config.yml,dnd-minimal.config.yml" {
|
||||
t.Fatalf("maintained configuration examples = %q, want only the minimal and complete D&D examples", got)
|
||||
}
|
||||
|
||||
profileEntries, err := os.ReadDir(repositoryPath("examples", "profiles"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names = names[:0]
|
||||
for _, entry := range profileEntries {
|
||||
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".yml") {
|
||||
names = append(names, entry.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
if got := strings.Join(names, ","); got != "dnd-extraction.yml" {
|
||||
t.Fatalf("maintained operator profiles = %q, want dnd-extraction.yml", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintainedExamplesValidateEffectiveProfilesOffline(t *testing.T) {
|
||||
t.Chdir(repositoryPath())
|
||||
t.Setenv("OPENROUTER_API_KEY", "")
|
||||
for _, example := range maintainedExampleFiles(t) {
|
||||
t.Run(example.name, func(t *testing.T) {
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"config", "validate", "--config", example.path, "--pipeline", "dnd-session",
|
||||
}, &stdout, &stderr, Options{})
|
||||
if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `valid for pipeline "dnd-session"`) {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
|
||||
result := make([]string, 0, len(resolved.Steps))
|
||||
for _, step := range resolved.Steps {
|
||||
laneIDs := make([]string, 0, len(step.ArtifactLanes))
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
laneIDs = append(laneIDs, lane.ID)
|
||||
}
|
||||
sort.Strings(laneIDs)
|
||||
result = append(result, step.ID+":"+strings.Join(laneIDs, ","))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
fake := &productionFakeLLMClient{}
|
||||
options := productionRunOptions(t, fake)
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session",
|
||||
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
||||
}, &stdout, &stderr, options)
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), `pipeline "dnd-session"`) || !strings.Contains(stdout.String(), "outputs=1 rejected=0") {
|
||||
t.Fatalf("stdout=%q, want completed pipeline and counts", stdout.String())
|
||||
}
|
||||
|
||||
runRoot := filepath.Join(outputRoot, productionRunID)
|
||||
index := readProductionJSON[exampleOutputIndex](t, filepath.Join(runRoot, "index.json"))
|
||||
if index.ManifestFile != "manifest.json" || index.RejectedFile != "rejected.json" || index.WarningsFile != "warnings.json" || len(index.OutputFiles) != 1 {
|
||||
t.Fatalf("index = %#v, want one spells output and fixed companion files", index)
|
||||
}
|
||||
entry := index.OutputFiles[0]
|
||||
if entry.LaneID != "spells" || entry.File != "lanes/spells.json" || entry.MediaType != "application/json" || entry.SchemaID != "notarius.dnd.spells" || entry.SchemaVersion != "v1" {
|
||||
t.Fatalf("index output entry = %#v, want spells JSON contract", entry)
|
||||
}
|
||||
|
||||
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
|
||||
if manifest.PipelineID != "dnd-session" || manifest.InputModule != "seriatim" || manifest.Chunker != "generic" || manifest.OutputEncoder != "json" || manifest.ValidationStatus != "approved" || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" {
|
||||
t.Fatalf("manifest = %#v, want approved minimal run", manifest)
|
||||
}
|
||||
if len(manifest.ArtifactLanes) != 1 {
|
||||
t.Fatalf("manifest lanes = %#v, want exactly spells", manifest.ArtifactLanes)
|
||||
}
|
||||
lane := manifest.ArtifactLanes[0]
|
||||
if lane.ID != "spells" || lane.Extractor != "dnd/spells" || lane.Merger != "appendorder" || lane.Normalizer != spellnormalize.Key {
|
||||
t.Fatalf("manifest lane = %#v, want production spells composition", lane)
|
||||
}
|
||||
if len(manifest.References) != 0 {
|
||||
t.Fatalf("base-only manifest references = %#v, want no overlay provenance", manifest.References)
|
||||
}
|
||||
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
||||
if !ok || len(stringValues(extractorMetadata["catalog_overlay_ids"])) != 0 {
|
||||
t.Fatalf("base-only extractor metadata = %#v, want no overlay IDs", lane.Metadata)
|
||||
}
|
||||
|
||||
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, entry.File))
|
||||
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Cure Wounds" || artifact.SpellCasts[0].SourceRefs[0].SourceID != "session-alpha" {
|
||||
t.Fatalf("artifact = %#v, want one source-linked Cure Wounds cast", artifact)
|
||||
}
|
||||
rejected := readProductionJSON[struct {
|
||||
Rejected []json.RawMessage `json:"rejected"`
|
||||
}](t, filepath.Join(runRoot, "rejected.json"))
|
||||
if len(rejected.Rejected) != 0 {
|
||||
t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected)
|
||||
}
|
||||
warnings := readProductionJSON[struct {
|
||||
Warnings []json.RawMessage `json:"warnings"`
|
||||
}](t, filepath.Join(runRoot, "warnings.json"))
|
||||
if len(warnings.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want empty warning list", warnings.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing.T) {
|
||||
malformed := filepath.Join(t.TempDir(), "malformed.json")
|
||||
if err := os.WriteFile(malformed, []byte("{not valid json"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, debug := range []bool{false, true} {
|
||||
name := "without debug"
|
||||
if debug {
|
||||
name = "with debug"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
debugRoot := filepath.Join(t.TempDir(), "debug")
|
||||
options := productionRunOptions(t, &productionFakeLLMClient{})
|
||||
args := []string{
|
||||
"run", "dnd-session",
|
||||
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
||||
"--input", malformed, "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
||||
}
|
||||
if debug {
|
||||
args = append(args, "--debug", "--debug-dir", debugRoot)
|
||||
}
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions(args, &stdout, &stderr, options)
|
||||
if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "parse input") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertAbsent(t, outputRoot)
|
||||
if !debug {
|
||||
assertAbsent(t, debugRoot)
|
||||
return
|
||||
}
|
||||
bundle := onlyChildDir(t, debugRoot)
|
||||
report := readProductionJSON[debugbundle.RunReport](t, filepath.Join(bundle, "summary", "run-report.json"))
|
||||
if report.Succeeded || report.PipelineID != "dnd-session" {
|
||||
t.Fatalf("failure report = %#v, want failed dnd-session report", report)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type exampleOutputIndex struct {
|
||||
ManifestFile string `json:"manifest_file"`
|
||||
OutputFiles []exampleOutputIndexEntry `json:"output_files"`
|
||||
RejectedFile string `json:"rejected_file"`
|
||||
WarningsFile string `json:"warnings_file"`
|
||||
}
|
||||
|
||||
type exampleOutputIndexEntry struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
MediaType string `json:"media_type"`
|
||||
File string `json:"file"`
|
||||
SchemaID string `json:"schema_id"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
}
|
||||
|
||||
func resolveInputForMaintainedExample(components productionComponents, pipelineID string) config.ResolveInput {
|
||||
return config.ResolveInput{PipelineID: pipelineID, Catalog: catalogFromRegistries(components.registries)}
|
||||
}
|
||||
80
internal/cli/npc_registry_contract_test.go
Normal file
80
internal/cli/npc_registry_contract_test.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestOversizedNPCRegistryFailsBeforeRuntimeAndCheckpointConstruction(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
npcPath := filepath.Join(t.TempDir(), "npcs.json")
|
||||
if err := os.WriteFile(npcPath, []byte(strings.Repeat("x", 1048577)), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
|
||||
content := fmt.Sprintf(`version: 4
|
||||
cache:
|
||||
chunk_plans:
|
||||
mode: bypass
|
||||
checkpoints:
|
||||
enabled: true
|
||||
directory: %q
|
||||
pipelines:
|
||||
dnd-session:
|
||||
input: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract:
|
||||
module: dnd/spells
|
||||
references:
|
||||
npcs: %q
|
||||
normalize: dnd/spells
|
||||
`, checkpointRoot, npcPath)
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
llmConstructed := false
|
||||
chunkStoreConstructed := false
|
||||
options := Options{
|
||||
Catalog: catalogFromRegistries(components.registries),
|
||||
Registries: components.registries,
|
||||
LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
llmConstructed = true
|
||||
return nil, nil, errors.New("LLM client must not be constructed")
|
||||
},
|
||||
ChunkPlanStoreFactory: func(string) (pipeline.ChunkPlanStore, error) {
|
||||
chunkStoreConstructed = true
|
||||
return nil, errors.New("chunk-plan store must not be constructed")
|
||||
},
|
||||
}
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session", "--config", configPath,
|
||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||
"--chunk_cache", "bypass", "--output-dir", t.TempDir(),
|
||||
}, &stdout, &stderr, options)
|
||||
for _, fragment := range []string{`pipeline "dnd-session"`, `reference slot "npcs"`, "1048577 bytes", "limit 1048576"} {
|
||||
if code == 0 || !strings.Contains(stderr.String(), fragment) {
|
||||
t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, stderr.String(), fragment)
|
||||
}
|
||||
}
|
||||
if llmConstructed || chunkStoreConstructed {
|
||||
t.Fatalf("runtime construction = LLM %t, chunk store %t; want materialization failure first", llmConstructed, chunkStoreConstructed)
|
||||
}
|
||||
if _, err := os.Stat(checkpointRoot); !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Fatalf("checkpoint root stat error = %v, want no checkpoint allocation", err)
|
||||
}
|
||||
}
|
||||
1077
internal/cli/production_contract_test.go
Normal file
1077
internal/cli/production_contract_test.go
Normal file
File diff suppressed because it is too large
Load Diff
46
internal/cli/promptkit_profiles.go
Normal file
46
internal/cli/promptkit_profiles.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string, assets *llm.AssetRegistry) error {
|
||||
if len(profileIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
inspector, err := llm.NewPromptKitProfileInspector(llm.PromptKitProfileInspectorConfig{
|
||||
Source: promptKitProfileSourceConfig(cfg),
|
||||
Assets: assets,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("load PromptKit profiles: %w", err)
|
||||
}
|
||||
for _, profileID := range profileIDs {
|
||||
if _, err := inspector.InspectProfile(ctx, profileID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func promptKitProfileSourceConfig(cfg config.Config) llm.PromptKitProfileSourceConfig {
|
||||
return llm.PromptKitProfileSourceConfig{
|
||||
ProfileDir: cfg.PromptKit.ProfileDir,
|
||||
ProfileFile: cfg.PromptKit.ProfileFile,
|
||||
LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend),
|
||||
}
|
||||
}
|
||||
|
||||
func mapPromptKitLocalBackend(cfg *config.PromptKitLocalBackendConfig) *llm.PromptKitLocalBackendConfig {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
return &llm.PromptKitLocalBackendConfig{
|
||||
Endpoint: cfg.Endpoint,
|
||||
ConcurrencyLimit: cfg.ConcurrencyLimit,
|
||||
}
|
||||
}
|
||||
159
internal/cli/promptkit_profiles_test.go
Normal file
159
internal/cli/promptkit_profiles_test.go
Normal file
@@ -0,0 +1,159 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) {
|
||||
var providerCalls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
||||
providerCalls.Add(1)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
writeProfile := func(t *testing.T, name, content string) string {
|
||||
t.Helper()
|
||||
profilePath := filepath.Join(t.TempDir(), name+".yaml")
|
||||
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return profilePath
|
||||
}
|
||||
localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n"
|
||||
credentialProfile := `id: credential-profile
|
||||
endpoint: ` + server.URL + `/v1
|
||||
model: credential-model
|
||||
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
`
|
||||
t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
profilePath string
|
||||
profileID string
|
||||
profileDir bool
|
||||
localBackend bool
|
||||
canceled bool
|
||||
wantErr []string
|
||||
rejectErr []string
|
||||
}{
|
||||
{
|
||||
name: "configured local backend",
|
||||
profilePath: writeProfile(t, "local-profile", localProfile),
|
||||
profileID: "local-profile",
|
||||
profileDir: true,
|
||||
localBackend: true,
|
||||
},
|
||||
{
|
||||
name: "missing local backend registration",
|
||||
profilePath: writeProfile(t, "local-profile", localProfile),
|
||||
profileID: "local-profile",
|
||||
wantErr: []string{`PromptKit profile "local-profile" is invalid or unreadable`},
|
||||
},
|
||||
{
|
||||
name: "absent profile",
|
||||
profilePath: writeProfile(t, "local-profile", localProfile),
|
||||
profileID: "absent-profile",
|
||||
localBackend: true,
|
||||
wantErr: []string{`PromptKit profile "absent-profile" is not configured`},
|
||||
},
|
||||
{
|
||||
name: "malformed profile",
|
||||
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
|
||||
profileID: "malformed-profile",
|
||||
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
|
||||
rejectErr: []string{"malformed-profile.yaml", "backend: ["},
|
||||
},
|
||||
{
|
||||
name: "invalid profile source",
|
||||
profilePath: filepath.Join(t.TempDir(), "missing-profile.yaml"),
|
||||
profileID: "missing-profile",
|
||||
wantErr: []string{"load PromptKit profiles", "profile configuration is invalid or unreadable"},
|
||||
},
|
||||
{
|
||||
name: "credential environment intentionally unset",
|
||||
profilePath: writeProfile(t, "credential-profile", credentialProfile),
|
||||
profileID: "credential-profile",
|
||||
},
|
||||
{
|
||||
name: "canceled inspection",
|
||||
profilePath: writeProfile(t, "local-profile", localProfile),
|
||||
profileID: "local-profile",
|
||||
localBackend: true,
|
||||
canceled: true,
|
||||
wantErr: []string{"context canceled"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
if tt.profileDir {
|
||||
cfg.PromptKit.ProfileDir = filepath.Dir(tt.profilePath)
|
||||
} else {
|
||||
cfg.PromptKit.ProfileFile = tt.profilePath
|
||||
}
|
||||
if tt.localBackend {
|
||||
cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
|
||||
Endpoint: server.URL + "/v1",
|
||||
ConcurrencyLimit: 2,
|
||||
}
|
||||
}
|
||||
ctx := context.Background()
|
||||
if tt.canceled {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithCancel(ctx)
|
||||
cancel()
|
||||
}
|
||||
err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID}, nil)
|
||||
if len(tt.wantErr) == 0 {
|
||||
if err != nil {
|
||||
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("validateExplicitPromptKitProfiles() error = nil, want failure")
|
||||
}
|
||||
if tt.canceled && !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled inspection error = %v, want context canceled", err)
|
||||
}
|
||||
for _, want := range tt.wantErr {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("validation error = %q, want %q", err, want)
|
||||
}
|
||||
}
|
||||
for _, rejected := range append(tt.rejectErr, tt.profilePath) {
|
||||
if rejected != "" && strings.Contains(err.Error(), rejected) {
|
||||
t.Fatalf("validation error = %q, must not expose %q", err, rejected)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if providerCalls.Load() != 0 {
|
||||
t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
||||
assets := llm.NewAssetRegistry()
|
||||
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"profiles/fallback.yaml": {Data: []byte("id: fallback-profile\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
|
||||
}, "profiles"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"fallback-profile"}, assets); err != nil {
|
||||
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
323
internal/cli/recompute_execution_contract_test.go
Normal file
323
internal/cli/recompute_execution_contract_test.go
Normal file
@@ -0,0 +1,323 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRecomputeStepRecoversThroughFilesystemCheckpoints(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
invalidateOutput bool
|
||||
wantCode int
|
||||
}{
|
||||
{name: "accepted producer is hydrated", wantCode: 0},
|
||||
{name: "invalid producer stops dependents", invalidateOutput: true, wantCode: 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newRecomputeTestRoots(t)
|
||||
harness := newRecomputeTestHarness()
|
||||
fresh := runRecomputeCommand(roots, harness.options(), false)
|
||||
if fresh.code != 0 {
|
||||
t.Fatalf("fresh run code=%d stderr=%q", fresh.code, fresh.stderr)
|
||||
}
|
||||
removeCheckpointLaneStage(t, roots.checkpoints, "extract", "first", "producer")
|
||||
removeCheckpointLaneStage(t, roots.checkpoints, "merge", "first", "producer")
|
||||
if tt.invalidateOutput {
|
||||
path := findCheckpointFile(t, roots.checkpoints, "normalize", "first", "producer", "output.json")
|
||||
if err := os.WriteFile(path, []byte("{"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
harness.resetCalls()
|
||||
|
||||
resumed := runRecomputeCommand(roots, harness.options(), true)
|
||||
if resumed.code != tt.wantCode {
|
||||
t.Fatalf("resumed code=%d stdout=%q stderr=%q", resumed.code, resumed.stdout, resumed.stderr)
|
||||
}
|
||||
events := readLatestCheckpointEvents(t, roots.debug)
|
||||
if tt.invalidateOutput {
|
||||
if harness.callsFor("test/extract/middle") != 0 || harness.callsFor("test/extract/dependent") != 0 {
|
||||
t.Fatalf("dependent calls after invalid producer = %#v", harness.callsSnapshot())
|
||||
}
|
||||
if !strings.Contains(resumed.stderr, string(pipeline.CheckpointReasonDecodeFailed)) {
|
||||
t.Fatalf("stderr=%q, want stable checkpoint reason", resumed.stderr)
|
||||
}
|
||||
assertNormalizeDecisionSequence(t, events, []checkpointDecisionExpectation{{"first", "producer", pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonDecodeFailed}})
|
||||
return
|
||||
}
|
||||
|
||||
if got := harness.callsSnapshot(); !reflect.DeepEqual(got, map[string]int{"test/extract/dependent": 1, "test/extract/middle": 1}) {
|
||||
t.Fatalf("resumed extractor calls = %#v", got)
|
||||
}
|
||||
outputPath := filepath.Join(latestChildDir(t, roots.output), "result.json")
|
||||
data, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "[\"producer\",\"unrelated\",\"middle\",\"dependent\"]\n" {
|
||||
t.Fatalf("ordered output = %q", data)
|
||||
}
|
||||
assertNormalizeDecisionSequence(t, events, []checkpointDecisionExpectation{
|
||||
{"first", "producer", pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonAcceptedArtifactReused},
|
||||
{"first", "unrelated", pipeline.CheckpointDecisionReused, pipeline.CheckpointReasonReused},
|
||||
{"second", "middle", pipeline.CheckpointDecisionForcedRecompute, pipeline.CheckpointReasonRecomputeStep},
|
||||
{"third", "dependent", pipeline.CheckpointDecisionForcedRecompute, pipeline.CheckpointReasonRecomputeStep},
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type checkpointDecisionExpectation struct {
|
||||
step, lane string
|
||||
action pipeline.CheckpointDecisionCategory
|
||||
reason pipeline.CheckpointReasonCode
|
||||
}
|
||||
|
||||
func assertNormalizeDecisionSequence(t *testing.T, events []pipeline.CheckpointEvent, want []checkpointDecisionExpectation) {
|
||||
t.Helper()
|
||||
var got []checkpointDecisionExpectation
|
||||
for _, event := range events {
|
||||
if event.Stage == string(pipeline.StageNormalize) {
|
||||
got = append(got, checkpointDecisionExpectation{event.StepID, event.LaneID, event.Action, event.ReasonCode})
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("normalize decisions = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
type recomputeTestHarness struct {
|
||||
base *stateTestHarness
|
||||
mu sync.Mutex
|
||||
calls map[string]int
|
||||
}
|
||||
|
||||
func newRecomputeTestHarness() *recomputeTestHarness {
|
||||
return &recomputeTestHarness{base: newStateTestHarness(), calls: make(map[string]int)}
|
||||
}
|
||||
|
||||
func (h *recomputeTestHarness) options() Options {
|
||||
opts := h.base.options()
|
||||
for _, key := range []string{"test/extract/producer", "test/extract/unrelated", "test/extract/middle", "test/extract/dependent"} {
|
||||
moduleKey := key
|
||||
spec := pipeline.ModuleSpec{
|
||||
Key: moduleKey, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind,
|
||||
ReferenceSlots: []contracts.ReferenceSlot{{Name: "upstream", AcceptedMediaTypes: []string{"application/json"}, AcceptedArtifactKinds: []contracts.ArtifactKind{stateTestArtifactKind}}},
|
||||
}
|
||||
if err := pipeline.RegisterExtractor(opts.Registries.Extractors, spec, func() (contracts.Extractor[stateTestArtifact], error) {
|
||||
return recomputeTestExtractor{key: moduleKey, harness: h}, nil
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
if err := opts.Registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/recompute-output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
|
||||
return recomputeTestOutput{}, nil
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
opts.Catalog = catalogFromRegistries(opts.Registries)
|
||||
return opts
|
||||
}
|
||||
|
||||
func (h *recomputeTestHarness) record(key string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.calls[key]++
|
||||
}
|
||||
|
||||
func (h *recomputeTestHarness) resetCalls() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.calls = make(map[string]int)
|
||||
}
|
||||
|
||||
func (h *recomputeTestHarness) callsFor(key string) int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.calls[key]
|
||||
}
|
||||
|
||||
func (h *recomputeTestHarness) callsSnapshot() map[string]int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
result := make(map[string]int, len(h.calls))
|
||||
for key, value := range h.calls {
|
||||
result[key] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type recomputeTestExtractor struct {
|
||||
key string
|
||||
harness *recomputeTestHarness
|
||||
}
|
||||
|
||||
func (e recomputeTestExtractor) Key() string { return e.key }
|
||||
func (e recomputeTestExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return []contracts.ReferenceSlot{{Name: "upstream", AcceptedMediaTypes: []string{"application/json"}, AcceptedArtifactKinds: []contracts.ArtifactKind{stateTestArtifactKind}}}
|
||||
}
|
||||
func (e recomputeTestExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[stateTestArtifact], error) {
|
||||
e.harness.record(e.key)
|
||||
return contracts.TypedExtractionResult[stateTestArtifact]{Value: stateTestArtifact{Value: e.key}}, nil
|
||||
}
|
||||
|
||||
type recomputeTestOutput struct{}
|
||||
|
||||
func (recomputeTestOutput) Key() string { return "test/recompute-output" }
|
||||
func (recomputeTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
lanes := make([]string, 0, len(req.NormalizeOutputs))
|
||||
for _, output := range req.NormalizeOutputs {
|
||||
lanes = append(lanes, output.LaneID)
|
||||
}
|
||||
data, err := json.Marshal(lanes)
|
||||
if err != nil {
|
||||
return contracts.OutputResult{}, err
|
||||
}
|
||||
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: append(data, '\n')}}}, nil
|
||||
}
|
||||
|
||||
func newRecomputeTestRoots(t *testing.T) stateTestRoots {
|
||||
t.Helper()
|
||||
roots := newStateTestRoots(t)
|
||||
config := fmt.Sprintf(`version: 4
|
||||
output:
|
||||
directory: %q
|
||||
cache:
|
||||
chunk_plans:
|
||||
directory: %q
|
||||
mode: bypass
|
||||
checkpoints:
|
||||
enabled: true
|
||||
directory: %q
|
||||
debug:
|
||||
directory: %q
|
||||
pipelines:
|
||||
sample:
|
||||
input: test/input
|
||||
chunk: test/chunk
|
||||
steps:
|
||||
- id: first
|
||||
artifacts:
|
||||
producer:
|
||||
extract: test/extract/producer
|
||||
merge: test/merge
|
||||
normalize: test/normalize
|
||||
unrelated:
|
||||
extract: test/extract/unrelated
|
||||
merge: test/merge
|
||||
normalize: test/normalize
|
||||
- id: second
|
||||
references:
|
||||
upstream:
|
||||
artifact:
|
||||
step: first
|
||||
lane: producer
|
||||
artifacts:
|
||||
middle:
|
||||
extract: test/extract/middle
|
||||
merge: test/merge
|
||||
normalize: test/normalize
|
||||
- id: third
|
||||
references:
|
||||
upstream:
|
||||
artifact:
|
||||
step: second
|
||||
lane: middle
|
||||
artifacts:
|
||||
dependent:
|
||||
extract: test/extract/dependent
|
||||
merge: test/merge
|
||||
normalize: test/normalize
|
||||
output: test/recompute-output
|
||||
`, roots.output, roots.plans, roots.checkpoints, roots.debug)
|
||||
if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return roots
|
||||
}
|
||||
|
||||
func runRecomputeCommand(roots stateTestRoots, opts Options, recompute bool) stateTestResult {
|
||||
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
|
||||
if recompute {
|
||||
args = append(args, "--resume", "--recompute-step", "second", "--debug")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
return stateTestResult{code: RunWithOptions(args, &stdout, &stderr, opts), stdout: stdout.String(), stderr: stderr.String()}
|
||||
}
|
||||
|
||||
func removeCheckpointLaneStage(t *testing.T, root, stage, step, lane string) {
|
||||
t.Helper()
|
||||
dir := filepath.Dir(findCheckpointFile(t, root, stage, step, lane, "manifest.json"))
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func findCheckpointFile(t *testing.T, root, stage, step, lane, name string) string {
|
||||
t.Helper()
|
||||
want := filepath.Join(stage, step, lane, name)
|
||||
var matches []string
|
||||
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !entry.IsDir() && strings.HasSuffix(path, want) {
|
||||
matches = append(matches, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(matches) != 1 {
|
||||
t.Fatalf("checkpoint files ending in %q = %v", want, matches)
|
||||
}
|
||||
return matches[0]
|
||||
}
|
||||
|
||||
func latestChildDir(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var dirs []string
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
dirs = append(dirs, filepath.Join(root, entry.Name()))
|
||||
}
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
t.Fatal("no child directory")
|
||||
}
|
||||
sort.Strings(dirs)
|
||||
return dirs[len(dirs)-1]
|
||||
}
|
||||
|
||||
func readLatestCheckpointEvents(t *testing.T, root string) []pipeline.CheckpointEvent {
|
||||
t.Helper()
|
||||
var events []pipeline.CheckpointEvent
|
||||
data, err := os.ReadFile(filepath.Join(latestChildDir(t, root), "summary", "checkpoint-events.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &events); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return events
|
||||
}
|
||||
56
internal/cli/recompute_policy_test.go
Normal file
56
internal/cli/recompute_policy_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRecomputePolicyIncludesDependentLanesAndReusablePredecessors(t *testing.T) {
|
||||
producer := pipeline.ResolvedArtifactLane{StepID: "first", ID: "producer"}
|
||||
unrelated := pipeline.ResolvedArtifactLane{StepID: "first", ID: "unrelated"}
|
||||
consumer := pipeline.ResolvedArtifactLane{
|
||||
StepID: "second", ID: "consumer",
|
||||
ExtractReferences: pipeline.ResolvedReferenceTarget{Bindings: []pipeline.ReferenceBinding{{Artifact: &pipeline.ArtifactReference{Step: "first", Lane: "producer"}}}},
|
||||
}
|
||||
downstream := pipeline.ResolvedArtifactLane{
|
||||
StepID: "third", ID: "downstream",
|
||||
ExtractReferences: pipeline.ResolvedReferenceTarget{Bindings: []pipeline.ReferenceBinding{{Artifact: &pipeline.ArtifactReference{Step: "second", Lane: "consumer"}}}},
|
||||
}
|
||||
independent := pipeline.ResolvedArtifactLane{StepID: "third", ID: "independent"}
|
||||
resolved := pipeline.ResolvedPipeline{Steps: []pipeline.ResolvedPipelineStep{
|
||||
{ID: "first", ArtifactLanes: []pipeline.ResolvedArtifactLane{producer, unrelated}},
|
||||
{ID: "second", ArtifactLanes: []pipeline.ResolvedArtifactLane{consumer}},
|
||||
{ID: "third", ArtifactLanes: []pipeline.ResolvedArtifactLane{downstream, independent}},
|
||||
}}
|
||||
|
||||
policy, err := recomputePolicy(resolved, "second")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("second", "consumer")]; !ok {
|
||||
t.Fatal("selected lane was not forced")
|
||||
}
|
||||
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("third", "downstream")]; !ok {
|
||||
t.Fatal("transitive dependent lane was not forced")
|
||||
}
|
||||
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("first", "producer")]; ok {
|
||||
t.Fatal("predecessor was implicitly forced")
|
||||
}
|
||||
if _, ok := policy.RequireReusableLanes[pipeline.CheckpointLaneKey("first", "producer")]; !ok {
|
||||
t.Fatal("required predecessor was not marked reusable")
|
||||
}
|
||||
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("first", "unrelated")]; ok {
|
||||
t.Fatal("unrelated lane was forced")
|
||||
}
|
||||
if _, ok := policy.ForcedLanes[pipeline.CheckpointLaneKey("third", "independent")]; ok {
|
||||
t.Fatal("unrelated later lane was forced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecomputePolicyRejectsUnknownStep(t *testing.T) {
|
||||
_, err := recomputePolicy(pipeline.ResolvedPipeline{Steps: []pipeline.ResolvedPipelineStep{{ID: "known"}}}, "missing")
|
||||
if err == nil {
|
||||
t.Fatal("unknown step was accepted")
|
||||
}
|
||||
}
|
||||
456
internal/cli/reference_contract_test.go
Normal file
456
internal/cli/reference_contract_test.go
Normal file
@@ -0,0 +1,456 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestReferenceSelectorsParseAndApplyAllDocumentedForms(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
selector string
|
||||
only []string
|
||||
wantStage pipeline.ModuleStage
|
||||
wantLane string
|
||||
wantSlot string
|
||||
}{
|
||||
{name: "flat", selector: "alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"},
|
||||
{name: "chunk", selector: "chunk.chunk-slot", wantStage: pipeline.StageChunk, wantSlot: "chunk-slot"},
|
||||
{name: "merge", selector: "merge.alpha-merge", only: []string{"alpha"}, wantStage: pipeline.StageMerge, wantLane: "alpha", wantSlot: "alpha-merge"},
|
||||
{name: "lane", selector: "alpha.alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"},
|
||||
{name: "lane extract", selector: "alpha.extract.alpha-slot", wantStage: pipeline.StageExtract, wantLane: "alpha", wantSlot: "alpha-slot"},
|
||||
{name: "lane merge", selector: "alpha.merge.alpha-merge", wantStage: pipeline.StageMerge, wantLane: "alpha", wantSlot: "alpha-merge"},
|
||||
{name: "lane normalize", selector: "alpha.normalize.alpha-normalize", wantStage: pipeline.StageNormalize, wantLane: "alpha", wantSlot: "alpha-normalize"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
selector, err := parseReferenceSelector(tt.selector, "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
overrides, _, err := resolveCLIReferenceRequests(cfg, "demo", tt.only, catalog, []cliReferenceRequest{{Selector: selector, Source: "reference.txt"}}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve selector: %v", err)
|
||||
}
|
||||
if len(overrides) != 1 {
|
||||
t.Fatalf("overrides = %#v, want one binding", overrides)
|
||||
}
|
||||
got := overrides[0]
|
||||
if got.Stage != tt.wantStage || got.LaneID != tt.wantLane || got.SlotName != tt.wantSlot || got.BindingSource != contracts.ReferenceBindingSourceCLI {
|
||||
t.Fatalf("binding = %#v, want %s/%s/%s from CLI", got, tt.wantStage, tt.wantLane, tt.wantSlot)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceSelectorsRejectAmbiguityWithSpecificSuggestions(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
selector string
|
||||
want []string
|
||||
}{
|
||||
{name: "flat shared slot", selector: "shared", want: []string{"alpha.extract.shared", "beta.extract.shared"}},
|
||||
{name: "lane shared slot", selector: "alpha.shared", want: []string{"alpha.extract.shared", "alpha.merge.shared", "alpha.normalize.shared"}},
|
||||
{name: "all mergers", selector: "merge.shared", want: []string{"alpha.merge.shared", "beta.merge.shared"}},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
selector, err := parseReferenceSelector(tt.selector, "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err = resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{{Selector: selector, Source: "reference.txt"}}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("resolve selector succeeded, want ambiguity error")
|
||||
}
|
||||
for _, fragment := range tt.want {
|
||||
if !strings.Contains(err.Error(), fragment) {
|
||||
t.Fatalf("error = %q, want suggestion %q", err, fragment)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceSelectorsRespectSelectedLanesBeforeMaterialization(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
selector string
|
||||
want string
|
||||
}{
|
||||
{name: "unselected lane", selector: "beta.extract.beta-slot", want: `reference lane "beta" is not selected`},
|
||||
{name: "unknown lane", selector: "missing.extract.beta-slot", want: `reference lane "missing" is not selected`},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
selector, err := parseReferenceSelector(tt.selector, "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err = resolveCLIReferenceRequests(cfg, "demo", []string{"alpha"}, catalog, []cliReferenceRequest{{Selector: selector, Source: filepath.Join(t.TempDir(), "missing.txt")}}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) || strings.Contains(err.Error(), "missing.txt") {
|
||||
t.Fatalf("error = %v, want selection failure before file access", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceSyntaxErrorsReturnTwo(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
}{
|
||||
{name: "reference missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference"}},
|
||||
{name: "reference missing selector", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "=path.txt"}},
|
||||
{name: "reference missing separator", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "slot"}},
|
||||
{name: "reference missing path", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "slot="}},
|
||||
{name: "reference excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "a.b.c.d=path.txt"}},
|
||||
{name: "unbind with path", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference", "slot=path.txt"}},
|
||||
{name: "unbind excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference", "a.b.c.d"}},
|
||||
{name: "unbind missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference"}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(tt.args, &stdout, &stderr, Options{LookupEnv: emptyLookup})
|
||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceOverridesUseFinalExactTargetBinding(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
alphaShared, err := parseReferenceSelector("alpha.extract.shared", "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
betaShared, err := parseReferenceSelector("beta.extract.shared", "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{
|
||||
{Selector: alphaShared, Source: "alpha-first.txt"},
|
||||
{Selector: alphaShared, Source: "alpha-final.txt"},
|
||||
{Selector: betaShared, Source: "beta-only.txt"},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(unbinds) != 0 {
|
||||
t.Fatalf("unbinds = %#v, want none", unbinds)
|
||||
}
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve pipeline: %v", err)
|
||||
}
|
||||
alpha := referenceContractLane(t, effective.ResolvedPipeline, "alpha")
|
||||
beta := referenceContractLane(t, effective.ResolvedPipeline, "beta")
|
||||
if source := referenceContractBindingSource(alpha.ExtractReferences.Bindings, "shared"); source != "alpha-final.txt" {
|
||||
t.Fatalf("alpha shared source = %q, want final exact-target override", source)
|
||||
}
|
||||
if source := referenceContractBindingSource(beta.ExtractReferences.Bindings, "shared"); source != "beta-only.txt" {
|
||||
t.Fatalf("beta shared source = %q, want target-specific override", source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceUnbindsRemoveOptionalAndProtectRequiredSlots(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
optional, err := parseReferenceSelector("alpha.extract.alpha-slot", "--without-reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, without, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, nil, []cliReferenceUnbindRequest{{Selector: optional}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceUnbinds: without})
|
||||
if err != nil {
|
||||
t.Fatalf("optional unbind: %v", err)
|
||||
}
|
||||
if binding := referenceContractFindBinding(referenceContractLane(t, effective.ResolvedPipeline, "alpha").ExtractReferences.Bindings, "alpha-slot"); binding != nil {
|
||||
t.Fatalf("optional binding after unbind = %#v, want absent", binding)
|
||||
}
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
selector string
|
||||
}{
|
||||
{name: "chunk", selector: "chunk.required-chunk"},
|
||||
{name: "extract", selector: "alpha.extract.required-extract"},
|
||||
{name: "merge", selector: "alpha.merge.required-merge"},
|
||||
{name: "normalize", selector: "alpha.normalize.required-normalize"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
selector, err := parseReferenceSelector(tt.selector, "--without-reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, nil, []cliReferenceUnbindRequest{{Selector: selector}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceUnbinds: unbinds})
|
||||
if err == nil || !strings.Contains(err.Error(), "required reference slot") {
|
||||
t.Fatalf("resolve error = %v, want required-slot failure", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceMaterializationSeparatesCLIAndConfigPathOrigins(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
workingDir := t.TempDir()
|
||||
cfg := referenceContractConfig()
|
||||
configPath := filepath.Join(configDir, "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte("version: 4\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(configDir, "required.txt"), []byte("config reference"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(configDir, "optional.txt"), []byte("optional reference"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(workingDir, "cli-reference.txt"), []byte("CLI reference"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
selector, err := parseReferenceSelector("alpha.extract.alpha-slot", "--reference")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "demo", nil, catalog, []cliReferenceRequest{{Selector: selector, Source: "cli-reference.txt"}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides, ReferenceUnbinds: unbinds})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve pipeline: %v", err)
|
||||
}
|
||||
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: workingDir})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize references: %v", err)
|
||||
}
|
||||
alpha := referenceContractLane(t, materialized, "alpha")
|
||||
cliItem := alpha.ExtractReferences.ReferenceSet.Slots["alpha-slot"].Items[0]
|
||||
if string(cliItem.Content) != "CLI reference" || cliItem.BindingSource != contracts.ReferenceBindingSourceCLI || cliItem.Origin.URI != referenceContractFileURI(filepath.Join(workingDir, "cli-reference.txt")) {
|
||||
t.Fatalf("CLI materialization = %#v, want working-directory provenance", cliItem)
|
||||
}
|
||||
configItem := alpha.ExtractReferences.ReferenceSet.Slots["required-extract"].Items[0]
|
||||
if string(configItem.Content) != "config reference" || configItem.BindingSource != contracts.ReferenceBindingSourceConfig || configItem.Origin.URI != referenceContractFileURI(filepath.Join(configDir, "required.txt")) {
|
||||
t.Fatalf("config materialization = %#v, want config-directory provenance", configItem)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceTargetLookupUsesArtifactVariantsAndReportsMissingContext(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
full := referenceContractCatalog(t, true, true)
|
||||
targets, err := selectedReferenceTargets(cfg, "demo", nil, full)
|
||||
if err != nil {
|
||||
t.Fatalf("select reference targets: %v", err)
|
||||
}
|
||||
var alphaMerge, betaMerge selectedReferenceTarget
|
||||
for _, target := range targets {
|
||||
if target.stage == pipeline.StageMerge && target.laneID == "alpha" {
|
||||
alphaMerge = target
|
||||
}
|
||||
if target.stage == pipeline.StageMerge && target.laneID == "beta" {
|
||||
betaMerge = target
|
||||
}
|
||||
}
|
||||
if _, ok := alphaMerge.slots["alpha-merge"]; !ok {
|
||||
t.Fatalf("alpha merger slots = %#v, want alpha artifact variant", alphaMerge.slots)
|
||||
}
|
||||
if _, ok := betaMerge.slots["beta-merge"]; !ok {
|
||||
t.Fatalf("beta merger slots = %#v, want beta artifact variant", betaMerge.slots)
|
||||
}
|
||||
if _, ok := betaMerge.slots["alpha-merge"]; ok {
|
||||
t.Fatalf("beta merger slots = %#v, must not use alpha variant", betaMerge.slots)
|
||||
}
|
||||
|
||||
missingMerger := referenceContractCatalog(t, false, true)
|
||||
_, err = selectedReferenceTargets(cfg, "demo", nil, missingMerger)
|
||||
if err == nil || !strings.Contains(err.Error(), "merger") || !strings.Contains(err.Error(), string(referenceContractKindBeta)) {
|
||||
t.Fatalf("missing merger error = %v, want artifact variant context", err)
|
||||
}
|
||||
missingNormalizer := referenceContractCatalog(t, true, false)
|
||||
_, err = selectedReferenceTargets(cfg, "demo", nil, missingNormalizer)
|
||||
if err == nil || !strings.Contains(err.Error(), "normalizer") || !strings.Contains(err.Error(), string(referenceContractKindBeta)) {
|
||||
t.Fatalf("missing normalizer error = %v, want artifact variant context", err)
|
||||
}
|
||||
missingExtractor := referenceContractCatalog(t, true, true)
|
||||
missingExtractor.Extractors = pipeline.NewExtractorRegistry()
|
||||
_, err = selectedReferenceTargets(cfg, "demo", nil, missingExtractor)
|
||||
if err == nil || !strings.Contains(err.Error(), `lane "alpha" extract module`) || !strings.Contains(err.Error(), "not registered") {
|
||||
t.Fatalf("missing extractor error = %v, want lane/module context", err)
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
referenceContractKindAlpha contracts.ArtifactKind = "reference/alpha"
|
||||
referenceContractKindBeta contracts.ArtifactKind = "reference/beta"
|
||||
)
|
||||
|
||||
func referenceContractConfig() config.Config {
|
||||
cfg := config.Default()
|
||||
cfg.Pipelines = map[string]pipeline.PipelineProfile{
|
||||
"demo": {
|
||||
ID: "demo",
|
||||
Input: pipeline.Binding("reference/input"),
|
||||
Chunk: pipeline.Binding("reference/chunk"),
|
||||
Output: pipeline.Binding("reference/output"),
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"alpha": {
|
||||
Extract: pipeline.Binding("reference/extract-alpha"),
|
||||
Merge: pipeline.Binding("reference/shared-merge"),
|
||||
Normalize: pipeline.Binding("reference/shared-normalize"),
|
||||
References: pipeline.ExternalReferenceMap(map[string]string{"required-extract": "required.txt"}),
|
||||
},
|
||||
"beta": {
|
||||
Extract: pipeline.Binding("reference/extract-beta"),
|
||||
Merge: pipeline.Binding("reference/shared-merge"),
|
||||
Normalize: pipeline.Binding("reference/shared-normalize"),
|
||||
References: pipeline.ExternalReferenceMap(map[string]string{"required-extract": "required.txt"}),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
profile := cfg.Pipelines["demo"]
|
||||
profile.Chunk.References = pipeline.ExternalReferenceMap(map[string]string{"required-chunk": "required.txt"})
|
||||
alpha := profile.Artifacts["alpha"]
|
||||
alpha.Extract.References = pipeline.ExternalReferenceMap(map[string]string{"required-extract": "required.txt", "alpha-slot": "optional.txt"})
|
||||
alpha.Merge.References = pipeline.ExternalReferenceMap(map[string]string{"required-merge": "required.txt"})
|
||||
alpha.Normalize.References = pipeline.ExternalReferenceMap(map[string]string{"required-normalize": "required.txt"})
|
||||
profile.Artifacts["alpha"] = alpha
|
||||
beta := profile.Artifacts["beta"]
|
||||
beta.Extract.References = pipeline.ExternalReferenceMap(map[string]string{"required-extract": "required.txt"})
|
||||
beta.Merge.References = pipeline.ExternalReferenceMap(map[string]string{"required-merge": "required.txt"})
|
||||
beta.Normalize.References = pipeline.ExternalReferenceMap(map[string]string{"required-normalize": "required.txt"})
|
||||
profile.Artifacts["beta"] = beta
|
||||
cfg.Pipelines["demo"] = profile
|
||||
return cfg
|
||||
}
|
||||
|
||||
func referenceContractCatalog(t *testing.T, includeBetaMerger, includeBetaNormalizer bool) pipeline.ModuleCatalog {
|
||||
t.Helper()
|
||||
registries := pipeline.Registries{
|
||||
Inputs: pipeline.NewInputAdapterRegistry(),
|
||||
Chunkers: pipeline.NewChunkerRegistry(),
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
|
||||
Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(),
|
||||
Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(),
|
||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
}
|
||||
register := func(err error) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register(registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "reference/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }))
|
||||
register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) { return stateTestChunker{}, nil }))
|
||||
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecA{}))
|
||||
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecB{}))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
|
||||
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
|
||||
if includeBetaMerger {
|
||||
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
|
||||
}
|
||||
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil }))
|
||||
if includeBetaNormalizer {
|
||||
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil }))
|
||||
}
|
||||
register(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) { return stateTestOutput{}, nil }))
|
||||
return catalogFromRegistries(registries)
|
||||
}
|
||||
|
||||
type referenceContractCodecB struct{}
|
||||
|
||||
type referenceContractCodecA struct{}
|
||||
|
||||
func (referenceContractCodecA) Kind() contracts.ArtifactKind { return referenceContractKindAlpha }
|
||||
func (referenceContractCodecA) Schema() contracts.ArtifactSchema {
|
||||
return contracts.ArtifactSchema{ID: "reference.alpha", Name: "reference_alpha", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
|
||||
}
|
||||
func (referenceContractCodecA) MediaType() string { return "application/json" }
|
||||
func (referenceContractCodecA) EncodeCandidate(stateTestArtifact) ([]byte, error) {
|
||||
return []byte(`{"value":"ok"}`), nil
|
||||
}
|
||||
func (referenceContractCodecA) Encode(stateTestArtifact) ([]byte, error) {
|
||||
return []byte(`{"value":"ok"}`), nil
|
||||
}
|
||||
func (referenceContractCodecA) Decode([]byte) (stateTestArtifact, error) {
|
||||
return stateTestArtifact{Value: "ok"}, nil
|
||||
}
|
||||
|
||||
func (referenceContractCodecB) Kind() contracts.ArtifactKind { return referenceContractKindBeta }
|
||||
func (referenceContractCodecB) Schema() contracts.ArtifactSchema {
|
||||
return contracts.ArtifactSchema{ID: "reference.beta", Name: "reference_beta", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
|
||||
}
|
||||
func (referenceContractCodecB) MediaType() string { return "application/json" }
|
||||
func (referenceContractCodecB) EncodeCandidate(stateTestArtifact) ([]byte, error) {
|
||||
return []byte(`{"value":"ok"}`), nil
|
||||
}
|
||||
func (referenceContractCodecB) Encode(stateTestArtifact) ([]byte, error) {
|
||||
return []byte(`{"value":"ok"}`), nil
|
||||
}
|
||||
func (referenceContractCodecB) Decode([]byte) (stateTestArtifact, error) {
|
||||
return stateTestArtifact{Value: "ok"}, nil
|
||||
}
|
||||
|
||||
func referenceContractLane(t *testing.T, resolved pipeline.ResolvedPipeline, id string) pipeline.ResolvedArtifactLane {
|
||||
t.Helper()
|
||||
for _, step := range resolved.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
if lane.ID == id {
|
||||
return lane
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("lane %q not found", id)
|
||||
return pipeline.ResolvedArtifactLane{}
|
||||
}
|
||||
|
||||
func referenceContractBindingSource(bindings []pipeline.ReferenceBinding, slot string) string {
|
||||
for _, binding := range bindings {
|
||||
if binding.SlotName == slot {
|
||||
return binding.Source
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func referenceContractFindBinding(bindings []pipeline.ReferenceBinding, slot string) *pipeline.ReferenceBinding {
|
||||
for i := range bindings {
|
||||
if bindings[i].SlotName == slot {
|
||||
return &bindings[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func referenceContractFileURI(path string) string {
|
||||
absolute, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
absolute = path
|
||||
}
|
||||
return "file://" + filepath.ToSlash(absolute)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
654
internal/cli/run_contract_test.go
Normal file
654
internal/cli/run_contract_test.go
Normal file
@@ -0,0 +1,654 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRunControlsRejectSyntaxWithoutAllocatingState(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args func(stateTestRoots) []string
|
||||
}{
|
||||
{name: "missing pipeline", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "--config", roots.config, "--input", roots.input}
|
||||
}},
|
||||
{name: "missing input", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config}
|
||||
}},
|
||||
{name: "unknown flag", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--unknown"}
|
||||
}},
|
||||
{name: "blank output directory", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--output-dir", ""}
|
||||
}},
|
||||
{name: "blank debug directory", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", ""}
|
||||
}},
|
||||
{name: "debug directory without debug", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", filepath.Join(filepath.Dir(roots.debug), "requested-debug")}
|
||||
}},
|
||||
{name: "blank session ID", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--session-id", ""}
|
||||
}},
|
||||
{name: "multiple pipeline IDs", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "extra", "--config", roots.config, "--input", roots.input}
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertAbsent(t, roots.output)
|
||||
assertAbsent(t, roots.debug)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecomputeStepCLIContract(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
configure func(*testing.T, stateTestRoots)
|
||||
flags []string
|
||||
wantCode int
|
||||
wantOutput string
|
||||
wantError string
|
||||
}{
|
||||
{
|
||||
name: "explicit step",
|
||||
configure: func(t *testing.T, roots stateTestRoots) {
|
||||
replaceStateTestConfigLine(t, roots.config, " artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n", " steps:\n - id: chosen\n artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n")
|
||||
},
|
||||
flags: []string{"--resume", "--recompute-step", "chosen"},
|
||||
wantCode: 0,
|
||||
wantOutput: "outputs=1",
|
||||
},
|
||||
{name: "implicit default step", flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 0, wantOutput: "outputs=1"},
|
||||
{name: "repeated flag", flags: []string{"--resume", "--recompute-step", "default", "--recompute-step", "default"}, wantCode: 2, wantError: "specified only once"},
|
||||
{name: "empty step", flags: []string{"--resume", "--recompute-step", ""}, wantCode: 2, wantError: "must not be empty"},
|
||||
{name: "unknown step", flags: []string{"--resume", "--recompute-step", "missing"}, wantCode: 1, wantError: "unknown pipeline step"},
|
||||
{name: "without resume", flags: []string{"--recompute-step", "default"}, wantCode: 2, wantError: "requires --resume"},
|
||||
{
|
||||
name: "checkpoint recording disabled",
|
||||
configure: func(t *testing.T, roots stateTestRoots) {
|
||||
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
|
||||
},
|
||||
flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 1, wantError: "cache.checkpoints.enabled",
|
||||
},
|
||||
{name: "with only", flags: []string{"--resume", "--recompute-step", "default", "--only", "items"}, wantCode: 2, wantError: "cannot be combined with --only"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
if tt.configure != nil {
|
||||
tt.configure(t, roots)
|
||||
}
|
||||
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
|
||||
args = append(args, tt.flags...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != tt.wantCode || (tt.wantOutput != "" && !strings.Contains(stdout.String(), tt.wantOutput)) || (tt.wantError != "" && !strings.Contains(stderr.String(), tt.wantError)) {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunValidFailuresClassifyAndReportDebug(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args func(stateTestRoots) []string
|
||||
wantError string
|
||||
wantDebug bool
|
||||
}{
|
||||
{name: "unknown pipeline", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "missing", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
|
||||
}, wantError: `pipeline "missing"`},
|
||||
{name: "unknown lane", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "missing", "--chunk_cache", "bypass", "--debug"}
|
||||
}, wantError: `lane "missing"`, wantDebug: true},
|
||||
{name: "unreadable input", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", filepath.Join(filepath.Dir(roots.input), "unreadable.txt"), "--chunk_cache", "bypass", "--debug"}
|
||||
}, wantError: "read input", wantDebug: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if tt.wantDebug {
|
||||
if !strings.Contains(stderr.String(), "debug=") {
|
||||
t.Fatalf("stderr=%q, want debug path", stderr.String())
|
||||
}
|
||||
onlyChildDir(t, roots.debug)
|
||||
} else {
|
||||
assertAbsent(t, roots.debug)
|
||||
}
|
||||
assertAbsent(t, roots.output)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnlyExecutesSelectedLanes(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data = []byte(replaceRequiredOnce(t, string(data), " output: test/output\n", " other:\n extract: test/extract\n output: test/output\n"))
|
||||
if err := os.WriteFile(roots.config, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
harness := newStateTestHarness()
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "items", "--chunk_cache", "bypass"}, &stdout, &stderr, harness.options())
|
||||
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
harness.mu.Lock()
|
||||
extractCalls := harness.extractCalls
|
||||
harness.mu.Unlock()
|
||||
if extractCalls != 1 {
|
||||
t.Fatalf("extract calls = %d, want only the selected lane", extractCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStateRootsHonorEnvironmentFlagsAndDefaults(t *testing.T) {
|
||||
t.Run("environment roots", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
environmentOutput := filepath.Join(t.TempDir(), "environment-output")
|
||||
environmentDebug := filepath.Join(t.TempDir(), "environment-debug")
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LookupEnv = lookupRunContractEnv(map[string]string{
|
||||
"NOTARIUS_OUTPUT_DIR": environmentOutput,
|
||||
"NOTARIUS_DEBUG_DIR": environmentDebug,
|
||||
})
|
||||
result := runWithStateRoots(t, roots, opts, nil)
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertFile(t, filepath.Join(environmentOutput, filepath.Base(onlyChildDir(t, environmentOutput)), "result.json"))
|
||||
onlyChildDir(t, environmentDebug)
|
||||
assertAbsent(t, roots.output)
|
||||
assertAbsent(t, roots.debug)
|
||||
})
|
||||
|
||||
t.Run("command flags override environment", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
environmentOutput := filepath.Join(t.TempDir(), "environment-output")
|
||||
environmentDebug := filepath.Join(t.TempDir(), "environment-debug")
|
||||
flagOutput := filepath.Join(t.TempDir(), "flag-output")
|
||||
flagDebug := filepath.Join(t.TempDir(), "flag-debug")
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LookupEnv = lookupRunContractEnv(map[string]string{
|
||||
"NOTARIUS_OUTPUT_DIR": environmentOutput,
|
||||
"NOTARIUS_DEBUG_DIR": environmentDebug,
|
||||
})
|
||||
result := runWithStateRoots(t, roots, opts, []string{"--output-dir", flagOutput, "--debug-dir", flagDebug})
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertFile(t, filepath.Join(flagOutput, filepath.Base(onlyChildDir(t, flagOutput)), "result.json"))
|
||||
onlyChildDir(t, flagDebug)
|
||||
assertAbsent(t, environmentOutput)
|
||||
assertAbsent(t, environmentDebug)
|
||||
})
|
||||
|
||||
t.Run("built-in roots", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(data)
|
||||
text = replaceRequiredOnce(t, text, fmt.Sprintf(" directory: %q\n", roots.output), "")
|
||||
text = replaceRequiredOnce(t, text, fmt.Sprintf(" directory: %q\n", roots.debug), "")
|
||||
if err := os.WriteFile(roots.config, []byte(text), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workDir := t.TempDir()
|
||||
t.Chdir(workDir)
|
||||
opts := newStateTestHarness().options()
|
||||
result := runWithStateRoots(t, roots, opts, nil)
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertFile(t, filepath.Join(workDir, "notarius-output", filepath.Base(onlyChildDir(t, filepath.Join(workDir, "notarius-output"))), "result.json"))
|
||||
onlyChildDir(t, filepath.Join(workDir, "notarius-debug"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
|
||||
t.Run("one effective profile reaches the factory and modules", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
profileDir := writeRunContractProfiles(t, "override-profile")
|
||||
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
||||
harness := newStateTestHarness()
|
||||
var factoryProfiles []string
|
||||
opts := harness.options()
|
||||
var factoryOverrides []LLMRuntimeOverrides
|
||||
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryProfiles = append(factoryProfiles, profileID)
|
||||
factoryOverrides = append(factoryOverrides, overrides)
|
||||
return nil, nil, nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "override-profile"}, &stdout, &stderr, opts)
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
|
||||
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
|
||||
}
|
||||
if len(factoryOverrides) != 1 || factoryOverrides[0].ReasoningEffort != nil {
|
||||
t.Fatalf("factory overrides = %#v, want inherited reasoning", factoryOverrides)
|
||||
}
|
||||
harness.mu.Lock()
|
||||
profiles := append([]string(nil), harness.moduleProfiles...)
|
||||
harness.mu.Unlock()
|
||||
if len(profiles) < 4 {
|
||||
t.Fatalf("module profiles = %#v, want chunk and lane stage requests", profiles)
|
||||
}
|
||||
for _, profile := range profiles {
|
||||
if profile != "override-profile" {
|
||||
t.Fatalf("module profiles = %#v, want override on every request", profiles)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("runtime override applies to validators", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
profileDir := writeRunContractProfiles(t, "override-profile", "validator-profile")
|
||||
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
||||
harness := newStateTestHarness()
|
||||
var validatorProfiles []string
|
||||
opts := harness.options()
|
||||
registerRunContractValidator(t, &opts, &validatorProfiles)
|
||||
factoryProfiles := []string{}
|
||||
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, _ LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryProfiles = append(factoryProfiles, profileID)
|
||||
return nil, nil, nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "override-profile"}, &stdout, &stderr, opts)
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
|
||||
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
|
||||
}
|
||||
if len(validatorProfiles) != 1 || validatorProfiles[0] != "override-profile" {
|
||||
t.Fatalf("validator profiles = %#v, want runtime override", validatorProfiles)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown profile is rejected without factory access", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
profileDir := writeRunContractProfiles(t, "override-profile")
|
||||
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
||||
factoryCalls := 0
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryCalls++
|
||||
return nil, nil, nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "missing-profile"}, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("pipeline default is rejected before factory access", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
profileDir := writeRunContractProfiles(t, "configured-profile")
|
||||
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
||||
replaceStateTestConfigLine(t, roots.config, " sample:\n", " sample:\n llm_profile: missing-profile\n")
|
||||
factoryCalls := 0
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryCalls++
|
||||
return nil, nil, nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunReasoningEffortOverrideReachesFactory(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
flags []string
|
||||
wantValue string
|
||||
wantSet bool
|
||||
}{
|
||||
{name: "inherit"},
|
||||
{name: "replace", flags: []string{"--reasoning-effort", " focused "}, wantValue: "focused", wantSet: true},
|
||||
{name: "clear", flags: []string{"--clear-reasoning-effort"}, wantSet: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
var got []LLMRuntimeOverrides
|
||||
opts.LLMClientFactory = func(_ context.Context, _ config.Config, _ string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
got = append(got, overrides)
|
||||
return nil, nil, nil
|
||||
}
|
||||
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.flags...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("factory overrides = %#v, want one call", got)
|
||||
}
|
||||
if !tt.wantSet {
|
||||
if got[0].ReasoningEffort != nil {
|
||||
t.Fatalf("reasoning effort = %q, want inherit", *got[0].ReasoningEffort)
|
||||
}
|
||||
return
|
||||
}
|
||||
if got[0].ReasoningEffort == nil || *got[0].ReasoningEffort != tt.wantValue {
|
||||
t.Fatalf("reasoning effort = %#v, want %q", got[0].ReasoningEffort, tt.wantValue)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReasoningEffortOverrideRejectsInvalidSyntax(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
flags []string
|
||||
wantError string
|
||||
}{
|
||||
{
|
||||
name: "mutually exclusive controls",
|
||||
flags: []string{"--reasoning-effort", "focused", "--clear-reasoning-effort"},
|
||||
wantError: "cannot be combined",
|
||||
},
|
||||
{
|
||||
name: "empty replacement",
|
||||
flags: []string{"--reasoning-effort", " "},
|
||||
wantError: "must not be empty",
|
||||
},
|
||||
{
|
||||
name: "duplicate replacement",
|
||||
flags: []string{"--reasoning-effort", "low", "--reasoning-effort", "high"},
|
||||
wantError: "may be specified only once",
|
||||
},
|
||||
{
|
||||
name: "missing replacement",
|
||||
flags: []string{"--reasoning-effort"},
|
||||
wantError: "flag needs an argument",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, tt.flags...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReasoningEffortOverrideSeparatesCheckpointIdentities(t *testing.T) {
|
||||
replacement := " focused "
|
||||
cleared := ""
|
||||
states := []struct {
|
||||
name string
|
||||
overrides LLMRuntimeOverrides
|
||||
wantValue string
|
||||
wantSet bool
|
||||
}{
|
||||
{name: "inherit"},
|
||||
{name: "replace", overrides: LLMRuntimeOverrides{ReasoningEffort: &replacement}, wantValue: "focused", wantSet: true},
|
||||
{name: "clear", overrides: LLMRuntimeOverrides{ReasoningEffort: &cleared}, wantValue: "<cleared>", wantSet: true},
|
||||
}
|
||||
digests := make(map[string]string, len(states))
|
||||
for _, state := range states {
|
||||
fingerprints := runtimeOverrideFingerprints("", "", state.overrides)
|
||||
var value string
|
||||
var found bool
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Name == "reasoning_effort_override" {
|
||||
value, found = fingerprint.Value, true
|
||||
}
|
||||
}
|
||||
if found != state.wantSet || (found && value != state.wantValue) {
|
||||
t.Fatalf("%s fingerprint found=%t value=%q, want found=%t value=%q", state.name, found, value, state.wantSet, state.wantValue)
|
||||
}
|
||||
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
||||
Pipeline: pipeline.ResolvedPipeline{ID: "sample", Digest: "sha256:pipeline", Input: pipeline.Binding("test/input")},
|
||||
RawInputDigest: "sha256:input",
|
||||
RuntimeOverrides: fingerprints,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digests[state.name] = identity.Digest
|
||||
}
|
||||
if digests["inherit"] == digests["replace"] || digests["inherit"] == digests["clear"] || digests["replace"] == digests["clear"] {
|
||||
t.Fatalf("checkpoint identity digests are not distinct: %#v", digests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveLLMProfileIDsAreSortedDeduplicatedAndLLMOnly(t *testing.T) {
|
||||
resolved := pipeline.ResolvedPipeline{
|
||||
Input: pipeline.ModuleBinding{LLMProfile: "input-profile"},
|
||||
InputExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Chunk: pipeline.ModuleBinding{LLMProfile: " zeta "},
|
||||
ChunkExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Steps: []pipeline.ResolvedPipelineStep{{
|
||||
ID: "default",
|
||||
ArtifactLanes: []pipeline.ResolvedArtifactLane{{
|
||||
Extract: pipeline.ModuleBinding{LLMProfile: "alpha"},
|
||||
ExtractExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Merge: pipeline.ModuleBinding{LLMProfile: "deterministic-merge"},
|
||||
MergeExecutionClass: contracts.ExecutionClassDeterministic,
|
||||
Normalize: pipeline.ModuleBinding{LLMProfile: " gamma "},
|
||||
NormalizeExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
}},
|
||||
}},
|
||||
ValidatorChains: []pipeline.ResolvedValidatorChain{{Validators: []pipeline.ResolvedValidator{
|
||||
{Binding: pipeline.ModuleBinding{LLMProfile: "deterministic-profile"}, ExecutionClass: contracts.ExecutionClassDeterministic},
|
||||
{Binding: pipeline.ModuleBinding{LLMProfile: "beta"}, ExecutionClass: contracts.ExecutionClassLLMBacked},
|
||||
}}},
|
||||
Output: pipeline.ModuleBinding{LLMProfile: "output-profile"},
|
||||
OutputExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
}
|
||||
got := effectiveLLMProfileIDs(resolved)
|
||||
want := []string{"alpha", "beta", "gamma", "input-profile", "output-profile", "zeta"}
|
||||
if strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("effective profiles = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSessionIDUsesExplicitValueOrSourceDocumentID(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{name: "source document", want: "source"},
|
||||
{name: "explicit trimmed value", args: []string{"--session-id", " explicit-session "}, want: "explicit-session"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.args...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(args, &stdout, &stderr, harness.options())
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
harness.mu.Lock()
|
||||
sessions := append([]string(nil), harness.sessionIDs...)
|
||||
harness.mu.Unlock()
|
||||
if len(sessions) < 4 {
|
||||
t.Fatalf("session IDs = %#v, want all prompt-facing module requests", sessions)
|
||||
}
|
||||
for _, session := range sessions {
|
||||
if session != tt.want {
|
||||
t.Fatalf("session IDs = %#v, want %q", sessions, tt.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) {
|
||||
t.Run("LLM factory", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
return nil, nil, errors.New("injected LLM factory failure")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "injected LLM factory failure") || stdout.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("pipeline preparation", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data = []byte(replaceRequiredOnce(t, string(data), "extract: test/extract", "extract: test/failing-extract"))
|
||||
if err := os.WriteFile(roots.config, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := newStateTestHarness().options()
|
||||
if err := pipeline.RegisterExtractorBuilder(opts.Registries.Extractors, pipeline.ModuleSpec{Key: "test/failing-extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Extractor[stateTestArtifact], error) {
|
||||
return nil, errors.New("injected extractor construction failure")
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.Catalog = catalogFromRegistries(opts.Registries)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "injected extractor construction failure") || stdout.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.includeWarnings = true
|
||||
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
|
||||
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning(s)") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")
|
||||
output, err := os.ReadFile(outputPath)
|
||||
if err != nil || !strings.Contains(string(output), "contract-warning") {
|
||||
t.Fatalf("durable output = %q, %v", output, err)
|
||||
}
|
||||
bundle := onlyChildDir(t, roots.debug)
|
||||
var warnings []contracts.Warning
|
||||
readStateTestSummaryJSON(t, bundle, "warnings.json", &warnings)
|
||||
if len(warnings) != 1 || warnings[0].ReasonCode != "contract-warning" {
|
||||
t.Fatalf("debug warnings = %#v", warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func runWithStateRoots(t *testing.T, roots stateTestRoots, opts Options, extra []string) stateTestResult {
|
||||
t.Helper()
|
||||
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}
|
||||
args = append(args, extra...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
return stateTestResult{code: RunWithOptions(args, &stdout, &stderr, opts), stdout: stdout.String(), stderr: stderr.String()}
|
||||
}
|
||||
|
||||
func lookupRunContractEnv(values map[string]string) func(string) (string, bool) {
|
||||
return func(name string) (string, bool) {
|
||||
value, ok := values[name]
|
||||
return value, ok
|
||||
}
|
||||
}
|
||||
|
||||
func prependRunContractConfig(t *testing.T, roots stateTestRoots, prefix string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(roots.config, append([]byte(prefix), data...), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeRunContractProfiles(t *testing.T, ids ...string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for _, id := range ids {
|
||||
profile := fmt.Sprintf("id: %s\nendpoint: http://127.0.0.1:1/v1\nmodel: %s-model\n", id, id)
|
||||
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(profile), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func registerRunContractValidator(t *testing.T, opts *Options, profiles *[]string) {
|
||||
t.Helper()
|
||||
if err := pipeline.RegisterTypedValidatorBuilder(opts.Registries.Validators, stateTestArtifactKind, pipeline.ValidatorSpec{Key: "run-contract-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.TypedValidator[stateTestArtifact], error) {
|
||||
return runContractValidator{profiles: profiles}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := opts.Registries.ValidatorChains.Register(pipeline.ValidatorChainMapping{Stage: pipeline.StageExtract, Module: "test/extract", Validators: []pipeline.ModuleBinding{{Module: "run-contract-validator", LLMProfile: "validator-profile"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.Catalog = catalogFromRegistries(opts.Registries)
|
||||
}
|
||||
|
||||
type runContractValidator struct {
|
||||
profiles *[]string
|
||||
}
|
||||
|
||||
func (v runContractValidator) Name() string { return "run-contract-validator" }
|
||||
|
||||
func (v runContractValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
|
||||
func (v runContractValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[stateTestArtifact]) (contracts.ValidationResult, error) {
|
||||
*v.profiles = append(*v.profiles, req.LLMProfile)
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
34
internal/cli/run_id.go
Normal file
34
internal/cli/run_id.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RunIDGenerator func(time.Time) (string, error)
|
||||
|
||||
func defaultRunIDGenerator(startedAt time.Time) (string, error) {
|
||||
var suffix [16]byte
|
||||
if _, err := io.ReadFull(rand.Reader, suffix[:]); err != nil {
|
||||
return "", fmt.Errorf("read random run ID suffix: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("run-%d-%s", startedAt.UnixNano(), hex.EncodeToString(suffix[:])), nil
|
||||
}
|
||||
|
||||
func validateRunID(runID string) error {
|
||||
if runID == "" {
|
||||
return fmt.Errorf("run ID must not be empty")
|
||||
}
|
||||
if runID != strings.TrimSpace(runID) {
|
||||
return fmt.Errorf("run ID %q must not have surrounding whitespace", runID)
|
||||
}
|
||||
if strings.ContainsAny(runID, `/\\`) || filepath.IsAbs(runID) || filepath.Clean(runID) != runID || runID == "." || runID == ".." {
|
||||
return fmt.Errorf("run ID %q must be one safe path component", runID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
87
internal/cli/run_id_test.go
Normal file
87
internal/cli/run_id_test.go
Normal file
@@ -0,0 +1,87 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestDefaultRunIDGeneratorProducesUniqueSafeIDs(t *testing.T) {
|
||||
startedAt := time.Unix(0, 123456789).UTC()
|
||||
pattern := regexp.MustCompile(`^run-123456789-[0-9a-f]{32}$`)
|
||||
seen := make(map[string]struct{}, 256)
|
||||
for i := 0; i < 256; i++ {
|
||||
runID, err := defaultRunIDGenerator(startedAt)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !pattern.MatchString(runID) {
|
||||
t.Fatalf("run ID %q does not match production format", runID)
|
||||
}
|
||||
if err := validateRunID(runID); err != nil {
|
||||
t.Fatalf("run ID %q is not path-safe: %v", runID, err)
|
||||
}
|
||||
if _, exists := seen[runID]; exists {
|
||||
t.Fatalf("duplicate run ID %q", runID)
|
||||
}
|
||||
seen[runID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteOutputFilesSupportsNestedLogicalPaths(t *testing.T) {
|
||||
runPath := filepath.Join(t.TempDir(), "output", "run-safe")
|
||||
if err := writeOutputFiles(runPath, []contracts.OutputFile{{Name: "nested/result.json", Bytes: []byte("result")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(runPath, "nested", "result.json"))
|
||||
if err != nil || string(data) != "result" {
|
||||
t.Fatalf("nested output = %q, %v", data, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteOutputFilesRejectsUnsafeNamesBeforeAllocatingRunDirectory(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
runPath := filepath.Join(outputRoot, "run-safe")
|
||||
for _, name := range []string{"", "../outside", "/absolute", `nested\\outside`, "nested/../outside"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err := writeOutputFiles(runPath, []contracts.OutputFile{{Name: "safe.json"}, {Name: name}}); err == nil {
|
||||
t.Fatalf("writeOutputFiles accepted %q", name)
|
||||
}
|
||||
if _, err := os.Stat(outputRoot); !os.IsNotExist(err) {
|
||||
t.Fatalf("output root exists or stat failed after %q: %v", name, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteOutputFilesRetainsNewPartialDirectoryAndPreservesSibling(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
siblingPath := filepath.Join(outputRoot, "sibling")
|
||||
if err := os.MkdirAll(siblingPath, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sentinelPath := filepath.Join(siblingPath, "sentinel")
|
||||
if err := os.WriteFile(sentinelPath, []byte("preserve sibling"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
runPath := filepath.Join(outputRoot, "run-safe")
|
||||
err := writeOutputFiles(runPath, []contracts.OutputFile{
|
||||
{Name: "blocked", Bytes: []byte("partial output")},
|
||||
{Name: "blocked/nested.json", Bytes: []byte("unreachable")},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "create output directory") {
|
||||
t.Fatalf("writeOutputFiles() error = %v, want later directory failure", err)
|
||||
}
|
||||
if got, err := os.ReadFile(filepath.Join(runPath, "blocked")); err != nil || string(got) != "partial output" {
|
||||
t.Fatalf("partial output = %q, %v", got, err)
|
||||
}
|
||||
if got, err := os.ReadFile(sentinelPath); err != nil || string(got) != "preserve sibling" {
|
||||
t.Fatalf("sibling sentinel = %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
111
internal/cli/run_result.go
Normal file
111
internal/cli/run_result.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const runResultSchemaVersion = "notarius.run-result.v1"
|
||||
|
||||
type runResult struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
RunID string `json:"run_id"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
OutputDirectory string `json:"output_directory"`
|
||||
IndexFile string `json:"index_file,omitempty"`
|
||||
NormalizedOutputCount int `json:"normalized_output_count"`
|
||||
RejectedOutputCount int `json:"rejected_output_count"`
|
||||
WarningCount int `json:"warning_count"`
|
||||
ValidationStatus string `json:"validation_status"`
|
||||
DebugDirectory string `json:"debug_directory,omitempty"`
|
||||
}
|
||||
|
||||
func newRunResult(resolved pipeline.ResolvedPipeline, output pipeline.RunOutput, outputDirectory, debugDirectory string) (runResult, error) {
|
||||
if strings.TrimSpace(output.Manifest.RunID) == "" {
|
||||
return runResult{}, fmt.Errorf("run result requires a run ID")
|
||||
}
|
||||
if strings.TrimSpace(resolved.ID) == "" {
|
||||
return runResult{}, fmt.Errorf("run result requires a resolved pipeline ID")
|
||||
}
|
||||
if strings.TrimSpace(output.Manifest.PipelineID) == "" {
|
||||
return runResult{}, fmt.Errorf("run result requires a manifest pipeline ID")
|
||||
}
|
||||
if output.Manifest.PipelineID != resolved.ID {
|
||||
return runResult{}, fmt.Errorf("run result pipeline ID does not match resolved pipeline")
|
||||
}
|
||||
if strings.TrimSpace(output.Manifest.ValidationStatus) == "" {
|
||||
return runResult{}, fmt.Errorf("run result requires a validation status")
|
||||
}
|
||||
if strings.TrimSpace(outputDirectory) == "" {
|
||||
return runResult{}, fmt.Errorf("run result requires an output directory")
|
||||
}
|
||||
|
||||
absOutputDirectory, err := filepath.Abs(outputDirectory)
|
||||
if err != nil {
|
||||
return runResult{}, fmt.Errorf("make output directory absolute: %w", err)
|
||||
}
|
||||
|
||||
result := runResult{
|
||||
SchemaVersion: runResultSchemaVersion,
|
||||
RunID: output.Manifest.RunID,
|
||||
PipelineID: output.Manifest.PipelineID,
|
||||
OutputDirectory: absOutputDirectory,
|
||||
NormalizedOutputCount: len(output.NormalizeOutputs),
|
||||
RejectedOutputCount: len(output.Rejected),
|
||||
WarningCount: len(output.Warnings),
|
||||
ValidationStatus: output.Manifest.ValidationStatus,
|
||||
}
|
||||
|
||||
if strings.TrimSpace(debugDirectory) != "" {
|
||||
absDebugDirectory, err := filepath.Abs(debugDirectory)
|
||||
if err != nil {
|
||||
return runResult{}, fmt.Errorf("make debug directory absolute: %w", err)
|
||||
}
|
||||
result.DebugDirectory = absDebugDirectory
|
||||
}
|
||||
|
||||
if resolved.Output.Module == pipeline.DefaultOutputModule {
|
||||
indexCount := 0
|
||||
for _, file := range output.OutputFiles {
|
||||
if file.Name == "index.json" {
|
||||
indexCount++
|
||||
}
|
||||
}
|
||||
if indexCount != 1 {
|
||||
return runResult{}, fmt.Errorf("production JSON output must contain exactly one index.json file")
|
||||
}
|
||||
result.IndexFile = "index.json"
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func encodeRunResult(result runResult) ([]byte, error) {
|
||||
encoded, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode run result: %w", err)
|
||||
}
|
||||
return append(encoded, '\n'), nil
|
||||
}
|
||||
|
||||
func writeRunResult(writer io.Writer, content []byte) error {
|
||||
for len(content) > 0 {
|
||||
written, err := writer.Write(content)
|
||||
if written < 0 || written > len(content) {
|
||||
return io.ErrShortWrite
|
||||
}
|
||||
content = content[written:]
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if written == 0 {
|
||||
return io.ErrShortWrite
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
199
internal/cli/run_result_command_test.go
Normal file
199
internal/cli/run_result_command_test.go
Normal file
@@ -0,0 +1,199 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
|
||||
)
|
||||
|
||||
func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session",
|
||||
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json",
|
||||
}, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{}))
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["schema_version"]; got != "notarius.run-result.v1" {
|
||||
t.Fatalf("schema_version = %q", got)
|
||||
}
|
||||
if got := receipt["run_id"]; got != productionRunID {
|
||||
t.Fatalf("run_id = %q", got)
|
||||
}
|
||||
if got := receipt["pipeline_id"]; got != "dnd-session" {
|
||||
t.Fatalf("pipeline_id = %q", got)
|
||||
}
|
||||
if got := receipt["index_file"]; got != "index.json" {
|
||||
t.Fatalf("index_file = %q", got)
|
||||
}
|
||||
if got := receipt["normalized_output_count"]; got != float64(1) {
|
||||
t.Fatalf("normalized_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["rejected_output_count"]; got != float64(0) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["warning_count"]; got != float64(0) {
|
||||
t.Fatalf("warning_count = %v", got)
|
||||
}
|
||||
if got := receipt["validation_status"]; got != "approved" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
}
|
||||
|
||||
outputDirectory, ok := receipt["output_directory"].(string)
|
||||
if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) {
|
||||
t.Fatalf("output_directory = %q", receipt["output_directory"])
|
||||
}
|
||||
indexFile := receipt["index_file"].(string)
|
||||
assertFile(t, filepath.Join(outputDirectory, indexFile))
|
||||
}
|
||||
|
||||
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--debug", "--json",
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["warning_count"]; got != float64(1) {
|
||||
t.Fatalf("warning_count = %v", got)
|
||||
}
|
||||
debugDirectory, ok := receipt["debug_directory"].(string)
|
||||
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
|
||||
t.Fatalf("debug_directory = %q", receipt["debug_directory"])
|
||||
}
|
||||
if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") {
|
||||
t.Fatalf("machine stdout contains human reporting: %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultReportsSuccessfulRejection(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
configBytes, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n"))
|
||||
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
harness := newStateTestHarness()
|
||||
opts := harness.options()
|
||||
if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.Catalog = catalogFromRegistries(opts.Registries)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--json",
|
||||
}, &stdout, &stderr, opts)
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["normalized_output_count"]; got != float64(0) {
|
||||
t.Fatalf("normalized_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["rejected_output_count"]; got != float64(1) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["validation_status"]; got != "rejected" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) {
|
||||
t.Run("syntax", func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("runtime", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.extractErr = errors.New("injected extraction failure")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--json",
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
writerErr := errors.New("result writer sentinel")
|
||||
stdout := &resultDeliveryWriter{err: writerErr}
|
||||
var stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--debug", "--json",
|
||||
}, stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
if stdout.accepted.Len() != 0 {
|
||||
t.Fatalf("accepted stdout = %q", stdout.accepted.String())
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
debugBundle := onlyChildDir(t, roots.debug)
|
||||
report := readStateTestRunReport(t, debugBundle)
|
||||
if !report.Succeeded {
|
||||
t.Fatalf("debug report = %#v, want successful persisted run", report)
|
||||
}
|
||||
if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) {
|
||||
t.Fatalf("debug bundle contains result writer error")
|
||||
}
|
||||
}
|
||||
|
||||
func decodeRunResultDocument(t *testing.T, stdout string) map[string]any {
|
||||
t.Helper()
|
||||
if strings.Count(stdout, "\n") != 1 {
|
||||
t.Fatalf("stdout = %q, want one JSON document", stdout)
|
||||
}
|
||||
var receipt map[string]any
|
||||
if err := json.Unmarshal([]byte(stdout), &receipt); err != nil {
|
||||
t.Fatalf("decode run result: %v; stdout=%q", err, stdout)
|
||||
}
|
||||
return receipt
|
||||
}
|
||||
|
||||
type resultDeliveryWriter struct {
|
||||
err error
|
||||
accepted bytes.Buffer
|
||||
}
|
||||
|
||||
func (w *resultDeliveryWriter) Write(content []byte) (int, error) {
|
||||
if w.err != nil {
|
||||
return 0, w.err
|
||||
}
|
||||
return w.accepted.Write(content)
|
||||
}
|
||||
200
internal/cli/run_result_test.go
Normal file
200
internal/cli/run_result_test.go
Normal file
@@ -0,0 +1,200 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRunResultEncodesRequiredFieldsAndCounts(t *testing.T) {
|
||||
result, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), testRunOutput(), "relative-output", "relative-debug")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
encoded, err := encodeRunResult(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if encoded[len(encoded)-1] != '\n' || bytes.Count(encoded, []byte{'\n'}) != 1 {
|
||||
t.Fatalf("encoded result is not one newline-terminated object: %q", encoded)
|
||||
}
|
||||
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := decoded["schema_version"]; got != runResultSchemaVersion {
|
||||
t.Fatalf("schema_version = %q", got)
|
||||
}
|
||||
if got := decoded["run_id"]; got != "run-123" {
|
||||
t.Fatalf("run_id = %q", got)
|
||||
}
|
||||
if got := decoded["pipeline_id"]; got != "sample" {
|
||||
t.Fatalf("pipeline_id = %q", got)
|
||||
}
|
||||
if got := decoded["validation_status"]; got != "rejected" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
}
|
||||
if got := decoded["index_file"]; got != "index.json" {
|
||||
t.Fatalf("index_file = %q", got)
|
||||
}
|
||||
if got := decoded["normalized_output_count"]; got != float64(2) {
|
||||
t.Fatalf("normalized_output_count = %v", got)
|
||||
}
|
||||
if got := decoded["rejected_output_count"]; got != float64(1) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := decoded["warning_count"]; got != float64(1) {
|
||||
t.Fatalf("warning_count = %v", got)
|
||||
}
|
||||
if got := decoded["output_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-output") {
|
||||
t.Fatalf("output_directory = %q", got)
|
||||
}
|
||||
if got := decoded["debug_directory"]; got != filepath.Join(mustWorkingDirectory(t), "relative-debug") {
|
||||
t.Fatalf("debug_directory = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultRejectsInvalidRequiredValues(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
resolved pipeline.ResolvedPipeline
|
||||
output pipeline.RunOutput
|
||||
directory string
|
||||
}{
|
||||
{name: "blank run ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.RunID = " " }), directory: "output"},
|
||||
{name: "blank resolved pipeline ID", resolved: pipeline.ResolvedPipeline{Output: pipeline.ModuleBinding{Module: pipeline.DefaultOutputModule}}, output: testRunOutput(), directory: "output"},
|
||||
{name: "blank manifest pipeline ID", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "" }), directory: "output"},
|
||||
{name: "mismatched pipeline IDs", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.PipelineID = "other" }), directory: "output"},
|
||||
{name: "blank validation status", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutputWithout(func(output *pipeline.RunOutput) { output.Manifest.ValidationStatus = " " }), directory: "output"},
|
||||
{name: "blank output directory", resolved: testResolvedPipeline(pipeline.DefaultOutputModule), output: testRunOutput(), directory: " "},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if _, err := newRunResult(tt.resolved, tt.output, tt.directory, ""); err == nil {
|
||||
t.Fatal("newRunResult() succeeded")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultOmitsIndexFileForOtherOutputModules(t *testing.T) {
|
||||
result, err := newRunResult(testResolvedPipeline("test/output"), testRunOutputWithout(func(output *pipeline.RunOutput) {
|
||||
output.OutputFiles = nil
|
||||
}), "output", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.IndexFile != "" {
|
||||
t.Fatalf("index_file = %q", result.IndexFile)
|
||||
}
|
||||
encoded, err := encodeRunResult(result)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := decoded["index_file"]; ok {
|
||||
t.Fatalf("encoded non-JSON result contains index_file: %s", encoded)
|
||||
}
|
||||
if _, ok := decoded["debug_directory"]; ok {
|
||||
t.Fatalf("encoded result without debug capture contains debug_directory: %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultRequiresOneProductionIndexFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
files []contracts.OutputFile
|
||||
}{
|
||||
{name: "missing", files: nil},
|
||||
{name: "duplicate", files: []contracts.OutputFile{{Name: "index.json"}, {Name: "index.json"}}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
output := testRunOutput()
|
||||
output.OutputFiles = tt.files
|
||||
if _, err := newRunResult(testResolvedPipeline(pipeline.DefaultOutputModule), output, "output", ""); err == nil {
|
||||
t.Fatal("newRunResult() succeeded")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteRunResultCompletesAndReportsWriterFailure(t *testing.T) {
|
||||
content := []byte("result\n")
|
||||
var target bytes.Buffer
|
||||
if err := writeRunResult(partialResultWriter{writer: &target, limit: 2}, content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := target.String(); got != string(content) {
|
||||
t.Fatalf("written result = %q", got)
|
||||
}
|
||||
|
||||
writerErr := errors.New("result writer failed")
|
||||
if err := writeRunResult(failingResultWriter{err: writerErr}, content); !errors.Is(err, writerErr) {
|
||||
t.Fatalf("writeRunResult() error = %v", err)
|
||||
}
|
||||
if err := writeRunResult(zeroResultWriter{}, content); !errors.Is(err, io.ErrShortWrite) {
|
||||
t.Fatalf("zero-progress error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func testResolvedPipeline(outputModule string) pipeline.ResolvedPipeline {
|
||||
return pipeline.ResolvedPipeline{ID: "sample", Output: pipeline.ModuleBinding{Module: outputModule}}
|
||||
}
|
||||
|
||||
func testRunOutput() pipeline.RunOutput {
|
||||
return pipeline.RunOutput{
|
||||
Manifest: artifacts.RunManifest{RunID: "run-123", PipelineID: "sample", ValidationStatus: "rejected"},
|
||||
NormalizeOutputs: []contracts.SerializedOutput{{}, {}},
|
||||
Rejected: []contracts.RejectedOutput{{}},
|
||||
Warnings: []contracts.Warning{{}},
|
||||
OutputFiles: []contracts.OutputFile{{Name: "index.json"}},
|
||||
}
|
||||
}
|
||||
|
||||
func testRunOutputWithout(change func(*pipeline.RunOutput)) pipeline.RunOutput {
|
||||
output := testRunOutput()
|
||||
change(&output)
|
||||
return output
|
||||
}
|
||||
|
||||
func mustWorkingDirectory(t *testing.T) string {
|
||||
t.Helper()
|
||||
workingDirectory, err := filepath.Abs(".")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return workingDirectory
|
||||
}
|
||||
|
||||
type partialResultWriter struct {
|
||||
writer io.Writer
|
||||
limit int
|
||||
}
|
||||
|
||||
func (w partialResultWriter) Write(content []byte) (int, error) {
|
||||
if len(content) > w.limit {
|
||||
content = content[:w.limit]
|
||||
}
|
||||
return w.writer.Write(content)
|
||||
}
|
||||
|
||||
type failingResultWriter struct{ err error }
|
||||
|
||||
func (w failingResultWriter) Write([]byte) (int, error) { return 0, w.err }
|
||||
|
||||
type zeroResultWriter struct{}
|
||||
|
||||
func (zeroResultWriter) Write([]byte) (int, error) { return 0, nil }
|
||||
90
internal/cli/run_terminal.go
Normal file
90
internal/cli/run_terminal.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
type DebugTerminalWriter interface {
|
||||
WriteRunReport(debugbundle.RunReport) error
|
||||
WriteError(string) error
|
||||
}
|
||||
|
||||
type pipelineCommandState struct {
|
||||
report debugbundle.RunReport
|
||||
terminalized bool
|
||||
}
|
||||
|
||||
func newPipelineCommandState(runID, pipelineID, outputPath string) *pipelineCommandState {
|
||||
return &pipelineCommandState{report: debugbundle.RunReport{
|
||||
RunID: runID,
|
||||
PipelineID: pipelineID,
|
||||
OutputPath: outputPath,
|
||||
}}
|
||||
}
|
||||
|
||||
func (s *pipelineCommandState) setDebugPath(debugPath string) {
|
||||
if s != nil {
|
||||
s.report.DebugPath = debugPath
|
||||
}
|
||||
}
|
||||
|
||||
func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.report.OutputCount = len(output.NormalizeOutputs)
|
||||
s.report.RejectedCount = len(output.Rejected)
|
||||
s.report.WarningCount = len(output.Warnings)
|
||||
s.report.ValidationStatus = output.Manifest.ValidationStatus
|
||||
}
|
||||
|
||||
func (s *pipelineCommandState) terminalize(writer DebugTerminalWriter, primaryErr error) (error, error) {
|
||||
if s == nil || s.terminalized {
|
||||
return primaryErr, nil
|
||||
}
|
||||
s.terminalized = true
|
||||
if writer == nil {
|
||||
return primaryErr, nil
|
||||
}
|
||||
|
||||
report := s.report
|
||||
report.Succeeded = primaryErr == nil
|
||||
reportErr := writer.WriteRunReport(report)
|
||||
if reportErr != nil {
|
||||
reportErr = fmt.Errorf("write debug run report: %w", reportErr)
|
||||
if primaryErr == nil {
|
||||
primaryErr = reportErr
|
||||
reportErr = nil
|
||||
}
|
||||
}
|
||||
|
||||
var errorLogErr error
|
||||
if primaryErr != nil {
|
||||
if err := writer.WriteError(primaryErr.Error()); err != nil {
|
||||
errorLogErr = fmt.Errorf("write debug error log: %w", err)
|
||||
}
|
||||
}
|
||||
return primaryErr, errors.Join(reportErr, errorLogErr)
|
||||
}
|
||||
|
||||
func failPipelineCommand(stderr io.Writer, state *pipelineCommandState, writer DebugTerminalWriter, primaryErr error, persistenceErrs ...error) int {
|
||||
primaryErr, terminalErr := state.terminalize(writer, primaryErr)
|
||||
persistenceErrs = append(persistenceErrs, terminalErr)
|
||||
return writePipelineCommandFailure(stderr, state, primaryErr, errors.Join(persistenceErrs...))
|
||||
}
|
||||
|
||||
func writePipelineCommandFailure(stderr io.Writer, state *pipelineCommandState, primaryErr, persistenceErr error) int {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", primaryErr)
|
||||
if persistenceErr != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", persistenceErr)
|
||||
}
|
||||
if state != nil && state.report.DebugPath != "" {
|
||||
fmt.Fprintf(stderr, "notarius: debug=%s\n", state.report.DebugPath)
|
||||
}
|
||||
return 1
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,68 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
const profileCheckPromptID = "notarius.profile.check"
|
||||
|
||||
var profileCheckPromptFS = fstest.MapFS{
|
||||
"prompts/profile-check.yaml": &fstest.MapFile{Data: []byte(`id: notarius.profile.check
|
||||
version: "1.0.0"
|
||||
default_profile: mistral-small-3
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: "{{input \"transcript\"}}"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
repair_attempts: 0
|
||||
`)},
|
||||
}
|
||||
|
||||
func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
|
||||
if len(profileIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
engine, err := newProfileValidationEngine(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load Scriptorium profiles: %w", err)
|
||||
}
|
||||
for _, profileID := range profileIDs {
|
||||
if _, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||
PromptID: profileCheckPromptID,
|
||||
ProfileID: profileID,
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("profile check"),
|
||||
},
|
||||
}); err != nil {
|
||||
if errors.Is(err, scriptorium.ErrProfileNotFound) {
|
||||
return fmt.Errorf("Scriptorium profile %q is not configured", profileID)
|
||||
}
|
||||
return fmt.Errorf("validate Scriptorium profile %q: %w", profileID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newProfileValidationEngine(cfg config.Config) (*scriptorium.Engine, error) {
|
||||
opts := []scriptorium.Option{
|
||||
scriptorium.WithPromptFS(profileCheckPromptFS, "prompts"),
|
||||
}
|
||||
if cfg.Scriptorium.ProfileFile != "" {
|
||||
opts = append(opts, scriptorium.WithProfileFile(cfg.Scriptorium.ProfileFile))
|
||||
}
|
||||
return scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "unused",
|
||||
ProfileDir: cfg.Scriptorium.ProfileDir,
|
||||
}, opts...)
|
||||
}
|
||||
503
internal/cli/spell_catalog_identity_contract_test.go
Normal file
503
internal/cli/spell_catalog_identity_contract_test.go
Normal file
@@ -0,0 +1,503 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||
)
|
||||
|
||||
func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := writeProductionSpellCatalogContractConfig(t)
|
||||
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve production configuration: %v", err)
|
||||
}
|
||||
overlayPath := filepath.Join(t.TempDir(), "catalog.json")
|
||||
resolved := effective.ResolvedPipeline
|
||||
bindings := resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings
|
||||
catalogBindingIndex := -1
|
||||
for index, binding := range bindings {
|
||||
if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
||||
catalogBindingIndex = index
|
||||
break
|
||||
}
|
||||
}
|
||||
if catalogBindingIndex < 0 {
|
||||
t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings)
|
||||
}
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath
|
||||
normalizeBindings := resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings
|
||||
normalizeCatalogBindingIndex := -1
|
||||
for index, binding := range normalizeBindings {
|
||||
if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
||||
normalizeCatalogBindingIndex = index
|
||||
break
|
||||
}
|
||||
}
|
||||
if normalizeCatalogBindingIndex < 0 {
|
||||
t.Fatalf("normalize spell catalog bindings = %#v, want catalog binding", normalizeBindings)
|
||||
}
|
||||
resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings[normalizeCatalogBindingIndex].Source = overlayPath
|
||||
|
||||
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize first catalog: %v", err)
|
||||
}
|
||||
identityA := catalogCheckpointIdentity(t, materializedA)
|
||||
metadataA := catalogExtractorMetadata(t, materializedA)
|
||||
normalizerMetadataA := catalogNormalizerMetadata(t, materializedA)
|
||||
referenceA := catalogReference(t, materializedA)
|
||||
|
||||
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize reordered catalog: %v", err)
|
||||
}
|
||||
identityB := catalogCheckpointIdentity(t, materializedB)
|
||||
metadataB := catalogExtractorMetadata(t, materializedB)
|
||||
normalizerMetadataB := catalogNormalizerMetadata(t, materializedB)
|
||||
referenceB := catalogReference(t, materializedB)
|
||||
|
||||
if identityA.Digest == identityB.Digest {
|
||||
t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest)
|
||||
}
|
||||
if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI {
|
||||
t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB)
|
||||
}
|
||||
digestA, ok := metadataA["catalog_digest"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA)
|
||||
}
|
||||
digestB, ok := metadataB["catalog_digest"].(string)
|
||||
if !ok || digestA != digestB {
|
||||
t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB)
|
||||
}
|
||||
if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) {
|
||||
t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want)
|
||||
}
|
||||
normalizerDigestA, ok := normalizerMetadataA["catalog_digest"].(string)
|
||||
normalizerDigestB, okB := normalizerMetadataB["catalog_digest"].(string)
|
||||
if !ok || !okB || normalizerDigestA != digestA || normalizerDigestB != digestB {
|
||||
t.Fatalf("normalizer catalog digests = %#v and %#v, want extractor semantic digests %q and %q", normalizerMetadataA["catalog_digest"], normalizerMetadataB["catalog_digest"], digestA, digestB)
|
||||
}
|
||||
if got, want := normalizerMetadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(normalizerMetadataB["catalog_overlay_ids"], want) {
|
||||
t.Fatalf("normalizer overlay IDs = %#v and %#v, want %#v", got, normalizerMetadataB["catalog_overlay_ids"], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) {
|
||||
base := productionSpellCatalogContractConfig(t)
|
||||
changed := strings.Replace(base, repositoryPath("examples", "dnd-spell-catalog.json"), filepath.Join(t.TempDir(), "alternate-spell-catalog.json"), 1)
|
||||
if changed == base {
|
||||
t.Fatal("production configuration did not contain the maintained catalog binding")
|
||||
}
|
||||
root := t.TempDir()
|
||||
firstPath := filepath.Join(root, "first.yml")
|
||||
secondPath := filepath.Join(root, "second.yml")
|
||||
if err := os.WriteFile(firstPath, []byte(base), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(secondPath, []byte(changed), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
components := productionTestComponents(t)
|
||||
first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve first configuration: %v", err)
|
||||
}
|
||||
second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve changed configuration: %v", err)
|
||||
}
|
||||
if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest {
|
||||
t.Fatalf("resolved pipeline digest = %q for different catalog bindings, want change", first.ResolvedPipeline.Digest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := writeProductionSpellCatalogContractConfig(t)
|
||||
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fingerprints := prepared.CheckpointFingerprints()
|
||||
wantNames := map[string]struct{}{
|
||||
"extract:spells:" + spells.Key + ":effective_catalog": {},
|
||||
"extract:spells:" + spells.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {},
|
||||
"normalize:spells:" + spellnormalize.Key + ":effective_catalog": {},
|
||||
"normalize:spells:" + spellnormalize.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {},
|
||||
}
|
||||
seen := make(map[string]string, len(fingerprints))
|
||||
for _, fingerprint := range fingerprints {
|
||||
if _, ok := wantNames[fingerprint.Name]; ok {
|
||||
seen[fingerprint.Name] = fingerprint.Value
|
||||
}
|
||||
}
|
||||
if len(seen) != len(wantNames) {
|
||||
t.Fatalf("prepared fingerprints = %#v, want scoped extractor and normalize catalog identities", fingerprints)
|
||||
}
|
||||
var catalogDigest string
|
||||
for name, value := range seen {
|
||||
if catalogDigest == "" {
|
||||
catalogDigest = value
|
||||
} else if value != catalogDigest {
|
||||
t.Fatalf("prepared fingerprint %q = %q, want shared semantic catalog digest %q", name, value, catalogDigest)
|
||||
}
|
||||
}
|
||||
|
||||
identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity {
|
||||
identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
||||
Pipeline: materialized,
|
||||
InputKey: materialized.Input.Module,
|
||||
RawInputDigest: "sha256:unchanged-input",
|
||||
References: pipeline.ReferenceProvenance(materialized),
|
||||
ProvenanceFingerprints: checkpointIdentityFingerprints(values),
|
||||
})
|
||||
if identityErr != nil {
|
||||
t.Fatal(identityErr)
|
||||
}
|
||||
return identity
|
||||
}
|
||||
first := identityFor(fingerprints)
|
||||
changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
|
||||
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
|
||||
second := identityFor(changed)
|
||||
if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) {
|
||||
t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := writeProductionSpellCatalogContractConfig(t)
|
||||
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fingerprints := prepared.CheckpointFingerprints()
|
||||
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}}
|
||||
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
|
||||
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"}
|
||||
doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
||||
doc.Digest, err = source.DigestDocument(&doc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
normalizeDependencies := []pipeline.CheckpointFingerprint{{Name: "artifact[0]", Value: "sha256:merged-artifact"}}
|
||||
normalizeSchema := contracts.ArtifactSchema{ID: "notarius.dnd.spells", Name: "notarius_dnd_spells", Version: "v1"}
|
||||
normalizeArtifact := pipeline.CheckpointArtifact{
|
||||
LaneID: "spells", ModuleKey: spellnormalize.Key, SourceID: doc.ID,
|
||||
SchemaDigest: contracts.DigestArtifactSchema(normalizeSchema),
|
||||
Artifact: contracts.SerializedArtifact{
|
||||
Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
|
||||
},
|
||||
}
|
||||
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused {
|
||||
t.Fatalf("same fingerprint decision = %#v, want reuse", decision)
|
||||
}
|
||||
if restored, decision := sameLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); !decision.Reused || string(restored.Output.Artifact.Content) != `{"spell_casts":[]}` {
|
||||
t.Fatalf("same normalize checkpoint = %#v, decision=%#v, want reuse", restored, decision)
|
||||
}
|
||||
changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
|
||||
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
|
||||
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused {
|
||||
t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision)
|
||||
}
|
||||
if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
||||
t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision)
|
||||
}
|
||||
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v3")
|
||||
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName())
|
||||
_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, decision := mappingLoader.Source(materialized.Input.Module); decision.Reused {
|
||||
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
|
||||
}
|
||||
|
||||
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}}
|
||||
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
|
||||
t.Fatalf("changed PromptKit profile source decision = %#v, want cold miss", decision)
|
||||
}
|
||||
if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
||||
t.Fatalf("changed PromptKit profile normalize decision = %#v, want cold miss", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeSpellCatalogFingerprintName() string {
|
||||
return "normalize:spells:" + spellnormalize.Key + ":effective_catalog"
|
||||
}
|
||||
|
||||
func extractSpellMappingFingerprintName() string {
|
||||
return "extract:spells:" + spells.Key + ":mapping_policy"
|
||||
}
|
||||
|
||||
func replaceCheckpointFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, value string) []pipeline.CheckpointFingerprint {
|
||||
t.Helper()
|
||||
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
|
||||
matches := 0
|
||||
for index := range changed {
|
||||
if changed[index].Name == name {
|
||||
changed[index].Value = value
|
||||
matches++
|
||||
}
|
||||
}
|
||||
if matches != 1 {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want exactly one fingerprint named %q", fingerprints, name)
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
func assertOnlyCheckpointFingerprintChanged(t *testing.T, before, after []pipeline.CheckpointFingerprint, changedName string) {
|
||||
t.Helper()
|
||||
if len(before) != len(after) {
|
||||
t.Fatalf("fingerprint lengths = %d and %d, want equal", len(before), len(after))
|
||||
}
|
||||
changes := 0
|
||||
for index := range before {
|
||||
if before[index].Name != after[index].Name {
|
||||
t.Fatalf("fingerprint[%d] name changed from %q to %q", index, before[index].Name, after[index].Name)
|
||||
}
|
||||
if before[index].Value == after[index].Value {
|
||||
continue
|
||||
}
|
||||
changes++
|
||||
if before[index].Name != changedName {
|
||||
t.Fatalf("fingerprint %q changed unexpectedly", before[index].Name)
|
||||
}
|
||||
}
|
||||
if changes != 1 {
|
||||
t.Fatalf("fingerprints changed %d values, want exactly %q", changes, changedName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}
|
||||
options := productionRunOptions(t, fake)
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session",
|
||||
"--config", writeProductionSpellCatalogContractConfig(t),
|
||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
||||
}, &stdout, &stderr, options)
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
runRoot := filepath.Join(outputRoot, productionRunID)
|
||||
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
|
||||
if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 {
|
||||
t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest)
|
||||
}
|
||||
lane := manifest.ArtifactLanes[0]
|
||||
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata)
|
||||
}
|
||||
if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") {
|
||||
t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata)
|
||||
}
|
||||
if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) {
|
||||
t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got)
|
||||
}
|
||||
normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("lane metadata = %#v, want normalizer metadata", lane.Metadata)
|
||||
}
|
||||
if normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), []string{"notarius.example-campaign"}) {
|
||||
t.Fatalf("normalizer catalog metadata = %#v, want base ID, semantic digest, and overlay IDs", normalizerMetadata)
|
||||
}
|
||||
if normalizerMetadata["catalog_digest"] != extractorMetadata["catalog_digest"] || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), stringValues(extractorMetadata["catalog_overlay_ids"])) {
|
||||
t.Fatalf("extractor metadata = %#v, normalizer metadata = %#v, want shared catalog identity", extractorMetadata, normalizerMetadata)
|
||||
}
|
||||
|
||||
var catalogProvenances []artifacts.ReferenceProvenance
|
||||
for index := range manifest.References {
|
||||
reference := &manifest.References[index]
|
||||
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
||||
catalogProvenances = append(catalogProvenances, *reference)
|
||||
}
|
||||
}
|
||||
if len(catalogProvenances) != 2 {
|
||||
t.Fatalf("manifest references = %#v, want independently materialized extract and normalize catalog provenance", manifest.References)
|
||||
}
|
||||
overlayBytes := readRepositoryFile(t, "examples", "dnd-spell-catalog.json")
|
||||
for _, catalogProvenance := range catalogProvenances {
|
||||
if catalogProvenance.Stage != "extract" && catalogProvenance.Stage != "normalize" {
|
||||
t.Fatalf("catalog provenance = %#v, want extract or normalize scope", catalogProvenance)
|
||||
}
|
||||
if catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spell-catalog.json") {
|
||||
t.Fatalf("catalog provenance = %#v, want raw overlay provenance in both scopes", catalogProvenance)
|
||||
}
|
||||
}
|
||||
manifestBytes, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} {
|
||||
if strings.Contains(string(manifestBytes), leaked) {
|
||||
t.Fatalf("manifest leaked overlay content %q", leaked)
|
||||
}
|
||||
}
|
||||
|
||||
requests := fake.requestsFor(spells.PromptID)
|
||||
if len(requests) != 1 {
|
||||
t.Fatalf("spell requests = %d, want one", len(requests))
|
||||
}
|
||||
catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
||||
if !ok || !strings.Contains(string(catalogInput.Content), "Aegis of Emberfall") || strings.Contains(string(catalogInput.Content), "Emberfall Aegis") {
|
||||
t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name without alias", catalogInput)
|
||||
}
|
||||
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json"))
|
||||
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
|
||||
t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact)
|
||||
}
|
||||
rejected := readProductionJSON[struct {
|
||||
Rejected []json.RawMessage `json:"rejected"`
|
||||
}](t, filepath.Join(runRoot, "rejected.json"))
|
||||
if len(rejected.Rejected) != 0 {
|
||||
t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity {
|
||||
t.Helper()
|
||||
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
||||
Pipeline: resolved,
|
||||
InputKey: resolved.Input.Module,
|
||||
RawInputDigest: "sha256:catalog-test-input",
|
||||
References: pipeline.ReferenceProvenance(resolved),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create checkpoint identity: %v", err)
|
||||
}
|
||||
return identity
|
||||
}
|
||||
|
||||
func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
|
||||
t.Helper()
|
||||
lane := resolved.Steps[0].ArtifactLanes[0]
|
||||
extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet)
|
||||
if err != nil {
|
||||
t.Fatalf("construct extractor: %v", err)
|
||||
}
|
||||
return extractor.ManifestMetadata()
|
||||
}
|
||||
|
||||
func catalogNormalizerMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
|
||||
t.Helper()
|
||||
lane := resolved.Steps[0].ArtifactLanes[0]
|
||||
normalizer, err := spellnormalize.New(spellnormalize.Options{}, lane.NormalizeReferences.ReferenceSet)
|
||||
if err != nil {
|
||||
t.Fatalf("construct normalizer: %v", err)
|
||||
}
|
||||
return normalizer.ManifestMetadata()
|
||||
}
|
||||
|
||||
func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance {
|
||||
t.Helper()
|
||||
for _, reference := range pipeline.ReferenceProvenance(resolved) {
|
||||
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" {
|
||||
return reference
|
||||
}
|
||||
}
|
||||
t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved))
|
||||
return artifacts.ReferenceProvenance{}
|
||||
}
|
||||
|
||||
func stringValue(value any) string {
|
||||
result, _ := value.(string)
|
||||
return result
|
||||
}
|
||||
|
||||
func stringValues(value any) []string {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var values []string
|
||||
if err := json.Unmarshal(raw, &values); err != nil {
|
||||
return nil
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func digestBytes(value []byte) string {
|
||||
sum := sha256.Sum256(value)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
const reorderedOverlayA = `{
|
||||
"schema_version": "notarius.dnd.spell-catalog-overlay.v1",
|
||||
"catalogs": [
|
||||
{"id":"campaign.a","ruleset":"dnd-5e-2014","source":{"title":"Campaign A"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]},
|
||||
{"id":"campaign.b","ruleset":"dnd-5e-2014","source":{"title":"Campaign B"},"spells":[{"name":"Cinder Veil","aliases":["Veil of Cinder","Cinder Shroud"]}]}
|
||||
]
|
||||
}`
|
||||
|
||||
const reorderedOverlayB = `{"catalogs":[{"spells":[{"aliases":["Cinder Shroud","Veil of Cinder"],"name":"Cinder Veil"}],"source":{"title":"Campaign B"},"ruleset":"dnd-5e-2014","id":"campaign.b"},{"spells":[{"aliases":["Emberfall Aegis"],"name":"Aegis of Emberfall"}],"source":{"title":"Campaign A"},"ruleset":"dnd-5e-2014","id":"campaign.a"}],"schema_version":"notarius.dnd.spell-catalog-overlay.v1"}`
|
||||
160
internal/cli/spell_catalog_retry_contract_test.go
Normal file
160
internal/cli/spell_catalog_retry_contract_test.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
)
|
||||
|
||||
func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
const retries = 2
|
||||
tests := []struct {
|
||||
name string
|
||||
responses []string
|
||||
wantCalls int
|
||||
wantRejected bool
|
||||
wantSpell string
|
||||
wantWarningCode string
|
||||
}{
|
||||
{
|
||||
name: "unknown spell remains rejected after exhaustion",
|
||||
responses: []string{
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
},
|
||||
wantCalls: retries + 1,
|
||||
wantRejected: true,
|
||||
},
|
||||
{
|
||||
name: "overlay spell becomes valid on retry",
|
||||
responses: []string{
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
productionSpellResponse("Aegis of Emberfall"),
|
||||
},
|
||||
wantCalls: 2,
|
||||
wantSpell: "Aegis of Emberfall",
|
||||
wantWarningCode: "spell_not_near_source",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := writeProductionSpellCatalogContractConfig(t)
|
||||
cfg := loadMaintainedExample(t, configPath)
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve production configuration: %v", err)
|
||||
}
|
||||
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
|
||||
ConfigPath: configPath,
|
||||
WorkingDir: filepath.Dir(configPath),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize production references: %v", err)
|
||||
}
|
||||
materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries
|
||||
|
||||
llmClient := &catalogRetryLLMClient{responses: tt.responses}
|
||||
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare production pipeline: %v", err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if calls := llmClient.CallCount(); calls > retries+1 || calls != tt.wantCalls {
|
||||
t.Fatalf("LLM calls = %d, want %d and no more than %d", calls, tt.wantCalls, retries+1)
|
||||
}
|
||||
|
||||
if tt.wantRejected {
|
||||
if len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 {
|
||||
t.Fatalf("rejected = %#v normalized = %#v, want one nonfatal rejection and no merge output", output.Rejected, output.NormalizeOutputs)
|
||||
}
|
||||
rejection := output.Rejected[0]
|
||||
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
|
||||
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
|
||||
}
|
||||
if len(output.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want no warnings from rejected attempts", output.Warnings)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("rejected = %#v normalized = %#v, want only accepted output", output.Rejected, output.NormalizeOutputs)
|
||||
}
|
||||
var value dnd.SpellList
|
||||
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &value); err != nil {
|
||||
t.Fatalf("decode normalized spell list: %v", err)
|
||||
}
|
||||
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
|
||||
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
|
||||
}
|
||||
if len(output.Warnings) != 2 || output.Warnings[0].ReasonCode != tt.wantWarningCode || output.Warnings[1].ReasonCode != tt.wantWarningCode {
|
||||
t.Fatalf("warnings = %#v, want accepted-attempt warnings from extract and normalize validation", output.Warnings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type catalogRetryLLMClient struct {
|
||||
mu sync.Mutex
|
||||
responses []string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (client *catalogRetryLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
if req.PromptID != spells.PromptID {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
|
||||
}
|
||||
client.mu.Lock()
|
||||
index := client.calls
|
||||
client.calls++
|
||||
client.mu.Unlock()
|
||||
if index >= len(client.responses) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index)
|
||||
}
|
||||
content := []byte(client.responses[index])
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
|
||||
}
|
||||
|
||||
func (client *catalogRetryLLMClient) CallCount() int {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
return client.calls
|
||||
}
|
||||
|
||||
func productionSpellResponse(name string) string {
|
||||
content, err := json.Marshal(dnd.SpellList{SpellCasts: []dnd.SpellCast{{
|
||||
Caster: "Aria",
|
||||
Spell: name,
|
||||
SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}},
|
||||
}}})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return string(content)
|
||||
}
|
||||
988
internal/cli/state_hardening_test.go
Normal file
988
internal/cli/state_hardening_test.go
Normal file
@@ -0,0 +1,988 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const stateTestDigest = "sha256:e511d8906649b78eb639b11215fa57a9652a1a64f4aefa3ed68320dbda46f439"
|
||||
|
||||
func TestRunStateSurfaceMatrix(t *testing.T) {
|
||||
for _, debug := range []bool{false, true} {
|
||||
for _, resume := range []bool{false, true} {
|
||||
for _, mode := range []string{"auto", "bypass", "refresh"} {
|
||||
name := fmt.Sprintf("debug=%t/resume=%t/cache=%s", debug, resume, mode)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
opts := harness.options()
|
||||
var storeRoots []string
|
||||
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
||||
storeRoots = append(storeRoots, root)
|
||||
return chunkplan.NewFilesystemStore(root)
|
||||
}
|
||||
result := runStateTest(t, roots, opts, debug, resume, mode)
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
if mode == "bypass" {
|
||||
assertAbsent(t, roots.plans)
|
||||
if len(storeRoots) != 0 {
|
||||
t.Fatalf("chunk plan store roots = %v, want none", storeRoots)
|
||||
}
|
||||
} else {
|
||||
assertFile(t, filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
|
||||
if len(storeRoots) != 1 || storeRoots[0] != roots.plans {
|
||||
t.Fatalf("chunk plan store roots = %v, want [%q]", storeRoots, roots.plans)
|
||||
}
|
||||
}
|
||||
assertAnyFile(t, roots.checkpoints)
|
||||
assertRestrictedTree(t, roots.checkpoints)
|
||||
if debug {
|
||||
bundle := onlyChildDir(t, roots.debug)
|
||||
assertFile(t, filepath.Join(bundle, "summary", "invocation.json"))
|
||||
assertAnyFile(t, filepath.Join(bundle, "trace"))
|
||||
assertRestrictedTree(t, roots.debug)
|
||||
} else {
|
||||
assertAbsent(t, roots.debug)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunKeepsStateRootsIndependentAndReusesSelectedCheckpointRoot(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
first := runStateTest(t, roots, harness.options(), true, false, "auto")
|
||||
if first.code != 0 {
|
||||
t.Fatalf("first run code=%d stderr=%q", first.code, first.stderr)
|
||||
}
|
||||
planPath := filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json")
|
||||
initialPlan, err := os.ReadFile(planPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
firstBundle := onlyChildDir(t, roots.debug)
|
||||
|
||||
second := runStateTest(t, roots, harness.options(), false, false, "auto")
|
||||
if second.code != 0 {
|
||||
t.Fatalf("second run code=%d stderr=%q", second.code, second.stderr)
|
||||
}
|
||||
if harness.chunkCalls != 1 {
|
||||
t.Fatalf("chunk calls after debug toggle = %d, want 1", harness.chunkCalls)
|
||||
}
|
||||
if harness.extractCalls != 2 {
|
||||
t.Fatalf("extract calls after two recording-only runs = %d, want 2", harness.extractCalls)
|
||||
}
|
||||
if got, err := os.ReadFile(planPath); err != nil || !bytes.Equal(got, initialPlan) {
|
||||
t.Fatalf("chunk plan changed after debug toggle: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(firstBundle); err != nil {
|
||||
t.Fatalf("initial debug bundle was removed: %v", err)
|
||||
}
|
||||
|
||||
checkpointRoot := roots.checkpoints
|
||||
extractCallsBeforeResume := harness.extractCalls
|
||||
seed := runStateTest(t, roots, harness.options(), false, true, "auto")
|
||||
if seed.code != 0 {
|
||||
t.Fatalf("checkpoint seed code=%d stderr=%q", seed.code, seed.stderr)
|
||||
}
|
||||
if harness.extractCalls != extractCallsBeforeResume {
|
||||
t.Fatalf("extract calls after reusing recording-only checkpoint = %d, want %d", harness.extractCalls, extractCallsBeforeResume)
|
||||
}
|
||||
extractCalls := harness.extractCalls
|
||||
checkpointFiles := readTree(t, checkpointRoot)
|
||||
reused := runStateTest(t, roots, harness.options(), false, true, "auto")
|
||||
if reused.code != 0 {
|
||||
t.Fatalf("checkpoint reuse code=%d stderr=%q", reused.code, reused.stderr)
|
||||
}
|
||||
if harness.extractCalls != extractCalls {
|
||||
t.Fatalf("extract calls after checkpoint reuse = %d, want %d", harness.extractCalls, extractCalls)
|
||||
}
|
||||
if got := readTree(t, checkpointRoot); !sameFiles(got, checkpointFiles) {
|
||||
t.Fatal("reused checkpoint was rewritten")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecomputesOnlyAfterExplicitChunkPlanRemoval(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
first := runStateTest(t, roots, harness.options(), true, false, "auto")
|
||||
if first.code != 0 {
|
||||
t.Fatalf("first run code=%d stderr=%q", first.code, first.stderr)
|
||||
}
|
||||
firstOutput := onlyChildDir(t, roots.output)
|
||||
firstBundle := onlyChildDir(t, roots.debug)
|
||||
entry := filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"))
|
||||
if err := os.RemoveAll(entry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := runStateTest(t, roots, harness.options(), false, false, "auto")
|
||||
if second.code != 0 {
|
||||
t.Fatalf("second run code=%d stderr=%q", second.code, second.stderr)
|
||||
}
|
||||
if harness.chunkCalls != 2 {
|
||||
t.Fatalf("chunk calls = %d, want 2 after removing exact cache entry", harness.chunkCalls)
|
||||
}
|
||||
assertFile(t, filepath.Join(firstOutput, "result.json"))
|
||||
assertFile(t, filepath.Join(firstBundle, "summary", "run-report.json"))
|
||||
}
|
||||
|
||||
func TestRunRetainsDebugBundlesAcrossFailures(t *testing.T) {
|
||||
t.Run("configuration failure precedes allocation", func(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "debug")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", filepath.Join(t.TempDir(), "missing.yml"), "--input", "missing", "--debug", "--debug-dir", root}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 1 || !strings.Contains(stderr.String(), "config file") {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
assertAbsent(t, root)
|
||||
})
|
||||
|
||||
for _, failure := range []struct {
|
||||
name string
|
||||
expected string
|
||||
setup func(*testing.T, stateTestRoots, *stateTestHarness) Options
|
||||
}{
|
||||
{"resolution", "pipeline \"missing\"", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { return h.options() }},
|
||||
{"pipeline", "synthetic extraction failure", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options {
|
||||
h.extractErr = errors.New("synthetic extraction failure")
|
||||
return h.options()
|
||||
}},
|
||||
{"output", "create output parent", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options {
|
||||
if err := os.WriteFile(roots.output, []byte("not a directory"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h.options()
|
||||
}},
|
||||
{"summary", "write debug invocation metadata", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options {
|
||||
opts := h.options()
|
||||
opts.DebugRecorderFactory = func(traceRoot string) (pipeline.DebugRecorder, error) {
|
||||
if err := os.RemoveAll(filepath.Join(filepath.Dir(traceRoot), "summary")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(filepath.Dir(traceRoot), "summary"), []byte("blocked"), 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return frameworkdebug.NewFilesystemRecorder(traceRoot)
|
||||
}
|
||||
return opts
|
||||
}},
|
||||
{"trace", "trace unavailable", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options {
|
||||
opts := h.options()
|
||||
opts.DebugRecorderFactory = func(string) (pipeline.DebugRecorder, error) { return failingDebugRecorder{}, nil }
|
||||
return opts
|
||||
}},
|
||||
} {
|
||||
t.Run(failure.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
opts := failure.setup(t, roots, harness)
|
||||
failureStderr := ""
|
||||
if failure.name == "resolution" {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "missing", "--config", roots.config, "--input", roots.input, "--debug"}, &stdout, &stderr, opts)
|
||||
if code != 1 {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
failureStderr = stderr.String()
|
||||
} else {
|
||||
result := runStateTest(t, roots, opts, true, false, "bypass")
|
||||
if result.code != 1 {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
failureStderr = result.stderr
|
||||
}
|
||||
if !strings.Contains(failureStderr, failure.expected) || !strings.Contains(failureStderr, "debug=") {
|
||||
t.Fatalf("stderr=%q, want %q and debug path", failureStderr, failure.expected)
|
||||
}
|
||||
bundle := onlyChildDir(t, roots.debug)
|
||||
if !strings.Contains(readAllFiles(t, bundle), "synthetic") && failure.name == "pipeline" {
|
||||
t.Fatal("pipeline failure was not retained in debug bundle")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDebugArtifactsRedactSecretsButRetainApplicationData(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
t.Setenv("STATE_TEST_UNRELATED_ENV", "HOST_ONLY_SENTINEL")
|
||||
if err := os.WriteFile(filepath.Join(filepath.Dir(roots.input), "unrelated.txt"), []byte("HOST_ONLY_FILE_SENTINEL"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
harness := newStateTestHarness()
|
||||
result := runStateTest(t, roots, harness.options(), true, false, "bypass")
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
bundle := onlyChildDir(t, roots.debug)
|
||||
summary := readAllFiles(t, filepath.Join(bundle, "summary"))
|
||||
trace := readAllFiles(t, filepath.Join(bundle, "trace"))
|
||||
for _, forbidden := range []string{"sk-secretvalue", "Bearer secretvalue", "HOST_ONLY_SENTINEL", "HOST_ONLY_FILE_SENTINEL"} {
|
||||
if strings.Contains(summary, forbidden) || strings.Contains(trace, forbidden) {
|
||||
t.Fatalf("debug bundle contains %q", forbidden)
|
||||
}
|
||||
}
|
||||
if strings.Contains(summary, "application content") {
|
||||
t.Fatal("summary contains raw application input")
|
||||
}
|
||||
if !strings.Contains(trace, "application content") {
|
||||
t.Fatal("trace does not retain expected application input")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRedactsSensitiveModuleOptionsFromConfigAndPipelineSummaries(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configText := replaceRequiredOnce(t, string(data), " input: test/input\n", ` input:
|
||||
module: test/input
|
||||
options:
|
||||
api_key: CONFIG_SUMMARY_SECRET_SENTINEL
|
||||
safe: SAFE_OPTION_SENTINEL
|
||||
nested:
|
||||
- - password: PIPELINE_SUMMARY_SECRET_SENTINEL
|
||||
neighbor: SAFE_NESTED_OPTION_SENTINEL
|
||||
`)
|
||||
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result := runStateTest(t, roots, newStateTestHarness().options(), true, false, "bypass")
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
summaryRoot := filepath.Join(onlyChildDir(t, roots.debug), "summary")
|
||||
for _, name := range []string{"effective-config.json", "resolved-pipeline.json"} {
|
||||
contents, err := os.ReadFile(filepath.Join(summaryRoot, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(contents)
|
||||
for _, secret := range []string{"CONFIG_SUMMARY_SECRET_SENTINEL", "PIPELINE_SUMMARY_SECRET_SENTINEL"} {
|
||||
if strings.Contains(text, secret) {
|
||||
t.Fatalf("%s contains %q: %s", name, secret, text)
|
||||
}
|
||||
}
|
||||
for _, retained := range []string{"[REDACTED]", "SAFE_OPTION_SENTINEL", "SAFE_NESTED_OPTION_SENTINEL"} {
|
||||
if !strings.Contains(text, retained) {
|
||||
t.Fatalf("%s does not contain %q: %s", name, retained, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUsesOneInjectedIdentityForDebugOutputAndManifest(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
opts := harness.options()
|
||||
const runID = "run-1000000000-11111111111111111111111111111111"
|
||||
opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil }
|
||||
|
||||
result := runStateTest(t, roots, opts, true, false, "bypass")
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
outputPath := filepath.Join(roots.output, runID)
|
||||
debugPath := filepath.Join(roots.debug, runID)
|
||||
assertFile(t, filepath.Join(outputPath, "result.json"))
|
||||
assertFile(t, filepath.Join(debugPath, "summary", "run-manifest.json"))
|
||||
if !strings.Contains(result.stdout, "output="+outputPath) || !strings.Contains(result.stdout, "debug="+debugPath) {
|
||||
t.Fatalf("stdout=%q, want shared run identity", result.stdout)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(debugPath, "summary", "run-manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var manifest artifacts.RunManifest
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if manifest.RunID != runID {
|
||||
t.Fatalf("manifest run ID = %q, want %q", manifest.RunID, runID)
|
||||
}
|
||||
wantStartedAt := time.Unix(1, 0).UTC()
|
||||
if manifest.StartedAt == nil || !manifest.StartedAt.Equal(wantStartedAt) {
|
||||
t.Fatalf("manifest started at = %v, want %v", manifest.StartedAt, wantStartedAt)
|
||||
}
|
||||
var invocation debugbundle.Invocation
|
||||
readStateTestSummaryJSON(t, debugPath, "invocation.json", &invocation)
|
||||
if invocation.RunID != runID || !invocation.StartedAt.Equal(wantStartedAt) {
|
||||
t.Fatalf("debug invocation identity = %#v, want run %q at %v", invocation, runID, wantStartedAt)
|
||||
}
|
||||
report := readStateTestRunReport(t, debugPath)
|
||||
if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != "approved" {
|
||||
t.Fatalf("success report = %#v", report)
|
||||
}
|
||||
if !strings.Contains(result.stdout, "outputs=1 rejected=0") {
|
||||
t.Fatalf("stdout=%q, want report counts", result.stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
pipelineID string
|
||||
wantError string
|
||||
wantOutputs int
|
||||
wantValidation string
|
||||
configureFailure func(*testing.T, stateTestRoots, *stateTestHarness)
|
||||
}{
|
||||
{name: "resolution", pipelineID: "missing", wantError: `pipeline "missing"`},
|
||||
{name: "pipeline", pipelineID: "sample", wantError: "synthetic extraction failure", wantValidation: "failed", configureFailure: func(_ *testing.T, _ stateTestRoots, h *stateTestHarness) {
|
||||
h.extractErr = errors.New("synthetic extraction failure")
|
||||
}},
|
||||
{name: "output", pipelineID: "sample", wantError: "create output parent", wantOutputs: 1, wantValidation: "approved", configureFailure: func(t *testing.T, roots stateTestRoots, _ *stateTestHarness) {
|
||||
if err := os.WriteFile(roots.output, []byte("not a directory"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
if tc.configureFailure != nil {
|
||||
tc.configureFailure(t, roots, harness)
|
||||
}
|
||||
opts := harness.options()
|
||||
var stdout, stderr bytes.Buffer
|
||||
args := []string{"run", tc.pipelineID, "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}
|
||||
code := RunWithOptions(args, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), tc.wantError) {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
bundlePath := onlyChildDir(t, roots.debug)
|
||||
runID := filepath.Base(bundlePath)
|
||||
report := readStateTestRunReport(t, bundlePath)
|
||||
if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != tc.wantValidation {
|
||||
t.Fatalf("failure report = %#v", report)
|
||||
}
|
||||
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
|
||||
if err != nil || !strings.Contains(string(errorLog), tc.wantError) {
|
||||
t.Fatalf("error log = %q, %v", errorLog, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "partial-warning", Message: "warning retained before failure"}}
|
||||
harness.extractErr = errors.New("synthetic partial pipeline failure")
|
||||
|
||||
result := runStateTest(t, roots, harness.options(), true, true, "bypass")
|
||||
if result.code != 1 {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
bundlePath := onlyChildDir(t, roots.debug)
|
||||
report := readStateTestRunReport(t, bundlePath)
|
||||
if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningCount != 1 || report.ValidationStatus != "failed" {
|
||||
t.Fatalf("partial failure report = %#v", report)
|
||||
}
|
||||
|
||||
var manifest artifacts.RunManifest
|
||||
readStateTestSummaryJSON(t, bundlePath, "run-manifest.json", &manifest)
|
||||
if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" {
|
||||
t.Fatalf("partial manifest = %#v", manifest)
|
||||
}
|
||||
var warnings []contracts.Warning
|
||||
readStateTestSummaryJSON(t, bundlePath, "warnings.json", &warnings)
|
||||
if len(warnings) != 1 || warnings[0].ReasonCode != "partial-warning" {
|
||||
t.Fatalf("partial warnings = %#v", warnings)
|
||||
}
|
||||
var events []pipeline.CheckpointEvent
|
||||
readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events)
|
||||
if len(events) == 0 || events[0].Stage != "source" {
|
||||
t.Fatalf("partial checkpoint events = %#v, want retained source decision", events)
|
||||
}
|
||||
var chunkPlan artifacts.ChunkPlanSummary
|
||||
readStateTestSummaryJSON(t, bundlePath, "chunk-plan.json", &chunkPlan)
|
||||
if chunkPlan.Mode != "bypass" || chunkPlan.ValidationStatus == "not_run" {
|
||||
t.Fatalf("partial chunk plan = %#v", chunkPlan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunTerminalPersistenceFailuresDoNotRecurseOrHidePrimaryError(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
reportErr error
|
||||
errorLogErr error
|
||||
wantSecondary string
|
||||
}{
|
||||
{name: "run report", reportErr: errors.New("injected run report failure"), wantSecondary: "injected run report failure"},
|
||||
{name: "error log", errorLogErr: errors.New("injected error log failure"), wantSecondary: "injected error log failure"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.extractErr = errors.New("primary pipeline failure")
|
||||
opts := harness.options()
|
||||
var terminal *recordingTerminalWriter
|
||||
opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
|
||||
terminal = &recordingTerminalWriter{delegate: delegate, reportErr: tc.reportErr, errorLogErr: tc.errorLogErr}
|
||||
return terminal
|
||||
}
|
||||
|
||||
result := runStateTest(t, roots, opts, true, false, "bypass")
|
||||
if result.code != 1 {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
if terminal == nil {
|
||||
t.Fatal("terminal writer was not constructed")
|
||||
}
|
||||
if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 {
|
||||
t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls)
|
||||
}
|
||||
primaryIndex := strings.Index(result.stderr, "primary pipeline failure")
|
||||
secondaryIndex := strings.Index(result.stderr, tc.wantSecondary)
|
||||
debugIndex := strings.Index(result.stderr, "debug=")
|
||||
if primaryIndex < 0 || secondaryIndex <= primaryIndex || debugIndex <= secondaryIndex {
|
||||
t.Fatalf("stderr order = %q", result.stderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReportFailureOnSuccessIsTerminalizedWithoutRetry(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
var terminal *recordingTerminalWriter
|
||||
opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
|
||||
terminal = &recordingTerminalWriter{delegate: delegate, reportErr: errors.New("injected success report failure")}
|
||||
return terminal
|
||||
}
|
||||
|
||||
result := runStateTest(t, roots, opts, true, false, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "write debug run report") || !strings.Contains(result.stderr, "injected success report failure") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
if terminal == nil {
|
||||
t.Fatal("terminal writer was not constructed")
|
||||
}
|
||||
if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 {
|
||||
t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls)
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("stdout=%q, want no success message", result.stdout)
|
||||
}
|
||||
bundlePath := onlyChildDir(t, roots.debug)
|
||||
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
|
||||
if err != nil || !strings.Contains(string(errorLog), "injected success report failure") {
|
||||
t.Fatalf("error log = %q, %v", errorLog, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunWithoutDebugDoesNotUseTerminalSummaryWriter(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.extractErr = errors.New("non-debug pipeline failure")
|
||||
opts := harness.options()
|
||||
factoryCalls := 0
|
||||
opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
|
||||
factoryCalls++
|
||||
return delegate
|
||||
}
|
||||
|
||||
result := runStateTest(t, roots, opts, false, false, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "non-debug pipeline failure") {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
if factoryCalls != 0 {
|
||||
t.Fatalf("terminal summary factory calls = %d, want 0", factoryCalls)
|
||||
}
|
||||
assertAbsent(t, roots.debug)
|
||||
}
|
||||
|
||||
func TestRunRefusesExistingOutputDirectoryWithoutChangingIt(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
const runID = "run-1000000000-22222222222222222222222222222222"
|
||||
runPath := filepath.Join(roots.output, runID)
|
||||
if err := os.MkdirAll(filepath.Join(runPath, "nested"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(runPath, "sentinel"), []byte("existing output"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(runPath, "nested", "data"), []byte("preserve me"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := readTree(t, runPath)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil }
|
||||
|
||||
result := runStateTest(t, roots, opts, true, false, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "output run directory") || !strings.Contains(result.stderr, "already exists") {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
if after := readTree(t, runPath); !sameFiles(after, before) {
|
||||
t.Fatalf("existing output changed: before=%v after=%v", before, after)
|
||||
}
|
||||
bundlePath := filepath.Join(roots.debug, runID)
|
||||
report := readStateTestRunReport(t, bundlePath)
|
||||
if report.Succeeded || report.RunID != runID || report.OutputPath != runPath || report.DebugPath != bundlePath || report.OutputCount != 1 || report.ValidationStatus != "approved" {
|
||||
t.Fatalf("output collision report = %#v", report)
|
||||
}
|
||||
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
|
||||
if err != nil || !strings.Contains(string(errorLog), "already exists") {
|
||||
t.Fatalf("output collision error log = %q, %v", errorLog, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepeatedRunIdentityCannotOverwriteFirstOutput(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
const runID = "run-1000000000-33333333333333333333333333333333"
|
||||
harness := newStateTestHarness()
|
||||
opts := harness.options()
|
||||
opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil }
|
||||
|
||||
first := runStateTest(t, roots, opts, false, false, "bypass")
|
||||
if first.code != 0 {
|
||||
t.Fatalf("first code=%d stderr=%q", first.code, first.stderr)
|
||||
}
|
||||
runPath := filepath.Join(roots.output, runID)
|
||||
before := readTree(t, runPath)
|
||||
second := runStateTest(t, roots, opts, false, false, "bypass")
|
||||
if second.code != 1 || !strings.Contains(second.stderr, "already exists") {
|
||||
t.Fatalf("second code=%d stderr=%q", second.code, second.stderr)
|
||||
}
|
||||
if after := readTree(t, runPath); !sameFiles(after, before) {
|
||||
t.Fatalf("first output changed: before=%v after=%v", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRefusesExistingDebugBundleWithoutChangingIt(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
const runID = "run-1000000000-44444444444444444444444444444444"
|
||||
bundlePath := filepath.Join(roots.debug, runID)
|
||||
if err := os.MkdirAll(bundlePath, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sentinelPath := filepath.Join(bundlePath, "sentinel")
|
||||
if err := os.WriteFile(sentinelPath, []byte("existing debug"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := newStateTestHarness().options()
|
||||
opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil }
|
||||
|
||||
result := runStateTest(t, roots, opts, true, false, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "debug bundle") || !strings.Contains(result.stderr, "already exists") {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
if got, err := os.ReadFile(sentinelPath); err != nil || string(got) != "existing debug" {
|
||||
t.Fatalf("sentinel = %q, %v", got, err)
|
||||
}
|
||||
assertAbsent(t, roots.output)
|
||||
}
|
||||
|
||||
func TestRunIDGenerationFailurePrecedesDebugAllocation(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.RunIDGenerator = func(time.Time) (string, error) { return "", errors.New("random source unavailable") }
|
||||
|
||||
result := runStateTest(t, roots, opts, true, false, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "generate run ID: random source unavailable") {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
assertAbsent(t, roots.debug)
|
||||
assertAbsent(t, roots.output)
|
||||
}
|
||||
|
||||
func TestRunRejectsUnsafeGeneratedIdentityBeforePathUse(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.RunIDGenerator = func(time.Time) (string, error) { return "../outside", nil }
|
||||
|
||||
result := runStateTest(t, roots, opts, true, false, "bypass")
|
||||
if result.code != 1 || !strings.Contains(result.stderr, "invalid generated run ID") || !strings.Contains(result.stderr, "one safe path component") {
|
||||
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
assertAbsent(t, roots.debug)
|
||||
assertAbsent(t, roots.output)
|
||||
}
|
||||
|
||||
type stateTestRoots struct{ config, input, output, plans, checkpoints, debug string }
|
||||
|
||||
func newStateTestRoots(t *testing.T) stateTestRoots {
|
||||
t.Helper()
|
||||
base := t.TempDir()
|
||||
roots := stateTestRoots{input: filepath.Join(base, "input.txt"), output: filepath.Join(base, "output"), plans: filepath.Join(base, "plans"), checkpoints: filepath.Join(base, "checkpoints"), debug: filepath.Join(base, "debug")}
|
||||
if err := os.WriteFile(roots.input, []byte("application content Bearer secretvalue sk-secretvalue"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
roots.config = filepath.Join(base, "config.yml")
|
||||
config := fmt.Sprintf("version: 4\noutput:\n directory: %q\ncache:\n chunk_plans:\n directory: %q\n mode: auto\n checkpoints:\n enabled: true\n directory: %q\ndebug:\n directory: %q\npipelines:\n sample:\n input: test/input\n chunk: test/chunk\n artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n output: test/output\n", roots.output, roots.plans, roots.checkpoints, roots.debug)
|
||||
if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return roots
|
||||
}
|
||||
|
||||
type stateTestResult struct {
|
||||
code int
|
||||
stdout, stderr string
|
||||
}
|
||||
|
||||
func runStateTest(t *testing.T, roots stateTestRoots, opts Options, debug, resume bool, mode string) stateTestResult {
|
||||
t.Helper()
|
||||
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", mode}
|
||||
if debug {
|
||||
args = append(args, "--debug")
|
||||
}
|
||||
if resume {
|
||||
args = append(args, "--resume")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
return stateTestResult{RunWithOptions(args, &stdout, &stderr, opts), stdout.String(), stderr.String()}
|
||||
}
|
||||
|
||||
func assertStateTestOutput(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
output := onlyChildDir(t, root)
|
||||
data, err := os.ReadFile(filepath.Join(output, "result.json"))
|
||||
if err != nil || string(data) != "{\"ok\":true}\n" {
|
||||
t.Fatalf("output = %q, %v", data, err)
|
||||
}
|
||||
}
|
||||
|
||||
func onlyChildDir(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var dirs []string
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
dirs = append(dirs, filepath.Join(root, entry.Name()))
|
||||
}
|
||||
}
|
||||
if len(dirs) != 1 {
|
||||
t.Fatalf("directories in %q = %v, want one", root, dirs)
|
||||
}
|
||||
return dirs[0]
|
||||
}
|
||||
|
||||
func assertFile(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if info, err := os.Stat(path); err != nil || info.IsDir() {
|
||||
t.Fatalf("file %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
func assertAbsent(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("%q exists or stat failed: %v", path, err)
|
||||
}
|
||||
}
|
||||
func assertAnyFile(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
if text := readAllFiles(t, root); text == "" {
|
||||
t.Fatalf("no files under %q", root)
|
||||
}
|
||||
}
|
||||
|
||||
func readAllFiles(t *testing.T, root string) string {
|
||||
t.Helper()
|
||||
var content strings.Builder
|
||||
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
content.Write(data)
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return content.String()
|
||||
}
|
||||
|
||||
func readStateTestRunReport(t *testing.T, bundlePath string) debugbundle.RunReport {
|
||||
t.Helper()
|
||||
var report debugbundle.RunReport
|
||||
readStateTestSummaryJSON(t, bundlePath, "run-report.json", &report)
|
||||
return report
|
||||
}
|
||||
|
||||
func readStateTestSummaryJSON(t *testing.T, bundlePath, name string, target any) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(bundlePath, "summary", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(data, target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertRestrictedTree(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
if runtime.GOOS == "windows" {
|
||||
return
|
||||
}
|
||||
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
want := os.FileMode(0o600)
|
||||
if info.IsDir() {
|
||||
want = 0o700
|
||||
}
|
||||
if info.Mode().Perm() != want {
|
||||
return fmt.Errorf("%s has mode %o, want %o", path, info.Mode().Perm(), want)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func readTree(t *testing.T, root string) map[string][]byte {
|
||||
t.Helper()
|
||||
files := map[string][]byte{}
|
||||
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
relative, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
files[relative] = data
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return files
|
||||
}
|
||||
func sameFiles(left, right map[string][]byte) bool {
|
||||
if len(left) != len(right) {
|
||||
return false
|
||||
}
|
||||
for path, data := range left {
|
||||
if !bytes.Equal(data, right[path]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type stateTestHarness struct {
|
||||
mu sync.Mutex
|
||||
chunkCalls, extractCalls int
|
||||
runIDCalls uint64
|
||||
extractErr error
|
||||
chunkWarnings []contracts.Warning
|
||||
moduleProfiles []string
|
||||
sessionIDs []string
|
||||
outputWarnings []contracts.Warning
|
||||
includeWarnings bool
|
||||
}
|
||||
|
||||
func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} }
|
||||
func (h *stateTestHarness) options() Options {
|
||||
registries := pipeline.Registries{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
|
||||
if err := pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, stateTestCodec{}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := registries.Chunkers.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "cache-reference"}}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Chunker, error) { return stateTestChunker{h}, nil }); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "test/extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{h}, nil }); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "test/merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{harness: h}, nil }); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "test/normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{harness: h}, nil }); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
|
||||
return stateTestOutput{harness: h, includeWarnings: h.includeWarnings}, nil
|
||||
}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return Options{Catalog: catalogFromRegistries(registries), Registries: registries, LookupEnv: emptyLookup, Now: func() time.Time { return time.Unix(1, 0) }, RunIDGenerator: func(startedAt time.Time) (string, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.runIDCalls++
|
||||
return fmt.Sprintf("run-%d-%032x", startedAt.UnixNano(), h.runIDCalls), nil
|
||||
}, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
return nil, nil, nil
|
||||
}}
|
||||
}
|
||||
|
||||
type stateTestInput struct{}
|
||||
|
||||
func (stateTestInput) Key() string { return "test/input" }
|
||||
func (stateTestInput) Parse(_ context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
|
||||
doc := &source.SourceDocument{ID: "source", Kind: "text", Format: "text/plain", Units: []source.SourceUnit{{ID: 1, Kind: "text", Text: string(req.Raw), Ref: source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}}}
|
||||
digest, err := source.DigestDocument(doc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc.Digest = digest
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
type stateTestChunker struct{ harness *stateTestHarness }
|
||||
|
||||
func (stateTestChunker) Key() string { return "test/chunk" }
|
||||
func (stateTestChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
c.harness.mu.Lock()
|
||||
c.harness.moduleProfiles = append(c.harness.moduleProfiles, req.LLMProfile)
|
||||
c.harness.sessionIDs = append(c.harness.sessionIDs, req.SessionID)
|
||||
c.harness.mu.Unlock()
|
||||
c.harness.mu.Lock()
|
||||
c.harness.chunkCalls++
|
||||
c.harness.mu.Unlock()
|
||||
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Warnings: append([]contracts.Warning(nil), c.harness.chunkWarnings...)}, nil
|
||||
}
|
||||
|
||||
const stateTestArtifactKind contracts.ArtifactKind = "test/artifact"
|
||||
|
||||
type stateTestArtifact struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
type stateTestCodec struct{}
|
||||
|
||||
func (stateTestCodec) Kind() contracts.ArtifactKind { return stateTestArtifactKind }
|
||||
func (stateTestCodec) Schema() contracts.ArtifactSchema {
|
||||
return contracts.ArtifactSchema{ID: "test.artifact", Name: "test_artifact", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
|
||||
}
|
||||
func (stateTestCodec) MediaType() string { return "application/json" }
|
||||
func (stateTestCodec) EncodeCandidate(v stateTestArtifact) ([]byte, error) {
|
||||
return []byte(`{"value":"ok"}`), nil
|
||||
}
|
||||
func (stateTestCodec) Encode(v stateTestArtifact) ([]byte, error) {
|
||||
return []byte(`{"value":"ok"}`), nil
|
||||
}
|
||||
func (stateTestCodec) Decode([]byte) (stateTestArtifact, error) {
|
||||
return stateTestArtifact{Value: "ok"}, nil
|
||||
}
|
||||
|
||||
type stateTestExtractor struct{ harness *stateTestHarness }
|
||||
|
||||
func (stateTestExtractor) Key() string { return "test/extract" }
|
||||
func (stateTestExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (e stateTestExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[stateTestArtifact], error) {
|
||||
e.harness.mu.Lock()
|
||||
defer e.harness.mu.Unlock()
|
||||
e.harness.extractCalls++
|
||||
e.harness.moduleProfiles = append(e.harness.moduleProfiles, req.LLMProfile)
|
||||
e.harness.sessionIDs = append(e.harness.sessionIDs, req.SessionID)
|
||||
if e.harness.extractErr != nil {
|
||||
return contracts.TypedExtractionResult[stateTestArtifact]{}, e.harness.extractErr
|
||||
}
|
||||
return contracts.TypedExtractionResult[stateTestArtifact]{Value: stateTestArtifact{Value: "ok"}}, nil
|
||||
}
|
||||
|
||||
type stateTestMerger struct{ harness *stateTestHarness }
|
||||
|
||||
func (stateTestMerger) Key() string { return "test/merge" }
|
||||
func (m stateTestMerger) Merge(_ context.Context, req contracts.TypedMergeRequest[stateTestArtifact]) (contracts.TypedMergeResult[stateTestArtifact], error) {
|
||||
m.harness.mu.Lock()
|
||||
m.harness.moduleProfiles = append(m.harness.moduleProfiles, req.LLMProfile)
|
||||
m.harness.sessionIDs = append(m.harness.sessionIDs, req.SessionID)
|
||||
m.harness.mu.Unlock()
|
||||
return contracts.TypedMergeResult[stateTestArtifact]{Value: req.ExtractOutputs[0].Value}, nil
|
||||
}
|
||||
|
||||
type stateTestNormalizer struct{ harness *stateTestHarness }
|
||||
|
||||
func (stateTestNormalizer) Key() string { return "test/normalize" }
|
||||
func (stateTestNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (n stateTestNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[stateTestArtifact]) (contracts.TypedNormalizeResult[stateTestArtifact], error) {
|
||||
n.harness.mu.Lock()
|
||||
n.harness.moduleProfiles = append(n.harness.moduleProfiles, req.LLMProfile)
|
||||
n.harness.sessionIDs = append(n.harness.sessionIDs, req.SessionID)
|
||||
n.harness.mu.Unlock()
|
||||
return contracts.TypedNormalizeResult[stateTestArtifact]{Value: req.MergeOutput.Value}, nil
|
||||
}
|
||||
|
||||
type stateTestOutput struct {
|
||||
harness *stateTestHarness
|
||||
includeWarnings bool
|
||||
}
|
||||
|
||||
func (o stateTestOutput) Key() string { return "test/output" }
|
||||
func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
o.harness.mu.Lock()
|
||||
o.harness.outputWarnings = append([]contracts.Warning(nil), req.Warnings...)
|
||||
o.harness.mu.Unlock()
|
||||
data := []byte("{\"ok\":true}\n")
|
||||
if o.includeWarnings && len(req.Warnings) > 0 {
|
||||
data = []byte(fmt.Sprintf("{\"ok\":true,\"warnings\":%q}\n", req.Warnings[0].ReasonCode))
|
||||
}
|
||||
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: data}}}, nil
|
||||
}
|
||||
|
||||
type failingDebugRecorder struct{}
|
||||
|
||||
func (failingDebugRecorder) Enabled() bool { return true }
|
||||
func (failingDebugRecorder) WriteJSON(string, any) error { return errors.New("trace unavailable") }
|
||||
func (failingDebugRecorder) WriteBytes(string, []byte) error { return errors.New("trace unavailable") }
|
||||
|
||||
type recordingTerminalWriter struct {
|
||||
delegate DebugTerminalWriter
|
||||
reportErr, errorLogErr error
|
||||
reportCalls, errorLogCalls int
|
||||
}
|
||||
|
||||
func (w *recordingTerminalWriter) WriteRunReport(report debugbundle.RunReport) error {
|
||||
w.reportCalls++
|
||||
if w.reportErr != nil {
|
||||
return w.reportErr
|
||||
}
|
||||
return w.delegate.WriteRunReport(report)
|
||||
}
|
||||
|
||||
func (w *recordingTerminalWriter) WriteError(message string) error {
|
||||
w.errorLogCalls++
|
||||
if w.errorLogErr != nil {
|
||||
return w.errorLogErr
|
||||
}
|
||||
return w.delegate.WriteError(message)
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
const name = "NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE"
|
||||
previous, existed := os.LookupEnv(name)
|
||||
if err := os.Setenv(name, "bypass"); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
code := m.Run()
|
||||
if existed {
|
||||
_ = os.Setenv(name, previous)
|
||||
} else {
|
||||
_ = os.Unsetenv(name)
|
||||
}
|
||||
os.Exit(code)
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ArtifactLaneManifest struct {
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Extractor string `json:"extractor"`
|
||||
Merger string `json:"merger"`
|
||||
@@ -25,21 +27,54 @@ type ValidatorManifest struct {
|
||||
}
|
||||
|
||||
type LLMProfileManifest struct {
|
||||
ID string `json:"id"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
BackendID string `json:"backend_id,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
}
|
||||
|
||||
// Normalized returns the canonical representation used for manifest identity
|
||||
// and publication.
|
||||
func (profile LLMProfileManifest) Normalized() LLMProfileManifest {
|
||||
profile.ID = strings.TrimSpace(profile.ID)
|
||||
profile.Provider = strings.TrimSpace(profile.Provider)
|
||||
profile.Model = strings.TrimSpace(profile.Model)
|
||||
profile.BackendID = strings.TrimSpace(profile.BackendID)
|
||||
profile.ReasoningEffort = strings.TrimSpace(profile.ReasoningEffort)
|
||||
return profile
|
||||
}
|
||||
|
||||
// IdentityKey returns an opaque, deterministic key for the effective profile.
|
||||
func (profile LLMProfileManifest) IdentityKey() string {
|
||||
profile = profile.Normalized()
|
||||
return profile.ID + "\x00" +
|
||||
profile.Provider + "\x00" +
|
||||
profile.Model + "\x00" +
|
||||
profile.BackendID + "\x00" +
|
||||
profile.ReasoningEffort
|
||||
}
|
||||
|
||||
type ReferenceProvenance struct {
|
||||
Stage string `json:"stage,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
SlotName string `json:"slot_name"`
|
||||
OriginType string `json:"origin_type"`
|
||||
OriginURI string `json:"origin_uri,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes,omitempty"`
|
||||
BindingSource string `json:"binding_source,omitempty"`
|
||||
Stage string `json:"stage,omitempty"`
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
SlotName string `json:"slot_name"`
|
||||
OriginType string `json:"origin_type"`
|
||||
OriginURI string `json:"origin_uri,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes,omitempty"`
|
||||
BindingSource string `json:"binding_source,omitempty"`
|
||||
ArtifactKind string `json:"artifact_kind,omitempty"`
|
||||
SchemaID string `json:"schema_id,omitempty"`
|
||||
SchemaName string `json:"schema_name,omitempty"`
|
||||
SchemaVersion string `json:"schema_version,omitempty"`
|
||||
SchemaDigest string `json:"schema_digest,omitempty"`
|
||||
ProducerPipeline string `json:"producer_pipeline_id,omitempty"`
|
||||
ProducerStep string `json:"producer_step_id,omitempty"`
|
||||
ProducerLane string `json:"producer_lane_id,omitempty"`
|
||||
ProducerModule string `json:"producer_module_key,omitempty"`
|
||||
}
|
||||
|
||||
type OutputSchemaProvenance struct {
|
||||
@@ -49,6 +84,7 @@ type OutputSchemaProvenance struct {
|
||||
}
|
||||
|
||||
type NormalizedOutputManifest struct {
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
@@ -58,6 +94,7 @@ type NormalizedOutputManifest struct {
|
||||
|
||||
type RejectedOutputManifest struct {
|
||||
Stage string `json:"stage"`
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
@@ -69,6 +106,16 @@ type RejectedOutputManifest struct {
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
|
||||
type CheckpointDecisionManifest struct {
|
||||
Stage string `json:"stage"`
|
||||
StepID string `json:"step_id,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
Category string `json:"category"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
type ChunkPlanManifest struct {
|
||||
Mode string `json:"mode"`
|
||||
Action string `json:"action,omitempty"`
|
||||
@@ -99,27 +146,28 @@ type ChunkPlanSummary struct {
|
||||
}
|
||||
|
||||
type RunManifest struct {
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
PipelineID string `json:"pipeline_id,omitempty"`
|
||||
PipelineDigest string `json:"pipeline_digest,omitempty"`
|
||||
InputModule string `json:"input_module,omitempty"`
|
||||
Chunker string `json:"chunker,omitempty"`
|
||||
ChunkPlan *ChunkPlanManifest `json:"chunk_plan,omitempty"`
|
||||
SourceDigests []string `json:"source_digests,omitempty"`
|
||||
Extractors []string `json:"extractors,omitempty"`
|
||||
Merger string `json:"merger,omitempty"`
|
||||
Normalizer string `json:"normalizer,omitempty"`
|
||||
OutputEncoder string `json:"output_encoder,omitempty"`
|
||||
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
|
||||
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
||||
ValidatorChains []ValidatorChainManifest `json:"validator_chains,omitempty"`
|
||||
References []ReferenceProvenance `json:"references,omitempty"`
|
||||
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
||||
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
||||
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
SchemaVersion string `json:"schema_version,omitempty"`
|
||||
ValidationStatus string `json:"validation_status,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
PipelineID string `json:"pipeline_id,omitempty"`
|
||||
PipelineDigest string `json:"pipeline_digest,omitempty"`
|
||||
InputModule string `json:"input_module,omitempty"`
|
||||
Chunker string `json:"chunker,omitempty"`
|
||||
ChunkPlan *ChunkPlanManifest `json:"chunk_plan,omitempty"`
|
||||
SourceDigests []string `json:"source_digests,omitempty"`
|
||||
Extractors []string `json:"extractors,omitempty"`
|
||||
Merger string `json:"merger,omitempty"`
|
||||
Normalizer string `json:"normalizer,omitempty"`
|
||||
OutputEncoder string `json:"output_encoder,omitempty"`
|
||||
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
|
||||
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
||||
ValidatorChains []ValidatorChainManifest `json:"validator_chains,omitempty"`
|
||||
References []ReferenceProvenance `json:"references,omitempty"`
|
||||
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
||||
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
||||
CheckpointDecisions []CheckpointDecisionManifest `json:"checkpoint_decisions,omitempty"`
|
||||
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
SchemaVersion string `json:"schema_version,omitempty"`
|
||||
ValidationStatus string `json:"validation_status,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ func TestRunManifestOmitsEmptyOptionalFields(t *testing.T) {
|
||||
func TestRunManifestChunkPlanIsAdditiveAndOmitsPlanContent(t *testing.T) {
|
||||
manifest := RunManifest{ChunkPlan: &ChunkPlanManifest{
|
||||
Mode: "auto", Action: "reused", SourceDigest: "sha256:source", PlanDigest: "sha256:plan",
|
||||
PlanSchemaVersion: "notarius.chunk-plan.v1", RequestedModule: "chunk/current",
|
||||
PlanSchemaVersion: "notarius.chunk-plan.v2", RequestedModule: "chunk/current",
|
||||
ProducerInputModule: "input/original", ProducerModule: "chunk/original",
|
||||
}}
|
||||
encoded, err := json.Marshal(manifest)
|
||||
@@ -53,7 +53,13 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
||||
PipelineID: "pipeline-1",
|
||||
PipelineDigest: "sha256:abc123",
|
||||
LLMProfiles: []LLMProfileManifest{
|
||||
{ID: "default", Provider: "scriptorium", Model: "model-a"},
|
||||
{
|
||||
ID: "default",
|
||||
Provider: "promptkit",
|
||||
Model: "model-a",
|
||||
BackendID: "openrouter",
|
||||
ReasoningEffort: "high",
|
||||
},
|
||||
},
|
||||
ArtifactLanes: []ArtifactLaneManifest{
|
||||
{
|
||||
@@ -101,7 +107,13 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
||||
if !ok {
|
||||
t.Fatalf("llm_profiles[0] = %#v, want object", profiles[0])
|
||||
}
|
||||
assertHasKeys(t, profile, "id", "provider", "model")
|
||||
assertHasKeys(t, profile, "id", "provider", "model", "backend_id", "reasoning_effort")
|
||||
if profile["provider"] != "promptkit" {
|
||||
t.Fatalf("llm_profiles[0].provider = %#v, want promptkit", profile["provider"])
|
||||
}
|
||||
if profile["backend_id"] != "openrouter" || profile["reasoning_effort"] != "high" {
|
||||
t.Fatalf("llm_profiles[0] = %#v, want backend and reasoning provenance", profile)
|
||||
}
|
||||
|
||||
lanes, ok := got["artifact_lanes"].([]any)
|
||||
if !ok {
|
||||
|
||||
31
internal/core/config/cache.go
Normal file
31
internal/core/config/cache.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DefaultChunkPlanRoot resolves the existing per-user chunk-plan cache root.
|
||||
func DefaultChunkPlanRoot(userCacheDir func() (string, error)) (string, error) {
|
||||
return defaultCacheFamilyRoot(userCacheDir, "chunk-plans")
|
||||
}
|
||||
|
||||
func DefaultCheckpointRoot(userCacheDir func() (string, error)) (string, error) {
|
||||
return defaultCacheFamilyRoot(userCacheDir, "checkpoints")
|
||||
}
|
||||
|
||||
func defaultCacheFamilyRoot(userCacheDir func() (string, error), family string) (string, error) {
|
||||
if userCacheDir == nil {
|
||||
return "", fmt.Errorf("user cache directory resolver must not be nil")
|
||||
}
|
||||
root, err := userCacheDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve user cache directory: %w", err)
|
||||
}
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" {
|
||||
return "", fmt.Errorf("user cache directory must not be empty")
|
||||
}
|
||||
return filepath.Join(filepath.Clean(root), "notarius", family), nil
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestChunkCacheDefaults(t *testing.T) {
|
||||
cfg := Default()
|
||||
if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheAuto || cfg.Workspace.ChunkCache.Directory != "" {
|
||||
t.Fatalf("chunk cache defaults = %#v", cfg.Workspace.ChunkCache)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkCacheFileConfiguration(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
workspace:
|
||||
chunk_cache:
|
||||
mode: refresh
|
||||
directory: " ./state/../plans "
|
||||
`)
|
||||
if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheRefresh {
|
||||
t.Fatalf("mode = %q", cfg.Workspace.ChunkCache.Mode)
|
||||
}
|
||||
if got, want := cfg.Workspace.ChunkCache.Directory, filepath.Clean("./state/../plans"); got != want {
|
||||
t.Fatalf("directory = %q, want %q", got, want)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkCacheEnvironmentOverridesFile(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
workspace:
|
||||
chunk_cache:
|
||||
mode: bypass
|
||||
directory: /file/plans
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{
|
||||
"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "refresh",
|
||||
"NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR": " /environment/../cache/plans ",
|
||||
})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheRefresh || cfg.Workspace.ChunkCache.Directory != filepath.Clean("/environment/../cache/plans") {
|
||||
t.Fatalf("effective chunk cache = %#v", cfg.Workspace.ChunkCache)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkCacheEmptyDirectoryEnvironmentSelectsDefault(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Workspace.ChunkCache.Directory = "/file/plans"
|
||||
if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR": " \t "})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Workspace.ChunkCache.Directory != "" {
|
||||
t.Fatalf("directory = %q, want unset", cfg.Workspace.ChunkCache.Directory)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkCacheRejectsInvalidSuppliedModes(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte("version: 2\nworkspace:\n chunk_cache:\n mode: sometimes\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfigWithLookup(fileCfg, emptyLookup); err == nil || !strings.Contains(err.Error(), "workspace.chunk_cache.mode") {
|
||||
t.Fatalf("file mode error = %v", err)
|
||||
}
|
||||
|
||||
cfg = Default()
|
||||
if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "sometimes"})); err == nil || !strings.Contains(err.Error(), "NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE") {
|
||||
t.Fatalf("environment mode error = %v", err)
|
||||
}
|
||||
|
||||
cfg = Default()
|
||||
cfg.Workspace.ChunkCache.Mode = "sometimes"
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "chunk cache") {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkCacheConfigurationClonesAndRedacts(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Workspace.ChunkCache = WorkspaceChunkCacheConfig{Mode: pipeline.ChunkCacheRefresh, Directory: "/var/cache/notarius/chunk-plans"}
|
||||
cloned := cloneConfig(cfg)
|
||||
redacted := cfg.Redacted()
|
||||
if cloned.Workspace.ChunkCache != cfg.Workspace.ChunkCache || redacted.Workspace.ChunkCache != cfg.Workspace.ChunkCache {
|
||||
t.Fatalf("cloned=%#v redacted=%#v", cloned.Workspace.ChunkCache, redacted.Workspace.ChunkCache)
|
||||
}
|
||||
redacted.Workspace.ChunkCache.Directory = "/changed"
|
||||
if cfg.Workspace.ChunkCache.Directory != "/var/cache/notarius/chunk-plans" {
|
||||
t.Fatal("redacted mutation changed original")
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkCacheDirectoryValidation(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Workspace.ChunkCache.Directory = "/var/cache/notarius/chunk-plans"
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate(system root) error = %v", err)
|
||||
}
|
||||
cfg.Workspace.ChunkCache.Directory = "bad\x00path"
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "NUL") {
|
||||
t.Fatalf("Validate(NUL directory) error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,29 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const SupportedFileConfigVersion = 2
|
||||
const SupportedFileConfigVersion = 4
|
||||
|
||||
type Config struct {
|
||||
Scriptorium ScriptoriumConfig `json:"scriptorium,omitempty"`
|
||||
PromptKit PromptKitConfig `json:"promptkit,omitempty"`
|
||||
Pipelines map[string]pipeline.PipelineProfile `json:"pipelines"`
|
||||
Concurrency ConcurrencyConfig `json:"concurrency"`
|
||||
Diagnostics DiagnosticsConfig `json:"diagnostics"`
|
||||
Workspace WorkspaceConfig `json:"workspace"`
|
||||
Output OutputConfig `json:"output"`
|
||||
Cache CacheConfig `json:"cache"`
|
||||
Debug DebugConfig `json:"debug"`
|
||||
}
|
||||
|
||||
type ScriptoriumConfig struct {
|
||||
ProfileDir string `json:"profile_dir,omitempty"`
|
||||
ProfileFile string `json:"profile_file,omitempty"`
|
||||
type PromptKitConfig struct {
|
||||
ProfileDir string `json:"profile_dir,omitempty"`
|
||||
ProfileFile string `json:"profile_file,omitempty"`
|
||||
LocalBackend *PromptKitLocalBackendConfig `json:"local_backend,omitempty"`
|
||||
}
|
||||
|
||||
type PromptKitLocalBackendConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
ConcurrencyLimit int `json:"concurrency_limit"`
|
||||
}
|
||||
|
||||
type ConcurrencyConfig struct {
|
||||
@@ -31,37 +34,26 @@ type ConcurrencyConfig struct {
|
||||
defaultedExtractWorkers int
|
||||
}
|
||||
|
||||
type DiagnosticsConfig struct {
|
||||
WorkDir string `json:"work_dir"`
|
||||
Retention diagnostics.RetentionMode `json:"retention"`
|
||||
type OutputConfig struct {
|
||||
Directory string `json:"directory"`
|
||||
}
|
||||
|
||||
type WorkspaceConfig struct {
|
||||
Directory string `json:"directory,omitempty"`
|
||||
ChunkCache WorkspaceChunkCacheConfig `json:"chunk_cache"`
|
||||
Diagnostics WorkspaceDiagnosticsConfig `json:"diagnostics"`
|
||||
Resume WorkspaceResumeConfig `json:"resume"`
|
||||
Debug WorkspaceDebugConfig `json:"debug"`
|
||||
type CacheConfig struct {
|
||||
ChunkPlans ChunkPlanCacheConfig `json:"chunk_plans"`
|
||||
Checkpoints CheckpointCacheConfig `json:"checkpoints"`
|
||||
}
|
||||
|
||||
type WorkspaceChunkCacheConfig struct {
|
||||
Mode pipeline.ChunkCacheMode `json:"mode"`
|
||||
type ChunkPlanCacheConfig struct {
|
||||
Directory string `json:"directory,omitempty"`
|
||||
Mode pipeline.ChunkCacheMode `json:"mode"`
|
||||
}
|
||||
|
||||
type WorkspaceDiagnosticsConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Retention diagnostics.RetentionMode `json:"retention,omitempty"`
|
||||
enabledSet bool
|
||||
retentionSet bool
|
||||
type CheckpointCacheConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Directory string `json:"directory,omitempty"`
|
||||
}
|
||||
|
||||
type WorkspaceResumeConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
type WorkspaceDebugConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
type DebugConfig struct {
|
||||
Directory string `json:"directory"`
|
||||
}
|
||||
|
||||
func Default() Config {
|
||||
@@ -72,48 +64,18 @@ func Default() Config {
|
||||
StageWorkers: map[string]int{"extract": 1},
|
||||
defaultedExtractWorkers: 1,
|
||||
},
|
||||
Diagnostics: DiagnosticsConfig{
|
||||
WorkDir: "/tmp/notarius",
|
||||
Retention: diagnostics.RetentionAuto,
|
||||
},
|
||||
Workspace: WorkspaceConfig{
|
||||
ChunkCache: WorkspaceChunkCacheConfig{Mode: pipeline.ChunkCacheAuto},
|
||||
Diagnostics: WorkspaceDiagnosticsConfig{
|
||||
Enabled: true,
|
||||
},
|
||||
},
|
||||
Output: OutputConfig{Directory: "./notarius-output"},
|
||||
Cache: CacheConfig{ChunkPlans: ChunkPlanCacheConfig{Mode: pipeline.ChunkCacheAuto}},
|
||||
Debug: DebugConfig{Directory: "./notarius-debug"},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) RecomputeEffectiveDiagnostics() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
if dir := c.workspaceDirectory(); dir != "" {
|
||||
c.Diagnostics.WorkDir = filepath.Join(dir, "diagnostics")
|
||||
}
|
||||
if c.Workspace.Diagnostics.retentionSet {
|
||||
c.Diagnostics.Retention = c.Workspace.Diagnostics.Retention
|
||||
}
|
||||
}
|
||||
|
||||
func (c Config) DiagnosticsEnabled() bool {
|
||||
if !c.Workspace.Diagnostics.enabledSet {
|
||||
return true
|
||||
}
|
||||
return c.Workspace.Diagnostics.Enabled
|
||||
}
|
||||
|
||||
func (c Config) workspaceDirectory() string {
|
||||
dir := strings.TrimSpace(c.Workspace.Directory)
|
||||
if dir == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Clean(dir)
|
||||
}
|
||||
|
||||
func cloneConfig(in Config) Config {
|
||||
out := in
|
||||
if in.PromptKit.LocalBackend != nil {
|
||||
localBackend := *in.PromptKit.LocalBackend
|
||||
out.PromptKit.LocalBackend = &localBackend
|
||||
}
|
||||
out.Concurrency.StageWorkers = cloneIntMap(in.Concurrency.StageWorkers)
|
||||
out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines))
|
||||
for key, profile := range in.Pipelines {
|
||||
@@ -155,7 +117,26 @@ func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile
|
||||
out.Input = cloneModuleBinding(in.Input)
|
||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||
out.Output = cloneModuleBinding(in.Output)
|
||||
out.References = cloneStringMap(in.References)
|
||||
out.References = cloneReferenceSourceMap(in.References)
|
||||
if len(in.Artifacts) > 0 {
|
||||
out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts))
|
||||
for key, lane := range in.Artifacts {
|
||||
out.Artifacts[key] = cloneArtifactLaneProfile(lane)
|
||||
}
|
||||
}
|
||||
if in.Steps != nil {
|
||||
out.Steps = make([]pipeline.PipelineStepProfile, len(in.Steps))
|
||||
for i, step := range in.Steps {
|
||||
out.Steps[i] = clonePipelineStepProfile(step)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func clonePipelineStepProfile(in pipeline.PipelineStepProfile) pipeline.PipelineStepProfile {
|
||||
out := in
|
||||
out.ID = in.ID
|
||||
out.References = cloneReferenceSourceMap(in.References)
|
||||
if len(in.Artifacts) > 0 {
|
||||
out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts))
|
||||
for key, lane := range in.Artifacts {
|
||||
@@ -170,7 +151,7 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
|
||||
out.Extract = cloneModuleBinding(in.Extract)
|
||||
out.Merge = cloneModuleBinding(in.Merge)
|
||||
out.Normalize = cloneModuleBinding(in.Normalize)
|
||||
out.References = cloneStringMap(in.References)
|
||||
out.References = cloneReferenceSourceMap(in.References)
|
||||
if len(in.Validators) > 0 {
|
||||
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
|
||||
for i, binding := range in.Validators {
|
||||
@@ -191,12 +172,32 @@ func cloneStringMap(in map[string]string) map[string]string {
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneReferenceSourceMap(in map[string]pipeline.ReferenceSource) map[string]pipeline.ReferenceSource {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]pipeline.ReferenceSource, len(in))
|
||||
for key, source := range in {
|
||||
out[key] = cloneReferenceSource(source)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource {
|
||||
out := in
|
||||
if in.Artifact != nil {
|
||||
artifact := *in.Artifact
|
||||
out.Artifact = &artifact
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
||||
out := in
|
||||
if len(in.Options) > 0 {
|
||||
out.Options = cloneOptions(in.Options)
|
||||
}
|
||||
out.References = cloneStringMap(in.References)
|
||||
out.References = cloneReferenceSourceMap(in.References)
|
||||
out.Validators = cloneValidatorOverride(in.Validators)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
)
|
||||
|
||||
func TestDefaultValues(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("unexpected Scriptorium profile source defaults: %+v", cfg.Scriptorium)
|
||||
}
|
||||
if len(cfg.Pipelines) != 0 {
|
||||
t.Fatalf("expected no built-in pipeline profiles, got %v", cfg.Pipelines)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 1 {
|
||||
t.Fatalf("unexpected total LLM concurrency: %d", cfg.Concurrency.TotalLLM)
|
||||
}
|
||||
if got := cfg.Concurrency.StageWorkers["extract"]; got != 1 {
|
||||
t.Fatalf("unexpected extract workers: %d", got)
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/tmp/notarius" {
|
||||
t.Fatalf("unexpected diagnostics work dir: %q", cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionAuto {
|
||||
t.Fatalf("unexpected diagnostics retention: %q", cfg.Diagnostics.Retention)
|
||||
}
|
||||
if cfg.Workspace.Directory != "" {
|
||||
t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory)
|
||||
}
|
||||
if !cfg.Workspace.Diagnostics.Enabled || !cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected workspace diagnostics enabled by default: %+v", cfg.Workspace.Diagnostics)
|
||||
}
|
||||
if cfg.Workspace.Diagnostics.Retention != "" {
|
||||
t.Fatalf("unexpected workspace diagnostics retention: %q", cfg.Workspace.Diagnostics.Retention)
|
||||
}
|
||||
if cfg.Workspace.Resume.Enabled {
|
||||
t.Fatalf("workspace resume should be disabled by default")
|
||||
}
|
||||
if cfg.Workspace.Debug.Enabled {
|
||||
t.Fatalf("workspace debug should be disabled by default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigMergesWithDefaults(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
scriptorium:
|
||||
profile_dir: ./profiles
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
artifacts:
|
||||
events:
|
||||
extract: fake/extract
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
|
||||
t.Fatalf("ApplyFileConfig: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Scriptorium.ProfileDir != "./profiles" {
|
||||
t.Fatalf("expected Scriptorium profile dir, got %+v", cfg.Scriptorium)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 1 {
|
||||
t.Fatalf("expected default concurrency preserved, got %d", cfg.Concurrency.TotalLLM)
|
||||
}
|
||||
if got := cfg.Concurrency.StageWorkers["extract"]; got != 1 {
|
||||
t.Fatalf("expected default extract workers preserved, got %d", got)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionAuto {
|
||||
t.Fatalf("expected default diagnostics retention preserved, got %q", cfg.Diagnostics.Retention)
|
||||
}
|
||||
if _, ok := cfg.Pipelines["example"]; !ok {
|
||||
t.Fatalf("expected file pipeline to be applied")
|
||||
}
|
||||
}
|
||||
@@ -42,12 +42,10 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
|
||||
}
|
||||
profile = clonePipelineProfile(profile)
|
||||
profile.ID = pipelineID
|
||||
if override := strings.TrimSpace(input.LLMProfileOverride); override != "" {
|
||||
applyLLMProfileOverride(&profile, override)
|
||||
}
|
||||
|
||||
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{
|
||||
Only: input.Only,
|
||||
LLMProfileOverride: input.LLMProfileOverride,
|
||||
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
|
||||
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
|
||||
}, input.Catalog)
|
||||
@@ -65,16 +63,6 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func applyLLMProfileOverride(profile *pipeline.PipelineProfile, profileID string) {
|
||||
profile.Chunk.LLMProfile = profileID
|
||||
for laneID, lane := range profile.Artifacts {
|
||||
lane.Extract.LLMProfile = profileID
|
||||
lane.Merge.LLMProfile = profileID
|
||||
lane.Normalize.LLMProfile = profileID
|
||||
profile.Artifacts[laneID] = lane
|
||||
}
|
||||
}
|
||||
|
||||
func lookupPipelineProfile(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
|
||||
pipelineID = strings.TrimSpace(pipelineID)
|
||||
for rawID, profile := range profiles {
|
||||
|
||||
584
internal/core/config/effective_config_contract_test.go
Normal file
584
internal/core/config/effective_config_contract_test.go
Normal file
@@ -0,0 +1,584 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestEffectiveConfigRejectsEmptyAndUnknownPipelineIDs(t *testing.T) {
|
||||
cfg := configForEffectiveTests(t, effectiveProfile())
|
||||
for _, pipelineID := range []string{"", "missing"} {
|
||||
name := pipelineID
|
||||
if name == "" {
|
||||
name = "empty"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := cfg.Resolve(ResolveInput{PipelineID: pipelineID, Catalog: effectiveCatalog(t)})
|
||||
if err == nil || !strings.Contains(err.Error(), "pipeline") {
|
||||
t.Fatalf("Resolve(%q) error = %v, want pipeline context", pipelineID, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigResolvesTrimmedPipelineMapKeys(t *testing.T) {
|
||||
profile := effectiveProfile()
|
||||
profile.ID = " main "
|
||||
cfg := Default()
|
||||
cfg.Pipelines = map[string]pipeline.PipelineProfile{" main ": profile}
|
||||
effective, err := cfg.Resolve(ResolveInput{PipelineID: "main", Catalog: effectiveCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if effective.PipelineID != "main" || effective.ResolvedPipeline.ID != "main" {
|
||||
t.Fatalf("resolved IDs = %q, %q", effective.PipelineID, effective.ResolvedPipeline.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigOnlySelectsRequestedLanesWithoutMutatingSource(t *testing.T) {
|
||||
profile := effectiveProfile()
|
||||
profile.Artifacts["other"] = pipeline.ArtifactLaneProfile{Extract: pipeline.Binding("extract")}
|
||||
cfg := configForEffectiveTests(t, profile)
|
||||
effective, err := cfg.Resolve(ResolveInput{
|
||||
PipelineID: "main",
|
||||
Only: []string{"other"},
|
||||
Catalog: effectiveCatalog(t),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if len(effective.ResolvedPipeline.Steps[0].ArtifactLanes) != 1 || effective.ResolvedPipeline.Steps[0].ArtifactLanes[0].ID != "other" {
|
||||
t.Fatalf("resolved lanes = %#v", effective.ResolvedPipeline.Steps[0].ArtifactLanes)
|
||||
}
|
||||
if len(cfg.Pipelines["main"].Artifacts) != 2 {
|
||||
t.Fatalf("source lanes were mutated: %#v", cfg.Pipelines["main"].Artifacts)
|
||||
}
|
||||
|
||||
_, err = cfg.Resolve(ResolveInput{
|
||||
PipelineID: "main",
|
||||
Only: []string{"missing"},
|
||||
Catalog: effectiveCatalog(t),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "lane \"missing\"") {
|
||||
t.Fatalf("unknown lane error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigMaterializesDefaultBindingsThroughCatalog(t *testing.T) {
|
||||
effective, err := resolveEffectiveProfile(t, effectiveProfile(), ResolveInput{})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
resolved := effective.ResolvedPipeline
|
||||
if resolved.Chunk.Module != pipeline.DefaultChunkModule || resolved.Output.Module != pipeline.DefaultOutputModule {
|
||||
t.Fatalf("default pipeline bindings = %#v, %#v", resolved.Chunk, resolved.Output)
|
||||
}
|
||||
if len(resolved.Steps[0].ArtifactLanes) != 1 || resolved.Steps[0].ArtifactLanes[0].Merge.Module != pipeline.DefaultMergeModule || resolved.Steps[0].ArtifactLanes[0].Normalize.Module != pipeline.DefaultNormalizeModule {
|
||||
t.Fatalf("default lane bindings = %#v", resolved.Steps[0].ArtifactLanes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigPreservesPromptKitProfileSource(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
profileSource PromptKitConfig
|
||||
}{
|
||||
{name: "profile directory", profileSource: PromptKitConfig{ProfileDir: "./profiles"}},
|
||||
{name: "profile file", profileSource: PromptKitConfig{ProfileFile: "./profiles.yml"}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := configForEffectiveTests(t, effectiveProfile())
|
||||
cfg.PromptKit = tt.profileSource
|
||||
effective, err := cfg.Resolve(ResolveInput{PipelineID: "main", Catalog: effectiveCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if effective.Config.PromptKit != cfg.PromptKit {
|
||||
t.Fatalf("effective PromptKit config = %#v, want %#v", effective.Config.PromptKit, cfg.PromptKit)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigOwnsPromptKitLocalBackend(t *testing.T) {
|
||||
cfg := configForEffectiveTests(t, effectiveProfile())
|
||||
cfg.PromptKit.LocalBackend = &PromptKitLocalBackendConfig{
|
||||
Endpoint: "http://localhost:8000/v1",
|
||||
ConcurrencyLimit: 2,
|
||||
}
|
||||
effective, err := cfg.Resolve(ResolveInput{PipelineID: "main", Catalog: effectiveCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if effective.Config.PromptKit.LocalBackend == nil {
|
||||
t.Fatal("effective local backend = nil")
|
||||
}
|
||||
if effective.Config.PromptKit.LocalBackend == cfg.PromptKit.LocalBackend {
|
||||
t.Fatal("effective local backend aliases input config")
|
||||
}
|
||||
|
||||
cfg.PromptKit.LocalBackend.Endpoint = "http://changed-input.example/v1"
|
||||
if effective.Config.PromptKit.LocalBackend.Endpoint != "http://localhost:8000/v1" {
|
||||
t.Fatalf("input mutation changed effective config: %#v", effective.Config.PromptKit.LocalBackend)
|
||||
}
|
||||
effective.Config.PromptKit.LocalBackend.ConcurrencyLimit = 9
|
||||
if cfg.PromptKit.LocalBackend.ConcurrencyLimit != 2 {
|
||||
t.Fatalf("effective mutation changed input config: %#v", cfg.PromptKit.LocalBackend)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigResolutionFailuresRetainContext(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*pipeline.PipelineProfile)
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "unknown module",
|
||||
mutate: func(profile *pipeline.PipelineProfile) {
|
||||
profile.Input.Module = "missing-input"
|
||||
},
|
||||
want: []string{"pipeline \"main\"", "input"},
|
||||
},
|
||||
{
|
||||
name: "missing capability",
|
||||
mutate: func(profile *pipeline.PipelineProfile) {
|
||||
profile.Chunk.Module = "needs-capability"
|
||||
},
|
||||
want: []string{"pipeline \"main\"", "chunk"},
|
||||
},
|
||||
{
|
||||
name: "missing artifact variant",
|
||||
mutate: func(profile *pipeline.PipelineProfile) {
|
||||
profile.Artifacts["lane"] = pipeline.ArtifactLaneProfile{
|
||||
Extract: pipeline.Binding("extract"),
|
||||
Merge: pipeline.Binding("other-merge"),
|
||||
}
|
||||
},
|
||||
want: []string{"pipeline \"main\"", "lane \"lane\"", "merge"},
|
||||
},
|
||||
{
|
||||
name: "invalid module options",
|
||||
mutate: func(profile *pipeline.PipelineProfile) {
|
||||
profile.Chunk = pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"unknown": true}}
|
||||
},
|
||||
want: []string{"pipeline \"main\"", "chunk", "generic", "options"},
|
||||
},
|
||||
{
|
||||
name: "invalid validator options",
|
||||
mutate: func(profile *pipeline.PipelineProfile) {
|
||||
lane := profile.Artifacts["lane"]
|
||||
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []pipeline.ModuleBinding{{
|
||||
Module: "option-validator",
|
||||
Options: map[string]any{"invalid": true},
|
||||
}},
|
||||
}
|
||||
profile.Artifacts["lane"] = lane
|
||||
},
|
||||
want: []string{"pipeline \"main\"", "lane \"lane\"", "extract", "option-validator", "options"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
profile := effectiveProfile()
|
||||
tt.mutate(&profile)
|
||||
_, err := resolveEffectiveProfile(t, profile, ResolveInput{})
|
||||
if err == nil {
|
||||
t.Fatal("Resolve() error = nil, want failure")
|
||||
}
|
||||
for _, fragment := range tt.want {
|
||||
if !strings.Contains(err.Error(), fragment) {
|
||||
t.Fatalf("Resolve() error = %v, want context %q", err, fragment)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigLLMProfileOverrideChangesDigestAndOverridesValidators(t *testing.T) {
|
||||
profile := effectiveProfile()
|
||||
profile.Chunk.LLMProfile = "chunk-profile"
|
||||
lane := profile.Artifacts["lane"]
|
||||
lane.Extract.LLMProfile = "extract-profile"
|
||||
lane.Merge.LLMProfile = "merge-profile"
|
||||
lane.Normalize.LLMProfile = "normalize-profile"
|
||||
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []pipeline.ModuleBinding{{
|
||||
Module: "llm-validator",
|
||||
LLMProfile: "validator-profile",
|
||||
}},
|
||||
}
|
||||
profile.Artifacts["lane"] = lane
|
||||
|
||||
base, err := resolveEffectiveProfile(t, profile, ResolveInput{})
|
||||
if err != nil {
|
||||
t.Fatalf("base Resolve() error = %v", err)
|
||||
}
|
||||
overridden, err := resolveEffectiveProfile(t, profile, ResolveInput{LLMProfileOverride: "override-profile"})
|
||||
if err != nil {
|
||||
t.Fatalf("overridden Resolve() error = %v", err)
|
||||
}
|
||||
if base.ResolvedPipeline.Digest == overridden.ResolvedPipeline.Digest {
|
||||
t.Fatal("LLM profile override did not change the pipeline digest")
|
||||
}
|
||||
resolved := overridden.ResolvedPipeline
|
||||
if resolved.Chunk.LLMProfile != "override-profile" || resolved.Steps[0].ArtifactLanes[0].Extract.LLMProfile != "override-profile" ||
|
||||
resolved.Steps[0].ArtifactLanes[0].Merge.LLMProfile != "override-profile" || resolved.Steps[0].ArtifactLanes[0].Normalize.LLMProfile != "override-profile" {
|
||||
t.Fatalf("pipeline profile override was not applied: %#v", resolved)
|
||||
}
|
||||
validators := findEffectiveValidatorChain(resolved, pipeline.StageExtract, "lane")
|
||||
if len(validators.Validators) != 1 || validators.Validators[0].Binding.LLMProfile != "override-profile" {
|
||||
t.Fatalf("validator profile = %#v, want runtime override", validators)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigPipelineLLMProfileIsInheritedWithoutMutatingConfig(t *testing.T) {
|
||||
profile := effectiveProfile()
|
||||
profile.LLMProfile = " configured-profile "
|
||||
effective, err := resolveEffectiveProfile(t, profile, ResolveInput{})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
if got := effective.Config.Pipelines["main"].LLMProfile; got != " configured-profile " {
|
||||
t.Fatalf("effective config pipeline llm profile = %q, want preserved programmatic value", got)
|
||||
}
|
||||
resolved := effective.ResolvedPipeline
|
||||
if got := resolved.Chunk.LLMProfile; got != "configured-profile" {
|
||||
t.Fatalf("resolved chunk profile = %q, want inherited profile", got)
|
||||
}
|
||||
if got := resolved.Steps[0].ArtifactLanes[0].Extract.LLMProfile; got != "configured-profile" {
|
||||
t.Fatalf("resolved extract profile = %q, want inherited profile", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigValidatorOverridesRemainDistinctAndOrdered(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
value pipeline.ValidatorOverride
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "omitted uses default",
|
||||
want: []string{"default-validator"},
|
||||
},
|
||||
{
|
||||
name: "explicit empty",
|
||||
value: pipeline.ValidatorOverride{Set: true},
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "configured order",
|
||||
value: pipeline.ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []pipeline.ModuleBinding{
|
||||
pipeline.Binding("configured-a"),
|
||||
pipeline.Binding("configured-b"),
|
||||
},
|
||||
},
|
||||
want: []string{"configured-a", "configured-b"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
profile := effectiveProfile()
|
||||
lane := profile.Artifacts["lane"]
|
||||
lane.Extract.Validators = tt.value
|
||||
profile.Artifacts["lane"] = lane
|
||||
effective, err := resolveEffectiveProfile(t, profile, ResolveInput{})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
chain := findEffectiveValidatorChain(effective.ResolvedPipeline, pipeline.StageExtract, "lane")
|
||||
got := make([]string, len(chain.Validators))
|
||||
for i, validator := range chain.Validators {
|
||||
got[i] = validator.Binding.Module
|
||||
}
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("validator chain = %#v, want %v", got, tt.want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Fatalf("validator chain = %#v, want %v", got, tt.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigAndResolutionInputsDoNotAliasSource(t *testing.T) {
|
||||
profile := effectiveProfile()
|
||||
profile.Chunk.Options = map[string]any{"nested": map[string]any{"safe": "source"}}
|
||||
profile.Chunk.References = pipeline.ExternalReferenceMap(map[string]string{"chunk-ref": "chunk.txt"})
|
||||
lane := profile.Artifacts["lane"]
|
||||
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []pipeline.ModuleBinding{{
|
||||
Module: "configured-a",
|
||||
Options: map[string]any{"nested": map[string]any{"safe": "validator-source"}},
|
||||
}},
|
||||
}
|
||||
profile.Artifacts["lane"] = lane
|
||||
cfg := configForEffectiveTests(t, profile)
|
||||
only := []string{"lane"}
|
||||
overrides := []pipeline.ReferenceBinding{{Stage: pipeline.StageChunk, SlotName: "chunk-ref", Source: "source.txt"}}
|
||||
effective, err := cfg.Resolve(ResolveInput{
|
||||
PipelineID: "main",
|
||||
Only: only,
|
||||
ReferenceOverrides: overrides,
|
||||
Catalog: effectiveCatalog(t),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
|
||||
effective.Config.Pipelines["main"].Chunk.Options["nested"].(map[string]any)["safe"] = "effective-config"
|
||||
effective.ResolvedPipeline.Chunk.Options["nested"].(map[string]any)["safe"] = "resolved-pipeline"
|
||||
effective.ResolvedPipeline.ChunkReferences.Bindings[0].Source = "resolved-reference"
|
||||
effective.ResolvedPipeline.ValidatorChains[1].Validators[0].Binding.Options["nested"].(map[string]any)["safe"] = "resolved-validator"
|
||||
effective.Only[0] = "mutated-only"
|
||||
effective.ReferenceOverrides[0].Source = "mutated-override"
|
||||
|
||||
if got := cfg.Pipelines["main"].Chunk.Options["nested"].(map[string]any)["safe"]; got != "source" {
|
||||
t.Fatalf("source config option was aliased: %v", got)
|
||||
}
|
||||
if got := cfg.Pipelines["main"].Chunk.References["chunk-ref"].Path; got != "chunk.txt" {
|
||||
t.Fatalf("source config references were aliased: %v", got)
|
||||
}
|
||||
if only[0] != "lane" || overrides[0].Source != "source.txt" {
|
||||
t.Fatal("resolution inputs were aliased")
|
||||
}
|
||||
}
|
||||
|
||||
func effectiveProfile() pipeline.PipelineProfile {
|
||||
return pipeline.PipelineProfile{
|
||||
ID: "main",
|
||||
Input: pipeline.Binding("input"),
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"lane": {Extract: pipeline.Binding("extract")},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func configForEffectiveTests(t *testing.T, profile pipeline.PipelineProfile) Config {
|
||||
t.Helper()
|
||||
cfg := Default()
|
||||
cfg.Pipelines = map[string]pipeline.PipelineProfile{"main": profile}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func resolveEffectiveProfile(t *testing.T, profile pipeline.PipelineProfile, input ResolveInput) (EffectiveConfig, error) {
|
||||
t.Helper()
|
||||
cfg := configForEffectiveTests(t, profile)
|
||||
if input.PipelineID == "" {
|
||||
input.PipelineID = "main"
|
||||
}
|
||||
if input.Catalog.Inputs == nil {
|
||||
input.Catalog = effectiveCatalog(t)
|
||||
}
|
||||
return cfg.Resolve(input)
|
||||
}
|
||||
|
||||
func findEffectiveValidatorChain(resolved pipeline.ResolvedPipeline, stage pipeline.ModuleStage, laneID string) pipeline.ResolvedValidatorChain {
|
||||
for _, chain := range resolved.ValidatorChains {
|
||||
if chain.Stage == stage && chain.LaneID == laneID {
|
||||
return chain
|
||||
}
|
||||
}
|
||||
return pipeline.ResolvedValidatorChain{}
|
||||
}
|
||||
|
||||
type effectiveArtifact struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
const effectiveArtifactKind contracts.ArtifactKind = "test/effective"
|
||||
|
||||
type effectiveCodec struct{}
|
||||
|
||||
func (effectiveCodec) Kind() contracts.ArtifactKind { return effectiveArtifactKind }
|
||||
func (effectiveCodec) Schema() contracts.ArtifactSchema {
|
||||
return contracts.ArtifactSchema{
|
||||
ID: "effective-schema",
|
||||
Name: "Effective artifact",
|
||||
Version: "1",
|
||||
JSONSchema: []byte(`{"type":"object"}`),
|
||||
}
|
||||
}
|
||||
func (effectiveCodec) MediaType() string { return "application/json" }
|
||||
func (effectiveCodec) EncodeCandidate(value effectiveArtifact) ([]byte, error) {
|
||||
return json.Marshal(value)
|
||||
}
|
||||
func (effectiveCodec) Encode(value effectiveArtifact) ([]byte, error) {
|
||||
return json.Marshal(value)
|
||||
}
|
||||
func (effectiveCodec) Decode(content []byte) (effectiveArtifact, error) {
|
||||
var value effectiveArtifact
|
||||
err := json.Unmarshal(content, &value)
|
||||
return value, err
|
||||
}
|
||||
|
||||
type effectiveInput struct{ key string }
|
||||
|
||||
func (m effectiveInput) Key() string { return m.key }
|
||||
func (m effectiveInput) Parse(context.Context, contracts.ParseRequest) (*source.SourceDocument, error) {
|
||||
return &source.SourceDocument{}, nil
|
||||
}
|
||||
|
||||
type effectiveChunker struct{ key string }
|
||||
|
||||
func (m effectiveChunker) Key() string { return m.key }
|
||||
func (m effectiveChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
if m.key == pipeline.DefaultChunkModule {
|
||||
return []contracts.ReferenceSlot{{Name: "chunk-ref"}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (m effectiveChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
return contracts.ChunkPlanResult{}, nil
|
||||
}
|
||||
|
||||
type effectiveExtractor struct{ key string }
|
||||
|
||||
func (m effectiveExtractor) Key() string { return m.key }
|
||||
func (m effectiveExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (m effectiveExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[effectiveArtifact], error) {
|
||||
return contracts.TypedExtractionResult[effectiveArtifact]{}, nil
|
||||
}
|
||||
|
||||
type effectiveMerger struct{ key string }
|
||||
|
||||
func (m effectiveMerger) Key() string { return m.key }
|
||||
func (m effectiveMerger) Merge(context.Context, contracts.TypedMergeRequest[effectiveArtifact]) (contracts.TypedMergeResult[effectiveArtifact], error) {
|
||||
return contracts.TypedMergeResult[effectiveArtifact]{}, nil
|
||||
}
|
||||
|
||||
type effectiveNormalizer struct{ key string }
|
||||
|
||||
func (m effectiveNormalizer) Key() string { return m.key }
|
||||
func (m effectiveNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (m effectiveNormalizer) Normalize(context.Context, contracts.TypedNormalizeRequest[effectiveArtifact]) (contracts.TypedNormalizeResult[effectiveArtifact], error) {
|
||||
return contracts.TypedNormalizeResult[effectiveArtifact]{}, nil
|
||||
}
|
||||
|
||||
type effectiveOutput struct{ key string }
|
||||
|
||||
func (m effectiveOutput) Key() string { return m.key }
|
||||
func (m effectiveOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{}, nil
|
||||
}
|
||||
|
||||
type effectiveValidator struct {
|
||||
name string
|
||||
class contracts.ExecutionClass
|
||||
}
|
||||
|
||||
func (v effectiveValidator) Name() string { return v.name }
|
||||
func (v effectiveValidator) ExecutionClass() contracts.ExecutionClass { return v.class }
|
||||
func (v effectiveValidator) Validate(context.Context, contracts.TypedValidationRequest[effectiveArtifact]) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
func effectiveCatalog(t *testing.T) pipeline.ModuleCatalog {
|
||||
t.Helper()
|
||||
catalog := pipeline.ModuleCatalog{
|
||||
Inputs: pipeline.NewInputAdapterRegistry(),
|
||||
Chunkers: pipeline.NewChunkerRegistry(),
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
|
||||
Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(),
|
||||
Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(),
|
||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
}
|
||||
if err := pipeline.RegisterArtifactCodec(catalog.ArtifactCodecs, effectiveCodec{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := catalog.Inputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func() (contracts.InputAdapter, error) {
|
||||
return effectiveInput{key: "input"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chunkSpec := pipeline.ModuleSpec{
|
||||
Key: pipeline.DefaultChunkModule,
|
||||
Stage: pipeline.StageChunk,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Requires: []string{"source"},
|
||||
Provides: []string{"chunk"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-ref"}},
|
||||
}
|
||||
chunkOptions := func(options map[string]any) error { return pipeline.RejectUnknownOptions(options, "size", "nested") }
|
||||
if err := catalog.Chunkers.RegisterBuilderWithSpec(chunkSpec, chunkOptions, func(pipeline.BuildRequest) (contracts.Chunker, error) {
|
||||
return effectiveChunker{key: pipeline.DefaultChunkModule}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := catalog.Chunkers.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "needs-capability", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"missing"}}, chunkOptions, func(pipeline.BuildRequest) (contracts.Chunker, error) {
|
||||
return effectiveChunker{key: "needs-capability"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pipeline.RegisterExtractor(catalog.Extractors, pipeline.ModuleSpec{Key: "extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: effectiveArtifactKind, Requires: []string{"chunk"}, Provides: []string{"candidate"}}, func() (contracts.Extractor[effectiveArtifact], error) {
|
||||
return effectiveExtractor{key: "extract"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pipeline.RegisterMerger(catalog.Mergers, pipeline.ModuleSpec{Key: pipeline.DefaultMergeModule, Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: effectiveArtifactKind, Requires: []string{"candidate"}, Provides: []string{"merged"}}, func() (contracts.Merger[effectiveArtifact], error) {
|
||||
return effectiveMerger{key: pipeline.DefaultMergeModule}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pipeline.RegisterMerger(catalog.Mergers, pipeline.ModuleSpec{Key: "other-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: "other-kind", Requires: []string{"candidate"}, Provides: []string{"merged"}}, func() (contracts.Merger[effectiveArtifact], error) {
|
||||
return effectiveMerger{key: "other-merge"}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pipeline.RegisterNormalizer(catalog.Normalizers, pipeline.ModuleSpec{Key: pipeline.DefaultNormalizeModule, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: effectiveArtifactKind, Requires: []string{"merged"}, Provides: []string{"normalized"}}, func() (contracts.Normalizer[effectiveArtifact], error) {
|
||||
return effectiveNormalizer{key: pipeline.DefaultNormalizeModule}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := catalog.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}}, func() (contracts.OutputEncoder, error) {
|
||||
return effectiveOutput{key: pipeline.DefaultOutputModule}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, validator := range []struct {
|
||||
key string
|
||||
class contracts.ExecutionClass
|
||||
}{
|
||||
{key: "default-validator", class: contracts.ExecutionClassDeterministic},
|
||||
{key: "configured-a", class: contracts.ExecutionClassDeterministic},
|
||||
{key: "configured-b", class: contracts.ExecutionClassDeterministic},
|
||||
{key: "llm-validator", class: contracts.ExecutionClassLLMBacked},
|
||||
} {
|
||||
if err := pipeline.RegisterTypedValidatorBuilder(catalog.Validators, effectiveArtifactKind, pipeline.ValidatorSpec{Key: validator.key, ExecutionClass: validator.class}, func(options map[string]any) error {
|
||||
return pipeline.RejectUnknownOptions(options, "nested")
|
||||
}, func(pipeline.BuildRequest) (contracts.TypedValidator[effectiveArtifact], error) {
|
||||
return effectiveValidator{name: validator.key, class: validator.class}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := pipeline.RegisterTypedValidatorBuilder(catalog.Validators, effectiveArtifactKind, pipeline.ValidatorSpec{Key: "option-validator", ExecutionClass: contracts.ExecutionClassDeterministic}, func(options map[string]any) error {
|
||||
return pipeline.RejectUnknownOptions(options, "allowed")
|
||||
}, func(pipeline.BuildRequest) (contracts.TypedValidator[effectiveArtifact], error) {
|
||||
return effectiveValidator{name: "option-validator", class: contracts.ExecutionClassDeterministic}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := catalog.ValidatorChains.Register(pipeline.ValidatorChainMapping{Stage: pipeline.StageExtract, Module: "extract", Validators: []pipeline.ModuleBinding{pipeline.Binding("default-validator")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
@@ -1,241 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestResolveRejectsEmptyAndUnknownPipelineID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pipelineID string
|
||||
want string
|
||||
}{
|
||||
{name: "empty", pipelineID: " ", want: "pipeline id"},
|
||||
{name: "unknown", pipelineID: "missing", want: "not configured"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := validConfig().Resolve(ResolveInput{PipelineID: tc.pipelineID, Catalog: fakeCatalog(t)})
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveMaterializesDefaultExtractWorkersFromEffectiveTotal(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
cfg.Concurrency.TotalLLM = 4
|
||||
|
||||
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
if got := effective.Config.Concurrency.StageWorkers["extract"]; got != 4 {
|
||||
t.Fatalf("effective extract workers = %d, want total concurrency 4", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveLaneFilteringSuccessAndFailure(t *testing.T) {
|
||||
effective, err := validConfig().Resolve(ResolveInput{
|
||||
PipelineID: " example ",
|
||||
Only: []string{" notes "},
|
||||
Catalog: fakeCatalog(t),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
}
|
||||
|
||||
if effective.PipelineID != "example" {
|
||||
t.Fatalf("unexpected pipeline ID: %q", effective.PipelineID)
|
||||
}
|
||||
if len(effective.ResolvedPipeline.ArtifactLanes) != 1 || effective.ResolvedPipeline.ArtifactLanes[0].ID != "notes" {
|
||||
t.Fatalf("unexpected resolved lanes: %+v", effective.ResolvedPipeline.ArtifactLanes)
|
||||
}
|
||||
if effective.ResolvedPipeline.Digest == "" {
|
||||
t.Fatalf("expected digest")
|
||||
}
|
||||
|
||||
_, err = validConfig().Resolve(ResolveInput{
|
||||
PipelineID: "example",
|
||||
Only: []string{"missing"},
|
||||
Catalog: fakeCatalog(t),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "selected artifact lane") {
|
||||
t.Fatalf("expected invalid lane error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveUsesTrimmedPipelineMapKeys(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
cfg.Pipelines[" example "] = cfg.Pipelines["example"]
|
||||
delete(cfg.Pipelines, "example")
|
||||
|
||||
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
}
|
||||
if effective.PipelineID != "example" {
|
||||
t.Fatalf("unexpected pipeline ID: %q", effective.PipelineID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveSurfacesUnknownModuleKeyThroughCatalog(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
lane := cfg.Pipelines["example"].Artifacts["events"]
|
||||
lane.Extract = pipeline.Binding("missing/extract")
|
||||
cfg.Pipelines["example"].Artifacts["events"] = lane
|
||||
|
||||
_, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
||||
if err == nil || !strings.Contains(err.Error(), "missing/extract") || !strings.Contains(err.Error(), "events") {
|
||||
t.Fatalf("expected unknown module error with lane context, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveSurfacesMissingCapabilityThroughCatalog(t *testing.T) {
|
||||
_, err := validConfig().Resolve(ResolveInput{
|
||||
PipelineID: "example",
|
||||
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
|
||||
Key: "json",
|
||||
Stage: pipeline.StageOutput,
|
||||
Requires: []string{"missing-capability"},
|
||||
}),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "missing capability") || !strings.Contains(err.Error(), "json") {
|
||||
t.Fatalf("expected missing capability error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.Chunk = pipeline.Binding("dnd/scenes")
|
||||
lane := profile.Artifacts["events"]
|
||||
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"events": lane}
|
||||
cfg.Pipelines["example"] = profile
|
||||
|
||||
catalog := fakeCatalog(t,
|
||||
pipeline.ModuleSpec{
|
||||
Key: "fake/input",
|
||||
Stage: pipeline.StageInput,
|
||||
Provides: []string{"source.transcript"},
|
||||
},
|
||||
pipeline.ModuleSpec{
|
||||
Key: "fake/extract",
|
||||
Stage: pipeline.StageExtract,
|
||||
Requires: []string{"chunks", "source.transcript"},
|
||||
Provides: []string{"artifact"},
|
||||
},
|
||||
)
|
||||
mustRegisterChunker(t, catalog.Chunkers, pipeline.ModuleSpec{
|
||||
Key: "dnd/scenes",
|
||||
Stage: pipeline.StageChunk,
|
||||
Requires: []string{"source.transcript"},
|
||||
Provides: []string{"chunks"},
|
||||
})
|
||||
|
||||
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
if got := effective.ResolvedPipeline.Chunk.Module; got != "dnd/scenes" {
|
||||
t.Fatalf("Chunk.Module = %q, want dnd/scenes", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve first: %v", err)
|
||||
}
|
||||
|
||||
lane := cfg.Pipelines["example"].Artifacts["events"]
|
||||
lane.Extract.Options = map[string]any{"temperature": 0.2}
|
||||
cfg.Pipelines["example"].Artifacts["events"] = lane
|
||||
second, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve second: %v", err)
|
||||
}
|
||||
|
||||
if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest {
|
||||
t.Fatalf("expected digest to change, got %q", first.ResolvedPipeline.Digest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.Input.LLMProfile = "input-profile"
|
||||
profile.Output.LLMProfile = "output-profile"
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Merge.LLMProfile = "merge-profile"
|
||||
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []pipeline.ModuleBinding{
|
||||
{Module: "fake/llm-validator", LLMProfile: "validator-profile"},
|
||||
},
|
||||
}
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
|
||||
base, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve base: %v", err)
|
||||
}
|
||||
effective, err := cfg.Resolve(ResolveInput{
|
||||
PipelineID: "example",
|
||||
Catalog: fakeCatalog(t),
|
||||
LLMProfileOverride: "runtime",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve override: %v", err)
|
||||
}
|
||||
|
||||
if base.ResolvedPipeline.Digest == effective.ResolvedPipeline.Digest {
|
||||
t.Fatalf("expected digest to change after LLM profile override")
|
||||
}
|
||||
for _, binding := range llmCapableBindings(effective.ResolvedPipeline) {
|
||||
if binding.LLMProfile != "runtime" {
|
||||
t.Fatalf("LLM-capable binding profile = %q, want runtime", binding.LLMProfile)
|
||||
}
|
||||
}
|
||||
if effective.ResolvedPipeline.Input.LLMProfile != "input-profile" {
|
||||
t.Fatalf("input profile = %q, want original input-profile", effective.ResolvedPipeline.Input.LLMProfile)
|
||||
}
|
||||
if effective.ResolvedPipeline.Output.LLMProfile != "output-profile" {
|
||||
t.Fatalf("output profile = %q, want original output-profile", effective.ResolvedPipeline.Output.LLMProfile)
|
||||
}
|
||||
eventLane := effective.ResolvedPipeline.ArtifactLanes[0]
|
||||
if eventLane.Merge.LLMProfile != "runtime" {
|
||||
t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile)
|
||||
}
|
||||
validatorChain := findEffectiveValidatorChain(effective.ResolvedPipeline.ValidatorChains, pipeline.StageExtract, "events", "fake/extract")
|
||||
if validatorChain == nil || len(validatorChain.Validators) != 1 {
|
||||
t.Fatalf("validator chain = %#v, want one extract validator", effective.ResolvedPipeline.ValidatorChains)
|
||||
}
|
||||
if validatorChain.Validators[0].Binding.LLMProfile != "validator-profile" {
|
||||
t.Fatalf("validator profile = %q, want original validator-profile", validatorChain.Validators[0].Binding.LLMProfile)
|
||||
}
|
||||
}
|
||||
|
||||
func findEffectiveValidatorChain(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID string, module string) *pipeline.ResolvedValidatorChain {
|
||||
for i := range chains {
|
||||
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
|
||||
return &chains[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
|
||||
bindings := []pipeline.ModuleBinding{resolved.Chunk}
|
||||
for _, lane := range resolved.ArtifactLanes {
|
||||
bindings = append(bindings, lane.Extract, lane.Merge, lane.Normalize)
|
||||
}
|
||||
return bindings
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
@@ -49,52 +48,49 @@ func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool))
|
||||
c.Concurrency.extractWorkersConfigured = true
|
||||
}
|
||||
c.Concurrency.recomputeStageWorkerDefaults()
|
||||
if raw, ok := lookup("NOTARIUS_WORK_DIR"); ok {
|
||||
c.Diagnostics.WorkDir = strings.TrimSpace(raw)
|
||||
if raw, ok := lookup("NOTARIUS_OUTPUT_DIR"); ok {
|
||||
c.Output.Directory = strings.TrimSpace(raw)
|
||||
if c.Output.Directory == "" {
|
||||
return fmt.Errorf("NOTARIUS_OUTPUT_DIR: must not be empty")
|
||||
}
|
||||
if strings.ContainsRune(c.Output.Directory, '\x00') {
|
||||
return fmt.Errorf("NOTARIUS_OUTPUT_DIR: must not contain NUL")
|
||||
}
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_DIAGNOSTICS_RETENTION"); ok {
|
||||
c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(raw))
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIR"); ok {
|
||||
c.Workspace.Directory = strings.TrimSpace(raw)
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE"); ok {
|
||||
if raw, ok := lookup("NOTARIUS_CACHE_CHUNK_PLANS_MODE"); ok {
|
||||
mode, err := pipeline.ParseChunkCacheMode(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE: %w", err)
|
||||
return fmt.Errorf("NOTARIUS_CACHE_CHUNK_PLANS_MODE: %w", err)
|
||||
}
|
||||
c.Workspace.ChunkCache.Mode = mode
|
||||
c.Cache.ChunkPlans.Mode = mode
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR"); ok {
|
||||
c.Workspace.ChunkCache.Directory = cleanOptionalPath(raw)
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED"); ok {
|
||||
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
if raw, ok := lookup("NOTARIUS_CACHE_CHUNK_PLANS_DIR"); ok {
|
||||
c.Cache.ChunkPlans.Directory = cleanOptionalPath(raw)
|
||||
if c.Cache.ChunkPlans.Directory == "" {
|
||||
return fmt.Errorf("NOTARIUS_CACHE_CHUNK_PLANS_DIR: must not be empty")
|
||||
}
|
||||
c.Workspace.Diagnostics.Enabled = value
|
||||
c.Workspace.Diagnostics.enabledSet = true
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION"); ok {
|
||||
c.Workspace.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(raw))
|
||||
c.Workspace.Diagnostics.retentionSet = true
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_RESUME_ENABLED"); ok {
|
||||
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_RESUME_ENABLED", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
if strings.ContainsRune(c.Cache.ChunkPlans.Directory, '\x00') {
|
||||
return fmt.Errorf("NOTARIUS_CACHE_CHUNK_PLANS_DIR: must not contain NUL")
|
||||
}
|
||||
c.Workspace.Resume.Enabled = value
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DEBUG_ENABLED"); ok {
|
||||
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_DEBUG_ENABLED", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
if raw, ok := lookup("NOTARIUS_CACHE_CHECKPOINTS_DIR"); ok {
|
||||
c.Cache.Checkpoints.Directory = cleanOptionalPath(raw)
|
||||
if c.Cache.Checkpoints.Directory == "" {
|
||||
return fmt.Errorf("NOTARIUS_CACHE_CHECKPOINTS_DIR: must not be empty")
|
||||
}
|
||||
if strings.ContainsRune(c.Cache.Checkpoints.Directory, '\x00') {
|
||||
return fmt.Errorf("NOTARIUS_CACHE_CHECKPOINTS_DIR: must not contain NUL")
|
||||
}
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_DEBUG_DIR"); ok {
|
||||
c.Debug.Directory = strings.TrimSpace(raw)
|
||||
if c.Debug.Directory == "" {
|
||||
return fmt.Errorf("NOTARIUS_DEBUG_DIR: must not be empty")
|
||||
}
|
||||
if strings.ContainsRune(c.Debug.Directory, '\x00') {
|
||||
return fmt.Errorf("NOTARIUS_DEBUG_DIR: must not contain NUL")
|
||||
}
|
||||
c.Workspace.Debug.Enabled = value
|
||||
}
|
||||
c.RecomputeEffectiveDiagnostics()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -105,11 +101,3 @@ func parseIntEnv(name string, raw string) (int, error) {
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func parseBoolEnv(name string, raw string) (bool, error) {
|
||||
value, err := strconv.ParseBool(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s: must be a boolean", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
239
internal/core/config/env_contract_test.go
Normal file
239
internal/core/config/env_contract_test.go
Normal file
@@ -0,0 +1,239 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestPrecedenceFileValuesOverrideBuiltInDefaults(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 4
|
||||
concurrency:
|
||||
total_llm: 4
|
||||
stage_workers:
|
||||
extract: 2
|
||||
output:
|
||||
directory: ./file-output
|
||||
cache:
|
||||
chunk_plans:
|
||||
directory: ./file-plans
|
||||
mode: refresh
|
||||
checkpoints:
|
||||
directory: ./file-checkpoints
|
||||
debug:
|
||||
directory: ./file-debug
|
||||
`)
|
||||
if cfg.Concurrency.TotalLLM != 4 || cfg.Concurrency.StageWorkers["extract"] != 2 ||
|
||||
cfg.Output.Directory != "./file-output" || cfg.Cache.ChunkPlans.Directory != "file-plans" ||
|
||||
cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheRefresh || cfg.Cache.Checkpoints.Directory != "file-checkpoints" ||
|
||||
cfg.Debug.Directory != "./file-debug" {
|
||||
t.Fatalf("file values did not override defaults: %#v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrecedenceOperationalEnvironmentOverridesFileValues(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 4
|
||||
concurrency:
|
||||
total_llm: 2
|
||||
stage_workers:
|
||||
extract: 1
|
||||
output:
|
||||
directory: ./file-output
|
||||
cache:
|
||||
chunk_plans:
|
||||
directory: ./file-plans
|
||||
mode: refresh
|
||||
checkpoints:
|
||||
directory: ./file-checkpoints
|
||||
debug:
|
||||
directory: ./file-debug
|
||||
`)
|
||||
env := map[string]string{
|
||||
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "8",
|
||||
"NOTARIUS_STAGE_WORKERS_EXTRACT": "6",
|
||||
"NOTARIUS_OUTPUT_DIR": "/env/output",
|
||||
"NOTARIUS_CACHE_CHUNK_PLANS_MODE": "bypass",
|
||||
"NOTARIUS_CACHE_CHUNK_PLANS_DIR": "/env/plans",
|
||||
"NOTARIUS_CACHE_CHECKPOINTS_DIR": "/env/checkpoints",
|
||||
"NOTARIUS_DEBUG_DIR": "/env/debug",
|
||||
}
|
||||
if err := cfg.ApplyEnvOverridesWithLookup(lookupValues(env)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 8 || cfg.Concurrency.StageWorkers["extract"] != 6 ||
|
||||
cfg.Output.Directory != "/env/output" || cfg.Cache.ChunkPlans.Directory != "/env/plans" ||
|
||||
cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheBypass || cfg.Cache.Checkpoints.Directory != "/env/checkpoints" ||
|
||||
cfg.Debug.Directory != "/env/debug" {
|
||||
t.Fatalf("environment values did not override file values: %#v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrecedenceExtractWorkersFollowEffectiveConcurrencyUnlessExplicit(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
file string
|
||||
env map[string]string
|
||||
wantTotal int
|
||||
wantWorker int
|
||||
}{
|
||||
{
|
||||
name: "default follows environment total",
|
||||
file: "version: 4\n",
|
||||
env: map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "5"},
|
||||
wantTotal: 5,
|
||||
wantWorker: 5,
|
||||
},
|
||||
{
|
||||
name: "file worker is retained",
|
||||
file: "version: 4\nconcurrency:\n total_llm: 3\n stage_workers:\n extract: 2\n",
|
||||
env: map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6"},
|
||||
wantTotal: 6,
|
||||
wantWorker: 2,
|
||||
},
|
||||
{
|
||||
name: "environment worker is retained",
|
||||
file: "version: 4\nconcurrency:\n total_llm: 2\n",
|
||||
env: map[string]string{
|
||||
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6",
|
||||
"NOTARIUS_STAGE_WORKERS_EXTRACT": "4",
|
||||
},
|
||||
wantTotal: 6,
|
||||
wantWorker: 4,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := applyFileConfig(t, tt.file)
|
||||
if err := cfg.ApplyEnvOverridesWithLookup(lookupValues(tt.env)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != tt.wantTotal || cfg.Concurrency.StageWorkers["extract"] != tt.wantWorker {
|
||||
t.Fatalf("concurrency = %#v, want total %d and extract %d", cfg.Concurrency, tt.wantTotal, tt.wantWorker)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrecedenceEmptyFileCacheDirectoriesDeferPerUserResolution(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 4
|
||||
cache:
|
||||
chunk_plans:
|
||||
directory: ""
|
||||
checkpoints:
|
||||
directory: ""
|
||||
`)
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("empty file cache directories should be valid: %v", err)
|
||||
}
|
||||
if cfg.Cache.ChunkPlans.Directory != "" || cfg.Cache.Checkpoints.Directory != "" {
|
||||
t.Fatalf("empty cache directories were not preserved for deferred resolution: %#v", cfg.Cache)
|
||||
}
|
||||
resolver := func() (string, error) { return "/user/cache", nil }
|
||||
chunkPlans, err := DefaultChunkPlanRoot(resolver)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkpoints, err := DefaultCheckpointRoot(resolver)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if chunkPlans != "/user/cache/notarius/chunk-plans" || checkpoints != "/user/cache/notarius/checkpoints" {
|
||||
t.Fatalf("deferred cache roots = %q, %q", chunkPlans, checkpoints)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultCacheRootsRejectInvalidUserCacheResolvers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
resolver func() (string, error)
|
||||
want string
|
||||
}{
|
||||
{name: "nil resolver", want: "must not be nil"},
|
||||
{
|
||||
name: "resolver failure",
|
||||
resolver: func() (string, error) {
|
||||
return "", errors.New("cache home unavailable")
|
||||
},
|
||||
want: "resolve user cache directory",
|
||||
},
|
||||
{name: "empty directory", resolver: func() (string, error) { return " ", nil }, want: "must not be empty"},
|
||||
}
|
||||
families := []struct {
|
||||
name string
|
||||
root func(func() (string, error)) (string, error)
|
||||
}{
|
||||
{name: "chunk plans", root: DefaultChunkPlanRoot},
|
||||
{name: "checkpoints", root: DefaultCheckpointRoot},
|
||||
}
|
||||
|
||||
for _, family := range families {
|
||||
for _, tt := range tests {
|
||||
t.Run(family.name+"/"+tt.name, func(t *testing.T) {
|
||||
_, err := family.root(tt.resolver)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("error = %v, want substring %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvEmptyDirectoryOverridesAreErrors(t *testing.T) {
|
||||
tests := []string{
|
||||
"NOTARIUS_OUTPUT_DIR",
|
||||
"NOTARIUS_CACHE_CHUNK_PLANS_DIR",
|
||||
"NOTARIUS_CACHE_CHECKPOINTS_DIR",
|
||||
"NOTARIUS_DEBUG_DIR",
|
||||
}
|
||||
for _, name := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
err := cfg.ApplyEnvOverridesWithLookup(lookupValues(map[string]string{name: " \t"}))
|
||||
if err == nil || !strings.Contains(err.Error(), name) {
|
||||
t.Fatalf("error = %v, want responsible environment variable", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvInvalidIntegersAndChunkCacheModesReportTheirNames(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "not-an-integer",
|
||||
"NOTARIUS_STAGE_WORKERS_EXTRACT": "not-an-integer",
|
||||
"NOTARIUS_CACHE_CHUNK_PLANS_MODE": "not-a-cache-mode",
|
||||
}
|
||||
for name, value := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
err := cfg.ApplyEnvOverridesWithLookup(lookupValues(map[string]string{name: value}))
|
||||
if err == nil || !strings.Contains(err.Error(), name) {
|
||||
t.Fatalf("error = %v, want responsible environment variable", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvRemovedProviderVariablesAreIgnored(t *testing.T) {
|
||||
before := Default()
|
||||
cfg := Default()
|
||||
removed := map[string]string{
|
||||
"NOTARIUS_LLM_DEFAULT_ENDPOINT": "ignored-provider-setting",
|
||||
"NOTARIUS_LLM_DEFAULT_MODEL": "ignored-provider-setting",
|
||||
}
|
||||
if err := cfg.ApplyEnvOverridesWithLookup(lookupValues(removed)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(cfg, before) {
|
||||
t.Fatalf("removed provider variables changed configuration: %#v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func lookupValues(values map[string]string) func(string) (string, bool) {
|
||||
return func(name string) (string, bool) {
|
||||
value, ok := values[name]
|
||||
return value, ok
|
||||
}
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestApplyEnvOverridesOperationalValues(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Pipelines["example"] = pipeline.PipelineProfile{ID: "example", Input: pipeline.Binding("before")}
|
||||
|
||||
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
|
||||
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "3",
|
||||
"NOTARIUS_STAGE_WORKERS_EXTRACT": "2",
|
||||
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
|
||||
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
|
||||
"NOTARIUS_WORKSPACE_DIR": "/var/lib/notarius-env",
|
||||
"NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED": "false",
|
||||
"NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION": "always",
|
||||
"NOTARIUS_WORKSPACE_RESUME_ENABLED": "true",
|
||||
"NOTARIUS_WORKSPACE_DEBUG_ENABLED": "true",
|
||||
"NOTARIUS_PIPELINE_INPUT": "after",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyEnvOverrides: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("LLM environment overrides must not change Scriptorium config: %+v", cfg.Scriptorium)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 3 {
|
||||
t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM)
|
||||
}
|
||||
if got := cfg.Concurrency.StageWorkers["extract"]; got != 2 {
|
||||
t.Fatalf("extract workers = %d, want 2", got)
|
||||
}
|
||||
if cfg.Workspace.Directory != "/var/lib/notarius-env" {
|
||||
t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory)
|
||||
}
|
||||
if cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected workspace diagnostics disabled")
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/var/lib/notarius-env/diagnostics" || cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
|
||||
t.Fatalf("unexpected diagnostics config: %+v", cfg.Diagnostics)
|
||||
}
|
||||
if !cfg.Workspace.Resume.Enabled {
|
||||
t.Fatalf("expected workspace resume enabled")
|
||||
}
|
||||
if !cfg.Workspace.Debug.Enabled {
|
||||
t.Fatalf("expected workspace debug enabled")
|
||||
}
|
||||
if cfg.Pipelines["example"].Input.Module != "before" {
|
||||
t.Fatalf("environment overrides must not change pipeline wiring: %+v", cfg.Pipelines["example"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyEnvOverridesRejectsInvalidIntegers(t *testing.T) {
|
||||
for _, name := range []string{"NOTARIUS_TOTAL_LLM_CONCURRENCY", "NOTARIUS_STAGE_WORKERS_EXTRACT"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{name: "many"}))
|
||||
if err == nil || !strings.Contains(err.Error(), name) {
|
||||
t.Fatalf("expected named integer error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageWorkerEnvironmentPrecedenceAndDefaulting(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
concurrency:
|
||||
total_llm: 4
|
||||
stage_workers:
|
||||
extract: 2
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML() error = %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
|
||||
t.Fatalf("ApplyFileConfig() error = %v", err)
|
||||
}
|
||||
if err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
|
||||
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "5",
|
||||
"NOTARIUS_STAGE_WORKERS_EXTRACT": "3",
|
||||
})); err != nil {
|
||||
t.Fatalf("ApplyEnvOverrides() error = %v", err)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 5 || cfg.Concurrency.StageWorkers["extract"] != 3 {
|
||||
t.Fatalf("effective concurrency = %#v, want total 5 and extract 3", cfg.Concurrency)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate(overridden) error = %v, want nil", err)
|
||||
}
|
||||
|
||||
defaulted := Default()
|
||||
if err := defaulted.applyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6"})); err != nil {
|
||||
t.Fatalf("ApplyEnvOverrides(defaulted) error = %v", err)
|
||||
}
|
||||
if got := defaulted.Concurrency.StageWorkers["extract"]; got != 6 {
|
||||
t.Fatalf("defaulted extract workers = %d, want effective total 6", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageWorkerRangeValidationUsesFinalEnvironmentTotal(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
concurrency:
|
||||
total_llm: 4
|
||||
stage_workers:
|
||||
extract: 5
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML() error = %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
|
||||
t.Fatalf("ApplyFileConfig() error = %v", err)
|
||||
}
|
||||
if err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6"})); err != nil {
|
||||
t.Fatalf("ApplyEnvOverrides() error = %v", err)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v, want final total to make extract workers valid", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyEnvOverridesRejectsInvalidBooleans(t *testing.T) {
|
||||
for _, name := range []string{
|
||||
"NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED",
|
||||
"NOTARIUS_WORKSPACE_RESUME_ENABLED",
|
||||
"NOTARIUS_WORKSPACE_DEBUG_ENABLED",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{name: "maybe"}))
|
||||
if err == nil || !strings.Contains(err.Error(), name) {
|
||||
t.Fatalf("expected named boolean error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyEnvOverridesLegacyDiagnosticsRemainCompatibleWithoutWorkspace(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
|
||||
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
|
||||
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyEnvOverrides: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Diagnostics.WorkDir != "/tmp/notarius-env" {
|
||||
t.Fatalf("diagnostics work dir = %q, want legacy env", cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
|
||||
t.Fatalf("diagnostics retention = %q, want legacy env", cfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvUsesDefaultConfig(t *testing.T) {
|
||||
t.Setenv("NOTARIUS_TOTAL_LLM_CONCURRENCY", "2")
|
||||
|
||||
cfg, err := LoadFromEnv()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFromEnv: %v", err)
|
||||
}
|
||||
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("unexpected Scriptorium config from env: %+v", cfg.Scriptorium)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 2 {
|
||||
t.Fatalf("expected env concurrency override, got %+v", cfg.Concurrency)
|
||||
}
|
||||
if got := cfg.Concurrency.StageWorkers["extract"]; got != 2 {
|
||||
t.Fatalf("expected extract workers to default to total, got %d", got)
|
||||
}
|
||||
}
|
||||
@@ -8,39 +8,117 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type FileConfig struct {
|
||||
Version int `yaml:"version"`
|
||||
Scriptorium *FileScriptoriumConfig `yaml:"scriptorium,omitempty"`
|
||||
PromptKit *FilePromptKitConfig `yaml:"promptkit,omitempty"`
|
||||
Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
|
||||
Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"`
|
||||
Diagnostics *FileDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
||||
Workspace *FileWorkspaceConfig `yaml:"workspace,omitempty"`
|
||||
Output *FileOutputConfig `yaml:"output,omitempty"`
|
||||
Cache *FileCacheConfig `yaml:"cache,omitempty"`
|
||||
Debug *FileDebugConfig `yaml:"debug,omitempty"`
|
||||
}
|
||||
|
||||
type FileScriptoriumConfig struct {
|
||||
ProfileDir *string `yaml:"profile_dir,omitempty"`
|
||||
ProfileFile *string `yaml:"profile_file,omitempty"`
|
||||
type FilePromptKitConfig struct {
|
||||
ProfileDir *string `yaml:"profile_dir,omitempty"`
|
||||
ProfileFile *string `yaml:"profile_file,omitempty"`
|
||||
LocalBackend *FilePromptKitLocalBackendConfig `yaml:"local_backend,omitempty"`
|
||||
}
|
||||
|
||||
type FilePromptKitLocalBackendConfig struct {
|
||||
Endpoint *string `yaml:"endpoint,omitempty"`
|
||||
ConcurrencyLimit *int `yaml:"concurrency_limit,omitempty"`
|
||||
}
|
||||
|
||||
type FilePipelineProfile struct {
|
||||
Input fileModuleBinding `yaml:"input"`
|
||||
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
||||
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
||||
Output *fileModuleBinding `yaml:"output,omitempty"`
|
||||
References map[string]string `yaml:"references,omitempty"`
|
||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||
Input fileModuleBinding `yaml:"input"`
|
||||
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
||||
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
||||
Steps []FilePipelineStepProfile `yaml:"steps,omitempty"`
|
||||
Output *fileModuleBinding `yaml:"output,omitempty"`
|
||||
References map[string]fileReferenceSource `yaml:"references,omitempty"`
|
||||
artifactsSet bool `yaml:"-"`
|
||||
stepsSet bool `yaml:"-"`
|
||||
llmProfileSet bool `yaml:"-"`
|
||||
}
|
||||
|
||||
func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
|
||||
type plainFilePipelineProfile FilePipelineProfile
|
||||
var decoded plainFilePipelineProfile
|
||||
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
|
||||
"llm_profile": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
|
||||
}, "pipeline profile")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*p = FilePipelineProfile(decoded)
|
||||
_, p.artifactsSet = seen["artifacts"]
|
||||
_, p.stepsSet = seen["steps"]
|
||||
_, p.llmProfileSet = seen["llm_profile"]
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *FilePipelineStepProfile) UnmarshalYAML(node *yaml.Node) error {
|
||||
type plainFilePipelineStepProfile FilePipelineStepProfile
|
||||
var decoded plainFilePipelineStepProfile
|
||||
if _, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
|
||||
"id": {}, "artifacts": {}, "references": {},
|
||||
}, "pipeline step"); err != nil {
|
||||
return err
|
||||
}
|
||||
*s = FilePipelineStepProfile(decoded)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *FileArtifactLaneProfile) UnmarshalYAML(node *yaml.Node) error {
|
||||
type plainFileArtifactLaneProfile FileArtifactLaneProfile
|
||||
var decoded plainFileArtifactLaneProfile
|
||||
if _, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
|
||||
"extract": {}, "merge": {}, "normalize": {}, "validators": {}, "references": {},
|
||||
}, "artifact lane"); err != nil {
|
||||
return err
|
||||
}
|
||||
*l = FileArtifactLaneProfile(decoded)
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodeKnownMapping(node *yaml.Node, target any, allowed map[string]struct{}, context string) (map[string]struct{}, error) {
|
||||
if node.Kind != yaml.MappingNode {
|
||||
return nil, fmt.Errorf("%s must be an object", context)
|
||||
}
|
||||
if err := node.Decode(target); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := make(map[string]struct{}, len(node.Content)/2)
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
key := node.Content[i].Value
|
||||
if _, exists := seen[key]; exists {
|
||||
return nil, fmt.Errorf("%s field %q is duplicated", context, key)
|
||||
}
|
||||
if _, ok := allowed[key]; !ok {
|
||||
return nil, fmt.Errorf("field %s not found in %s", key, context)
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
return seen, nil
|
||||
}
|
||||
|
||||
type FilePipelineStepProfile struct {
|
||||
ID string `yaml:"id"`
|
||||
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts"`
|
||||
References map[string]fileReferenceSource `yaml:"references,omitempty"`
|
||||
}
|
||||
|
||||
type FileArtifactLaneProfile struct {
|
||||
Extract fileModuleBinding `yaml:"extract"`
|
||||
Merge *fileModuleBinding `yaml:"merge,omitempty"`
|
||||
Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
|
||||
Validators []fileModuleBinding `yaml:"validators,omitempty"`
|
||||
References map[string]string `yaml:"references,omitempty"`
|
||||
Extract fileModuleBinding `yaml:"extract"`
|
||||
Merge *fileModuleBinding `yaml:"merge,omitempty"`
|
||||
Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
|
||||
Validators []fileModuleBinding `yaml:"validators,omitempty"`
|
||||
References map[string]fileReferenceSource `yaml:"references,omitempty"`
|
||||
}
|
||||
|
||||
type FileConcurrencyConfig struct {
|
||||
@@ -48,31 +126,23 @@ type FileConcurrencyConfig struct {
|
||||
StageWorkers map[string]int `yaml:"stage_workers,omitempty"`
|
||||
}
|
||||
|
||||
type FileDiagnosticsConfig struct {
|
||||
WorkDir *string `yaml:"work_dir,omitempty"`
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
}
|
||||
|
||||
type FileWorkspaceConfig struct {
|
||||
Directory *string `yaml:"directory,omitempty"`
|
||||
ChunkCache *FileWorkspaceChunkCacheConfig `yaml:"chunk_cache,omitempty"`
|
||||
Diagnostics *FileWorkspaceDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
||||
Resume *FileWorkspaceEnabledConfig `yaml:"resume,omitempty"`
|
||||
Debug *FileWorkspaceEnabledConfig `yaml:"debug,omitempty"`
|
||||
}
|
||||
|
||||
type FileWorkspaceChunkCacheConfig struct {
|
||||
Mode *string `yaml:"mode,omitempty"`
|
||||
type FileOutputConfig struct {
|
||||
Directory *string `yaml:"directory,omitempty"`
|
||||
}
|
||||
|
||||
type FileWorkspaceDiagnosticsConfig struct {
|
||||
Enabled *bool `yaml:"enabled,omitempty"`
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
type FileCacheConfig struct {
|
||||
ChunkPlans *FileChunkPlanCacheConfig `yaml:"chunk_plans,omitempty"`
|
||||
Checkpoints *FileCheckpointCacheConfig `yaml:"checkpoints,omitempty"`
|
||||
}
|
||||
|
||||
type FileWorkspaceEnabledConfig struct {
|
||||
Enabled *bool `yaml:"enabled,omitempty"`
|
||||
type FileChunkPlanCacheConfig struct {
|
||||
Directory *string `yaml:"directory,omitempty"`
|
||||
Mode *string `yaml:"mode,omitempty"`
|
||||
}
|
||||
type FileCheckpointCacheConfig struct {
|
||||
Enabled *bool `yaml:"enabled,omitempty"`
|
||||
Directory *string `yaml:"directory,omitempty"`
|
||||
}
|
||||
type FileDebugConfig struct {
|
||||
Directory *string `yaml:"directory,omitempty"`
|
||||
}
|
||||
|
||||
type fileModuleBinding struct {
|
||||
@@ -80,10 +150,90 @@ type fileModuleBinding struct {
|
||||
LLMProfile string
|
||||
Retries int
|
||||
Options map[string]any
|
||||
References map[string]string
|
||||
References map[string]fileReferenceSource
|
||||
Validators pipeline.ValidatorOverride
|
||||
}
|
||||
|
||||
type fileReferenceSource struct {
|
||||
path string
|
||||
artifact *pipeline.ArtifactReference
|
||||
}
|
||||
|
||||
func (source *fileReferenceSource) UnmarshalYAML(node *yaml.Node) error {
|
||||
if source == nil {
|
||||
return fmt.Errorf("reference source must not be nil")
|
||||
}
|
||||
switch node.Kind {
|
||||
case yaml.ScalarNode:
|
||||
if node.Tag != "!!str" {
|
||||
return fmt.Errorf("external reference path must be a string")
|
||||
}
|
||||
path := strings.TrimSpace(node.Value)
|
||||
if path == "" {
|
||||
return fmt.Errorf("external reference path must not be empty")
|
||||
}
|
||||
source.path = path
|
||||
source.artifact = nil
|
||||
return nil
|
||||
case yaml.MappingNode:
|
||||
if len(node.Content) != 2 || node.Content[0].Value != "artifact" {
|
||||
return fmt.Errorf("reference source mapping must contain only artifact")
|
||||
}
|
||||
artifactNode := node.Content[1]
|
||||
if artifactNode.Kind != yaml.MappingNode {
|
||||
return fmt.Errorf("artifact reference must be an object")
|
||||
}
|
||||
var step, lane string
|
||||
seen := map[string]bool{}
|
||||
for i := 0; i < len(artifactNode.Content); i += 2 {
|
||||
key := artifactNode.Content[i].Value
|
||||
value := artifactNode.Content[i+1]
|
||||
if seen[key] {
|
||||
return fmt.Errorf("artifact reference field %q is duplicated", key)
|
||||
}
|
||||
seen[key] = true
|
||||
if value.Tag != "!!str" {
|
||||
return fmt.Errorf("artifact reference field %q must be a string", key)
|
||||
}
|
||||
switch key {
|
||||
case "step":
|
||||
step = strings.TrimSpace(value.Value)
|
||||
case "lane":
|
||||
lane = strings.TrimSpace(value.Value)
|
||||
default:
|
||||
return fmt.Errorf("field %s not found in artifact reference", key)
|
||||
}
|
||||
}
|
||||
if step == "" || lane == "" {
|
||||
return fmt.Errorf("artifact reference step and lane must not be empty")
|
||||
}
|
||||
source.path = ""
|
||||
source.artifact = &pipeline.ArtifactReference{Step: step, Lane: lane}
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("reference source must be a string or object")
|
||||
}
|
||||
}
|
||||
|
||||
func (source fileReferenceSource) toPipelineSource() pipeline.ReferenceSource {
|
||||
if source.artifact != nil {
|
||||
artifact := *source.artifact
|
||||
return pipeline.ReferenceSource{Artifact: &artifact}
|
||||
}
|
||||
return pipeline.ExternalReference(source.path)
|
||||
}
|
||||
|
||||
func fileReferenceSourcesToPipeline(values map[string]fileReferenceSource) map[string]pipeline.ReferenceSource {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]pipeline.ReferenceSource, len(values))
|
||||
for key, value := range values {
|
||||
out[strings.TrimSpace(key)] = value.toPipelineSource()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||
switch node.Kind {
|
||||
case yaml.ScalarNode:
|
||||
@@ -110,6 +260,9 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||
return err
|
||||
}
|
||||
b.LLMProfile = strings.TrimSpace(llmProfile)
|
||||
if b.LLMProfile == "" {
|
||||
return fmt.Errorf("llm_profile must not be empty when set")
|
||||
}
|
||||
case "retries":
|
||||
var retries int
|
||||
if err := valueNode.Decode(&retries); err != nil {
|
||||
@@ -123,7 +276,7 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||
}
|
||||
b.Options = normalizeOptions(options)
|
||||
case "references":
|
||||
var references map[string]string
|
||||
var references map[string]fileReferenceSource
|
||||
if err := valueNode.Decode(&references); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -154,7 +307,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
||||
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
||||
Retries: b.Retries,
|
||||
Options: cloneOptions(b.Options),
|
||||
References: normalizedStringMap(b.References),
|
||||
References: fileReferenceSourcesToPipeline(b.References),
|
||||
Validators: b.Validators,
|
||||
}
|
||||
}
|
||||
@@ -172,18 +325,30 @@ func LoadFileConfig(path string) (FileConfig, error) {
|
||||
}
|
||||
|
||||
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
|
||||
var header struct {
|
||||
Version int `yaml:"version"`
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &header); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("decode yaml version header: %w", err)
|
||||
}
|
||||
if header.Version == 0 {
|
||||
return FileConfig{}, fmt.Errorf("config version is required")
|
||||
}
|
||||
if header.Version == 2 {
|
||||
return FileConfig{}, fmt.Errorf("config version 2 is no longer supported; migrate the file using the version 2-to-3 migration in docs/config.md")
|
||||
}
|
||||
if header.Version == 3 {
|
||||
return FileConfig{}, fmt.Errorf("config version 3 is no longer supported; change \"version: 3\" to \"version: 4\" and rename \"scriptorium:\" to \"promptkit:\"")
|
||||
}
|
||||
if header.Version != SupportedFileConfigVersion {
|
||||
return FileConfig{}, fmt.Errorf("unsupported config version %d (supported version is %d)", header.Version, SupportedFileConfigVersion)
|
||||
}
|
||||
var fileCfg FileConfig
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||
decoder.KnownFields(true)
|
||||
if err := decoder.Decode(&fileCfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
|
||||
}
|
||||
if fileCfg.Version == 0 {
|
||||
return FileConfig{}, fmt.Errorf("config version is required")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
return fileCfg, nil
|
||||
}
|
||||
|
||||
@@ -213,10 +378,49 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
}
|
||||
for _, pipelineID := range pipelineIDs {
|
||||
filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]]
|
||||
hasArtifacts := filePipeline.artifactsSet || filePipeline.Artifacts != nil
|
||||
hasSteps := filePipeline.stepsSet || filePipeline.Steps != nil
|
||||
if hasArtifacts && hasSteps {
|
||||
return fmt.Errorf("pipeline %q must not declare both artifacts and steps", pipelineID)
|
||||
}
|
||||
if hasSteps && len(filePipeline.Steps) == 0 {
|
||||
return fmt.Errorf("pipeline %q must declare at least one ordered step", pipelineID)
|
||||
}
|
||||
if hasSteps {
|
||||
seenSteps := make(map[string]struct{}, len(filePipeline.Steps))
|
||||
seenLanes := make(map[string]struct{})
|
||||
for index, step := range filePipeline.Steps {
|
||||
stepID := strings.TrimSpace(step.ID)
|
||||
if stepID == "" {
|
||||
return fmt.Errorf("pipeline %q step[%d] id must not be empty", pipelineID, index)
|
||||
}
|
||||
if _, ok := seenSteps[stepID]; ok {
|
||||
return fmt.Errorf("pipeline %q step id %q is duplicated after trimming", pipelineID, stepID)
|
||||
}
|
||||
seenSteps[stepID] = struct{}{}
|
||||
laneIDs, rawLaneIDs, err := normalizedMapKeys(step.Artifacts, fmt.Sprintf("pipeline %q step %q artifact lane id", pipelineID, stepID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, laneID := range laneIDs {
|
||||
if _, ok := seenLanes[laneID]; ok {
|
||||
return fmt.Errorf("pipeline %q artifact lane id %q is duplicated across steps", pipelineID, laneID)
|
||||
}
|
||||
seenLanes[laneID] = struct{}{}
|
||||
fileLane := step.Artifacts[rawLaneIDs[laneID]]
|
||||
if err := validateFileLaneReferences(pipelineID, stepID, laneID, fileLane); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := validateFileReferenceSources(step.References, fmt.Sprintf("pipeline %q step %q reference slot", pipelineID, stepID)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
|
||||
if err := validateFileReferenceSources(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if filePipeline.Chunk != nil {
|
||||
@@ -261,34 +465,57 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Scriptorium != nil {
|
||||
if fileCfg.Scriptorium.ProfileDir != nil {
|
||||
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileDir)
|
||||
if fileCfg.PromptKit != nil {
|
||||
if fileCfg.PromptKit.ProfileDir != nil {
|
||||
value := strings.TrimSpace(*fileCfg.PromptKit.ProfileDir)
|
||||
if value == "" {
|
||||
return fmt.Errorf("scriptorium.profile_dir must not be empty when set")
|
||||
return fmt.Errorf("promptkit.profile_dir must not be empty when set")
|
||||
}
|
||||
c.Scriptorium.ProfileDir = value
|
||||
c.PromptKit.ProfileDir = value
|
||||
}
|
||||
if fileCfg.Scriptorium.ProfileFile != nil {
|
||||
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileFile)
|
||||
if fileCfg.PromptKit.ProfileFile != nil {
|
||||
value := strings.TrimSpace(*fileCfg.PromptKit.ProfileFile)
|
||||
if value == "" {
|
||||
return fmt.Errorf("scriptorium.profile_file must not be empty when set")
|
||||
return fmt.Errorf("promptkit.profile_file must not be empty when set")
|
||||
}
|
||||
c.Scriptorium.ProfileFile = value
|
||||
c.PromptKit.ProfileFile = value
|
||||
}
|
||||
if fileCfg.PromptKit.LocalBackend != nil {
|
||||
if fileCfg.PromptKit.LocalBackend.Endpoint == nil {
|
||||
return fmt.Errorf("promptkit.local_backend.endpoint must not be empty when set")
|
||||
}
|
||||
endpoint := strings.TrimSpace(*fileCfg.PromptKit.LocalBackend.Endpoint)
|
||||
if endpoint == "" {
|
||||
return fmt.Errorf("promptkit.local_backend.endpoint must not be empty when set")
|
||||
}
|
||||
localBackend := PromptKitLocalBackendConfig{Endpoint: endpoint}
|
||||
if fileCfg.PromptKit.LocalBackend.ConcurrencyLimit != nil {
|
||||
localBackend.ConcurrencyLimit = *fileCfg.PromptKit.LocalBackend.ConcurrencyLimit
|
||||
}
|
||||
c.PromptKit.LocalBackend = &localBackend
|
||||
}
|
||||
}
|
||||
|
||||
for _, pipelineID := range pipelineIDs {
|
||||
filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]]
|
||||
llmProfile := ""
|
||||
if filePipeline.llmProfileSet || filePipeline.LLMProfile != nil {
|
||||
if filePipeline.LLMProfile == nil || strings.TrimSpace(*filePipeline.LLMProfile) == "" {
|
||||
return fmt.Errorf("pipeline %q llm_profile must not be empty when set", pipelineID)
|
||||
}
|
||||
llmProfile = strings.TrimSpace(*filePipeline.LLMProfile)
|
||||
}
|
||||
hasSteps := filePipeline.stepsSet || filePipeline.Steps != nil
|
||||
laneIDs, rawLaneIDs, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
profile := pipeline.PipelineProfile{
|
||||
ID: pipelineID,
|
||||
LLMProfile: llmProfile,
|
||||
Input: filePipeline.Input.toPipelineBinding(),
|
||||
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
||||
References: normalizedStringMap(filePipeline.References),
|
||||
References: fileReferenceSourcesToPipeline(filePipeline.References),
|
||||
}
|
||||
if filePipeline.Chunk != nil {
|
||||
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
|
||||
@@ -299,10 +526,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
for _, laneID := range laneIDs {
|
||||
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
|
||||
extract := fileLane.Extract.toPipelineBinding()
|
||||
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
|
||||
extract.References = mergeReferenceSources(fileReferenceSourcesToPipeline(fileLane.References), extract.References)
|
||||
lane := pipeline.ArtifactLaneProfile{
|
||||
Extract: extract,
|
||||
References: normalizedStringMap(fileLane.References),
|
||||
References: fileReferenceSourcesToPipeline(fileLane.References),
|
||||
}
|
||||
if fileLane.Merge != nil {
|
||||
lane.Merge = fileLane.Merge.toPipelineBinding()
|
||||
@@ -318,6 +545,42 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
}
|
||||
profile.Artifacts[laneID] = lane
|
||||
}
|
||||
if hasSteps {
|
||||
profile.Artifacts = nil
|
||||
profile.Steps = make([]pipeline.PipelineStepProfile, len(filePipeline.Steps))
|
||||
for i, fileStep := range filePipeline.Steps {
|
||||
stepID := strings.TrimSpace(fileStep.ID)
|
||||
step := pipeline.PipelineStepProfile{
|
||||
ID: stepID,
|
||||
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(fileStep.Artifacts)),
|
||||
References: fileReferenceSourcesToPipeline(fileStep.References),
|
||||
}
|
||||
stepLaneIDs, stepRawLaneIDs, err := normalizedMapKeys(fileStep.Artifacts, fmt.Sprintf("pipeline %q step %q artifact lane id", pipelineID, stepID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, laneID := range stepLaneIDs {
|
||||
fileLane := fileStep.Artifacts[stepRawLaneIDs[laneID]]
|
||||
extract := fileLane.Extract.toPipelineBinding()
|
||||
extract.References = mergeReferenceSources(fileReferenceSourcesToPipeline(fileLane.References), extract.References)
|
||||
lane := pipeline.ArtifactLaneProfile{Extract: extract, References: fileReferenceSourcesToPipeline(fileLane.References)}
|
||||
if fileLane.Merge != nil {
|
||||
lane.Merge = fileLane.Merge.toPipelineBinding()
|
||||
}
|
||||
if fileLane.Normalize != nil {
|
||||
lane.Normalize = fileLane.Normalize.toPipelineBinding()
|
||||
}
|
||||
if len(fileLane.Validators) > 0 {
|
||||
lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators))
|
||||
for index, validator := range fileLane.Validators {
|
||||
lane.Validators[index] = validator.toPipelineBinding()
|
||||
}
|
||||
}
|
||||
step.Artifacts[laneID] = lane
|
||||
}
|
||||
profile.Steps[i] = step
|
||||
}
|
||||
}
|
||||
c.Pipelines[pipelineID] = profile
|
||||
}
|
||||
|
||||
@@ -333,48 +596,52 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
c.Concurrency.extractWorkersConfigured = configured
|
||||
}
|
||||
c.Concurrency.recomputeStageWorkerDefaults()
|
||||
if fileCfg.Diagnostics != nil {
|
||||
if fileCfg.Diagnostics.WorkDir != nil {
|
||||
c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir)
|
||||
if fileCfg.Output != nil && fileCfg.Output.Directory != nil {
|
||||
c.Output.Directory = strings.TrimSpace(*fileCfg.Output.Directory)
|
||||
if c.Output.Directory == "" {
|
||||
return fmt.Errorf("output.directory must not be empty")
|
||||
}
|
||||
if fileCfg.Diagnostics.Retention != nil {
|
||||
c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Diagnostics.Retention))
|
||||
if strings.ContainsRune(c.Output.Directory, '\x00') {
|
||||
return fmt.Errorf("output.directory must not contain NUL")
|
||||
}
|
||||
}
|
||||
if fileCfg.Workspace != nil {
|
||||
if fileCfg.Workspace.Directory != nil {
|
||||
c.Workspace.Directory = strings.TrimSpace(*fileCfg.Workspace.Directory)
|
||||
}
|
||||
if fileCfg.Workspace.ChunkCache != nil {
|
||||
if fileCfg.Workspace.ChunkCache.Mode != nil {
|
||||
mode, err := pipeline.ParseChunkCacheMode(*fileCfg.Workspace.ChunkCache.Mode)
|
||||
if fileCfg.Cache != nil {
|
||||
if fileCfg.Cache.ChunkPlans != nil {
|
||||
if fileCfg.Cache.ChunkPlans.Mode != nil {
|
||||
mode, err := pipeline.ParseChunkCacheMode(*fileCfg.Cache.ChunkPlans.Mode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("workspace.chunk_cache.mode: %w", err)
|
||||
return fmt.Errorf("cache.chunk_plans.mode: %w", err)
|
||||
}
|
||||
c.Workspace.ChunkCache.Mode = mode
|
||||
c.Cache.ChunkPlans.Mode = mode
|
||||
}
|
||||
if fileCfg.Workspace.ChunkCache.Directory != nil {
|
||||
c.Workspace.ChunkCache.Directory = cleanOptionalPath(*fileCfg.Workspace.ChunkCache.Directory)
|
||||
if fileCfg.Cache.ChunkPlans.Directory != nil {
|
||||
c.Cache.ChunkPlans.Directory = cleanOptionalPath(*fileCfg.Cache.ChunkPlans.Directory)
|
||||
if strings.ContainsRune(c.Cache.ChunkPlans.Directory, '\x00') {
|
||||
return fmt.Errorf("cache.chunk_plans.directory must not contain NUL")
|
||||
}
|
||||
}
|
||||
}
|
||||
if fileCfg.Workspace.Diagnostics != nil {
|
||||
if fileCfg.Workspace.Diagnostics.Enabled != nil {
|
||||
c.Workspace.Diagnostics.Enabled = *fileCfg.Workspace.Diagnostics.Enabled
|
||||
c.Workspace.Diagnostics.enabledSet = true
|
||||
if fileCfg.Cache.Checkpoints != nil {
|
||||
if fileCfg.Cache.Checkpoints.Enabled != nil {
|
||||
c.Cache.Checkpoints.Enabled = *fileCfg.Cache.Checkpoints.Enabled
|
||||
}
|
||||
if fileCfg.Workspace.Diagnostics.Retention != nil {
|
||||
c.Workspace.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Workspace.Diagnostics.Retention))
|
||||
c.Workspace.Diagnostics.retentionSet = true
|
||||
if fileCfg.Cache.Checkpoints.Directory != nil {
|
||||
c.Cache.Checkpoints.Directory = cleanOptionalPath(*fileCfg.Cache.Checkpoints.Directory)
|
||||
if strings.ContainsRune(c.Cache.Checkpoints.Directory, '\x00') {
|
||||
return fmt.Errorf("cache.checkpoints.directory must not contain NUL")
|
||||
}
|
||||
}
|
||||
}
|
||||
if fileCfg.Workspace.Resume != nil && fileCfg.Workspace.Resume.Enabled != nil {
|
||||
c.Workspace.Resume.Enabled = *fileCfg.Workspace.Resume.Enabled
|
||||
}
|
||||
if fileCfg.Workspace.Debug != nil && fileCfg.Workspace.Debug.Enabled != nil {
|
||||
c.Workspace.Debug.Enabled = *fileCfg.Workspace.Debug.Enabled
|
||||
}
|
||||
}
|
||||
c.RecomputeEffectiveDiagnostics()
|
||||
if fileCfg.Debug != nil && fileCfg.Debug.Directory != nil {
|
||||
c.Debug.Directory = strings.TrimSpace(*fileCfg.Debug.Directory)
|
||||
if c.Debug.Directory == "" {
|
||||
return fmt.Errorf("debug.directory must not be empty")
|
||||
}
|
||||
if strings.ContainsRune(c.Debug.Directory, '\x00') {
|
||||
return fmt.Errorf("debug.directory must not contain NUL")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -425,30 +692,72 @@ func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, ma
|
||||
return keys, rawByNormalized, nil
|
||||
}
|
||||
|
||||
func normalizedStringMap(values map[string]string) map[string]string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
func validateFileReferenceSources(values map[string]fileReferenceSource, context string) error {
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
for rawSlot, source := range values {
|
||||
slot := strings.TrimSpace(rawSlot)
|
||||
if slot == "" {
|
||||
return fmt.Errorf("%s must not be empty", context)
|
||||
}
|
||||
if _, ok := seen[slot]; ok {
|
||||
return fmt.Errorf("%s %q is duplicated after trimming", context, slot)
|
||||
}
|
||||
seen[slot] = struct{}{}
|
||||
if source.artifact != nil {
|
||||
if strings.TrimSpace(source.artifact.Step) == "" || strings.TrimSpace(source.artifact.Lane) == "" {
|
||||
return fmt.Errorf("%s %q artifact selector step and lane must not be empty", context, slot)
|
||||
}
|
||||
if strings.TrimSpace(source.path) != "" {
|
||||
return fmt.Errorf("%s %q must contain either an external path or artifact selector", context, slot)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(source.path) == "" {
|
||||
return fmt.Errorf("%s %q source must not be empty", context, slot)
|
||||
}
|
||||
}
|
||||
out := make(map[string]string, len(values))
|
||||
keys := make([]string, 0, len(values))
|
||||
rawByNormalized := make(map[string]string, len(values))
|
||||
for rawKey := range values {
|
||||
key := strings.TrimSpace(rawKey)
|
||||
rawByNormalized[key] = rawKey
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
|
||||
}
|
||||
return out
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
|
||||
func validateFileLaneReferences(pipelineID, stepID, laneID string, lane FileArtifactLaneProfile) error {
|
||||
prefix := fmt.Sprintf("pipeline %q step %q lane %q", pipelineID, stepID, laneID)
|
||||
references := []struct {
|
||||
label string
|
||||
values map[string]fileReferenceSource
|
||||
}{
|
||||
{label: "reference slot", values: lane.References},
|
||||
{label: "extract reference slot", values: lane.Extract.References},
|
||||
}
|
||||
if lane.Merge != nil {
|
||||
references = append(references, struct {
|
||||
label string
|
||||
values map[string]fileReferenceSource
|
||||
}{label: "merge reference slot", values: lane.Merge.References})
|
||||
}
|
||||
if lane.Normalize != nil {
|
||||
references = append(references, struct {
|
||||
label string
|
||||
values map[string]fileReferenceSource
|
||||
}{label: "normalize reference slot", values: lane.Normalize.References})
|
||||
}
|
||||
for _, item := range references {
|
||||
if err := validateFileReferenceSources(item.values, prefix+" "+item.label); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for index, validator := range lane.Validators {
|
||||
if err := validateFileReferenceSources(validator.References, fmt.Sprintf("%s validator[%d] reference slot", prefix, index)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeReferenceSources(base map[string]pipeline.ReferenceSource, override map[string]pipeline.ReferenceSource) map[string]pipeline.ReferenceSource {
|
||||
if len(base) == 0 && len(override) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(base)+len(override))
|
||||
out := make(map[string]pipeline.ReferenceSource, len(base)+len(override))
|
||||
for key, value := range base {
|
||||
out[key] = value
|
||||
}
|
||||
|
||||
744
internal/core/config/file_config_contract_test.go
Normal file
744
internal/core/config/file_config_contract_test.go
Normal file
@@ -0,0 +1,744 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestDefaultReturnsDocumentedValuesAndIndependentMaps(t *testing.T) {
|
||||
first := Default()
|
||||
if first.Concurrency.TotalLLM != 1 || first.Concurrency.StageWorkers["extract"] != 1 {
|
||||
t.Fatalf("concurrency defaults = %#v", first.Concurrency)
|
||||
}
|
||||
if first.Output.Directory != "./notarius-output" || first.Debug.Directory != "./notarius-debug" {
|
||||
t.Fatalf("output/debug defaults = %#v, %#v", first.Output, first.Debug)
|
||||
}
|
||||
if first.Cache.ChunkPlans.Mode != pipeline.ChunkCacheAuto || first.Cache.ChunkPlans.Directory != "" || first.Cache.Checkpoints.Enabled || first.Cache.Checkpoints.Directory != "" {
|
||||
t.Fatalf("cache defaults = %#v", first.Cache)
|
||||
}
|
||||
if len(first.Pipelines) != 0 {
|
||||
t.Fatalf("pipeline defaults = %#v", first.Pipelines)
|
||||
}
|
||||
|
||||
first.Concurrency.StageWorkers["extract"] = 99
|
||||
first.Concurrency.StageWorkers["other"] = 100
|
||||
first.Pipelines["changed"] = pipeline.PipelineProfile{}
|
||||
second := Default()
|
||||
if second.Concurrency.StageWorkers["extract"] != 1 || len(second.Concurrency.StageWorkers) != 1 || len(second.Pipelines) != 0 {
|
||||
t.Fatalf("Default() returned state shared with an earlier result: %#v", second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigMinimalVersion4AppliesOverDefaults(t *testing.T) {
|
||||
file := parseFileConfig(t, "version: 4\n")
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Output.Directory != "./notarius-output" || cfg.Debug.Directory != "./notarius-debug" || cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheAuto {
|
||||
t.Fatalf("minimal file changed unrelated defaults: %#v", cfg)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 1 || cfg.Concurrency.StageWorkers["extract"] != 1 || len(cfg.Pipelines) != 0 {
|
||||
t.Fatalf("minimal file did not retain defaults: %#v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilePipelineLLMProfileIsPresenceAwareAndDetached(t *testing.T) {
|
||||
const pipelineYAML = `version: 4
|
||||
pipelines:
|
||||
main:
|
||||
%s
|
||||
input: input
|
||||
artifacts:
|
||||
lane:
|
||||
extract: extract
|
||||
`
|
||||
|
||||
t.Run("omitted", func(t *testing.T) {
|
||||
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, ""))
|
||||
if file.Pipelines["main"].LLMProfile != nil || file.Pipelines["main"].llmProfileSet {
|
||||
t.Fatalf("parsed pipeline profile = %#v, want omitted llm profile", file.Pipelines["main"])
|
||||
}
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := cfg.Pipelines["main"].LLMProfile; got != "" {
|
||||
t.Fatalf("pipeline llm profile = %q, want empty", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("trimmed and detached", func(t *testing.T) {
|
||||
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "llm_profile: ' configured-profile '"))
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := cfg.Pipelines["main"].LLMProfile; got != "configured-profile" {
|
||||
t.Fatalf("pipeline llm profile = %q, want trimmed value", got)
|
||||
}
|
||||
*file.Pipelines["main"].LLMProfile = "changed-profile"
|
||||
if got := cfg.Pipelines["main"].LLMProfile; got != "configured-profile" {
|
||||
t.Fatalf("effective config aliases parsed file: %q", got)
|
||||
}
|
||||
if got := cloneConfig(cfg).Pipelines["main"].LLMProfile; got != "configured-profile" {
|
||||
t.Fatalf("cloned pipeline llm profile = %q", got)
|
||||
}
|
||||
data, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var roundTripped Config
|
||||
if err := json.Unmarshal(data, &roundTripped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := roundTripped.Pipelines["main"].LLMProfile; got != "configured-profile" {
|
||||
t.Fatalf("round-tripped pipeline llm profile = %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
for _, value := range []string{"''", "' '", "null"} {
|
||||
t.Run("explicit empty "+value, func(t *testing.T) {
|
||||
file := parseFileConfig(t, fmt.Sprintf(pipelineYAML, "llm_profile: "+value))
|
||||
cfg := Default()
|
||||
err := cfg.ApplyFileConfig(file)
|
||||
if err == nil || !strings.Contains(err.Error(), `pipeline "main" llm_profile must not be empty`) {
|
||||
t.Fatalf("ApplyFileConfig() error = %v, want explicit-empty rejection", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileModuleBindingRejectsExplicitEmptyLLMProfile(t *testing.T) {
|
||||
const configYAML = `version: 4
|
||||
pipelines:
|
||||
main:
|
||||
input:
|
||||
module: input
|
||||
llm_profile: %s
|
||||
artifacts:
|
||||
lane:
|
||||
extract: extract
|
||||
`
|
||||
for _, value := range []string{"''", "' '", "null"} {
|
||||
t.Run(value, func(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(fmt.Sprintf(configYAML, value)))
|
||||
if err == nil || !strings.Contains(err.Error(), "llm_profile must not be empty when set") {
|
||||
t.Fatalf("ParseFileConfigYAML() error = %v, want explicit-empty binding profile rejection", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilePromptKitProfileSourcesSurviveConfigBoundaries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want PromptKitConfig
|
||||
}{
|
||||
{
|
||||
name: "profile directory",
|
||||
yaml: "version: 4\npromptkit:\n profile_dir: ' ./profiles '\n",
|
||||
want: PromptKitConfig{ProfileDir: "./profiles"},
|
||||
},
|
||||
{
|
||||
name: "profile file",
|
||||
yaml: "version: 4\npromptkit:\n profile_file: ' ./profiles.yml '\n",
|
||||
want: PromptKitConfig{ProfileFile: "./profiles.yml"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := applyFileConfig(t, tt.yaml)
|
||||
if cfg.PromptKit != tt.want {
|
||||
t.Fatalf("PromptKit config = %#v, want %#v", cfg.PromptKit, tt.want)
|
||||
}
|
||||
if got := cloneConfig(cfg).PromptKit; got != tt.want {
|
||||
t.Fatalf("cloned PromptKit config = %#v, want %#v", got, tt.want)
|
||||
}
|
||||
if got := cfg.Redacted().PromptKit; got != tt.want {
|
||||
t.Fatalf("redacted PromptKit config = %#v, want %#v", got, tt.want)
|
||||
}
|
||||
|
||||
data, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
var payload map[string]json.RawMessage
|
||||
if err := json.Unmarshal(data, &payload); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
if _, ok := payload["promptkit"]; !ok {
|
||||
t.Fatalf("runtime JSON keys = %v, want promptkit", payload)
|
||||
}
|
||||
if _, ok := payload["scriptorium"]; ok {
|
||||
t.Fatalf("runtime JSON keys = %v, must not contain removed section", payload)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilePromptKitLocalBackendSurvivesConfigBoundaries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
concurrencyYAML string
|
||||
wantConcurrency int
|
||||
}{
|
||||
{name: "omitted concurrency defaults to zero"},
|
||||
{name: "positive concurrency is preserved", concurrencyYAML: " concurrency_limit: 2\n", wantConcurrency: 2},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
file := parseFileConfig(t, "version: 4\npromptkit:\n local_backend:\n endpoint: ' http://localhost:8000/v1 '\n"+tt.concurrencyYAML)
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||
t.Fatalf("ApplyFileConfig() error = %v", err)
|
||||
}
|
||||
want := PromptKitLocalBackendConfig{
|
||||
Endpoint: "http://localhost:8000/v1",
|
||||
ConcurrencyLimit: tt.wantConcurrency,
|
||||
}
|
||||
if cfg.PromptKit.LocalBackend == nil || *cfg.PromptKit.LocalBackend != want {
|
||||
t.Fatalf("local backend config = %#v, want %#v", cfg.PromptKit.LocalBackend, want)
|
||||
}
|
||||
|
||||
*file.PromptKit.LocalBackend.Endpoint = "http://changed.example/v1"
|
||||
if file.PromptKit.LocalBackend.ConcurrencyLimit != nil {
|
||||
*file.PromptKit.LocalBackend.ConcurrencyLimit = 99
|
||||
}
|
||||
if *cfg.PromptKit.LocalBackend != want {
|
||||
t.Fatalf("effective config aliases parsed file model: %#v", cfg.PromptKit.LocalBackend)
|
||||
}
|
||||
|
||||
cloned := cloneConfig(cfg)
|
||||
if cloned.PromptKit.LocalBackend == cfg.PromptKit.LocalBackend || *cloned.PromptKit.LocalBackend != want {
|
||||
t.Fatalf("cloned local backend = %#v, want detached %#v", cloned.PromptKit.LocalBackend, want)
|
||||
}
|
||||
cloned.PromptKit.LocalBackend.Endpoint = "http://clone.example/v1"
|
||||
if *cfg.PromptKit.LocalBackend != want {
|
||||
t.Fatalf("mutating clone changed source config: %#v", cfg.PromptKit.LocalBackend)
|
||||
}
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
if redacted.PromptKit.LocalBackend == cfg.PromptKit.LocalBackend || *redacted.PromptKit.LocalBackend != want {
|
||||
t.Fatalf("redacted local backend = %#v, want detached %#v", redacted.PromptKit.LocalBackend, want)
|
||||
}
|
||||
|
||||
data, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
var payload struct {
|
||||
PromptKit map[string]json.RawMessage `json:"promptkit"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &payload); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
localJSON, ok := payload.PromptKit["local_backend"]
|
||||
if !ok {
|
||||
t.Fatalf("runtime PromptKit JSON keys = %v, want local_backend", payload.PromptKit)
|
||||
}
|
||||
var localPayload map[string]json.RawMessage
|
||||
if err := json.Unmarshal(localJSON, &localPayload); err != nil {
|
||||
t.Fatalf("unmarshal local_backend JSON: %v", err)
|
||||
}
|
||||
if _, ok := localPayload["endpoint"]; !ok {
|
||||
t.Fatalf("runtime local_backend JSON keys = %v, want endpoint", localPayload)
|
||||
}
|
||||
if _, ok := localPayload["concurrency_limit"]; !ok {
|
||||
t.Fatalf("runtime local_backend JSON keys = %v, want concurrency_limit", localPayload)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilePromptKitLocalBackendRequiresEndpoint(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
yaml string
|
||||
}{
|
||||
{name: "missing", yaml: "version: 4\npromptkit:\n local_backend: {}\n"},
|
||||
{name: "empty", yaml: "version: 4\npromptkit:\n local_backend:\n endpoint: ''\n"},
|
||||
{name: "blank", yaml: "version: 4\npromptkit:\n local_backend:\n endpoint: ' '\n"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
file := parseFileConfig(t, tt.yaml)
|
||||
cfg := Default()
|
||||
err := cfg.ApplyFileConfig(file)
|
||||
if err == nil || !strings.Contains(err.Error(), "promptkit.local_backend.endpoint") {
|
||||
t.Fatalf("ApplyFileConfig() error = %v, want endpoint field context", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilePromptKitExplicitEmptyProfileSourcesAreRejected(t *testing.T) {
|
||||
for _, field := range []string{"profile_dir", "profile_file"} {
|
||||
t.Run(field, func(t *testing.T) {
|
||||
file := parseFileConfig(t, "version: 4\npromptkit:\n "+field+": ''\n")
|
||||
cfg := Default()
|
||||
err := cfg.ApplyFileConfig(file)
|
||||
if err == nil || !strings.Contains(err.Error(), "promptkit."+field+" must not be empty") {
|
||||
t.Fatalf("ApplyFileConfig() error = %v, want explicit-empty rejection", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilePromptKitProfileSourcesRemainMutuallyExclusive(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 4
|
||||
promptkit:
|
||||
profile_dir: ./profiles
|
||||
profile_file: ./profiles.yml
|
||||
`)
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "promptkit profile_dir and profile_file are mutually exclusive") {
|
||||
t.Fatalf("Validate() error = %v, want mutually exclusive profile sources", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigMissingVersionIsReportedBeforeFieldDecoding(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte("workspace:\n directory: /tmp/old\n"))
|
||||
if err == nil || !strings.Contains(err.Error(), "config version is required") {
|
||||
t.Fatalf("missing version error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigVersion3ReportsPromptKitMigration(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte("version: 3\nscriptorium:\n profile_dir: ./profiles\n"))
|
||||
if err == nil ||
|
||||
!strings.Contains(err.Error(), `change "version: 3" to "version: 4"`) ||
|
||||
!strings.Contains(err.Error(), `rename "scriptorium:" to "promptkit:"`) {
|
||||
t.Fatalf("version 3 error = %v, want actionable version and section migration", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigRejectsUnknownCurrentAndRemovedFields(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "removed diagnostics",
|
||||
yaml: "version: 4\ndiagnostics: {}\n",
|
||||
want: "field diagnostics not found",
|
||||
},
|
||||
{
|
||||
name: "removed llm profiles",
|
||||
yaml: "version: 4\nllm_profiles: {}\n",
|
||||
want: "field llm_profiles not found",
|
||||
},
|
||||
{
|
||||
name: "removed scriptorium section",
|
||||
yaml: "version: 4\nscriptorium: {}\n",
|
||||
want: "field scriptorium not found",
|
||||
},
|
||||
{
|
||||
name: "version 2 migration",
|
||||
yaml: "version: 2\nworkspace:\n directory: /tmp/old\n",
|
||||
want: "version 2-to-3 migration",
|
||||
},
|
||||
{
|
||||
name: "pipeline field",
|
||||
yaml: "version: 4\npipelines:\n main:\n unknown: true\n",
|
||||
want: "field unknown not found",
|
||||
},
|
||||
{
|
||||
name: "lane field",
|
||||
yaml: "version: 4\npipelines:\n main:\n artifacts:\n spells:\n unknown: true\n",
|
||||
want: "field unknown not found",
|
||||
},
|
||||
{
|
||||
name: "module binding field",
|
||||
yaml: "version: 4\npipelines:\n main:\n input:\n module: seriatim\n unknown: true\n",
|
||||
want: "field unknown not found in module binding",
|
||||
},
|
||||
{
|
||||
name: "checkpoint field",
|
||||
yaml: "version: 4\ncache:\n checkpoints:\n unknown: true\n",
|
||||
want: "field unknown not found",
|
||||
},
|
||||
{
|
||||
name: "checkpoint enabled type",
|
||||
yaml: "version: 4\ncache:\n checkpoints:\n enabled: definitely\n",
|
||||
want: "cannot unmarshal",
|
||||
},
|
||||
{
|
||||
name: "local backend field",
|
||||
yaml: "version: 4\npromptkit:\n local_backend:\n endpoint: http://localhost:8000/v1\n unknown: true\n",
|
||||
want: "field unknown not found",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(tt.yaml))
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("error = %v, want context %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigModuleBindingsPreserveFormsAndValidatorPresence(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 4
|
||||
pipelines:
|
||||
main:
|
||||
input: seriatim
|
||||
chunk:
|
||||
module: generic
|
||||
llm_profile: chunk-profile
|
||||
retries: 2
|
||||
options:
|
||||
max_units: 25
|
||||
references:
|
||||
glossary: ./glossary.md
|
||||
validators: []
|
||||
artifacts:
|
||||
spells:
|
||||
extract:
|
||||
module: dnd/spells
|
||||
options:
|
||||
nested:
|
||||
enabled: true
|
||||
merge: appendorder
|
||||
normalize: noop
|
||||
`)
|
||||
profile := cfg.Pipelines["main"]
|
||||
if profile.Input.Module != "seriatim" || profile.Input.Validators.Set {
|
||||
t.Fatalf("shorthand binding = %#v", profile.Input)
|
||||
}
|
||||
if profile.Chunk.Module != "generic" || profile.Chunk.LLMProfile != "chunk-profile" || profile.Chunk.Retries != 2 ||
|
||||
!reflect.DeepEqual(profile.Chunk.Options, map[string]any{"max_units": 25}) ||
|
||||
!reflect.DeepEqual(profile.Chunk.References, pipeline.ExternalReferenceMap(map[string]string{"glossary": "./glossary.md"})) {
|
||||
t.Fatalf("object binding = %#v", profile.Chunk)
|
||||
}
|
||||
if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 {
|
||||
t.Fatalf("explicit empty validators = %#v", profile.Chunk.Validators)
|
||||
}
|
||||
if profile.Artifacts["spells"].Extract.Module != "dnd/spells" ||
|
||||
!reflect.DeepEqual(profile.Artifacts["spells"].Extract.Options, map[string]any{
|
||||
"nested": map[string]any{"enabled": true},
|
||||
}) {
|
||||
t.Fatalf("extract binding = %#v", profile.Artifacts["spells"].Extract)
|
||||
}
|
||||
if profile.Artifacts["spells"].Merge.Module != "appendorder" || profile.Artifacts["spells"].Normalize.Module != "noop" {
|
||||
t.Fatalf("stage shorthand bindings = %#v", profile.Artifacts["spells"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigReferencePrecedenceIsRetained(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 4
|
||||
pipelines:
|
||||
main:
|
||||
input: seriatim
|
||||
references:
|
||||
pipeline-only: ./pipeline.txt
|
||||
shared: ./pipeline-shared.txt
|
||||
chunk:
|
||||
module: generic
|
||||
references:
|
||||
chunk-only: ./chunk.txt
|
||||
artifacts:
|
||||
spells:
|
||||
references:
|
||||
lane-only: ./lane.txt
|
||||
shared: ./lane-shared.txt
|
||||
overridden: ./lane.txt
|
||||
extract:
|
||||
module: dnd/spells
|
||||
references:
|
||||
extract-only: ./extract.txt
|
||||
overridden: ./extract-overridden.txt
|
||||
merge:
|
||||
module: appendorder
|
||||
references:
|
||||
merge-only: ./merge.txt
|
||||
normalize:
|
||||
module: noop
|
||||
references:
|
||||
normalize-only: ./normalize.txt
|
||||
`)
|
||||
profile := cfg.Pipelines["main"]
|
||||
if !reflect.DeepEqual(profile.References, pipeline.ExternalReferenceMap(map[string]string{
|
||||
"pipeline-only": "./pipeline.txt",
|
||||
"shared": "./pipeline-shared.txt",
|
||||
})) {
|
||||
t.Fatalf("pipeline references = %#v", profile.References)
|
||||
}
|
||||
if !reflect.DeepEqual(profile.Chunk.References, pipeline.ExternalReferenceMap(map[string]string{"chunk-only": "./chunk.txt"})) {
|
||||
t.Fatalf("chunk references = %#v", profile.Chunk.References)
|
||||
}
|
||||
lane := profile.Artifacts["spells"]
|
||||
if !reflect.DeepEqual(lane.References, pipeline.ExternalReferenceMap(map[string]string{
|
||||
"lane-only": "./lane.txt",
|
||||
"shared": "./lane-shared.txt",
|
||||
"overridden": "./lane.txt",
|
||||
})) {
|
||||
t.Fatalf("lane compatibility references = %#v", lane.References)
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Extract.References, pipeline.ExternalReferenceMap(map[string]string{
|
||||
"lane-only": "./lane.txt",
|
||||
"shared": "./lane-shared.txt",
|
||||
"overridden": "./extract-overridden.txt",
|
||||
"extract-only": "./extract.txt",
|
||||
})) {
|
||||
t.Fatalf("extract references = %#v", lane.Extract.References)
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Merge.References, pipeline.ExternalReferenceMap(map[string]string{"merge-only": "./merge.txt"})) ||
|
||||
!reflect.DeepEqual(lane.Normalize.References, pipeline.ExternalReferenceMap(map[string]string{"normalize-only": "./normalize.txt"})) {
|
||||
t.Fatalf("merge/normalize references = %#v, %#v", lane.Merge.References, lane.Normalize.References)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigStageLocalValidatorsPreserveOrderAndFields(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 4
|
||||
pipelines:
|
||||
main:
|
||||
input: seriatim
|
||||
chunk:
|
||||
module: generic
|
||||
validators:
|
||||
- generic/always_accept
|
||||
- module: generic/valid_json
|
||||
llm_profile: validator-profile
|
||||
options:
|
||||
schema: compact
|
||||
artifacts:
|
||||
spells:
|
||||
extract:
|
||||
module: dnd/spells
|
||||
validators:
|
||||
- module: extract/dnd/spells/shape
|
||||
options:
|
||||
strict: true
|
||||
merge:
|
||||
module: appendorder
|
||||
validators:
|
||||
- generic/always_accept
|
||||
normalize:
|
||||
module: noop
|
||||
validators:
|
||||
- module: generic/valid_json
|
||||
options:
|
||||
mode: normalized
|
||||
`)
|
||||
profile := cfg.Pipelines["main"]
|
||||
chunkValidators := profile.Chunk.Validators.Validators
|
||||
if !profile.Chunk.Validators.Set || len(chunkValidators) != 2 || chunkValidators[0].Module != "generic/always_accept" ||
|
||||
chunkValidators[1].Module != "generic/valid_json" || chunkValidators[1].LLMProfile != "validator-profile" ||
|
||||
!reflect.DeepEqual(chunkValidators[1].Options, map[string]any{"schema": "compact"}) {
|
||||
t.Fatalf("chunk validators = %#v", profile.Chunk.Validators)
|
||||
}
|
||||
lane := profile.Artifacts["spells"]
|
||||
if len(lane.Extract.Validators.Validators) != 1 || lane.Extract.Validators.Validators[0].Module != "extract/dnd/spells/shape" ||
|
||||
!reflect.DeepEqual(lane.Extract.Validators.Validators[0].Options, map[string]any{"strict": true}) {
|
||||
t.Fatalf("extract validators = %#v", lane.Extract.Validators)
|
||||
}
|
||||
if len(lane.Merge.Validators.Validators) != 1 || lane.Merge.Validators.Validators[0].Module != "generic/always_accept" {
|
||||
t.Fatalf("merge validators = %#v", lane.Merge.Validators)
|
||||
}
|
||||
if len(lane.Normalize.Validators.Validators) != 1 || lane.Normalize.Validators.Validators[0].Module != "generic/valid_json" ||
|
||||
!reflect.DeepEqual(lane.Normalize.Validators.Validators[0].Options, map[string]any{"mode": "normalized"}) {
|
||||
t.Fatalf("normalize validators = %#v", lane.Normalize.Validators)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigStateSectionsApplyIndependently(t *testing.T) {
|
||||
cfg := applyFileConfig(t, `version: 4
|
||||
promptkit:
|
||||
profile_dir: ./profiles
|
||||
concurrency:
|
||||
total_llm: 7
|
||||
output:
|
||||
directory: ./output
|
||||
cache:
|
||||
chunk_plans:
|
||||
directory: ./plans
|
||||
mode: bypass
|
||||
checkpoints:
|
||||
enabled: true
|
||||
directory: ./checkpoints
|
||||
debug:
|
||||
directory: ./debug
|
||||
`)
|
||||
if cfg.PromptKit.ProfileDir != "./profiles" || cfg.PromptKit.ProfileFile != "" {
|
||||
t.Fatalf("promptkit = %#v", cfg.PromptKit)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 7 || cfg.Concurrency.StageWorkers["extract"] != 7 {
|
||||
t.Fatalf("concurrency = %#v", cfg.Concurrency)
|
||||
}
|
||||
if cfg.Output.Directory != "./output" || cfg.Cache.ChunkPlans.Directory != "plans" || cfg.Cache.ChunkPlans.Mode != pipeline.ChunkCacheBypass ||
|
||||
!cfg.Cache.Checkpoints.Enabled || cfg.Cache.Checkpoints.Directory != "checkpoints" || cfg.Debug.Directory != "./debug" {
|
||||
t.Fatalf("state sections = %#v, %#v, %#v, %#v", cfg.Output, cfg.Cache, cfg.Debug, cfg.PromptKit)
|
||||
}
|
||||
if cfg.Output.Directory == cfg.Cache.ChunkPlans.Directory || cfg.Cache.ChunkPlans.Directory == cfg.Cache.Checkpoints.Directory || cfg.Cache.Checkpoints.Directory == cfg.Debug.Directory {
|
||||
t.Fatal("state roots were coupled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigCheckpointEnabledCanBeExplicitlyDisabled(t *testing.T) {
|
||||
cfg := applyFileConfig(t, "version: 4\ncache:\n checkpoints:\n enabled: true\n")
|
||||
if !cfg.Cache.Checkpoints.Enabled || !cloneConfig(cfg).Cache.Checkpoints.Enabled {
|
||||
t.Fatalf("enabled checkpoint config was not retained: %#v", cfg.Cache.Checkpoints)
|
||||
}
|
||||
file := parseFileConfig(t, "version: 4\ncache:\n checkpoints:\n enabled: false\n")
|
||||
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Cache.Checkpoints.Enabled {
|
||||
t.Fatalf("explicit false checkpoint config was not applied: %#v", cfg.Cache.Checkpoints)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigRejectsTrimmedKeyCollisions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "pipeline ids",
|
||||
yaml: "version: 4\npipelines:\n main: {}\n ' main ': {}\n",
|
||||
want: "pipeline id \"main\" is duplicated after trimming",
|
||||
},
|
||||
{
|
||||
name: "lane ids",
|
||||
yaml: "version: 4\npipelines:\n main:\n artifacts:\n spells: {}\n ' spells ': {}\n",
|
||||
want: "artifact lane id \"spells\" is duplicated after trimming",
|
||||
},
|
||||
{
|
||||
name: "reference slots",
|
||||
yaml: "version: 4\npipelines:\n main:\n references:\n slot: ./one.txt\n ' slot ': ./two.txt\n",
|
||||
want: "reference slot \"slot\" is duplicated after trimming",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
file := parseFileConfig(t, tt.yaml)
|
||||
cfg := Default()
|
||||
err := cfg.ApplyFileConfig(file)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("ApplyFileConfig() error = %v, want context %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigParsesOrderedStepsAndReferenceSources(t *testing.T) {
|
||||
file := parseFileConfig(t, `version: 4
|
||||
pipelines:
|
||||
session:
|
||||
input: seriatim
|
||||
steps:
|
||||
- id: identify-npcs
|
||||
artifacts:
|
||||
npcs:
|
||||
extract: dnd/npcs
|
||||
- id: grounded-events
|
||||
references:
|
||||
npcs:
|
||||
artifact:
|
||||
step: identify-npcs
|
||||
lane: npcs
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`)
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfig(file); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := cfg.Pipelines["session"]
|
||||
if len(profile.Steps) != 2 || profile.Steps[0].ID != "identify-npcs" || profile.Steps[1].ID != "grounded-events" {
|
||||
t.Fatalf("steps = %#v", profile.Steps)
|
||||
}
|
||||
source := profile.Steps[1].References["npcs"]
|
||||
if source.Artifact == nil || source.Artifact.Step != "identify-npcs" || source.Artifact.Lane != "npcs" {
|
||||
t.Fatalf("generated source = %#v", source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigRejectsAmbiguousReferenceSourceForms(t *testing.T) {
|
||||
for _, source := range []string{
|
||||
"artifact: {step: a, lane: b, extra: c}",
|
||||
"artifact: {step: 1, lane: b}",
|
||||
"1",
|
||||
} {
|
||||
_, err := ParseFileConfigYAML([]byte("version: 4\npipelines:\n p:\n input: text\n references:\n slot: " + source + "\n"))
|
||||
if err == nil {
|
||||
t.Fatalf("ParseFileConfigYAML(%q) error = nil", source)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileConfigRejectsEmptyAndAmbiguousPipelineShapes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty steps",
|
||||
yaml: "version: 4\npipelines:\n p:\n input: text\n steps: []\n",
|
||||
want: "at least one ordered step",
|
||||
},
|
||||
{
|
||||
name: "both forms",
|
||||
yaml: "version: 4\npipelines:\n p:\n input: text\n artifacts: {}\n steps: []\n",
|
||||
want: "both artifacts and steps",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
file := parseFileConfig(t, tt.yaml)
|
||||
cfg := Default()
|
||||
err := cfg.ApplyFileConfig(file)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("ApplyFileConfig() error = %v, want context %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileConfigReportsPathAndOperationContext(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
missing := filepath.Join(dir, "missing.yml")
|
||||
_, err := LoadFileConfig(missing)
|
||||
if err == nil || !strings.Contains(err.Error(), "read config file") || !strings.Contains(err.Error(), missing) {
|
||||
t.Fatalf("missing-file error = %v", err)
|
||||
}
|
||||
|
||||
malformed := filepath.Join(dir, "malformed.yml")
|
||||
if err := os.WriteFile(malformed, []byte("version: [\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = LoadFileConfig(malformed)
|
||||
if err == nil || !strings.Contains(err.Error(), "parse config file") || !strings.Contains(err.Error(), malformed) {
|
||||
t.Fatalf("malformed-file error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func parseFileConfig(t *testing.T, source string) FileConfig {
|
||||
t.Helper()
|
||||
file, err := ParseFileConfigYAML([]byte(source))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML() error = %v", err)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
func applyFileConfig(t *testing.T, source string) Config {
|
||||
t.Helper()
|
||||
cfg := Default()
|
||||
if err := cfg.ApplyFileConfig(parseFileConfig(t, source)); err != nil {
|
||||
t.Fatalf("ApplyFileConfig() error = %v", err)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
@@ -1,723 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
)
|
||||
|
||||
func TestParseMinimalValidConfig(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
t.Fatalf("unexpected version: %d", fileCfg.Version)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileConfig(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(path, []byte("version: 2\n"), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
fileCfg, err := LoadFileConfig(path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFileConfig: %v", err)
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
t.Fatalf("unexpected version: %d", fileCfg.Version)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigRejectsUnknownYAMLFields(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
unexpected: true
|
||||
`))
|
||||
if err == nil || !strings.Contains(err.Error(), "field unexpected not found") {
|
||||
t.Fatalf("expected unknown field error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigRejectsUnknownModuleBindingFields(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input:
|
||||
module: fake/input
|
||||
unexpected: true
|
||||
artifacts:
|
||||
events:
|
||||
extract: fake/extract
|
||||
`))
|
||||
if err == nil || !strings.Contains(err.Error(), "field unexpected not found") {
|
||||
t.Fatalf("expected unknown binding field error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigRejectsMissingAndUnsupportedVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
want string
|
||||
}{
|
||||
{name: "missing", data: `scriptorium: {}`, want: "version is required"},
|
||||
{name: "unsupported", data: `version: 1`, want: "unsupported config version"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(tc.data))
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigRejectsStaleLLMProfiles(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
llm_profiles:
|
||||
default: {}
|
||||
`))
|
||||
if err == nil || !strings.Contains(err.Error(), "llm_profiles") {
|
||||
t.Fatalf("expected stale llm_profiles error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigScriptoriumProfileSources(t *testing.T) {
|
||||
t.Run("profile dir", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
scriptorium:
|
||||
profile_dir: ./profiles
|
||||
`)
|
||||
if cfg.Scriptorium.ProfileDir != "./profiles" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("Scriptorium = %+v, want profile_dir", cfg.Scriptorium)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("profile file", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
scriptorium:
|
||||
profile_file: ./profiles.yml
|
||||
`)
|
||||
if cfg.Scriptorium.ProfileFile != "./profiles.yml" || cfg.Scriptorium.ProfileDir != "" {
|
||||
t.Fatalf("Scriptorium = %+v, want profile_file", cfg.Scriptorium)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseFileConfigModuleBindingForms(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
chunk:
|
||||
module: generic
|
||||
retries: 2
|
||||
options:
|
||||
size: 10
|
||||
flags:
|
||||
- alpha
|
||||
nested:
|
||||
enabled: true
|
||||
artifacts:
|
||||
events:
|
||||
extract:
|
||||
module: fake/extract
|
||||
llm_profile: fast
|
||||
retries: 3
|
||||
options:
|
||||
temperature: 0
|
||||
merge:
|
||||
module: appendorder
|
||||
retries: 1
|
||||
normalize:
|
||||
module: noop
|
||||
output: json
|
||||
`)
|
||||
|
||||
profile := cfg.Pipelines["example"]
|
||||
if profile.Input.Module != "fake/input" {
|
||||
t.Fatalf("unexpected input binding: %+v", profile.Input)
|
||||
}
|
||||
if profile.Chunk.Module != "generic" {
|
||||
t.Fatalf("unexpected chunk binding: %+v", profile.Chunk)
|
||||
}
|
||||
if profile.Chunk.Retries != 2 {
|
||||
t.Fatalf("chunk retries = %d, want 2", profile.Chunk.Retries)
|
||||
}
|
||||
if profile.Chunk.Options["size"] != 10 {
|
||||
t.Fatalf("expected chunk options to preserve scalar, got %#v", profile.Chunk.Options)
|
||||
}
|
||||
if !reflect.DeepEqual(profile.Chunk.Options["flags"], []any{"alpha"}) {
|
||||
t.Fatalf("expected list option, got %#v", profile.Chunk.Options["flags"])
|
||||
}
|
||||
nested, ok := profile.Chunk.Options["nested"].(map[string]any)
|
||||
if !ok || nested["enabled"] != true {
|
||||
t.Fatalf("expected nested map option, got %#v", profile.Chunk.Options["nested"])
|
||||
}
|
||||
|
||||
lane := profile.Artifacts["events"]
|
||||
if lane.Extract.Module != "fake/extract" || lane.Extract.LLMProfile != "fast" {
|
||||
t.Fatalf("unexpected extract binding: %+v", lane.Extract)
|
||||
}
|
||||
if lane.Extract.Retries != 3 || lane.Merge.Retries != 1 {
|
||||
t.Fatalf("unexpected retries: extract=%d merge=%d", lane.Extract.Retries, lane.Merge.Retries)
|
||||
}
|
||||
if lane.Extract.Options["temperature"] != 0 {
|
||||
t.Fatalf("expected object options, got %#v", lane.Extract.Options)
|
||||
}
|
||||
if lane.Merge.Module != "appendorder" || lane.Normalize.Module != "noop" {
|
||||
t.Fatalf("unexpected lane defaults: %+v", lane)
|
||||
}
|
||||
if profile.Output.Module != "json" {
|
||||
t.Fatalf("unexpected output binding: %+v", profile.Output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigReferenceMaps(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
references:
|
||||
" roster ": " ./shared-roster.yml "
|
||||
artifacts:
|
||||
events:
|
||||
extract: fake/extract
|
||||
references:
|
||||
" lore ": " ./lore.md "
|
||||
`)
|
||||
|
||||
profile := cfg.Pipelines["example"]
|
||||
if !reflect.DeepEqual(profile.References, map[string]string{"roster": "./shared-roster.yml"}) {
|
||||
t.Fatalf("pipeline references = %#v, want trimmed map", profile.References)
|
||||
}
|
||||
gotLaneRefs := profile.Artifacts["events"].References
|
||||
if !reflect.DeepEqual(gotLaneRefs, map[string]string{"lore": "./lore.md"}) {
|
||||
t.Fatalf("lane references = %#v, want trimmed map", gotLaneRefs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigStageLocalReferenceMaps(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
chunk:
|
||||
module: generic
|
||||
references:
|
||||
" scene_guide ": " ./scenes.md "
|
||||
artifacts:
|
||||
events:
|
||||
extract:
|
||||
module: fake/extract
|
||||
references:
|
||||
" glossary ": " ./glossary.md "
|
||||
" roster ": " ./extract-roster.yml "
|
||||
references:
|
||||
roster: ./legacy-roster.yml
|
||||
lore: ./lore.md
|
||||
merge:
|
||||
module: appendorder
|
||||
references:
|
||||
" merge_notes ": " ./merge.md "
|
||||
normalize:
|
||||
module: noop
|
||||
references:
|
||||
" normalization_notes ": " ./normalization.md "
|
||||
`)
|
||||
|
||||
profile := cfg.Pipelines["example"]
|
||||
if !reflect.DeepEqual(profile.Chunk.References, map[string]string{"scene_guide": "./scenes.md"}) {
|
||||
t.Fatalf("chunk references = %#v, want trimmed map", profile.Chunk.References)
|
||||
}
|
||||
lane := profile.Artifacts["events"]
|
||||
if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) {
|
||||
t.Fatalf("lane references = %#v, want trimmed map", lane.References)
|
||||
}
|
||||
wantExtract := map[string]string{
|
||||
"glossary": "./glossary.md",
|
||||
"lore": "./lore.md",
|
||||
"roster": "./extract-roster.yml",
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Extract.References, wantExtract) {
|
||||
t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract)
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Merge.References, map[string]string{"merge_notes": "./merge.md"}) {
|
||||
t.Fatalf("merge references = %#v, want trimmed map", lane.Merge.References)
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) {
|
||||
t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
artifacts:
|
||||
events:
|
||||
extract: fake/extract
|
||||
validators:
|
||||
- fake/validator
|
||||
- module: fake/llm-validator
|
||||
llm_profile: careful
|
||||
options:
|
||||
threshold: 0.7
|
||||
`)
|
||||
|
||||
validators := cfg.Pipelines["example"].Artifacts["events"].Validators
|
||||
if len(validators) != 2 {
|
||||
t.Fatalf("expected two validators, got %d", len(validators))
|
||||
}
|
||||
if validators[0].Module != "fake/validator" {
|
||||
t.Fatalf("unexpected shorthand validator: %+v", validators[0])
|
||||
}
|
||||
if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" {
|
||||
t.Fatalf("unexpected object validator: %+v", validators[1])
|
||||
}
|
||||
if validators[1].Options["threshold"] != 0.7 {
|
||||
t.Fatalf("unexpected validator options: %#v", validators[1].Options)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigStageLocalValidatorOverrides(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
chunk:
|
||||
module: generic
|
||||
validators: []
|
||||
artifacts:
|
||||
events:
|
||||
extract:
|
||||
module: fake/extract
|
||||
validators:
|
||||
- fake/validator
|
||||
- module: fake/llm-validator
|
||||
llm_profile: careful
|
||||
options:
|
||||
threshold: 0.7
|
||||
merge:
|
||||
module: appendorder
|
||||
validators: []
|
||||
normalize:
|
||||
module: noop
|
||||
`)
|
||||
|
||||
profile := cfg.Pipelines["example"]
|
||||
if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 {
|
||||
t.Fatalf("chunk validator override = %#v, want explicit empty", profile.Chunk.Validators)
|
||||
}
|
||||
lane := profile.Artifacts["events"]
|
||||
if !lane.Extract.Validators.Set {
|
||||
t.Fatalf("extract validator override Set = false, want true")
|
||||
}
|
||||
validators := lane.Extract.Validators.Validators
|
||||
if len(validators) != 2 {
|
||||
t.Fatalf("extract validators = %#v, want two validators", validators)
|
||||
}
|
||||
if validators[0].Module != "fake/validator" {
|
||||
t.Fatalf("first validator = %#v, want fake/validator", validators[0])
|
||||
}
|
||||
if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" {
|
||||
t.Fatalf("second validator = %#v, want LLM validator with profile", validators[1])
|
||||
}
|
||||
if validators[1].Options["threshold"] != 0.7 {
|
||||
t.Fatalf("second validator options = %#v, want threshold", validators[1].Options)
|
||||
}
|
||||
if !lane.Merge.Validators.Set || len(lane.Merge.Validators.Validators) != 0 {
|
||||
t.Fatalf("merge validator override = %#v, want explicit empty", lane.Merge.Validators)
|
||||
}
|
||||
if lane.Normalize.Validators.Set {
|
||||
t.Fatalf("normalize validator override Set = true, want omitted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
" example ":
|
||||
input: fake/other-input
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
|
||||
if err == nil || !strings.Contains(err.Error(), "pipeline id") || !strings.Contains(err.Error(), "duplicated") {
|
||||
t.Fatalf("expected duplicate pipeline ID error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsDuplicateTrimmedArtifactLaneIDs(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
artifacts:
|
||||
events:
|
||||
extract: fake/extract
|
||||
" events ":
|
||||
extract: fake/other-extract
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
|
||||
if err == nil || !strings.Contains(err.Error(), `pipeline "example" artifact lane id`) || !strings.Contains(err.Error(), "duplicated") {
|
||||
t.Fatalf("expected duplicate artifact lane ID error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsDuplicateTrimmedReferenceSlots(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "pipeline",
|
||||
raw: `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
references:
|
||||
roster: ./first.yml
|
||||
" roster ": ./second.yml
|
||||
`,
|
||||
want: `pipeline "example" reference slot`,
|
||||
},
|
||||
{
|
||||
name: "lane",
|
||||
raw: `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
artifacts:
|
||||
events:
|
||||
extract: fake/extract
|
||||
references:
|
||||
roster: ./first.yml
|
||||
" roster ": ./second.yml
|
||||
`,
|
||||
want: `pipeline "example" lane "events" reference slot`,
|
||||
},
|
||||
{
|
||||
name: "chunk",
|
||||
raw: `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
chunk:
|
||||
module: generic
|
||||
references:
|
||||
roster: ./first.yml
|
||||
" roster ": ./second.yml
|
||||
`,
|
||||
want: `pipeline "example" chunk reference slot`,
|
||||
},
|
||||
{
|
||||
name: "extract",
|
||||
raw: `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
artifacts:
|
||||
events:
|
||||
extract:
|
||||
module: fake/extract
|
||||
references:
|
||||
roster: ./first.yml
|
||||
" roster ": ./second.yml
|
||||
`,
|
||||
want: `pipeline "example" lane "events" extract reference slot`,
|
||||
},
|
||||
{
|
||||
name: "normalize",
|
||||
raw: `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
artifacts:
|
||||
events:
|
||||
extract: fake/extract
|
||||
normalize:
|
||||
module: noop
|
||||
references:
|
||||
roster: ./first.yml
|
||||
" roster ": ./second.yml
|
||||
`,
|
||||
want: `pipeline "example" lane "events" normalize reference slot`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(tc.raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "duplicated") {
|
||||
t.Fatalf("expected duplicate reference slot error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsInvalidScriptoriumSources(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
}{
|
||||
{name: "empty profile dir", raw: "profile_dir: ' '", want: "profile_dir"},
|
||||
{name: "empty profile file", raw: "profile_file: ' '", want: "profile_file"},
|
||||
{name: "both sources", raw: "profile_dir: ./profiles\n profile_file: ./profiles.yml", want: "mutually exclusive"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
scriptorium:
|
||||
` + tc.raw + `
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
|
||||
if err == nil {
|
||||
err = cfg.Validate()
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigOperationalSections(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
concurrency:
|
||||
total_llm: 4
|
||||
diagnostics:
|
||||
work_dir: /tmp/notarius-test
|
||||
retention: always
|
||||
`)
|
||||
|
||||
if cfg.Concurrency.TotalLLM != 4 {
|
||||
t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM)
|
||||
}
|
||||
if got := cfg.Concurrency.StageWorkers["extract"]; got != 4 {
|
||||
t.Fatalf("default extract workers = %d, want total concurrency", got)
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/tmp/notarius-test" {
|
||||
t.Fatalf("unexpected work dir: %q", cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
|
||||
t.Fatalf("unexpected retention: %q", cfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigStageWorkers(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
concurrency:
|
||||
total_llm: 4
|
||||
stage_workers:
|
||||
extract: 3
|
||||
`)
|
||||
|
||||
if got := cfg.Concurrency.StageWorkers["extract"]; got != 3 {
|
||||
t.Fatalf("extract workers = %d, want 3", got)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigEmptyStageWorkersDefaultsExtractToTotal(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
concurrency:
|
||||
total_llm: 4
|
||||
stage_workers: {}
|
||||
`)
|
||||
if got := cfg.Concurrency.StageWorkers["extract"]; got != 4 {
|
||||
t.Fatalf("extract workers = %d, want total concurrency 4", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsUnsupportedStageWorkerKeys(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
key string
|
||||
want string
|
||||
}{
|
||||
{name: "empty", key: "' '", want: "must not be empty"},
|
||||
{name: "unknown", key: "merge", want: "not supported"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte("version: 2\nconcurrency:\n stage_workers:\n " + test.key + ": 1\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML() error = %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("ApplyFileConfig() error = %v, want %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigWorkspaceSection(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
enabled: false
|
||||
retention: never
|
||||
resume:
|
||||
enabled: true
|
||||
debug:
|
||||
enabled: true
|
||||
diagnostics:
|
||||
work_dir: /tmp/legacy
|
||||
retention: always
|
||||
`)
|
||||
|
||||
if cfg.Workspace.Directory != "/var/lib/notarius" {
|
||||
t.Fatalf("workspace directory = %q, want /var/lib/notarius", cfg.Workspace.Directory)
|
||||
}
|
||||
if cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected diagnostics disabled")
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/var/lib/notarius/diagnostics" {
|
||||
t.Fatalf("effective diagnostics work dir = %q, want workspace diagnostics root", cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
|
||||
t.Fatalf("effective diagnostics retention = %q, want workspace override", cfg.Diagnostics.Retention)
|
||||
}
|
||||
if !cfg.Workspace.Resume.Enabled {
|
||||
t.Fatalf("expected resume enabled")
|
||||
}
|
||||
if !cfg.Workspace.Debug.Enabled {
|
||||
t.Fatalf("expected debug enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigLegacyDiagnosticsRemainCompatible(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
diagnostics:
|
||||
work_dir: /tmp/legacy
|
||||
retention: always
|
||||
`)
|
||||
|
||||
if cfg.Workspace.Directory != "" {
|
||||
t.Fatalf("workspace directory = %q, want unset", cfg.Workspace.Directory)
|
||||
}
|
||||
if !cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected diagnostics enabled")
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/tmp/legacy" {
|
||||
t.Fatalf("effective diagnostics work dir = %q, want legacy", cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
|
||||
t.Fatalf("effective diagnostics retention = %q, want legacy", cfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigWorkspaceRetentionOverridesLegacyRetentionOnlyWhenSet(t *testing.T) {
|
||||
t.Run("legacy retained", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
retention: never
|
||||
`)
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
|
||||
t.Fatalf("effective diagnostics retention = %q, want legacy", cfg.Diagnostics.Retention)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("workspace overrides", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
retention: always
|
||||
diagnostics:
|
||||
retention: never
|
||||
`)
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
|
||||
t.Fatalf("effective diagnostics retention = %q, want workspace", cfg.Diagnostics.Retention)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func parseAndApplyConfig(t *testing.T, raw string) Config {
|
||||
t.Helper()
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
|
||||
t.Fatalf("ApplyFileConfig: %v", err)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func emptyLookup(string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
func mapLookup(values map[string]string) func(string) (string, bool) {
|
||||
return func(key string) (string, bool) {
|
||||
value, ok := values[key]
|
||||
return value, ok
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user