129 Commits

Author SHA1 Message Date
ef6926322d Remove obsolete sequential extraction path 2026-07-17 14:05:21 +00:00
2df7084d5d Enforce module family import boundaries 2026-07-17 13:58:50 +00:00
3d3cc0c08e Make typed module spec lookup artifact-aware 2026-07-17 13:53:55 +00:00
fbc3d9add6 Add retry-scoped merge and normalize debugging 2026-07-17 13:43:25 +00:00
35e45f0914 Include validator policy in pipeline identity 2026-07-17 13:33:16 +00:00
3e4fa923eb Update implementation plan to address remaining clean-up items 2026-07-17 08:28:18 -05:00
3013ee044d Finalize bounded typed pipeline implementation 2026-07-17 08:49:08 +00:00
adfd3bd052 Add bounded concurrent pipeline execution 2026-07-17 08:37:52 +00:00
4023c66508 Add extract worker configuration controls 2026-07-17 08:21:24 +00:00
adfe3825ee Remove legacy raw pipeline contracts 2026-07-17 08:13:08 +00:00
814fcdc6ba Serialize typed artifacts at durable boundaries 2026-07-17 07:33:49 +00:00
66de1a5520 Run D&D spell lanes through typed artifacts 2026-07-17 07:19:58 +00:00
52e6b31408 Add typed spell validation strategies 2026-07-17 07:02:30 +00:00
142ba36695 Introduce typed D&D spell artifacts 2026-07-17 06:50:08 +00:00
b949e9bbc0 Construct universal modules with decoded options 2026-07-17 06:30:56 +00:00
ce3a07512f Prepare pipelines before source execution 2026-07-17 06:18:46 +00:00
1c84d19e5f Add type-safe artifact lane resolution 2026-07-17 05:57:31 +00:00
fc1b57bde2 Add typed artifact codec foundation 2026-07-17 05:41:20 +00:00
075888c97f Move chunks into the canonical source model 2026-07-17 05:32:11 +00:00
40709e4ad8 Add canonical source unit provenance 2026-07-17 05:19:57 +00:00
15c369c509 Organize D&D extensions by domain 2026-07-17 05:05:48 +00:00
a81b9f1e1f Organize generic and Seriatim modules by domain 2026-07-17 04:57:47 +00:00
0327659355 Compose production modules through family registrars 2026-07-17 04:53:09 +00:00
c99bad19ae Add domain pipeline compatibility baselines 2026-07-17 04:45:01 +00:00
35f9446ed8 Add documentation and roadmap for a significant refactor around domain-focused module packages 2026-07-16 23:36:46 -05:00
21888d625f Minor documentation cleanup 2026-07-16 22:31:14 -05:00
feb03c3f8e Remove completed roadmap documentation 2026-07-16 22:25:39 -05:00
3b07b64a0f Consolidate documentation ownership 2026-07-17 03:12:35 +00:00
6e6375521d Refocus developer and internal documentation 2026-07-17 03:06:23 +00:00
b1fe9dc5a7 Reorganize user and operator documentation 2026-07-17 02:56:31 +00:00
6db2dc8d2a Refactor and clean up documentation 2026-07-16 21:42:24 -05:00
98b03a4629 Refactor the D&D spells module to apply deterministic fields where appropriate 2026-07-08 10:43:39 -05:00
610bdb4fea Enhance debug output to include response content files and update related metadata handling 2026-07-08 08:52:19 -05:00
68ec69f2e4 Rewrite the debug path to provide raw LLM prompt and response artifacts 2026-07-08 08:37:26 -05:00
451f6c0bb9 Stop base64 encoding LLM responses in debug output 2026-07-07 23:14:17 -05:00
3011dd91ca Update the debug workflow to provide raw LLM output 2026-07-07 23:08:14 -05:00
ae65b95374 Document completed workspace behavior 2026-07-08 03:18:50 +00:00
a5bbfea9b9 Write workspace debug artifacts during runs 2026-07-08 03:14:59 +00:00
ae9c2e1d5e Reuse valid workspace checkpoints on request 2026-07-08 03:02:50 +00:00
1d3a444df8 Write workspace checkpoints during runs 2026-07-08 02:46:38 +00:00
f044c00a7c Add checkpoint identity and manifest types 2026-07-08 02:31:07 +00:00
7d89c2702b Route diagnostics through workspace settings 2026-07-08 02:25:58 +00:00
93653cccb8 Add workspace filesystem helpers 2026-07-08 02:23:20 +00:00
a024492dbf Add workspace configuration support 2026-07-08 02:18:59 +00:00
304c68f9fc Add a planning roadmap and a staged implementation plan for the workspace configuration 2026-07-07 21:08:52 -05:00
c5f2b14ff4 Cleanup and complete the validator refactor 2026-07-07 19:36:48 -05:00
fc8e03f98c Document validation defaults and cleanup roadmap 2026-07-07 21:48:19 +00:00
16de4b6437 Register production validators and defaults 2026-07-07 21:43:32 +00:00
0f30888b00 Add D&D spell raw output validators 2026-07-07 21:37:58 +00:00
3e67be6ac3 Add generic raw output validators 2026-07-07 21:32:56 +00:00
5ef027b6f0 Wire resolved validator chains into runner 2026-07-07 21:27:28 +00:00
666b4bf801 Add validator chain config overrides 2026-07-07 21:21:12 +00:00
d593bfee0a Add validator chain provenance 2026-07-07 21:10:29 +00:00
b7ad66f0e0 Replace validator contracts with raw output validation 2026-07-07 20:55:49 +00:00
249e49c928 Update planning roadmap and add a staged imnplementation plan for the validator registry 2026-07-07 15:50:00 -05:00
e54e74ed88 Remove completed pipeline refactoring roadmap docs 2026-07-07 15:32:56 -05:00
582c5dceed Cleanup and complete the pipeline refactor 2026-07-07 15:32:35 -05:00
a9d8505cdb Document raw pipeline completion 2026-07-07 19:32:15 +00:00
7c95791e94 Update raw output files and manifests 2026-07-07 19:27:28 +00:00
aa14faa3cb Migrate production modules to raw outputs 2026-07-07 19:23:35 +00:00
cc6b050367 Implement runner retries and raw validation 2026-07-07 19:19:04 +00:00
bcedf19a08 Add merge references and retry config 2026-07-07 19:10:55 +00:00
c05ecb58d8 Implement raw module output contracts 2026-07-07 18:58:23 +00:00
9e3f8809b3 Implement integer source units and chunk payloads 2026-07-07 18:34:23 +00:00
4f057b99ac Add planning roadmaps for large-scale refactors of the pipleine and the validator registry, and add a staged imnplementation plan for the pipeline refactor 2026-07-07 13:20:10 -05:00
aec807fcb0 Update D&D schemas to require integer unit_id values 2026-07-06 14:41:24 -05:00
79a585d17e Remove a duplicated helper function 2026-07-06 14:26:40 -05:00
671ff6d132 Document completed DnD shared helper refactor 2026-07-06 17:26:38 +00:00
9b2d0297b7 Clean up DnD shared prompt asset tests 2026-07-06 17:24:15 +00:00
f91e643932 Use shared DnD prompt helpers in modules 2026-07-06 17:22:21 +00:00
35fe405448 Add DnD reference prompt helpers 2026-07-06 17:18:38 +00:00
524f2ffb8e Move DnD shared prompt assets into helper package 2026-07-06 17:15:27 +00:00
68b426cdb0 Make shared prompt filesystem composition generic 2026-07-06 17:10:59 +00:00
223f3751e8 Add reference slot clone helper 2026-07-06 17:06:24 +00:00
7861d040df Add a feature roadmap and staged implementation plan to refactor shared D&D module assets 2026-07-06 12:03:49 -05:00
47cf7e76ec Adjust the prompt FS layout and test strategy 2026-07-06 11:26:21 -05:00
8cafa64174 Added shared helper for module prompt FS 2026-07-06 07:29:41 -05:00
ecba0ad725 Add production prompt registry coverage 2026-07-06 01:36:55 +00:00
b3757dcf7b Remove legacy shared prompt package and update docs 2026-07-06 01:34:04 +00:00
3e456ec4d4 Update D&D prompt definitions to shallow asset paths 2026-07-06 01:32:27 +00:00
5b1efc89f6 Flatten D&D module prompt asset trees 2026-07-06 01:27:02 +00:00
aee48d011e Add shared D&D prompt assets package 2026-07-06 01:17:13 +00:00
3217bb3e12 Add a roadmap to refactor prompt assets 2026-07-05 20:13:26 -05:00
3df686f474 Bugfix in the built-in prompt references definition 2026-07-05 18:48:45 -05:00
7d4c027d09 Finish implementation of the scriptorium migration 2026-07-05 18:26:31 -05:00
31d70a2dd7 Finish Scriptorium runtime documentation 2026-07-05 18:34:21 +00:00
c9fbb331e2 Cut modules over to Scriptorium prompts 2026-07-05 18:27:07 +00:00
f6224dcbee Add Scriptorium-backed LLM runtime 2026-07-05 18:21:37 +00:00
de6689bc1d Add Scriptorium prompt assets 2026-07-05 18:13:10 +00:00
0fc740470f Switch config to Scriptorium profiles 2026-07-05 18:03:23 +00:00
49d94cc2e9 Add prompt input materials and session IDs 2026-07-05 17:51:36 +00:00
291298cf7b Add Scriptorium API grounding 2026-07-05 17:46:05 +00:00
9532ae8121 Add a feature roadmap and implementation plan to import the scriptorium LLM library 2026-07-05 12:42:15 -05:00
7601731a2c Clean up completed reference roadmap docs 2026-07-05 11:51:36 -05:00
3aa88ab9d3 Document completed LLM and reference expansion 2026-07-05 16:42:15 +00:00
c1ba94192d Cover reference prompts for chunk and normalize usage 2026-07-05 16:39:41 +00:00
22032dfd6d Deliver references to eligible runtime targets 2026-07-05 16:38:17 +00:00
4cafde2502 Materialize references for all eligible targets 2026-07-05 16:35:24 +00:00
8c623b7ad8 Support target-aware reference selectors 2026-07-05 16:31:23 +00:00
43dc954440 Resolve references across eligible pipeline targets 2026-07-05 16:24:49 +00:00
51053d390d Add stage-local reference config bindings 2026-07-05 16:18:00 +00:00
9278797aa9 Introduce target-aware resolved reference storage 2026-07-05 16:12:38 +00:00
39e49d7f77 Expand module contracts for references and normalizer LLM access 2026-07-05 16:06:47 +00:00
84c4c06712 Add a staged implementation plan to extend first-class LLM access and reference support across all chunk, extract, and normalize stages 2026-07-05 11:01:32 -05:00
eab640aa21 Update .gitignore to ignore build and testing output artifacts 2026-07-05 10:54:16 -05:00
a516944086 Finish the references implementation for the extraction module and update roadmap documentation 2026-07-05 10:53:16 -05:00
be6803ffa1 Document extraction reference support 2026-07-05 14:52:52 +00:00
ef4bdd4f9f Use references in D&D spell extraction 2026-07-05 14:49:17 +00:00
2f97895732 Record reference provenance in manifests 2026-07-05 14:44:44 +00:00
9e89b88efc Add prompt reference template functions 2026-07-05 14:37:39 +00:00
a57c6397e3 Materialize extraction reference files 2026-07-05 14:32:35 +00:00
39e071f5ca Add CLI reference binding flags 2026-07-05 14:27:23 +00:00
70d733edaf Resolve extraction reference bindings from config 2026-07-05 14:21:36 +00:00
1c31f56af1 Add reference contracts to extractor metadata 2026-07-05 14:13:48 +00:00
f9999a73df Add a staged implementation plan for background context references 2026-07-05 09:08:31 -05:00
11d8187052 Update documentation to reflect the implemented chunking module 2026-07-05 08:40:24 -05:00
86bff552c1 Update .gitgnore to ignore build artifacts 2026-07-05 08:39:34 -05:00
d3f790095e Trim and validate scene caveats 2026-07-05 13:30:32 +00:00
95218218e2 Canonicalize chunk units before extraction 2026-07-05 13:28:09 +00:00
e700df82d8 Record top-level module metadata in run manifests 2026-07-05 13:23:56 +00:00
e19cc02c4d Add a staged implementation plan to address gaps from the initial implementation of the scene chunking module 2026-07-05 08:15:46 -05:00
8a5419448f Update feature roadmaps to reflect future work 2026-07-05 08:11:54 -05:00
c8217549a8 Verify D&D scene chunker run output 2026-07-04 13:08:45 +00:00
2130414899 Register D&D scene chunker 2026-07-04 13:05:39 +00:00
7f83a20fa6 Implement D&D scene chunker 2026-07-04 13:02:02 +00:00
317ab0472d Add D&D scene chunking assets 2026-07-04 12:57:04 +00:00
e5eb0ba5c8 Make chunk validation a framework contract 2026-07-04 12:52:16 +00:00
b95af4f87d Add a roadmap to implement a D&D-specific chunk module 2026-07-04 07:48:36 -05:00
11073b613c Update AGENTS.md 2026-07-04 07:05:06 -05:00
258 changed files with 32126 additions and 11662 deletions

5
.gitignore vendored
View File

@@ -1,3 +1,7 @@
# build and testing artifacts
notarius
notarius-output
# ---> Go
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
@@ -49,6 +53,7 @@ go.work.sum
# Icon must end with two \r
Icon
# Thumbnails
._*

View File

@@ -1,3 +1,2 @@
Please carefully review the documents in `docs/policy` before making any changes to this repository.
- `architecture.md` provides the canonical high-level architecture policy for this repository.
- `documentation.md` provides the canonical documentation policy for this repository.
Please review `docs/development.md` for initial orientation in this repository
and follow its task-specific reading guide.

View File

@@ -4,33 +4,29 @@ Notarius is a Go CLI for extracting structured artifacts from source material
with explicit, configurable pipeline modules.
The current implementation reads Seriatim transcript JSON, chunks the source
units, extracts D&D spell-cast artifacts with an OpenAI-compatible LLM, and
writes JSON output plus diagnostics for each run.
units, extracts D&D spell-cast artifacts with a Scriptorium-backed LLM runtime,
and writes JSON output plus diagnostics when enabled.
```sh
NOTARIUS_LLM_DEFAULT_BASE_URL=http://127.0.0.1:8080/v1 \
NOTARIUS_LLM_DEFAULT_MODEL=your-model \
OPENROUTER_API_KEY=... \
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json
```
If the provider requires authentication, set
`NOTARIUS_LLM_DEFAULT_API_KEY` in the environment before running the command.
Outputs are written under `./notarius-output/<run-id>/` unless `--output-dir`
is provided.
This invocation uses the maintained example configuration and input. See the
configuration and operations references for profile selection, credentials, and
run artifacts.
Useful references:
- [CLI reference](docs/cli.md)
- [Configuration reference](docs/config.md)
- [Operations](docs/operations.md)
- [Troubleshooting](docs/troubleshooting.md)
- [Seriatim input contract](docs/integrations/seriatim.md)
- [OpenAI-compatible provider contract](docs/integrations/openai-compatible.md)
- [JSON output contract](docs/integrations/json-output.md)
- [D&D spell artifact contract](docs/integrations/dnd-spell-artifacts.md)
- [Developer workflow](docs/policy/development.md)
- [Internal architecture docs](docs/internal/overview.md)
- [Developer guide](docs/development.md)
- [Internal implementation docs](docs/internal/overview.md)
- [Maintained example config](examples/dnd-spells.config.yml)
- [Maintained example input](examples/seriatim-minimal-transcript.json)

View File

@@ -0,0 +1,23 @@
# ADR-0001: Record architecture decisions as ADRs
**Status:** Accepted
**Date:** 2026-07-13
## Context
Architectural reasoning made during design (pattern choices, rejected
alternatives, trigger conditions for revisiting) is lost if only the final
state is documented.
## Decision
We keep a living overview in docs/policy/architecture.md describing current
intended state, and immutable, numbered ADRs (Nygard format) in docs/adr/
recording each significant decision, its alternatives, and its consequences.
Changed decisions get a new ADR that marks the old one Superseded.
## Alternatives considered
- Overview doc only: loses the "why" and the rejected options.
- arc42 / RFC-style design docs: heavier than warranted for a solo repo.
## Consequences
Small ongoing writing cost; durable reasoning trail; cheap onboarding for
future contributors (including future-us).

View File

@@ -0,0 +1,51 @@
# ADR-0002: Linear pipes-and-filters pipeline, not a general DAG
**Status:** Accepted
**Date:** 2026-07-13
## Context
Notarius processes source material through one known workflow:
```text
input -> chunk -> extract -> merge -> normalize -> output
```
Input and chunking apply to the source as a whole. Each selected artifact lane
then performs extract, merge, and normalize, after which output aggregates the
lane outcomes. Chunk extraction has a natural scatter-gather shape, but no
current use case requires arbitrary branches, joins, or user-defined stage
topology.
## Decision
Notarius implements a fixed six-stage pipes-and-filters pipeline. Configuration
selects implementations for these stages but cannot add stages, reorder them,
or define an arbitrary graph.
The framework owns stage sequencing and the scatter-gather boundary between
chunk, extract, and merge. Extract results are handed to merge in deterministic
source-chunk order regardless of execution strategy. Each artifact lane remains
logically linear. Output runs after every selected lane has either produced an
accepted normalized artifact or reached a recorded rejection. A framework
execution failure aborts the pipeline.
The runner's concrete internal representation and stage-specific scheduling
policies are implementation details. Concurrency must preserve the pipeline's
deterministic handoffs, validation behavior, and provenance, and all execution
strategies must continue to honor context cancellation.
## Alternatives considered
- Build a general DAG engine now. This would support hypothetical branching
topologies, but would add scheduling, topology validation, configuration, and
state-management complexity without a current consumer. Revisit this choice
only when a concrete workflow requires a topology the fixed pipeline cannot
express.
## Consequences
The runner, configuration model, and operator mental model remain small. Stage
ownership stays visible, and general chunking, merging, or normalization cannot
be hidden inside extractors. A future DAG requirement will require an explicit
architectural change rather than incremental exceptions to the fixed pipeline.

View File

@@ -0,0 +1,119 @@
# ADR-0003: Strongly typed stage interfaces with a two-zone data model
**Status:** Accepted
**Date:** 2026-07-13
## Context
Pipeline stages must exchange source data and extracted artifacts. Universal
source data has one engine-wide meaning, while extracted artifacts have
domain-specific shapes. Passing opaque bytes or `any` between all stages would
make invalid wiring and merge behavior runtime concerns. Requiring JSON at
every handoff would preserve interoperability but discard useful Go type safety
while all modules are in-process.
The framework must also support multiple configured artifact domains, durable
checkpoints, diagnostics, and output encoders without making those consumers
depend on every domain's Go types.
## Decision
Notarius uses two typed data zones followed by one serialized boundary.
### Source zone
Input and chunk stages use conservative, engine-owned document, segment, chunk,
and source-reference types. Their exact Go names are implementation details.
Every segment carries engine-owned source provenance identifying the source
location from which it was produced. Chunks preserve the ordered provenance of
their segments.
Source-format-specific fields remain in input modules or explicitly namespaced
metadata; they do not become framework contracts.
### Domain artifact zone
Each artifact lane has one domain-owned Go artifact type `T`. Its extract,
merge, normalize, and domain-aware validation implementations use generic,
strongly typed contracts over the same `T`. Raw JSON, opaque bytes, and `any`
are not stage-handoff contracts within a lane.
Each registered domain artifact type supplies a codec for `T`. The codec owns:
- stable schema identity and an explicit schema version;
- JSON serialization and deserialization;
- the media type and schema metadata required at serialized boundaries; and
- rejection of data that cannot be represented by the declared artifact
schema.
An artifact type's JSON representation is a maintained domain contract.
Changing it incompatibly requires a new schema version.
Extract, merge, and normalize may change the contents of `T`, but they do not
change the lane's canonical Go artifact type or artifact schema identity. An
extractor maps any provider- or prompt-specific response type into `T` before
returning. A future lane that requires different artifact types at different
stages requires a new architectural decision.
### Serialized boundary
After normalization, each typed artifact is converted into an engine-owned
serialized artifact containing bytes, media type, and schema metadata. Output
aggregation and output encoders consume this type-erased form. Intermediate
checkpoint and debug encodings do not become stage-handoff contracts.
LLM transport, checkpoints, and opt-in debug recording are also explicit
serialization boundaries. They may encode or decode a typed artifact through
its domain codec, but they do not change the in-memory type used between
extract, merge, normalize, and typed validators. Checkpoint reuse requires a
compatible schema identity and version.
An LLM structured-response schema is a module transport contract and may differ
from the domain artifact schema. The calling module owns the response type and
maps it into the canonical `T`; the artifact codec remains authoritative for
artifact checkpoints and output serialization.
The framework may use private type-erased adapters to store heterogeneous lane
registrations and execute configured domains. Such an adapter must assemble a
type-consistent lane before execution and must not expose `any` or raw payloads
as module-facing handoffs inside the domain artifact zone.
### Construction and dependencies
Every module operation accepts `context.Context`. Modules receive stable runtime
collaborators through an injected dependency set at construction time. In
particular, LLM-using modules receive the application-provided structured LLM
client and do not construct provider clients or bypass shared scheduling.
The application boundary enforces one configurable global upper bound on
in-flight LLM calls across all stages, lanes, retries, and validators.
Configuration options are parsed and validated while a module is constructed,
before that module executes. Per-run data such as source material, references,
session identity, and lane identity remains operation input rather than a
construction dependency.
## Alternatives considered
- Pass raw bytes between stages. This maximizes decoupling but moves wiring,
parsing, and merge errors to runtime and prevents domain types from being the
canonical in-process contract.
- Require JSON plus schemas at every stage boundary. This is appropriate for an
out-of-process boundary, but adds serialization and parsing inside the current
in-process pipeline. The stable codec contract preserves this upgrade path if
remote plugins are introduced.
- Use a uniform `Process(any) (any, error)` contract. This simplifies a fully
dynamic engine but turns incompatible module composition into type assertions
and runtime failures. The fixed topology does not require that tradeoff.
## Consequences
Domain pipelines gain compile-time handoff safety and explicit merge semantics.
Serialization, schema compatibility, checkpoint decoding, and output erasure
have named owners. Dynamic registration requires a small erased adapter around
each typed lane, and generic stage implementations must be instantiated for a
specific artifact type or behavior rather than manipulating arbitrary JSON.
The engine-owned source model becomes a long-lived contract and must evolve
conservatively. Domain authors must maintain a codec and versioned schema in
addition to their Go artifact type.

View File

@@ -0,0 +1,81 @@
# ADR-0004: Package modules by domain, not by stage
**Status:** Accepted
**Date:** 2026-07-13
## Context
Module packages can be grouped first by pipeline stage, such as
`modules/chunk/dnd/scenes`, or first by domain, such as
`modules/dnd/chunk/scenes`. A domain's extract, merge, normalize, validation,
schema, prompt, and artifact-codec implementations collaborate around the same
artifact types and are likely to evolve together.
Go package dependencies also constrain registration. If shared types live in a
domain root package, that package cannot import child implementation packages
to register them because the children already import the root types.
## Decision
Production extensions are grouped by domain under:
```text
internal/modules/<domain>/<stage>/<name>
```
Shared artifact types live at the domain root, for example
`internal/modules/dnd/types.go`. Domain-specific validators, prompt fragments,
schemas, reference helpers, and codecs also live within that domain tree.
Each domain exposes one production registration entry point from a sibling
registrar package, for example `internal/modules/dnd/register`. The registrar
may import the domain root and its child implementations; the domain root does
not import its registrar or child packages. This keeps shared types available
as `dnd.SpellList` without creating a Go import cycle.
The `generic` tree is a peer extension family for reusable implementations that
contain no concrete source-format or artifact-domain knowledge. Source-format
and output-format families, such as Seriatim and JSON output, follow the same
domain-first organization even when they do not define a type in the
[domain artifact zone](0003-typed-interfaces-with-two-zone-data-model.md#domain-artifact-zone).
Concrete domain implementation packages do not import another concrete domain.
Generic extension packages never import concrete domains. A domain registrar
may import domain-neutral generic extension packages to instantiate a reusable
strategy for that domain's artifact type; the generic implementation remains
unaware of the concrete type's domain semantics. Reuse needed directly by a
domain implementation lives in a domain-neutral framework or helper package,
not in a peer extension package.
The application composition root may import multiple registrar packages, and
black-box integration tests may compose multiple domains. Other cross-domain
reuse occurs through engine contracts and composition-time registration rather
than concrete peer-domain imports.
A domain registrar owns registration of that domain's modules, validators,
default validator chains, artifact codecs, schemas, and prompt assets. It does
not take ownership of application execution or process behavior.
## Alternatives considered
- Group modules by stage. This keeps interchangeable strategies side by side,
but scatters a domain's shared artifact model and collaborating extensions
across the repository. It is preferable when generic strategy libraries
dominate or when the project is primarily a stage-extension framework rather
than an application composed from domain suites.
- Put both shared types and `Register` in the domain root. This gives the
shortest import path but creates an import cycle once child implementations
import the root artifact types.
## Consequences
The repository layout makes supported domains immediately visible, and adding
or extracting a domain affects one cohesive subtree. The CLI composition root
depends on a small set of domain registrars instead of every leaf package.
Package moves must preserve user-visible module and validator keys unless a
separate compatibility decision changes them. Shared behavior that cannot be
expressed through framework contracts may need to move into a domain-neutral
framework package rather than creating a concrete peer-domain import. Registrar
packages become explicit composition points for instantiating generic typed
strategies, in addition to registering domain-owned implementations.

View File

@@ -3,26 +3,15 @@
This is the canonical reference for the implemented Notarius command-line
interface.
## Quick Run
```sh
NOTARIUS_LLM_DEFAULT_BASE_URL=http://127.0.0.1:8080/v1 \
NOTARIUS_LLM_DEFAULT_MODEL=your-model \
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json
```
Set `NOTARIUS_LLM_DEFAULT_API_KEY` if the OpenAI-compatible provider requires
a bearer token.
For the minimal end-to-end invocation, see the [README](../README.md).
## Commands
```text
notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json]
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--output-dir path] [--diagnostics-dir path] [--llm-profile id] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list [--config path/to/config.yml] [--json]
```
Running `notarius` with no arguments, `notarius help`, `notarius --help`, or
@@ -36,26 +25,108 @@ file.
Flags:
- `--input path`: required source input file.
- `--config path`: config file path. If omitted, Notarius checks
`NOTARIUS_CONFIG`, then `/usr/local/etc/notarius/config.yml`.
- `--config path`: config file path. If omitted, Notarius uses the discovery
rules in [Configuration](config.md#discovery).
- `--only lane-a,lane-b`: run only the named artifact lanes. Values are
comma-separated and must be non-empty.
- `--output-dir path`: output root. The run writes to `<path>/<run-id>/`.
Defaults to `./notarius-output`.
- `--resume`: request checkpoint reuse for this invocation. See
[Operations](operations.md#checkpoints) for prerequisites and reuse behavior.
- `--output-dir path`: output root. Defaults to `./notarius-output`.
- `--diagnostics-dir path`: diagnostics work directory override for this
invocation.
- `--llm-profile id`: override every effective module binding to use one LLM
profile.
invocation. It does not change the workspace directory.
- `--llm-profile id`: override every effective LLM-capable pipeline module
binding with one Scriptorium profile ID. Validator-specific profiles are not
overridden.
- `--session-id id`: pass a stable prompt session identifier through LLM-backed
module calls.
- `--reference selector=path`: bind a reference path to a chunk, extractor,
merger, or normalizer reference slot. Repeatable.
- `--without-reference selector`: remove a configured optional reference binding.
Repeatable. It accepts the same selector forms as `--reference`, without
`=path`.
On success, the command prints the completed pipeline ID, approved and rejected
artifact counts, and the output directory. If the run completes with warnings,
On success, the command prints the completed pipeline ID, normalized output and
rejected output counts, and the output directory. If the run completes with warnings,
the warning count is printed to stderr.
For durable output, diagnostics, retention, and failure inspection, see
[Operations](operations.md).
Reference flags are resolved against selected chunk, extractor, merger, and
normalizer targets before the run starts. Flat slot names are accepted only
when exactly one selected target declares that slot. For configured reference
bindings, precedence, path resolution, and validation, see
[Configuration](config.md#pipelines).
The current `run` command requires the resolved pipeline to use exactly one
distinct LLM profile after defaults and overrides are applied.
`--reference` binds or replaces one slot for one selected target. Selectors are:
- `slot=path`: valid when exactly one selected target declares `slot`;
- `chunk.slot=path`: target the chunker;
- `merge.slot=path`: valid when exactly one selected merger declares `slot`;
- `lane.slot=path`: valid when exactly one selected extractor, merger, or
normalizer in that lane declares `slot`;
- `lane.extract.slot=path`: target a lane extractor;
- `lane.merge.slot=path`: target a lane merger;
- `lane.normalize.slot=path`: target a lane normalizer.
Use `slot=path` when the selected targets declare the slot unambiguously:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference roster=./campaign-roster.txt
```
Use an explicit selector when multiple selected targets declare the same slot or
when you want to target a specific target:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference spells.extract.glossary=./campaign-glossary.txt
```
The same grammar can target chunk, merge, and normalize slots when the configured
modules declare them:
```sh
go run ./cmd/notarius run dnd-session \
--config path/to/config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference chunk.scene_guide=./campaign-scenes.txt \
--reference spells.merge.merge_notes=./merge-notes.txt \
--reference spells.normalize.normalization_notes=./normalization-notes.txt
```
Use `--without-reference` to remove a configured optional binding for a run:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--without-reference glossary
```
Use `--session-id` when an external orchestrator needs all prompt calls from one
run to share an identifier:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--session-id campaign-17-session-04
```
The resume flag can be added to an otherwise identical run invocation:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--resume
```
For checkpoint behavior, durable output, diagnostics, retention, and failure
inspection, see [Operations](operations.md).
## `config validate`
@@ -63,8 +134,8 @@ distinct LLM profile after defaults and overrides are applied.
Flags:
- `--config path`: config file path. If omitted, discovery uses
`NOTARIUS_CONFIG`, then `/usr/local/etc/notarius/config.yml`.
- `--config path`: config file path. If omitted, Notarius uses the discovery
rules in [Configuration](config.md#discovery).
- `--pipeline pipeline-id`: additionally resolve one configured pipeline against
the production module catalog.
- `--only lane-a,lane-b`: validate resolution for selected artifact lanes. This
@@ -88,8 +159,8 @@ go run ./cmd/notarius config validate \
Flags:
- `--config path`: config file path. If omitted, discovery uses
`NOTARIUS_CONFIG`, then `/usr/local/etc/notarius/config.yml`.
- `--config path`: config file path. If omitted, Notarius uses the discovery
rules in [Configuration](config.md#discovery).
- `--json`: print `{"pipelines":[...]}` instead of one ID per line.
Examples:
@@ -112,18 +183,6 @@ go run ./cmd/notarius pipelines list \
- `2`: command syntax was invalid, a command was unknown, a required argument
was missing, or a flag value was malformed.
## Implemented Production Pipeline Modules
The production CLI currently registers these module keys:
- input: `seriatim`
- chunk: `generic`
- extract: `dnd/spells`
- merge: `appendorder`
- normalize: `noop`
- output: `json`
The production CLI does not currently register validator modules.
For YAML structure, defaults, environment overrides, and module binding syntax,
see [Configuration](config.md).
For YAML structure, defaults, Scriptorium profile sources, environment
overrides, and selectable module and validator keys, see
[Configuration](config.md).

View File

@@ -2,117 +2,136 @@
This is the canonical reference for implemented Notarius configuration.
Notarius reads YAML config files with `version: 1`. File config is applied over
Notarius reads YAML config files with `version: 2`. File config is applied over
built-in defaults, then environment overrides are applied.
## Discovery
Commands that accept `--config` load configuration in this order:
Commands that load configuration use this order:
1. the `--config` path, when provided;
1. an explicit path supplied through the CLI, when provided;
2. `NOTARIUS_CONFIG`, when set to a non-empty path;
3. `/usr/local/etc/notarius/config.yml`.
If none is available, the command fails with a config file not found error.
The explicit-path option is defined in the [CLI reference](cli.md).
## Minimal Example
## Maintained Examples
```yaml
version: 1
llm_profiles:
default:
provider: openai-compatible
base_url: http://127.0.0.1:8080/v1
model: your-model
pipelines:
dnd-session:
input: seriatim
chunk:
module: generic
options:
max_units: 50
artifacts:
spells:
extract: dnd/spells
```
- [Minimal D&D spell configuration](../examples/dnd-spells.config.yml)
- [Production-oriented D&D spell configuration](../examples/dnd-spells-production.config.yml)
The maintained fixture is [examples/dnd-spells.config.yml](../examples/dnd-spells.config.yml).
Both complete files are validated by the CLI test suite. The fragments below
illustrate individual fields and are not alternate complete configurations.
## Top-Level Fields
- `version`: required. The only supported value is `1`.
- `llm_profiles`: optional map of LLM profile IDs to profile settings.
- `version`: required. The only supported value is `2`.
- `scriptorium`: optional Scriptorium profile source settings.
- `pipelines`: optional map of pipeline IDs to pipeline definitions.
- `concurrency`: optional global concurrency settings.
- `workspace`: optional workspace settings for Notarius-owned local state.
- `diagnostics`: optional diagnostics settings.
Unknown YAML fields are rejected.
Unknown YAML fields are rejected. The removed top-level `llm_profiles` field is
rejected; execution profiles now come from Scriptorium.
## Defaults
Built-in defaults:
```yaml
llm_profiles:
default:
provider: openai-compatible
timeout: 600
max_retries: 3
max_concurrency: 1
concurrency:
total_llm: 1
diagnostics:
work_dir: /tmp/notarius
retention: auto
```
- `concurrency.total_llm`: `1`
- `concurrency.stage_workers.extract`: effective `concurrency.total_llm`
- `diagnostics.work_dir`: `/tmp/notarius`
- `diagnostics.retention`: `auto`
- `workspace.directory`: unset
- `workspace.diagnostics.enabled`: `true`
- `workspace.resume.enabled`: `false`
- `workspace.debug.enabled`: `false`
No pipelines are built in. A run requires a configured pipeline.
## LLM Profiles
If `scriptorium` is omitted, Notarius uses Scriptorium's built-in profile
catalog. Prompt definitions may also name default profile IDs. The current D&D
scene and spell prompts default to the built-in `mistral-small-3` profile when a
module binding does not set `llm_profile`. That built-in profile reads its
credential from `OPENROUTER_API_KEY`.
Each `llm_profiles` entry may contain:
## Scriptorium Profiles
- `provider`: optional provider key. Empty means `openai-compatible`; any other
non-empty value must be `openai-compatible`.
- `base_url`: provider base URL. Required for actual LLM calls.
- `model`: provider model name. Required for actual LLM calls.
- `api_key_env`: environment variable name to read for the API key.
- `timeout`: request timeout as whole seconds or a Go-style duration string such
as `10m`.
- `max_retries`: retry count for provider calls. Must be zero or greater.
- `max_concurrency`: per-profile LLM concurrency. Must be zero or greater; when
zero, Notarius uses `concurrency.total_llm`.
`scriptorium` fields:
Raw API keys are not accepted as file config fields. Use `api_key_env` or an
environment override.
- `profile_dir`: optional directory containing Scriptorium profile YAML files.
- `profile_file`: optional Scriptorium profile YAML file.
`profile_dir` and `profile_file` are mutually exclusive. Custom profiles
overlay Scriptorium built-in profiles by profile ID.
Scriptorium profile files use Scriptorium's profile schema. A minimal profile
looks like:
```yaml
id: local-fast
endpoint: http://127.0.0.1:8080/v1
model: your-model
api_key_env: SCRIPTORIUM_API_KEY
timeout_seconds: 180
```
Notarius does not accept raw API keys in Notarius config. For file-backed
Scriptorium profiles, store the environment variable name in `api_key_env` and
set that variable in the run environment. Scriptorium rejects raw `api_key`
fields in profile YAML.
## Environment Overrides
These environment variables are applied after the config file:
- `NOTARIUS_CONFIG`: config discovery path.
- `NOTARIUS_LLM_DEFAULT_API_KEY`: API key for the `default` LLM profile.
- `NOTARIUS_LLM_DEFAULT_BASE_URL`: base URL for the `default` LLM profile.
- `NOTARIUS_LLM_DEFAULT_MODEL`: model for the `default` LLM profile.
- `NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS`: integer timeout seconds for the
`default` LLM profile.
- `NOTARIUS_LLM_DEFAULT_MAX_RETRIES`: integer retry count for the `default` LLM
profile.
- `NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY`: integer max concurrency for the
`default` LLM profile.
- `NOTARIUS_TOTAL_LLM_CONCURRENCY`: integer global LLM concurrency.
- `NOTARIUS_WORK_DIR`: diagnostics work directory.
- `NOTARIUS_DIAGNOSTICS_RETENTION`: diagnostics retention mode.
- `NOTARIUS_STAGE_WORKERS_EXTRACT`: integer extract worker limit.
- `NOTARIUS_WORKSPACE_DIR`: workspace directory.
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED`: boolean diagnostics enablement.
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION`: workspace diagnostics retention
mode.
- `NOTARIUS_WORKSPACE_RESUME_ENABLED`: boolean resume checkpointing
enablement.
- `NOTARIUS_WORKSPACE_DEBUG_ENABLED`: boolean debug artifact enablement.
- `NOTARIUS_WORK_DIR`: deprecated diagnostics work directory compatibility
override.
- `NOTARIUS_DIAGNOSTICS_RETENTION`: deprecated diagnostics retention
compatibility override.
Integer environment values must parse as base-10 integers.
Integer environment values must parse as base-10 integers. Boolean environment
values must parse as Go booleans such as `true`, `false`, `1`, or `0`.
The removed `NOTARIUS_LLM_DEFAULT_*` variables are not read. Configure provider
endpoint, model, and credential environment variable names through Scriptorium
profiles.
## Concurrency
`concurrency` fields:
- `total_llm`: positive integer ceiling on concurrent provider calls.
- `stage_workers`: optional map of framework worker limits. The only supported
key is `extract`.
`stage_workers.extract` defaults to the effective `total_llm` value after file
and environment precedence. It must be between `1` and `total_llm`, inclusive.
Unknown or empty stage-worker keys are rejected. The environment override
`NOTARIUS_STAGE_WORKERS_EXTRACT` takes precedence over the file value, as does
`NOTARIUS_TOTAL_LLM_CONCURRENCY` for the global ceiling.
The worker value is present in effective and redacted configuration. It bounds
the fixed run-wide extract pool and its bounded dispatch queue. Extract jobs are
submitted by source chunk and then resolved lane; `total_llm` independently
bounds actual provider calls made by extracts, retries, and validators.
## Pipelines
A pipeline defines the fixed Notarius workflow:
```text
input -> chunk -> extract -> merge -> normalize -> output
```
A pipeline selects implementations for the fixed workflow defined by
[Architecture](policy/architecture.md#system-shape).
Pipeline fields:
@@ -121,18 +140,88 @@ Pipeline fields:
- `artifacts`: required for pipeline resolution. It maps artifact lane IDs to
lane definitions.
- `output`: optional module binding. Default module is `json`.
- `references`: optional map of reference slot names to reference paths. These
bindings are defaults for eligible pipeline targets that declare the matching
slot.
Artifact lane fields:
- `extract`: required module binding.
- `merge`: optional module binding. Default module is `appendorder`.
- `normalize`: optional module binding. Default module is `noop`.
- `validators`: optional list of module bindings. The production CLI currently
does not register validator modules.
- `validators`: deprecated lane-level validator list. Non-empty lists are
rejected; use `extract.validators`, `merge.validators`, or
`normalize.validators`.
- `references`: optional compatibility alias for extractor reference bindings.
Lane bindings override pipeline-level bindings for the same slot.
`notarius run` and `notarius config validate --pipeline` resolve the pipeline
against the production module catalog and fail fast for unknown or incompatible
module keys.
Commands that resolve a pipeline fail for unknown or incompatible module keys.
See [CLI Reference](cli.md) for command syntax.
Reference bindings are validated against reference slots declared by eligible
chunk, extract, merge, and normalize targets during pipeline resolution. Required slots
must be bound after config defaults, target-local references, lane-level
compatibility bindings, and command-line reference overrides are applied.
Config-relative paths are resolved relative to the config file; command-line
reference paths are resolved relative to the current working directory. Bound
files must be UTF-8 text. Reference media types are inferred from file
extensions and checked when a module restricts accepted types; unknown
extensions use `application/octet-stream`. See [CLI Reference](cli.md#run) for
command-line selectors and [Operations](operations.md) for recorded provenance
and sensitive-data handling.
Pipeline-level `references` are defaults. They are valid when at least one
eligible target in the full configured pipeline declares the slot, including
chunk, extractor, merger, and normalizer targets. During a run, they apply only
to the selected targets that declare the slot:
```yaml
pipelines:
dnd-session:
input: seriatim
references:
players: ./campaign/players.txt
party: ./campaign/party-roster.txt
glossary: ./campaign/glossary.txt
artifacts:
spells:
extract: dnd/spells
```
Extractor binding `references` are the canonical lane-local location. The
legacy lane-level `references` field remains supported as an alias; when both
bind the same slot, `extract.references` wins:
```yaml
pipelines:
dnd-session:
input: seriatim
references:
glossary: ./campaign/glossary.txt
artifacts:
spells:
references:
roster: ./campaign/legacy-roster.txt
extract:
module: dnd/spells
references:
party: ./campaign/session-party.txt
```
`chunk.references`, `merge.references`, and `normalize.references` are accepted
in object-form bindings. They override pipeline-level defaults for slots
declared by that target module. Extractor-local references apply only to the
extractor, merger-local references apply only to the merger, and
normalizer-local references apply only to the normalizer.
Target-local reference fields use the same map shape at:
- `pipelines.<id>.chunk.references`
- `pipelines.<id>.artifacts.<lane>.extract.references`
- `pipelines.<id>.artifacts.<lane>.merge.references`
- `pipelines.<id>.artifacts.<lane>.normalize.references`
Each binding is valid only when that target module declares the slot.
## Module Bindings
@@ -146,20 +235,46 @@ or object form:
```yaml
chunk:
module: generic
llm_profile: default
options:
max_units: 50
module: dnd/scenes
llm_profile: local-fast
```
Binding fields:
- `module`: module key.
- `llm_profile`: optional LLM profile ID. Empty means `default`.
- `llm_profile`: optional Scriptorium profile ID. Empty or omitted lets the
Scriptorium prompt default select the profile.
- `retries`: non-negative retry count for extra runtime attempts after the
first attempt. Default: `0`. Supported on `chunk`, `extract`, `merge`, and
`normalize` bindings.
- `options`: optional module-specific settings.
- `references`: optional reference bindings. Supported only for `chunk`,
`extract`, `merge`, and `normalize` bindings. `input` and `output` bindings
reject this field during validation.
- `validators`: optional stage-local validator chain override. Supported only
for `chunk`, `extract`, `merge`, and `normalize` bindings. Omit the field to
use the production default chain; set `validators: []` to force an empty
chain; set a non-empty list to use exactly those validators in configured
order.
The `--llm-profile` run flag overrides every effective module binding to use
one configured profile.
During resolution, each selected module's registered option validator runs.
Production input, chunk, and output bindings reject unknown or invalid options
with the affected binding context.
Validator bindings use the same shorthand or object module-binding form, but
only these fields are supported:
- `module`: validator key.
- `llm_profile`: optional Scriptorium profile ID for LLM-backed validators.
- `options`: optional validator-specific settings.
Validator bindings reject `references`, `retries`, and nested `validators`.
During resolution, deterministic validators reject explicit `llm_profile`
values.
Configured LLM-backed validators with explicit `llm_profile` values are
validated against the configured Scriptorium profile source. Deterministic
production validators do not call the LLM and must not set `llm_profile`.
## Implemented Production Modules
@@ -167,10 +282,37 @@ one configured profile.
| --- | --- | --- |
| input | `seriatim` | Reads Seriatim transcript JSON. |
| chunk | `generic` | Splits source units into ordered chunks. |
| extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. |
| merge | `appendorder` | Keeps candidates in append order. |
| normalize | `noop` | Passes merged artifacts through unchanged. |
| output | `json` | Produces JSON output files. |
| chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. |
| extract | `dnd/spells` | Extracts typed D&D spell-list artifacts. |
| merge | `appendorder` | Combines typed artifacts in chunk order. |
| normalize | `noop` | Passes merged typed artifacts through unchanged. |
| output | `json` | Produces JSON output files for normalized `application/json` lanes. |
## Implemented Production Validators
| Key | Execution | Notes |
| --- | --- | --- |
| `generic/always_accept` | deterministic | Accepts returned module output. |
| `generic/always_reject` | deterministic | Rejects returned module output with reason `always_reject`. |
| `generic/valid_json` | deterministic | Rejects payloads that are not syntactically valid JSON. |
| `generic/valid_json_schema` | deterministic | Rejects invalid JSON or JSON that does not conform to the module response schema. |
| `extract/dnd/spells/shape` | deterministic | Rejects malformed D&D spell-list artifacts. |
| `extract/dnd/spells/source_refs` | deterministic | Rejects missing or invalid D&D spell source references. |
| `extract/dnd/spells/source_relatedness` | deterministic | Emits warnings when a spell name is not found near its cited source text. |
The production default chain for the `dnd/spells` extractor is:
```yaml
validators:
- generic/valid_json
- generic/valid_json_schema
- extract/dnd/spells/shape
- extract/dnd/spells/source_refs
- extract/dnd/spells/source_relatedness
```
No other production module currently has a default validator chain. Empty
chains approve output by default.
The `generic` chunker accepts:
@@ -178,36 +320,90 @@ The `generic` chunker accepts:
- `overlap_units`: non-negative integer, default `0`, and must be less than
`max_units`.
The `dnd/scenes` chunker requires transcript source capabilities, calls the
configured structured LLM runtime, and does not accept module options. It
declares optional `players`, `party`, and `glossary` references for scene
disambiguation, and accepts `roster` as a deprecated compatibility alias for
`party`.
The `dnd/spells` extractor declares optional reference slots:
- `players`
- `party`
- `glossary`
- `roster` as a deprecated compatibility alias for `party`
Both modules accept UTF-8 plain text, Markdown, YAML, or JSON reference files.
The extractor uses references only as supporting disambiguation material; spell
casts still must be present in the source transcript.
## Workspace
`workspace` fields:
- `directory`: optional workspace root for Notarius-owned local state.
- `resume.enabled`: boolean resume checkpointing setting.
- `debug.enabled`: boolean debug artifact setting.
- `diagnostics`: optional diagnostics settings defined below.
`workspace.resume.enabled` and `workspace.debug.enabled` are independent.
Enabling one does not enable the other. For directory layout, state lifecycle,
permissions, and sensitive content, see [Operations](operations.md).
## Diagnostics
Preferred workspace diagnostics fields:
- `workspace.diagnostics.enabled`: set to `false` to skip creating diagnostics
run directories and diagnostics artifacts.
- `workspace.diagnostics.retention`: `auto`, `always`, or `never`.
Defaults for workspace and diagnostics fields are listed in
[Defaults](#defaults).
`workspace.diagnostics.retention` overrides legacy diagnostics retention when
set.
`diagnostics` fields:
- `work_dir`: directory for per-run diagnostics. Default: `/tmp/notarius`.
- `retention`: `auto`, `always`, or `never`. Empty uses `auto`.
- `work_dir`: deprecated compatibility directory for per-run diagnostics.
- `retention`: deprecated compatibility retention mode. `auto`, `always`, or
`never`.
`auto` retains diagnostics for failed runs and successful runs with warnings.
`always` retains diagnostics for every run. `never` removes diagnostics for
successful runs without regard to warnings; failed runs are retained.
Existing `diagnostics.work_dir`, `diagnostics.retention`, `NOTARIUS_WORK_DIR`,
and `NOTARIUS_DIAGNOSTICS_RETENTION` inputs remain supported for compatibility.
New configuration should use `workspace.directory` and
`workspace.diagnostics.retention` instead.
The `--diagnostics-dir` run flag overrides `diagnostics.work_dir` for that
invocation.
For retention behavior and the physical diagnostics layout, see
[Operations](operations.md#retention). For the invocation-specific diagnostics
override, see [CLI Reference](cli.md#run).
## Validation
Configuration validation checks:
- supported config version and known YAML fields;
- mutually exclusive `scriptorium.profile_dir` and `scriptorium.profile_file`;
- non-empty, non-duplicated IDs after trimming;
- supported LLM provider and non-negative profile limits;
- positive global LLM concurrency;
- supported stage-worker keys and an effective extract worker count in the
inclusive range `1..concurrency.total_llm`;
- supported diagnostics retention and non-empty work directory;
- module binding LLM profiles refer to configured profiles.
- stale removed fields such as `llm_profiles`.
Pipeline resolution additionally checks:
- the pipeline ID exists;
- at least one artifact lane is declared and selected;
- selected lanes exist when `--only` is used;
- lanes selected through the CLI exist in the resolved pipeline;
- required module keys are present;
- module keys are registered for the expected slot;
- module capability requirements are satisfied.
- module capability requirements are satisfied;
- non-empty validator overrides reference registered validator keys;
- deterministic validators do not set `llm_profile`;
- LLM-backed validators with explicit `llm_profile` values reference configured
Scriptorium profile IDs;
- bound reference slots are declared by selected chunk, extractor, merger, or
normalizer targets;
- required reference slots are bound for selected targets.

40
docs/development.md Normal file
View File

@@ -0,0 +1,40 @@
# Development
This is the first-read landing page for people and LLM coding agents working on
Notarius. It provides a concise repository orientation and routes each kind of
change to its canonical documentation.
Notarius is a Go CLI for configured structured extraction workflows. Start with
the [README](../README.md) for product context, [Architecture](policy/architecture.md)
for system boundaries, and [Internal Overview](internal/overview.md) for the
implemented component map.
## What To Read
| When working on | Read | Why |
| --- | --- | --- |
| Finding the package or component that owns current behavior | [Internal Overview](internal/overview.md) | It is the implemented component inventory and routes to focused internals. |
| Application shape, package boundaries, contracts, dependency direction, runtime guarantees, or safety properties | [Architecture](policy/architecture.md) and relevant [ADRs](adr/) | Architecture defines the intended system and its invariants; ADRs preserve significant decision rationale. |
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
| Production modules or validators | [Module Internals](internal/modules.md) | It documents implemented module contracts, capabilities, assets, and registration. |
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and Scriptorium integration. |
| Diagnostics, workspace state, resume, or debug artifacts | [Diagnostics Internals](internal/diagnostics.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
| CLI or user-visible configuration behavior | [CLI Reference](cli.md) and [Configuration](config.md) | These are the canonical user and operator references. |
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
For an existing subsystem, also inspect its focused tests and the package-local
types and contracts before changing behavior.
## Validation
Use focused package tests while iterating. Run the repository-wide checks when
a change affects shared contracts, application behavior, or maintained
documentation examples:
```sh
go test ./...
go vet ./...
go build ./cmd/notarius
```

View File

@@ -1,126 +1,69 @@
# D&D Spell-Cast Artifacts
# D&D Spell Artifact
This document is the durable artifact contract for approved
`dnd.spell_cast` artifacts produced by the implemented `dnd/spells` extractor.
This document is the durable serialized artifact contract for the production
D&D spell extractor. Selectable extractor keys are cataloged in
[Configuration](../config.md#implemented-production-modules).
## Artifact Identity
## Identity
- Extractor key: `dnd/spells`
- Artifact type: `dnd.spell_cast`
- Schema version: `v1`
- Artifact kind: `dnd/spell-list`
- Prompt ID: `dnd.spells`
- Response schema key: `dnd_spells`
- Response schema ID: `notarius.dnd.spells`
- Response schema name: `notarius_dnd_spells_v1`
- Response schema version: `v1`
- Media type: `application/json`
The extractor requires source chunks and transcript source capability. It
returns generic artifact candidates that are serialized by the JSON output
module.
The durable JSON Schema is owned by the D&D spell artifact codec. The
extractor's private LLM response schema is a separate transport contract: its
source-reference objects omit `source_id`, which the extractor assigns while
mapping the response to the canonical artifact. The LLM DTO and transport
schema are not part of this durable contract.
## Artifact Envelope
The output contains canonical spell casts derived from transcript evidence.
Source IDs are assigned from the input identity; source-unit ranges identify
the evidence location.
Approved artifacts use the generic artifact envelope documented in
[JSON Output](json-output.md#artifact-files):
## Output Shape
```json
{
"extractor_key": "dnd/spells",
"artifact_type": "dnd.spell_cast",
"schema_version": "v1",
"payload": {
"caster": "Aria",
"spell": "Cure Wounds",
"effect": "heals an injured ally",
"narrative_description": "Aria raises her holy symbol and casts Cure Wounds."
},
"source_refs": [
{
"source_id": "session-alpha",
"start_unit_id": "seg-001",
"end_unit_id": "seg-001"
}
]
}
The extractor payload is a JSON object with one required top-level array. Its
structure is:
```text
{"spell_casts": [<spell-cast object>, ...]}
```
## Payload Fields
`spell_casts` must be present. It may be empty when no spell casts are found.
When multiple chunk results are combined, spell casts remain in chunk order.
When the payload is written as durable output, its logical path is derived from
the configured artifact lane ID as defined by the
[JSON output contract](json-output.md#output-payload-files).
The `payload` object contains:
## Spell-Cast Fields
Each spell cast contains exactly these required fields:
- `caster`: in-world character or creature casting the spell;
- `spell`: spell name;
- `effect`: concise spell effect in the scene;
- `narrative_description`: short description of the spell cast in context.
- `narrative_description`: short description of the spell cast in context;
- `source_refs`: transcript source references with extractor-assigned source
IDs and evidence unit ranges. It must contain at least one entry.
All payload fields are strings and must be non-empty after trimming.
`caster` is the in-world caster, not the transcript speaker.
All four string fields must be non-empty. `caster` is the in-world caster, not
the transcript speaker. Unknown fields are rejected.
## Source References
Source references live on the artifact envelope as `source_refs`; they are not
duplicated inside the `payload`.
Each source reference contains exactly three required fields: `source_id`,
`start_unit_id`, and `end_unit_id`. The source ID must match the input identity.
The unit IDs must be positive integers present in the input, and the start unit
must not appear after the end unit. Unknown fields are rejected.
Each source reference uses the generic source-reference shape:
- `source_id`
- `start_unit_id`
- `end_unit_id`
Validation requires:
- at least one source reference;
- non-empty source ID and unit IDs;
- source ID matching the source document ID;
- start and end unit IDs existing in the source document;
- start unit appearing before or at the same position as end unit.
## Structured LLM Response Shape
The extractor asks the LLM for this top-level response shape:
```json
{
"spell_casts": [
{
"caster": "Aria",
"spell": "Cure Wounds",
"effect": "heals an injured ally",
"narrative_description": "Aria raises her holy symbol and casts Cure Wounds.",
"source_refs": [
{
"source_id": "session-alpha",
"start_unit_id": "seg-001",
"end_unit_id": "seg-001"
}
]
}
]
}
```
`spell_casts` must be present. It may be empty when no spell casts are found.
The response schema asset is embedded at
`internal/modules/extract/dnd/spells/assets/schemas/dnd_spells.v1.json`.
## Validators
The extractor supplies two deterministic validators by default:
- `dnd/spells/shape`
- `dnd/spells/source_refs`
Rejection reason codes:
- `invalid_payload`: payload JSON cannot be decoded as a spell-cast payload.
- `missing_required_field`: `caster`, `spell`, `effect`, or
`narrative_description` is blank.
- `missing_source_ref`: candidate has no source references.
- `invalid_source_ref`: at least one source reference fails generic source
reference validation.
Rejected candidates are written to `rejected.json` by the JSON output module.
Reference slot keys and accepted file types are defined in
[Configuration](../config.md#implemented-production-modules). References are
supporting disambiguation material, not source evidence, and are not
addressable through `source_refs`.
## Manifest Metadata
@@ -145,8 +88,3 @@ manifest metadata:
```
Raw prompt and schema content are not included in manifest metadata.
## Compatibility Limit
This contract covers only `dnd.spell_cast` artifacts produced by the
implemented spell-cast extractor.

View File

@@ -1,30 +1,26 @@
# JSON Output
This document is the durable JSON output file-format contract produced by the
implemented `json` output module and written by the CLI.
production JSON encoder and written by the CLI. Selectable output-encoder keys
are cataloged in
[Configuration](../config.md#implemented-production-modules).
## Output Directory
The CLI writes logical output files under:
```text
<output-root>/<run-id>/
```
The default output root is `./notarius-output`. Operational behavior is covered
in [Operations](../operations.md).
The output module produces the logical bundle described here. The CLI's
physical placement and lifecycle for that bundle are defined in
[Operations](../operations.md#output-directory).
## Files
The `json` output module writes:
The encoder writes:
- `index.json`
- `manifest.json`
- `artifacts/<artifact-type>.json`, one file per approved artifact type
- `lanes/<lane-id>.json`, one file per normalized serialized artifact
- `rejected.json`
- `warnings.json`
Files are pretty-printed JSON with a trailing newline.
Files are pretty-printed JSON with a trailing newline when the payload is JSON.
Logical file paths are relative, slash-separated, and may not contain `..`.
## `index.json`
@@ -33,10 +29,15 @@ Shape:
```json
{
"manifest_file": "manifest.json",
"artifact_files": [
"output_files": [
{
"artifact_type": "dnd.spell_cast",
"file": "artifacts/dnd.spell_cast.json"
"lane_id": "spells",
"media_type": "application/json",
"file": "lanes/spells.json",
"module_key": "noop",
"schema_id": "notarius.dnd.spells",
"schema_name": "notarius_dnd_spells_v1",
"schema_version": "v1"
}
],
"rejected_file": "rejected.json",
@@ -44,25 +45,30 @@ Shape:
}
```
`artifact_files` is sorted by artifact type. It is empty when no artifacts are
approved.
`output_files` is sorted by lane ID. Output file names are produced by
sanitizing the lane ID:
- characters outside `A-Z`, `a-z`, `0-9`, `.`, `_`, and `-` become `_`;
- repeated `..` sequences are replaced;
- leading and trailing `.`, `_`, and `-` are trimmed;
- empty sanitized names are rejected;
- two lanes that sanitize to the same output file are rejected.
`manifest_file`, `rejected_file`, and `warnings_file` contain the fixed paths
shown above. Each `output_files` entry requires `lane_id` and `file`. It also
contains the normalized payload `media_type`, normalizer `module_key`, and
response `schema_id`, `schema_name`, and `schema_version` when those values are
available.
## `manifest.json`
`manifest.json` contains a run manifest:
`manifest.json` contains a run manifest. This abridged example shows its core
structure:
```json
{
"run_id": "run-123",
"pipeline_id": "dnd-session",
"pipeline_digest": "sha256:...",
"input_module": "seriatim",
"chunker": "generic",
"source_digests": ["sha256:..."],
"extractors": ["dnd/spells"],
"merger": "appendorder",
"normalizer": "noop",
"output_encoder": "json",
"artifact_lanes": [
{
"id": "spells",
@@ -71,13 +77,6 @@ approved.
"normalizer": "noop"
}
],
"llm_profiles": [
{
"id": "default",
"provider": "openai-compatible",
"model": "configured-model"
}
],
"validation_status": "approved",
"started_at": "2026-01-01T00:00:00Z",
"completed_at": "2026-01-01T00:00:01Z"
@@ -86,52 +85,59 @@ approved.
Fields with empty values may be omitted by JSON encoding.
`validation_status` is `approved` when no candidates were rejected and
`rejected` when one or more candidates were rejected.
The manifest fields are:
## Artifact Files
- `run_id`, `pipeline_id`, and `pipeline_digest`: run and resolved-pipeline
identity;
- `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, and
`output_encoder`: resolved module keys;
- `module_metadata` and `artifact_lanes`: module and per-lane provenance,
including prompt and response-schema provenance when provided;
- `validator_chains`: resolved validation points and validators;
- `source_digests` and `references`: source and reference provenance;
- `normalized_outputs` and `rejected_outputs`: payload-free result summaries;
- `llm_profiles`: selected profile IDs and provider or model names when
available;
- `metadata`: the effective prompt `session_id`;
- `validation_status`: `approved` or `rejected`;
- `started_at` and `completed_at`: UTC run timestamps.
Each artifact file has this shape:
`source_digests` contains source document digests only. Bound references are
recorded separately under `references`, which contains provenance only: target
stage, lane ID when present, slot name, origin type and URI, digest, media
type, byte size, and binding source. Reference content is not written to
durable output.
```json
{
"artifact_type": "dnd.spell_cast",
"artifacts": [
{
"extractor_key": "dnd/spells",
"artifact_type": "dnd.spell_cast",
"schema_version": "v1",
"payload": {},
"source_refs": [
{
"source_id": "session-alpha",
"start_unit_id": "seg-001",
"end_unit_id": "seg-001"
}
]
}
]
}
```
Reference `stage` is `chunk`, `extract`, `merge`, or `normalize`. `lane_id` is
omitted for chunk references and present for extract, merge, and normalize
references.
Artifact envelope fields:
`validation_status` is `approved` when no outputs were rejected and `rejected`
when one or more outputs were rejected.
- `extractor_key`: extractor module key.
- `artifact_type`: artifact type.
- `schema_version`: artifact schema version.
- `payload`: artifact-type-specific JSON payload.
- `source_refs`: optional generic source references.
- `metadata`: optional artifact metadata.
`validator_chains` records the resolved validator chain for each validation
point. Entries include stage, lane ID when applicable, module key, and validators
with key and execution class. Empty chains are recorded with an empty
`validators` array, including chains resolved from explicit empty config
overrides.
Artifact file names are produced by sanitizing the artifact type:
`normalized_outputs` summarizes each normalized lane output without embedding
payload bytes. Entries include lane ID, normalizer module key, source ID, media
type, and response schema provenance where available.
- characters outside `A-Z`, `a-z`, `0-9`, `.`, `_`, and `-` become `_`;
- repeated `..` sequences are replaced;
- leading and trailing `.`, `_`, and `-` are trimmed;
- empty sanitized names are rejected.
`rejected_outputs` summarizes rejected module outputs without embedding raw
payload bytes. Entries include stage, lane, module, chunk, validator or reason,
message, attempt count, and optional diagnostic artifact path.
For current D&D spell-cast artifacts, the file is
`artifacts/dnd.spell_cast.json`.
## Output Payload Files
Each normalized serialized artifact is written to
`lanes/<sanitized-lane-id>.json`. The JSON output encoder is domain-neutral and
accepts only artifacts whose codec media type is `application/json`. The file
contains the codec-owned JSON bytes pretty-printed.
The schema of each lane payload is owned by that artifact contract. For the
current D&D spell lane, see [D&D Spell Artifact](dnd-spell-artifacts.md).
## `rejected.json`
@@ -139,25 +145,14 @@ Shape:
```json
{
"rejected": [
{
"candidate": {
"index": 0,
"extractor_key": "dnd/spells",
"artifact_type": "dnd.spell_cast",
"schema_version": "v1",
"payload": {},
"source_refs": []
},
"validator_name": "dnd/spells/source_refs",
"reason_code": "missing_source_ref",
"message": "spell cast candidate must include at least one source ref"
}
]
"rejected": []
}
```
`rejected` is an empty array when no candidates are rejected.
When output validation rejects an output, each entry contains `stage` and
`message`. It includes `lane_id`, `module_key`, `chunk_id`, `chunk_index`,
`validator_name`, `reason_code`, `attempt_count`, and
`diagnostic_artifact_path` when applicable.
## `warnings.json`
@@ -167,26 +162,14 @@ Shape:
{
"warnings": [
{
"scope": "output",
"reason_code": "example_warning",
"message": "warning message"
"scope": "extract",
"reason_code": "example",
"message": "human-readable warning"
}
]
}
```
`warnings` is an empty array when no warnings are reported.
## Path Safety
The output module returns slash-separated logical paths. The CLI also validates
logical output names before writing:
- names must be non-empty;
- names must be relative;
- names must be clean;
- names must use `/`, not `\`;
- names must not contain `..`;
- resolved paths must stay under the run output directory.
Durable writes are atomic per file.
Each warning requires `reason_code` and `message`; `scope` is omitted when it is
empty.

View File

@@ -1,128 +0,0 @@
# OpenAI-Compatible Structured Output
This document describes the external LLM provider contract implemented by the
production Notarius LLM client.
## Provider
- Provider key: `openai-compatible`
- HTTP method: `POST`
- Endpoint: `<base_url>/chat/completions`
- Request body: JSON
- Response mode: chat completions with structured JSON schema output
`base_url` is trimmed of trailing slashes before `/chat/completions` is
appended. Configure provider settings in [Configuration](../config.md).
## Request
The client sends a JSON object with:
```json
{
"model": "configured-model",
"messages": [
{
"role": "system",
"content": "..."
},
{
"role": "user",
"content": "..."
}
],
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "schema_name",
"strict": true,
"schema": {}
}
}
}
```
Implemented request behavior:
- `model` comes from the structured completion request when set, otherwise from
the configured LLM profile.
- `messages` must be non-empty; each role and content must be non-empty after
trimming.
- `response_format.type` is always `json_schema`.
- `response_format.json_schema.strict` is always `true`.
- `response_format.json_schema.name` and `schema` come from the extractor or
validator making the call.
If an API key is configured, the client sends:
```text
Authorization: Bearer <api-key>
```
The client always sends `Content-Type: application/json`.
## Response
The client expects a JSON response with at least one choice:
```json
{
"model": "provider-model",
"choices": [
{
"message": {
"content": "{\"field\":\"value\"}"
}
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15
}
}
```
`choices[0].message.content` may be either:
- a JSON string whose contents are valid JSON; or
- raw JSON.
The decoded content is unmarshaled into the caller-provided structured output
target. If `usage` is present, prompt, completion, and total token counts are
copied into the completion response.
## Errors And Retries
The client validates base URL, model, response schema name, response schema
JSON, messages, and output target before or during the call.
Retryable failures:
- HTTP request failure;
- response body read failure;
- HTTP `429`;
- HTTP `5xx`;
- malformed provider response envelope;
- missing choices;
- missing, empty, or invalid assistant JSON content;
- structured-output decode failure.
Non-retryable provider status codes include non-`429` `4xx` responses.
Provider error bodies are parsed for `error.message` or `message` when present.
Configured API key values and bearer-token values are redacted from returned
provider errors.
## Timeouts And Concurrency
The configured profile timeout is applied per provider request when greater
than zero. Context cancellation is respected.
The production CLI wraps the provider client with the LLM scheduler. Effective
concurrency is described in [LLM runtime internals](../internal/llm.md).
## Limits
This contract documents only the fields the implemented client sends and reads.
Provider-specific extensions are ignored unless they affect those fields.

View File

@@ -1,116 +1,65 @@
# Seriatim Transcript JSON
This document is the external input contract for the implemented `seriatim`
input adapter.
This document is the external input contract consumed by the production
Seriatim input adapter. Selectable input-adapter keys are cataloged in
[Configuration](../config.md#implemented-production-modules).
## Adapter
- Module key: `seriatim`
- Document kind: `transcript`
- Unit kind: `transcript_segment`
- Source format: `application/vnd.seriatim+json`
The adapter parses raw Seriatim JSON into a generic source document. It owns
transcript-specific JSON parsing and metadata mapping; core source and pipeline
code stay source-format agnostic.
## Accepted Shape
The input must be one JSON object with top-level `metadata` and `segments`
fields. This covers the maintained minimal fixture and Seriatim intermediate
output that provides the same required segment fields.
```json
{
"metadata": {
"id": "session-alpha",
"title": "Synthetic D&D spell session"
},
"segments": [
{
"id": "seg-001",
"start": 0,
"end": 4,
"speaker": "Aria",
"text": "Aria raises her holy symbol and casts Cure Wounds."
}
]
}
```
The maintained example is
[examples/seriatim-minimal-transcript.json](../../examples/seriatim-minimal-transcript.json).
Top-level metadata entries are preserved. Other segment fields, such as
`categories`, are ignored.
Required top-level fields:
- `metadata`: an object. Its entries are accepted as source metadata.
- `segments`: a non-empty array of segment objects.
Required segment fields:
- `id`: a positive integer JSON number or canonical decimal string without
leading zeros or surrounding whitespace;
- `start`: a finite, non-negative JSON number or numeric string;
- `end`: a finite, non-negative JSON number or numeric string that is not less
than `start`;
- `speaker`: a non-empty string;
- `text`: a non-empty string.
Other top-level and segment fields, such as `categories`, are ignored.
Multiple top-level JSON values are rejected.
## Validation
The adapter rejects:
- empty raw input;
- malformed JSON;
- top-level JSON that is not an object;
- missing, null, or non-object `metadata`;
- missing, null, non-array, or empty `segments`;
- segment values that are not objects;
- segment `id` values that are neither strings nor numbers;
- non-string `speaker` or `text`;
- empty segment IDs;
- segment IDs with leading or trailing whitespace;
- duplicate segment IDs;
- missing or empty `speaker`;
- missing, empty, invalid, non-finite, or negative `start`;
- missing, empty, invalid, non-finite, or negative `end`;
- `end` values before `start`;
- missing or empty `text`.
The adapter rejects empty input, malformed JSON, multiple top-level JSON values,
non-object segment values, duplicate segment IDs, and any violation of the
shape or field constraints above.
Segment text is preserved as provided, but it must not be empty after trimming.
## Source Mapping
## Derived Identity
The adapter maps input to `SourceDocument`:
Notarius identifies the parsed source in this order:
- `metadata` becomes `SourceDocument.Metadata`;
- `SourceDocument.Kind` is `transcript`;
- `SourceDocument.Format` is `application/vnd.seriatim+json`;
- `SourceDocument.Digest` is `sha256:<hex>` of the exact raw input bytes.
1. `metadata.id`, when it is a non-empty string after trimming;
2. `metadata.source_id`, when it is a non-empty string after trimming;
3. `seriatim:<first-16-hex-chars-of-raw-sha256>`.
`SourceDocument.ID` is selected in this order:
1. the parse request source ID, after trimming;
2. `metadata.id`, when it is a non-empty string after trimming;
3. `metadata.source_id`, when it is a non-empty string after trimming;
4. `seriatim:<first-16-hex-chars-of-raw-sha256>`.
Each segment becomes one `SourceUnit`:
- `segment.id` becomes `SourceUnit.ID`; numeric IDs are converted to their JSON
number text, so `1` becomes `"1"`;
- `segment.text` becomes `SourceUnit.Text`;
- `SourceUnit.Kind` is `transcript_segment`;
- `speaker`, `start`, and `end` are stored in source-unit metadata.
## Metadata Keys
Seriatim unit metadata uses these keys:
- `speaker`: string speaker label;
- `start`: `json.Number` start value;
- `end`: `json.Number` end value.
The `internal/modules/input/seriatim` package exposes typed accessors for these
values.
## Capabilities
The module declares these provided capabilities:
- `source.transcript`
- `transcript.speaker`
- `transcript.timestamps`
The exact raw input SHA-256 remains the basis of the fallback source ID. The
source digest recorded in output provenance is instead the SHA-256 of the
canonical generic source document, excluding the digest field itself. It covers
the derived source identity, document kind and format, ordered units and their
self-references, and accepted metadata. Segment IDs become the unit IDs used by
artifact source references; each produced unit carries a self-reference whose
source ID is the derived document ID and whose start and end IDs both equal the
segment ID.
## Compatibility Limit

View File

@@ -1,88 +1,93 @@
# Diagnostics Internals
Diagnostics internals live in `internal/core/diagnostics`. Operator-facing run
behavior is documented in [Operations](../operations.md).
## Purpose
Diagnostics provide local inspection artifacts for a run without becoming the
durable output contract. Durable user output is produced by output modules and
written by the CLI.
Diagnostics must not expose secrets.
`internal/core/diagnostics` provides the scoped writer and retention decision
used by `internal/cli`. The physical layout, artifact inventory, retention
semantics, failure inspection, and cleanup procedures are canonical in
[Operations](../operations.md#diagnostics-directory). Configuration fields and
defaults are canonical in [Configuration](../config.md#diagnostics).
## Run Directory
`NewRunDirectory(workDir, retention)` creates:
`NewRunDirectory` normalizes empty constructor inputs, creates the effective
diagnostics root when needed, and allocates a unique timestamp-based child
directory. It retries a bounded number of collisions before failing. The
resulting `RunDirectory` retains its creation time and retention mode for later
metadata and cleanup decisions.
```text
<workDir>/run-<unix-nanoseconds>/
```
The package does not resolve workspace configuration. `internal/cli` derives
effective workspace settings first and passes the diagnostics root into the
constructor.
If `workDir` is empty, it defaults to `/tmp/notarius`. Empty retention defaults
to `auto`.
## Scoped Writers
The writer makes the work directory if needed, then attempts to create a unique
run directory. It retries run ID creation a bounded number of times if a
collision occurs.
Typed methods on `RunDirectory` write invocation metadata, redacted effective
configuration, resolved pipeline/reference data, checkpoint events, source data
when explicitly requested, manifests, reports, warnings, and error text. The
current filenames and their operator-facing contents are listed in
[Operations](../operations.md#diagnostics-directory).
## Artifact Writers
JSON methods indent their payload and append a newline. All artifact writes use
a temporary file in the target directory, apply the requested permissions, and
rename it into place. Artifact resolution accepts only a single relative base
name; absolute paths, separators, and paths escaping the run directory fail
before writing.
Implemented artifact names:
## Redacted Configuration
- `invocation.json`
- `effective-config.json`
- `resolved-pipeline.json`
- `source-document.json`
- `run-manifest.json`
- `run-report.json`
- `warnings.json`
- `error.log`
`WriteRedactedEffectiveConfig` accepts a `RedactedDiagnosticsPayload` provider
rather than a raw config value. `internal/core/config` implements that contract
by cloning effective config data and removing secret-shaped values before JSON
encoding. The diagnostics package therefore never needs configuration-specific
field knowledge.
JSON artifacts are encoded with indentation and a trailing newline. Writes are
atomic through a temporary file in the target directory followed by rename.
## Retention Coordination
Artifact names must be single relative file names. Absolute paths, path
separators, and names resolving outside the run directory are rejected.
`ShouldRetainRunDirectory` is a pure decision over the effective retention mode,
run success, and warning presence. `ApplyRetention` uses that result to remove
only its own run directory. Unsupported modes retain data as a fail-safe, though
normal CLI execution rejects them during config validation.
## Redacted Effective Config
The meaning of each supported mode belongs in
[Operations](../operations.md#retention); this package implements that contract
without loading config or inspecting run artifacts.
Diagnostics writers accept payloads that implement
`RedactedDiagnosticsPayload`. `internal/core/config` uses this to redact API
keys in effective config diagnostics while preserving resolved pipeline context.
## CLI State Flow
The redaction path clones config data before replacing secret values.
When diagnostics are enabled, `internal/cli` creates the run directory after
configuration loading and before pipeline resolution. It then writes artifacts
as state becomes available: invocation data, effective resolution data,
pipeline results, and the final report. This ordering permits later failures to
retain the context already established.
## Retention
Failures before construction have no `RunDirectory`. Later failures write an
error log, preserve any available partial manifest, and apply a failed-run
retention decision. A diagnostics write failure is itself a command failure so
the CLI does not report success after losing requested inspection data.
Retention is decided by `ShouldRetainRunDirectory`.
When diagnostics are disabled, the CLI carries a nil run directory and the
shared `writeDiagnostics` helper turns writes into no-ops. User-facing errors
still go to stderr; that invocation behavior is documented in
[Operations](../operations.md#failures).
- Failed runs are always retained.
- `always` retains successful runs.
- `never` removes successful runs.
- `auto` retains successful runs only when warnings exist.
- Unknown retention values are treated as retain by the retention decision, but
config validation rejects unsupported values before normal runs.
## Package Guarantees
`ApplyRetention` removes only the specific run directory.
- A `RunDirectory` writes and removes only within its allocated directory.
- JSON and error artifacts use atomic replacement.
- Nil receivers and invalid typed payloads return errors rather than panicking.
- Retention never removes a failed run and never targets the diagnostics root.
- Diagnostics models contain inspection metadata, not the durable output
contract.
- Checkpoint and debug serializers remain separate framework components.
- Secret-handling follows the invariant in
[Architecture](../policy/architecture.md#state-output-and-safety).
## CLI Failure Behavior
## Tests To Inspect
The CLI creates the diagnostics run directory after config loading and before
pipeline resolution. Failures before that point do not have diagnostics.
After diagnostics creation, run failures call `WriteErrorLog` and apply
retention with `RunSucceeded: false`, so the run directory remains available.
When the pipeline returns a partial manifest on failure, the CLI writes that
manifest before logging the failure.
## Invariants
- Diagnostics paths must be narrow and run-directory scoped.
- Writes should be atomic where practical.
- Secrets must be redacted.
- Diagnostics write failures are command failures because they can hide the
information needed for recovery.
- Durable output file contracts belong to output modules and integration docs,
not to diagnostics.
- `internal/core/diagnostics/run_dir_test.go`: allocation, artifact confinement,
atomic writes, retention, and failure behavior.
- `internal/core/diagnostics/artifacts_test.go`: stable artifact identifiers.
- `internal/core/config/redaction_test.go`: clone-and-redact payload behavior.
- `internal/core/workspace/settings_test.go`: effective diagnostics-root and
enablement handoff.
- `internal/cli/run_test.go`: creation timing, artifact sequencing, disabled
diagnostics, overrides, failures, and retention integration.

View File

@@ -1,116 +1,148 @@
# LLM Runtime
# LLM Runtime Internals
The implemented LLM runtime lives in `internal/framework/llm`. It provides
transport-neutral structured completion contracts, an OpenAI-compatible HTTP
adapter, concurrency scheduling, schema registry helpers, retry behavior, and
secret redaction.
`internal/framework/llm` implements Notarius's transport boundary for structured
completion. It contains the Scriptorium adapter, concurrency scheduler,
prompt/schema registries, selected-profile recording, and provider-error
redaction.
## Contract
Provider-neutral ownership rules are defined in
[Architecture](../policy/architecture.md#llm-boundary). Profile sources,
credentials, and concurrency settings are defined in
[Configuration](../config.md).
Modules depend on `contracts.StructuredLLMClient`:
## Structured Contract
```go
CompleteStructured(ctx, request, out) (response, error)
```
Modules and LLM-backed validators depend on
`contracts.StructuredLLMClient.CompleteStructured`. A request identifies a
prompt and optional profile/session, supplies named input materials and
variables, and provides a caller-owned decoding target. A successful response
contains the validated raw structured bytes plus non-secret provider, model,
profile, and token metadata.
The request contains messages, optional model override, response schema name,
and response schema JSON. The caller supplies a pointer target for decoded
structured output.
The caller owns prompt selection, response-schema selection, and interpretation
of the decoded result. `LLMInputMaterial` keeps source and reference bytes with
their origin metadata so the adapter can pass named artifacts to Scriptorium
without exposing Scriptorium types through stage contracts.
Extractors own prompts and schemas. Provider adapters should not contain
domain-specific prompt logic.
## Production Construction
## Production Client Construction
`internal/cli` constructs the production runtime by:
`internal/cli` builds the production LLM client from the effective config:
1. allocating the asset registry populated by the generic, Seriatim, and D&D
package-family registrars;
2. creating a `ScriptoriumClient` from the effective profile source;
3. attaching an `LLMProfileRecorder`;
4. creating a scheduler from the effective concurrency limit;
5. returning a `ScheduledClient` wrapper;
6. decorating that shared client before preparation when debug recording is
enabled; and
7. injecting that one shared client into complete pipeline preparation before
the source file is read or the runner is invoked.
1. find the effective LLM profile;
2. build `OpenAICompatibleClientConfig`;
3. create an OpenAI-compatible client;
4. create a scheduler from profile or global concurrency;
5. wrap the client with `NewScheduledClient`;
6. return non-secret LLM profile manifest metadata.
The D&D scene chunker and spell extractor retain this injected client and use
it for every structured completion. Operation requests do not carry an LLM
client.
The current run command requires exactly one distinct effective LLM profile for
the resolved pipeline.
The CLI separately gathers explicit profile IDs from resolved LLM-capable stage
and validator bindings. It prepares a small internal check prompt for each ID so
missing or invalid profiles fail before pipeline execution. The runtime profile
override syntax and scope are defined in the
[CLI reference](../cli.md#run); binding rules are defined in
[Configuration](../config.md#module-bindings).
## OpenAI-Compatible Adapter
## Scriptorium Adapter
`OpenAICompatibleClient` posts JSON to:
`ScriptoriumClient` converts a Notarius request into a Scriptorium `RunRequest`.
It validates the decoding target and prompt identity, maps named input materials
to inline artifacts, forwards explicit profile and session context, delegates
rendering/provider execution/structured validation, and unmarshals successful
JSON into the caller target.
```text
<base_url>/chat/completions
```
Empty optional input material is represented by a single space so Scriptorium
retains the named input. The client returns Scriptorium's validated structured
bytes rather than re-encoding the caller target, allowing modules to preserve
the runtime result exactly.
It sends:
Selected profile, provider, model, and token metadata are mapped into the
Notarius response. The recorder deduplicates profiles by identity and supplies
manifest-safe profile summaries after actual calls; manifest population does
not guess the selected prompt default in advance.
- `model`
- `messages`
- `response_format.type = "json_schema"`
- `response_format.json_schema.name`
- `response_format.json_schema.strict = true`
- `response_format.json_schema.schema`
Generated-output validation failures and provider failures are wrapped with
prompt context. Error strings pass through bearer-token redaction before they
cross the runtime boundary.
If an API key is configured, the adapter sends an `Authorization: Bearer ...`
header.
## Scheduling
The adapter accepts assistant content either as a JSON string containing JSON or
as raw JSON content. It then unmarshals that content into the caller-provided
target.
`Scheduler` uses a bounded permit count and a FIFO waiter queue. Immediate
acquisition increments the in-flight count; queued acquisition waits for a
permit or context cancellation. Cancellation removes a queued waiter, while a
cancelled waiter that has already received a permit releases it.
External wire-contract details belong in the
[OpenAI-compatible integration doc](../integrations/openai-compatible.md).
`ScheduledClient` acquires a permit around each structured completion and
defers release on every result path. The effective limit and default are
configuration facts in [Configuration](../config.md#defaults).
## Retries And Timeouts
This provider-call ceiling is independent of the pipeline's extract worker
limit. Concurrent lanes, retries, and validators all use the same scheduled
client, so increasing framework workers cannot exceed `total_llm`. Pipeline
dispatch and cancellation mechanics are documented in
[Pipeline Internals](pipeline.md#execution-flow).
The adapter retries:
## Prompt And Schema Assets
- provider request failures;
- response read failures;
- HTTP `429`;
- HTTP `5xx`;
- malformed provider envelopes;
- malformed assistant JSON;
- structured-output decode failures.
`AssetRegistry` combines caller-owned prompt filesystems under stable prefixes
and rejects invalid or conflicting registrations. Production module packages
register their own prompt and schema assets; generic framework code contains no
D&D prompt content. `internal/framework/promptfs` provides the domain-neutral
filesystem composition helper used to combine module-owned files with shared
domain prompt fragments.
Non-retryable `4xx` responses are returned without retry. Request timeout comes
from the effective LLM profile. Context cancellation is respected.
Schema helpers load embedded JSON Schema with identity and digest metadata,
return defensive copies, and expose a diagnostics map that omits schema bytes.
The small framework registry contains only generic test schemas; production
schemas remain package-owned.
## Scheduler
## Debug And Redaction Boundaries
`Scheduler` bounds concurrent provider calls. It tracks in-flight calls and a
FIFO queue of waiters. Cancellation removes queued waiters or releases granted
permits.
The pipeline may wrap the client with a debug recorder that captures prepared
prompt/response material for an explicitly enabled debug run. Default
diagnostics and manifests receive identities, hashes, usage, and selected
profile summaries rather than prompt, source, reference, schema, or response
content.
`NewScheduledClient` wraps any structured LLM client and runs each completion
inside the scheduler.
The Scriptorium error wrapper removes bearer credential values from surfaced
provider errors; `RedactSecrets` and `ErrorWithSecretsRedacted` support known
secret values elsewhere in the runtime. Config diagnostics use a separate
clone-and-redact path in `internal/core/config`. These mechanisms implement the
security invariant in
[Architecture](../policy/architecture.md#state-output-and-safety); operator
handling of debug data is defined in [Operations](../operations.md#debug).
Effective concurrency is:
## Failure Behavior
1. `llm_profiles.<id>.max_concurrency`, when greater than zero;
2. `concurrency.total_llm`, when greater than zero;
3. `1`.
- Invalid targets, missing prompt IDs, malformed structured output, and
Scriptorium failures return contextual errors to the calling module.
- Scheduler construction rejects non-positive limits; acquisition respects
context cancellation.
- Asset registration rejects invalid roots, missing content, and path conflicts.
- Schema loading distinguishes missing assets, invalid JSON, and invalid
metadata.
- Profile validation errors occur during CLI preparation when an explicit
selected ID cannot be prepared.
## Schema Registry
## Tests To Inspect
The framework schema registry embeds generic test schemas. It also exposes
helpers for caller-owned schemas:
- `LoadResponseSchema`
- `LookupResponseSchema`
- `MustLookupResponseSchema`
- `ResponseSchema.DiagnosticsMap`
`DiagnosticsMap` omits raw schema content and includes metadata such as key,
ID, version, name, and SHA-256.
The D&D spell extractor owns and loads its own embedded response schema.
## Secret Redaction
Provider errors are passed through `ErrorWithSecretsRedacted` with the API key
and bearer-token value. Config diagnostics use redacted effective config
payloads.
Do not add raw provider request bodies, response bodies, API keys, or prompt
payloads to diagnostics by default.
- `internal/framework/llm/scriptorium_client_test.go` and
`scriptorium_api_test.go`: adapter mapping and local HTTP integration.
- `internal/framework/llm/scheduler_test.go` and
`scheduled_client_test.go`: permits, FIFO behavior, cancellation, and wrapper
release.
- `internal/framework/llm/asset_registry_test.go` and
`schema_registry_test.go`: asset composition, validation, and defensive
copies.
- `internal/framework/llm/secrets_test.go`: provider-error redaction.
- `internal/cli/run_test.go`: profile validation, production client wiring,
manifest recording, and debug integration.
- Module-local `scriptorium_assets_test.go` files: prompt inputs and package
asset registration.

View File

@@ -1,165 +1,231 @@
# Modules
# Module And Validator Internals
Production modules live under `internal/modules`. Each module implements one
contract from `internal/framework/contracts`, exposes a `ModuleSpec`, and
registers itself with the matching pipeline registry.
Production module and validator implementations live under their domain-first
trees in `internal/modules`.
The selectable keys, configuration options, reference slots, and default
validator chain are canonical in the
[module](../config.md#implemented-production-modules) and
[validator](../config.md#implemented-production-validators) catalogs in
Configuration.
The CLI production catalog currently registers only the modules listed here.
## Extension Pattern
## Contract Pattern
A stage module package provides a stable key, constructor, contract
implementation, `ModuleSpec`, `Register`, and focused behavior and registration
tests. A validator package follows the same pattern with `ValidatorSpec` and the
validator registry. Package-family registrars compose those leaf registrations
into the production catalog and own family-level policy such as default
validator chains and prompt asset collection.
A production module package should provide:
Production input, chunk, output, and D&D spell-extract packages register strict
option decoders and run-local builders. Preparation decodes their options into
implementation-owned values and injects dependencies. The spell extractor is
typed over the canonical D&D model. D&D validators, merge, and normalize use
typed variants; JSON representation validators use serialized requests; and
unconditional validators expose separate chunk and typed variants. The D&D
production registrar registers only the canonical typed spell implementations.
- a stable module key;
- a constructor such as `New`;
- the relevant contract implementation;
- `ModuleSpec`;
- `Register`;
- focused tests for registration, options, contract behavior, and errors.
Prepared extractors, extract validators, and codecs may be reused concurrently
by the run-wide extract pool. Production implementations are immutable after
construction: they retain only typed options, immutable assets, or the shared
concurrency-safe LLM client. Implementations that introduce mutable state must
synchronize that state without creating a separate provider scheduler.
Module specs should describe capabilities accurately. Resolution uses specs to
reject incompatible pipelines before execution.
Specs expose capability and execution metadata without constructing an
implementation. Registry entries separately expose option validation and
run-local construction. Chunk, extract, merge, and normalize modules that accept
auxiliary material declare identical reference slots from both
`ReferenceSlots()` and `ModuleSpec().ReferenceSlots`; registration tests enforce
that agreement. Runtime delivery uses the corresponding stage request's
`References` field.
## `seriatim` Input
LLM-backed extensions own their prompt definitions and response schemas under
package-local embedded assets. Shared filesystem composition belongs in
`internal/framework/promptfs`; reusable D&D prompt fragments, reference
declarations, prompt-input assembly, and source-unit helpers belong in
`internal/modules/dnd/shared`. Stage contracts expose only Notarius structured-
completion types, not Scriptorium public types.
Package: `internal/modules/input/seriatim`
Reference material may inform a module or prompt but must not become source
evidence. The resolver and materializer behavior is described in
[Pipeline Internals](pipeline.md#reference-materialization).
The `seriatim` adapter parses Seriatim transcript JSON into a generic source
document. It owns transcript JSON details, source ID selection, source digest
creation, transcript segment validation, and segment metadata mapping.
## Input Adapter
Provides:
### `internal/modules/seriatim/input/transcript`
- `source.transcript`
- `transcript.speaker`
- `transcript.timestamps`
The adapter decodes the supported transcript JSON, selects the source identity,
computes canonical source provenance, validates segments, and maps each segment
into a generic source unit with a self-reference plus speaker and timestamp
metadata. It accepts no module options. Its spec advertises the transcript
capabilities consumed by D&D modules.
External JSON shape belongs in the Seriatim integration doc.
Parsing is strict about required values and duplicate unit IDs but deliberately
ignores unrelated Seriatim fields. The external format and derived-identity
rules are defined in the
[Seriatim contract](../integrations/seriatim.md).
## `generic` Chunker
## Chunkers
Package: `internal/modules/chunk/generic`
### `internal/modules/generic/chunk/units`
The `generic` chunker splits source units into ordered chunks. It validates the
source document, clones source units, assigns chunk IDs such as `chunk-000001`,
and records chunk metadata for start unit, end unit, and unit count.
The generic chunker validates the source document, walks units in configured
windows, clones each selected unit, and emits deterministic ordered chunk IDs.
Overlap changes the next window start but never reorders units. It records the
first and last unit and unit count in chunk metadata, and derives the chunk's
canonical source reference from those unit references.
Options:
The accepted options and defaults are defined in
[Configuration](../config.md#implemented-production-modules). Generic
framework validation canonicalizes the returned unit slices before extraction.
The chunker decodes its options during construction and retains only the typed
window settings used by `Chunk`.
- `max_units`: positive integer, default `50`;
- `overlap_units`: non-negative integer, default `0`, and less than
`max_units`.
### `internal/modules/dnd/chunk/scenes`
Provides:
The scene chunker prepares a structured Scriptorium request from the full
transcript, session, and optional D&D reference inputs. It validates the model's
scene boundaries against source-unit IDs and converts them into deterministic
chunks with canonical source references spanning each scene's units.
Preparation injects the shared structured LLM client into the chunker; `Chunk`
supplies only the run-specific profile, session, source, references, and
metadata.
- `chunks`
Scene validation requires sequential, contiguous, non-overlapping coverage from
the first source unit through the last. Each chunk contains JSON scene content
and module-owned metadata for the scene description, boundaries, confidence,
participants, and unit count. Boundary caveats become warnings. Malformed
structured output is returned as an error; there is no fallback chunker.
## `dnd/spells` Extractor
The package embeds its prompt and response schema and reports their non-secret
identity and hashes through singleton module metadata. Shared D&D assets supply
reference declarations and prompt inputs; their user-facing keys and accepted
file types remain canonical in [Configuration](../config.md).
Package: `internal/modules/extract/dnd/spells`
## Extractor
The `dnd/spells` extractor owns D&D spell-cast artifact semantics. It renders
embedded prompts, loads the embedded structured response schema, calls the
structured LLM client, converts spell-cast responses into artifact candidates,
and supplies deterministic validators.
### `internal/modules/dnd/extract/spells`
Requires:
The spell extractor prepares a structured request from one chunk, the
chunk-scoped source input, the session, and optional D&D reference inputs. It
decodes the model response, assigns the generic source identity to every source
reference, canonicalizes duplicate references, orders spell casts by their
earliest cited unit, and returns `dnd.SpellList`.
- `chunks`
- `source.transcript`
The extractor owns its private model-response DTO, embedded prompt, LLM response
schema, strict option decoder, injected shared LLM client, and prompt/schema
manifest metadata. The separate `internal/modules/dnd/codec/spells` package
owns the durable schema and stable JSON representation for artifact kind
`dnd/spell-list`. The runner keeps the result typed through validators and later
stages, using the codec only for checkpoint, debug, and output boundaries.
Shared D&D helpers keep prompt input
names and source-unit reference conversion consistent with the scene chunker.
Provides:
- `dnd.spell_casts`
Artifact type and schema version:
- artifact type: `dnd.spell_cast`
- schema version: `v1`
The extractor adds prompt and response-schema provenance to lane manifest
metadata. Durable artifact payload details belong in the
The durable payload and manifest metadata shapes are defined in the
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
## Merger And Normalizer
### `internal/modules/generic/merge/appendorder`
The merger passes typed values to an injected combine function in framework
source-chunk order. The D&D registrar specializes it with a spell-list append
function.
### `internal/modules/generic/normalize/noop`
The normalizer returns the merged domain value unchanged and is reusable for
any registered artifact type.
## Output Encoder
### `internal/modules/generic/output/json`
The JSON encoder sorts normalized results by lane, derives collision-checked
safe logical names, pretty-prints JSON payloads, and assembles the logical index,
manifest, rejected-result, warning, and lane files. Invalid JSON, unsupported
media types, unsafe names, and sanitized-name collisions are errors.
The encoder returns logical files only. The CLI places them on disk, and the
[JSON output contract](../integrations/json-output.md) defines their external
paths and schemas.
## Generic Validators
The generic validator implementations live under
`internal/modules/generic/validate`.
The unconditional accept and reject validators provide explicit chunk and
typed-artifact variants used primarily for controlled composition and tests.
The serialized JSON syntax validator uses `encoding/json` to reject malformed
representation bytes. The serialized JSON Schema validator requires schema
bytes, parses the instance and schema with `jsonschema`, and distinguishes
payload rejection from schema loading or compilation errors. The framework
serialized-validation request carries either canonical chunk bytes or artifact
codec bytes according to its target context. Neither validator calls the LLM.
## D&D Spell Validators
The spell extractor returns two built-in validators:
All three validators receive `dnd.SpellList` directly. The shape validator
rejects missing or empty spell fields and empty reference lists. The
source-reference validator applies generic source-reference validation to every
cited range. The relatedness validator warns when a case-insensitive spell name
is absent from all cited source text.
- `dnd/spells/shape`: rejects malformed payloads and missing required fields.
- `dnd/spells/source_refs`: rejects candidates without valid source references.
Reason codes include:
- `invalid_payload`
- `missing_required_field`
- `missing_source_ref`
- `invalid_source_ref`
These validators are supplied by the extractor when no validators are configured
for the lane.
## `appendorder` Merger
Package: `internal/modules/merge/appendorder`
The `appendorder` merger clones and appends candidates in chunk order. It does
not deduplicate or reconcile candidates.
Provides:
- `merged`
## `noop` Normalizer
Package: `internal/modules/normalize/noop`
The `noop` normalizer clones merged candidates and returns them unchanged.
Requires:
- `merged`
Provides:
- `normalized`
## `json` Output
Package: `internal/modules/output/json`
The `json` output encoder converts approved artifacts, rejected artifacts,
warnings, and the run manifest into logical JSON output files. It groups
approved artifacts by artifact type and sanitizes artifact-type file names.
Requires:
- `normalized`
Provides:
- `encoded`
Durable output file shapes belong in the
[JSON output contract](../integrations/json-output.md). Operator behavior
belongs in [Operations](../operations.md).
These validators are deterministic. Their selectable keys and production order
are defined in
[Configuration](../config.md#implemented-production-validators); their durable
payload rules are defined in the
[artifact contract](../integrations/dnd-spell-artifacts.md).
## Production Registration
Production registration is centralized in `internal/cli/catalog.go`.
The CLI allocates one complete framework registry set and one LLM asset
registry. It invokes `internal/modules/generic/register`,
`internal/modules/seriatim/register`, and `internal/modules/dnd/register` in
that order, then exposes the matching catalog for resolution. The generic and
Seriatim registrars own their production leaf registrations. The D&D registrar
owns D&D leaf registrations, the spell default-validator chain, and D&D
prompt/schema asset collection.
Do not make framework code import production modules. The CLI wires production
modules at the application boundary; tests may provide fake registries or fake
catalogs directly.
Concrete implementation packages do not import generic implementation
packages directly. A concrete family's `register` package is its composition
point for specializing reusable generic implementations, while the generic
registrar composes only generic children.
## Adding A Module
Framework packages must not import production extensions. Tests may compose
registries and catalogs directly with fakes.
When adding a module, keep source-format and extraction-domain boundaries clear:
## Adding An Extension
- input modules may know external source formats;
- extract modules may know artifact semantics and prompt/schema assets;
- merge and normalize modules own candidate combination and reconciliation;
- output modules own serialization, not diagnostics or CLI reporting.
When adding a production module or validator:
Update [Development](../policy/development.md), [Configuration](../config.md),
internal docs, integration docs, and examples when the new module becomes
implemented production behavior.
1. implement the stage or validator contract and package-local key;
2. expose and test its spec, constructor, and registration function;
3. keep format or domain parsing inside the concrete package;
4. add package-owned prompt/schema assets when the extension is LLM-backed;
5. register it through its package-family registrar and add a default chain
there only when production policy requires one;
6. add resolution and composition coverage for capabilities, options,
references, and validation behavior;
7. update the selectable-key catalog in [Configuration](../config.md), the
relevant external contract, this inventory, and maintained examples when
user-visible behavior changes.
Do not add the extension to `docs/development.md`; that file routes by task and
does not inventory implementations.
## Tests To Inspect
- Package-local `*_test.go` files under the module or validator being changed.
- `internal/framework/pipeline/typed_resolution_test.go`: typed registry, spec,
and heterogeneous artifact composition.
- `internal/framework/pipeline/default_modules_test.go`: framework binding
defaults.
- `internal/cli/run_test.go`: production catalog, config resolution, and
end-to-end CLI composition.
- `internal/framework/promptfs/*_test.go` and
`internal/modules/dnd/shared/*_test.go`: shared prompt and reference assembly.
- `internal/modules/integration/*_test.go`: black-box composition across
production extension domains.

View File

@@ -1,86 +1,141 @@
# Internal Overview
This directory documents implemented Notarius internals for developers and LLM
coding agents. It complements [Architecture](../policy/architecture.md), which
is the durable policy for boundaries and invariants.
This document inventories the implemented Notarius components. Normative
boundaries and dependency direction belong in
[Architecture](../policy/architecture.md); external behavior belongs in the
[CLI](../cli.md), [Configuration](../config.md),
[Operations](../operations.md), and [integration contracts](../integrations/).
## Executable And CLI
## Execution Path
`cmd/notarius` calls the CLI package. `internal/cli` owns:
`cmd/notarius` delegates to `internal/cli`, the production composition root.
The CLI loads configuration, builds the production catalogs and runtime
collaborators, invokes `internal/framework/pipeline`, and places the logical
output files returned by the runner. Diagnostics, checkpoints, and debug
recorders are optional side-channel collaborators supplied at this boundary.
- command parsing and usage;
- config discovery and loading;
- production module catalog and registry wiring;
- production LLM client construction;
- run directory creation;
- durable output writes;
- user-facing stdout, stderr, and exit codes.
Resolution produces a fixed ordered workflow and a sorted set of artifact
lanes. Preparation constructs the complete module and validator set before the
runner receives source bytes. Source parsing and chunking are serial; extraction
uses a bounded run-wide worker pool, followed by serial per-lane merge and
normalize continuations that may overlap across lanes.
The CLI should stay thin around framework contracts. Domain extraction behavior
belongs in modules, not in command handlers.
## Application Boundary
| Package | Implemented responsibility |
| --- | --- |
| `cmd/notarius` | Executable entry point and process exit delegation. |
| `internal/cli` | Command parsing, config discovery, package-family registrar invocation, LLM client construction, reference materialization, workspace collaborator setup, durable writes, and user-facing results. |
## Core Packages
- `internal/core/artifacts`: artifact candidates, approved artifacts, rejected
artifacts, validation decisions, and run manifests.
- `internal/core/config`: defaults, YAML config parsing, environment overrides,
validation, redaction, and resolved pipeline config.
- `internal/core/diagnostics`: per-run diagnostics directory creation,
diagnostics artifact writers, atomic writes, and retention decisions.
- `internal/core/source`: source documents, source units, source references, and
validation.
Core packages should remain deterministic and concrete. They should not import
production modules.
| Package | Implemented responsibility |
| --- | --- |
| `internal/core/artifacts` | Run-manifest and provenance models. |
| `internal/core/config` | Defaults, YAML parsing, environment overrides, validation, redaction, and effective pipeline resolution. |
| `internal/core/diagnostics` | Scoped run directories, diagnostics writers, atomic writes, and retention decisions. |
| `internal/core/source` | Generic source documents, units, chunks, canonical references, lookup, validation, and deterministic source digests. |
| `internal/core/workspace` | Effective workspace settings, confined paths and writes, checkpoint identity, and checkpoint manifest models. |
## Framework Packages
- `internal/framework/contracts`: interfaces and request/result structs for
input adapters, chunkers, extractors, mergers, normalizers, validators, output
encoders, and structured LLM clients.
- `internal/framework/pipeline`: module registries, module specs, profile
resolution, capability checks, run orchestration, warnings, validation, and
manifest population.
- `internal/framework/llm`: OpenAI-compatible structured-output client,
scheduler, schema registry, retries, and secret redaction.
- `internal/framework/prompt`: embedded prompt registry and template rendering.
- `internal/framework/validate`: validator decision helpers and cardinality
enforcement.
| Package | Implemented responsibility |
| --- | --- |
| `internal/framework/contracts` | Source-stage contracts plus artifact identity, schema, serialized representation, codec, validator, reference, output, and structured-completion interfaces and data types. |
| `internal/framework/pipeline` | Module and artifact-codec registries, option validation, profile resolution, capability checks, reference materialization, complete pipeline preparation, retries, orchestration, warnings, and manifest population. |
| `internal/framework/validate` | Shared validator decision and cardinality helpers. |
| `internal/framework/llm` | Scriptorium-backed structured completions, prompt/schema registration, scheduling, profile recording, and secret redaction. |
| `internal/framework/promptfs` | Builds module prompt filesystems from module-owned and caller-provided shared prompt assets. |
| `internal/framework/checkpoint` | Workspace-backed checkpoint loading, recording, and payload serialization. |
| `internal/framework/debug` | Workspace-backed framework and LLM debug recording. |
Framework code should stay source-agnostic and domain-agnostic.
Framework contracts provide typed artifact, provenance-wrapper, chunk-validator,
serialized-validator, and
typed-validator interfaces. The runner owns handoff provenance, validation
sequencing, rejection handling, checkpoint and debug boundaries, and final
manifest assembly.
## Module Packages
Artifact registries support heterogeneous typed extraction entries and
kind-specific merger, normalizer, and validator variants. Resolution derives a
lane's kind from its extractor, requires the matching codec, verifies exact Go
type equality across the lane, and records schema identity in the resolved lane
and pipeline digest. Registry entries carry separate option-validation and
run-local construction closures. Preparation injects shared dependencies and
constructs input, chunk, validators, ordered lanes, and output before source
parsing. Production modules use strict construction-time option decoding, and
LLM-backed modules retain the injected shared client. The D&D family registers
the canonical `dnd/spell-list` codec, typed spell extractor and validators, and
kind-specific generic merge and normalize strategies; generic JSON validators
use the serialized-validation contract. The runner executes lanes through
private exact-type-checked closures, coordinates extract results independently
of completion timing, and serializes artifacts only through their codec at
checkpoint, debug, and output boundaries.
Production module packages live under `internal/modules`:
## Production Extensions
- `input/seriatim`
- `chunk/generic`
- `extract/dnd/spells`
- `merge/appendorder`
- `normalize/noop`
- `output/json`
The canonical catalogs of user-selectable
[module](../config.md#implemented-production-modules) and
[validator](../config.md#implemented-production-validators) keys are in
Configuration. The implemented module packages are:
Each module package owns its contract implementation, module spec,
registration, options, focused tests, and module-specific errors.
| Package | Implemented responsibility |
| --- | --- |
| `internal/modules/seriatim/input/transcript` | Parses the supported Seriatim transcript format into the generic source model. |
| `internal/modules/generic/chunk/units` | Splits ordered source units by unit count and overlap. |
| `internal/modules/dnd/chunk/scenes` | Produces contiguous D&D scene chunks from structured model output. |
| `internal/modules/dnd` | Owns the canonical D&D spell-list and spell-cast artifact types. |
| `internal/modules/dnd/codec/spells` | Strictly decodes and stably encodes the durable D&D spell-list representation. |
| `internal/modules/dnd/extract/spells` | Maps private structured model output to canonical source-grounded D&D spell lists. |
| `internal/modules/generic/merge/appendorder` | Combines accepted extraction results in chunk order. |
| `internal/modules/generic/normalize/noop` | Preserves accepted merged output. |
| `internal/modules/generic/output/json` | Encodes manifests, lane payloads, warnings, and rejections as logical JSON files. |
## Fixtures And Tests
`internal/modules/dnd/shared` owns reusable D&D prompt fragments,
reference declarations, prompt input assembly, and source-unit reference
helpers. Domain-neutral prompt filesystem composition lives in
`internal/framework/promptfs`.
The repository uses focused package tests plus a fixture-driven CLI workflow.
Generic validators under `internal/modules/generic/validate` provide
unconditional test decisions, JSON syntax validation, and JSON Schema
validation. D&D spell validators under `internal/modules/dnd/validate/spells`
consume the canonical spell-list type directly to provide shape,
source-reference, and source-relatedness decisions.
- CLI acceptance tests cover maintained examples under `examples/`.
- Pipeline tests cover registry composition and end-to-end framework behavior
with fakes.
- Module tests cover implemented module contracts without requiring real
provider calls.
- LLM tests use local test servers and fakes.
Production composition is grouped behind package-family registrars, and every
implemented production extension uses its domain-first tree:
Do not use real external services in tests. Use fakes, fixtures, or local test
servers.
| Package | Implemented responsibility |
| --- | --- |
| `internal/modules/generic/register` | Registers domain-neutral chunk, merge, normalize, output, and validator implementations. |
| `internal/modules/seriatim/register` | Registers the Seriatim input adapter. |
| `internal/modules/dnd/register` | Registers D&D modules, validators, default validator policy, and prompt/schema assets. |
## Boundary Reminders
The CLI allocates the framework registries and asset registry, then invokes
these registrars in generic, Seriatim, and D&D order.
- Source-format details stay in input modules and integration docs.
- Extraction-domain details stay in extract modules and artifact docs.
- Provider wire details stay in the LLM runtime and provider integration docs.
- Durable output contracts belong in integration docs.
- Operator procedures belong in `docs/operations.md`, not internal docs.
Implementation details for all production extensions are in
[Module Internals](modules.md).
## Run-State Components
| Surface | Implemented owners | Internal purpose |
| --- | --- | --- |
| Durable output | Output module, pipeline runner, and CLI writer | Return logical consumer files and place them for a run. |
| Diagnostics | `internal/core/diagnostics` and `internal/cli` | Record redacted invocation, resolution, result, and failure inspection data. |
| Checkpoints | `internal/framework/checkpoint` and `internal/core/workspace` | Validate and serialize reusable stage outcomes. |
| Debug artifacts | `internal/framework/debug` and pipeline instrumentation | Capture sensitive framework-boundary and LLM-call material. |
Physical layout, retention, recovery, and sensitive-data handling are defined
in [Operations](../operations.md). Concrete stage modules receive recorder
interfaces and request data, not workspace paths.
## Focused Documentation
- [Pipeline Internals](pipeline.md): resolution, execution, validation, retries,
checkpoint/debug hooks, and result assembly.
- [Module Internals](modules.md): production modules, validators, assets,
registration, and the contributor recipe for adding an extension.
- [LLM Runtime](llm.md): structured completion contracts, Scriptorium adapter,
assets, scheduling, profile recording, and redaction.
- [Diagnostics Internals](diagnostics.md): scoped writers, retention
coordination, CLI failure flow, and path safety.

View File

@@ -1,127 +1,287 @@
# Pipeline Internals
The implemented pipeline runner lives in `internal/framework/pipeline`. It
executes the fixed workflow defined by the architecture policy:
The implemented resolver and runner live in `internal/framework/pipeline`.
Their fixed workflow and ownership boundaries are defined by
[Architecture](../policy/architecture.md#system-shape). Configuration fields,
defaults, and selectable keys are defined in
[Configuration](../config.md#pipelines).
```text
input -> chunk -> extract -> merge -> normalize -> output
```
Resolution fixes the selected lanes and all stage bindings; preparation
constructs every selected implementation before the runner begins source work.
After serial input parsing and chunking, the runner dispatches extract work to
one bounded run-wide worker pool in chunk-first, lane-second order. Each lane's
merge and normalize operations remain serial and may overlap other lanes once
all extracts for that lane are terminal.
Pipeline execution is serial. The runner executes the resolved lanes one after
another in the fixed workflow order.
## Resolution
## Profile Resolution
`internal/core/config.Config.Resolve` validates the loaded configuration,
selects the named profile, applies the runtime inputs supplied by the CLI, and
calls `pipeline.ResolvePipeline`.
Config loading produces `pipeline.PipelineProfile` values. Resolution happens
before execution:
`ResolvePipeline`:
1. `internal/core/config.Config.Resolve` validates config and finds the named
pipeline.
2. The optional lane selection is passed to `pipeline.ResolvePipeline`.
3. Module bindings are defaulted:
- chunk: `generic`
- merge: `appendorder`
- normalize: `noop`
- output: `json`
- LLM profile: `default`
4. The module catalog is checked for each bound module key.
5. Module capabilities are checked in workflow order.
6. A digest is calculated from the resolved pipeline without the digest field.
1. selects and sorts artifact lanes;
2. completes omitted bindings using the documented configuration defaults;
3. looks up each module and validator spec without constructing it;
4. for a typed extractor, derives its artifact kind, requires the codec, and
selects exact-type merger, normalizer, and validator variants;
5. checks required and provided capabilities in workflow order;
6. resolves target-aware reference bindings and validator chains;
7. validates each selected module and validator option set through its registry
entry; and
8. calculates a digest over the resolved structure, including typed artifact
kind and schema identity and the effective validator policy in its resolved
execution order.
The CLI writes the resolved pipeline and digest to diagnostics.
Resolution returns a `ResolvedPipeline` containing ordered lanes, concrete
bindings, validator chains, reference targets, and the digest. It does not read
reference bytes or construct runtime modules. CLI lane and reference selector
syntax is defined in the [CLI reference](../cli.md#run).
## Registries And Module Specs
The digest includes each resolved validator chain's stage, lane, owning module,
ordered validator bindings, execution classes, targets, and artifact kinds.
Changing a default chain or an explicit override therefore changes pipeline
identity whenever it changes the effective validator policy.
`pipeline.Registries` holds concrete constructors for execution. A
`pipeline.ModuleCatalog` exposes module specs for config validation and
resolution.
## Reference Materialization
Every production module registers a `ModuleSpec` with:
The CLI calls `MaterializeReferences` after resolution and before constructing
the LLM client or running the pipeline. The materializer checks each binding
against its resolved target declaration, reads and validates the file, and
builds both a `contracts.ReferenceSet` and provenance-only metadata on the
corresponding `ResolvedReferenceTarget`.
- `Key`: module key used in config;
- `Stage`: module kind such as input, chunk, extract, merge, normalize,
validate, or output;
- `Provides`: capabilities added after that module runs;
- `Requires`: capabilities that must already be available.
The runner clones the resulting set into the chunk, extract, merge, or normalize
request that owns the target. LLM-backed extensions may convert those items into
named prompt inputs. Reference content remains separate from source evidence and
source digests.
Capability checks prevent incompatible pipeline composition before a run starts.
Binding precedence, path resolution, accepted content, and media-type behavior
are configuration contracts; see [Configuration](../config.md#pipelines).
Durable provenance is defined in the
[JSON output contract](../integrations/json-output.md#manifestjson), while
runtime sensitive-data handling belongs in [Operations](../operations.md).
## Runner Input And Output
## Registries And Specs
`pipeline.RunInput` carries:
`pipeline.Registries` holds option validators and run-local builders used during
resolution and preparation.
`pipeline.ModuleCatalog` exposes their specs during configuration validation and
resolution. Separate registries exist for every stage and for validators;
`ValidatorChainRegistry` stores production default-chain mappings. Both
containers also carry an `ArtifactCodecRegistry`. Generic registration records
one codec per stable artifact kind, validates its schema metadata and JSON
Schema, retains the exact schema digest and Go type, and safely encodes or
decodes framework-erased values with typed errors on incompatibility.
- a `ResolvedPipeline`;
- optional source ID, input path, and raw input bytes;
- a structured LLM client;
- run ID, start time, LLM profile manifest metadata, and CLI metadata.
Typed extractor entries are keyed by module key and declare one artifact kind.
Merger, normalizer, and typed-validator variants are keyed by module or
validator key plus artifact kind. Chunk and serialized validators occupy
separate target namespaces; serialized registrations declare whether they
support chunks, artifacts, or both. Duplicate variants and exact Go-type
mismatches are rejected deterministically.
`pipeline.RunOutput` carries:
Lane-sensitive merger and normalizer spec discovery always supplies the
extractor's artifact kind, so variants under one reusable key may declare
different capabilities and reference slots. Kind-neutral registry inspection
selects the first registered artifact kind in sorted order.
- run manifest;
- approved artifacts;
- rejected artifacts;
- warnings;
- logical output files returned by the output encoder.
Production composition registers the D&D spell-list codec and typed extractor,
matching typed merge, normalize, and semantic-validator variants, and
serialized JSON validators. Every artifact lane resolves through the typed
registries and a matching codec.
The CLI owns durable file writes and diagnostics writes after the runner returns.
A `ModuleSpec` declares its stage plus required and provided capabilities.
Chunk, extract, merge, and normalize specs may also declare reference slots.
Registry implementations defensively copy spec metadata, reject duplicate keys,
and verify that a constructed implementation reports the registered key.
Builder registrations accept `ModuleDependencies` and cloned configuration
options through one `BuildRequest`. Builders decode those options and retain
typed values or injected dependencies in the constructed implementation.
Extractors declare their artifact kind, and merger, normalizer, and validator
resolution selects the matching typed variant.
## Execution
A `ValidatorSpec` declares a validator key and execution class. Resolution uses
the execution class to reject incompatible profile bindings before execution.
The current production catalog and default chain are listed only in
[Configuration](../config.md#implemented-production-validators).
## Preparation And Runner Boundary
`pipeline.Prepare` receives a resolved pipeline, the registries, and shared
module dependencies. It constructs input; chunk and its validators; each lane's
extract, merge, and normalize modules and validator chains in resolved order;
then output. It stops at the first error with pipeline, stage, lane, module, and
validator context as applicable. It never invokes an operation method.
`PreparedPipeline` keeps private constructed executors and exposes cloned
resolved input, chunk, lane, and output identities. `pipeline.RunInput` carries
that prepared pipeline, raw source input, run identity and timing, optional
session and profile metadata, and checkpoint/debug collaborators. The runner
parses source bytes through the already constructed input adapter. Later stage
requests receive the generic source model; extract requests receive
chunk-scoped input material, while chunk, merge, and normalize requests retain
access to the original source material. Input, chunk, and output operation
requests do not carry raw module options. The chunk request also does not carry
an LLM client; an LLM-backed chunker receives the shared client during
preparation. Their operation requests retain run-specific source, reference,
profile, session, and metadata context as applicable.
Prepared lanes retain exact-type-checked erased operation closures. The runner
uses those closures to keep each value typed through extraction, validation,
merge, and normalization.
Source validation requires every unit to carry a canonical self-reference to
its containing document and its own unit ID. Explicit clone, checkpoint, and
debug boundaries retain that reference, and the canonical source digest covers
it deterministically. Chunks use the same source model and carry one canonical
reference spanning the first selected unit through the last.
`pipeline.RunOutput` carries the run manifest, accepted normalized serialized
artifacts with lane and normalizer provenance,
rejected results, warnings, checkpoint events, and logical files returned by the
output encoder. The CLI owns diagnostics and durable filesystem writes after the
runner returns.
## Execution Flow
The runner:
1. validates run input and registries;
2. builds the input adapter and parses the raw input into a source document;
3. validates the source document;
4. builds the chunker and produces source chunks;
5. runs each selected artifact lane in sorted resolved order;
6. builds the output encoder and validates logical output file names.
1. validates its prepared input;
2. parses the raw input with the prepared adapter and validates the generic
source document;
3. obtains or executes the chunk result;
4. validates and canonicalizes chunks;
5. dispatches extract jobs in source-chunk then resolved-lane order, starting a
bounded lane continuation when all extracts for that lane are terminal;
6. invokes the prepared output encoder and validates its logical file results;
7. returns the assembled manifest, outcomes, warnings, and files.
Within an artifact lane, the runner:
Within each artifact lane, it reuses the prepared extractor, merger, normalizer,
and validators while performing these transitions:
1. builds the extractor, merger, and normalizer;
2. records module manifest metadata when modules provide it;
3. extracts candidates from each chunk;
4. normalizes candidate envelope fields such as index, extractor key, artifact
type, and schema version;
5. merges candidates;
6. normalizes merged candidates;
7. validates candidate envelope consistency;
8. runs validators;
9. converts approved candidates to artifacts.
1. extract once per accepted chunk and add runner-owned lane, source, and chunk
provenance;
2. validate each extract result and omit rejected results from merge input;
3. skip the rest of the lane when no extract result is accepted;
4. merge accepted extract results in their existing order;
5. validate the merge result and skip normalization on rejection;
6. normalize the accepted merge result;
7. validate and append the accepted normalized result.
## Validators
Module-provided warnings and payload warnings are promoted only from attempts
whose results are accepted and used.
If a lane declares validators in config, the runner builds those validators from
the validator registry. Otherwise it uses validators returned by the extractor.
The extract job channel has the same capacity as the effective extract worker
count, so dispatch applies backpressure. A fixed continuation executor prevents
ready or checkpoint-reused lanes from creating one goroutine each. Workers and
continuations publish lane-local results; the coordinator is the only writer of
aggregate output and merges those results in resolved lane and source-chunk
order.
Each validator must return exactly one decision for each eligible candidate. The
runner enforces decision cardinality with `internal/framework/validate`.
Rejected candidates are removed before the next validator runs. Approved
candidates continue through the chain.
## Chunk Canonicalization
The production CLI currently registers no standalone validator modules. The
current D&D spell extractor supplies deterministic shape and source-reference
validators.
Before lane execution, generic validation requires unique chunk IDs, matching
source identity, indexes matching returned order, a valid canonical reference,
non-empty content and media type, and at least one valid source unit per chunk.
Units may not repeat inside a chunk and must form a contiguous range in
source-document order. The chunk reference must exactly match the source and
the first and last unit references.
## Warnings And Failures
The runner then rebuilds each chunk's unit slice from the source document by
unit ID. It preserves the canonical reference, content, media type, and cloned
metadata. The framework permits gaps and overlap between separate
chunks; stricter coverage policy belongs to the chunk implementation.
Warnings from chunking, extraction, merging, normalization, validation, and
output encoding are accumulated in `RunOutput.Warnings`.
## Validation And Retries
Errors wrap the operation and module key or lane context. If execution fails
after a manifest exists, the returned manifest is marked `failed` and receives a
completion timestamp.
Chunk, extract, merge, and normalize results pass through the resolved validator
chain for their stage and module. Chunk validators receive canonical chunks;
typed validators receive the domain value; and serialized validators receive
canonical chunk JSON or artifact codec bytes. Validators execute in resolved
order and stop at the first error or rejection. An empty chain approves the
result.
On successful execution, the manifest validation status is:
`runWithRetry` applies the effective retry policy around module execution and
its complete validation chain. A module or validator error becomes a framework
error when attempts are exhausted. A rejection becomes a recorded
`RejectedOutput` when attempts are exhausted. Cancellation stops retry
processing immediately.
- `approved` when no candidates were rejected;
- `rejected` when at least one candidate was rejected.
Rejected output is a non-fatal pipeline outcome and does not advance. Warnings
from discarded attempts are not promoted. Configuration owns retry counts and
validator overrides; see [Module Bindings](../config.md#module-bindings).
## Manifest Population
## Checkpoint And Debug Hooks
The manifest records run ID, pipeline ID, pipeline digest, module keys, artifact
lanes, LLM profile metadata, source digest, validation status, and timing.
The runner depends on recorder and loader interfaces, using no-op
implementations when collaborators are absent. Each checkpointed workflow
boundary records a running, succeeded, or failed transition. Reuse decisions
are consulted in workflow order and accepted payloads are cloned before
entering the normal handoff path. Typed extract, merge, and normalize
checkpoints store codec bytes with artifact kind, schema ID and version, exact
schema digest, and media type. Reuse compares that identity with the prepared
codec and decodes through the codec; missing identity, mismatches, corrupt
bytes, and decode failures become explicit reuse misses and execute the step
normally. Dependency fingerprints and debug content digests use the same stable
codec bytes that cross those boundaries.
Modules can add non-secret manifest metadata by implementing
`contracts.ManifestMetadataProvider`. The D&D spell extractor uses this for
prompt and response-schema provenance.
Debug instrumentation wraps run, stage, attempt, validator, and structured LLM
boundaries. Every executed module retry has an attempt envelope containing its
candidate, accepted-attempt warnings, rejection or error, and only the LLM
calls made by that module attempt. Validator attempts retain independent scopes
under `validate/`. Debug-write failures are framework errors; debug data is
never used as a checkpoint source. Typed artifact debug envelopes are
domain-neutral, redact sensitive metadata and bytes through the common debug
policy, and record codec identity plus schema and content digests.
Checkpoint identity, physical layout, reuse behavior, and debug artifact
handling are operator contracts in [Operations](../operations.md). Serialization
and recorder implementation are inventoried in
[Internal Overview](overview.md#run-state-components).
## Results And Failures
The runner owns manifest assembly and handoff summaries but not the durable JSON
schema. It records resolved module and lane provenance, validator chains,
source/reference identities, selected LLM profiles, normalized and rejected
summaries, status, and timing. Serialized artifact content remains outside the manifest.
Module metadata providers may add non-secret singleton or lane-scoped metadata.
Execution errors include stage, module, lane, or validator context. Once a
manifest exists, a failing run returns it with failed status and completion
time. Successful status reflects whether any result was rejected. The
durable manifest and logical file schemas are defined in the
[JSON output contract](../integrations/json-output.md).
On a framework failure, the runner cancels its derived context, stops submitting
new extract work, drains started tasks, and skips the output encoder. Parent
cancellation takes precedence. Otherwise context-cancellation fallout is
discarded when a substantive error exists, and the primary error is selected by
stage, resolved lane, and source chunk rather than completion time.
## Tests To Inspect
- `internal/core/config/effective_config_test.go`: config-to-resolution boundary.
- `internal/framework/pipeline/profile_test.go`: selection, defaults,
capabilities, validator chains, and digest behavior.
- `internal/framework/pipeline/artifact_codec_registry_test.go`: typed codec
metadata, registration, erasure safety, strict decoding, and cloning.
- `internal/framework/pipeline/typed_resolution_test.go`: heterogeneous typed
lane resolution and preparation, target-specific validators,
incompatibilities, ordering, and schema-sensitive pipeline identity.
- `internal/framework/pipeline/runner_concurrency_test.go`: bounded dispatch and
continuations, reverse completion, stable errors, rejection, cancellation,
retries, and independent provider-call limits.
- `internal/framework/pipeline/preparation_test.go`: option validation,
construction order, dependency failures, and the before-source-work boundary.
- `internal/framework/pipeline/references_test.go`: target resolution and
materialization.
- `internal/cli/run_test.go`: production stage transitions, retries, rejections,
warnings, debug hooks, manifests, and end-to-end composition.
- `internal/modules/integration/*_test.go` and
`internal/modules/seriatim/input/transcript/runner_test.go`: typed runner
composition across concrete module families.
- `internal/framework/checkpoint/*_test.go`: checkpoint serialization and reuse
collaborators.

View File

@@ -4,20 +4,16 @@ This is the canonical reference for operating implemented Notarius runs.
## Normal Run
A run reads one source file, resolves one configured pipeline, calls the
configured OpenAI-compatible LLM profile, writes durable JSON output, and writes
diagnostics for inspection.
A run reads one source file, resolves one configured pipeline, executes its
modules, writes durable output, and writes diagnostics when enabled. Start with
the [README quickstart](../README.md), then use the [CLI reference](cli.md) for
invocation options.
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--output-dir ./notarius-output \
--diagnostics-dir /tmp/notarius
```
The command prints a success line with the pipeline ID, approved and rejected
artifact counts, and the output path.
For production, configure an application-owned workspace such as
`/var/lib/notarius` and ensure the Notarius process can create files below it.
For local development, prefer an ignored project-local workspace such as
`./.notarius/workspace`. See [Configuration](config.md#workspace) for workspace
fields.
## Output Directory
@@ -27,23 +23,10 @@ Durable output is written to:
<output-root>/<run-id>/
```
The default output root is `./notarius-output`. Use `--output-dir` to choose a
different root.
The `json` output module writes these files:
- `index.json`: file index with paths to the manifest, artifact files,
rejected artifacts, and warnings.
- `manifest.json`: run manifest with resolved pipeline provenance, module keys,
validation status, and timing.
- `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact
type. For the current D&D spell extractor, this includes
`artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved.
- `rejected.json`: rejected candidates and validator decisions.
- `warnings.json`: warnings reported by pipeline modules or the output encoder.
Output writes are atomic per file. Logical output file names must be clean,
relative, slash-separated paths and must not contain `..`.
The output root and its invocation-specific override are defined in the
[CLI reference](cli.md#run). Output writes are atomic per file. The
[JSON output contract](integrations/json-output.md) defines the logical files,
paths, schemas, and media types inside each run directory.
## Diagnostics Directory
@@ -53,37 +36,138 @@ Diagnostics are written under:
<diagnostics-work-dir>/<run-id>/
```
The default diagnostics work directory is `/tmp/notarius`. It can be set with
`diagnostics.work_dir`, `NOTARIUS_WORK_DIR`, or `--diagnostics-dir`.
When a workspace directory is configured, diagnostics are written under
`<workspace.directory>/diagnostics/<run-id>/`. An invocation-specific override
changes only the diagnostics root, not the workspace root. Configuration and
environment controls are defined in [Configuration](config.md); the override
flag is defined in the [CLI reference](cli.md#run).
Diagnostics can be disabled through configuration. When disabled, Notarius
does not create a diagnostics run directory or write diagnostics artifacts;
concise failures are still printed to stderr.
Implemented diagnostics artifacts:
- `invocation.json`: command metadata such as operation, config path, input
path, selected lanes, run ID, and pipeline digest when available.
- `effective-config.json`: resolved config with API keys redacted.
- `effective-config.json`: resolved config without raw API keys.
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
- `resolved-references.json`: resolved reference provenance, including target
stage, lane ID when present, origin, digest, media type, byte size, and
binding source, without reference content.
- `checkpoint-events.json`: checkpoint steps that were reused or executed
during an explicit resume invocation.
- `run-manifest.json`: the same run manifest written to durable output when it
is available.
is available, including top-level module metadata when present.
- `warnings.json`: warning list.
- `run-report.json`: counts, status, output path, diagnostics path, and run ID.
- `error.log`: failure message, written after diagnostics directory creation
when a run fails.
`source-document.json` is supported by the diagnostics writer but is not written
by the current CLI run workflow.
## Checkpoints
When checkpoint writing is enabled for a configured workspace, runs write
checkpoints under:
```text
<workspace.directory>/checkpoints/<pipeline-id>/<input-key>-<source-or-input-digest>/<pipeline-digest>/<identity-digest>/
```
Each workflow step owns its own manifest and payload files. There is no
root-level checkpoint summary. Ordinary invocations execute the pipeline
normally and refresh checkpoints. An explicit resume invocation reuses valid
checkpoints and executes any missing, invalid, or incompatible step normally.
Configuration controls checkpoint writing, while the explicit resume option is
defined in the [Configuration](config.md#workspace) and
[CLI](cli.md#run) references.
Checkpoints do not include raw prompts, raw reference contents, raw LLM request
payloads, or debug traces. They can still contain source text, intermediate
extracted content, rejected outputs, metadata, warnings, and content digests.
Treat checkpoint directories as sensitive local state.
A checkpoint is reused only when its stored status, dependencies, payloads, and
digests match the current invocation. Changes to input bytes, the resolved
pipeline, selected lanes, the runtime LLM profile override, or bound reference
content invalidate reuse. The resolved pipeline identity includes effective
default and explicitly overridden validator chains, so adding, removing,
reordering, or reconfiguring a validator invalidates checkpoints even when the
pipeline profile itself is unchanged.
Typed artifact checkpoints additionally record codec-owned bytes, artifact
kind, schema ID and version, exact schema digest, and media type. A missing or
mismatched codec identity, or bytes the current codec cannot decode, is reported
as a checkpoint reuse miss. The affected operation executes normally and, when
checkpoint writing is enabled, replaces the incompatible checkpoint.
Current checkpoint manifests use workspace schema `notarius.workspace.v2`.
Manifests written with `notarius.workspace.v1` are incompatible because their
chunk provenance has an older shape. On the first explicit resume after an
upgrade, each affected checkpoint is treated as a reuse miss and its workflow
step executes normally. The compatibility check does not migrate or delete the
v1 files; when checkpoint writing is enabled, normal execution refreshes the
affected checkpoint files in the current schema.
Runs do not reuse checkpoints unless explicitly requested. Without reuse, the
workflow executes normally and refreshes checkpoint files when checkpointing is
enabled.
## Debug
When debug recording is enabled for a configured workspace, runs write debug
artifacts under:
```text
<workspace.directory>/debug/<run-id>/
```
Debug output is per invocation. It is independent of checkpointing and is not
used for resume. Enabling debug does not write checkpoints, and enabling resume
checkpointing does not write debug output.
Debug artifacts include inputs and outputs for source, chunk, extract, merge,
normalize, and output work, structured LLM request and response data, validator
requests and results, timing, and retry attempt metadata. LLM calls made inside
a retry or validator attempt
write `prompt-000N.json`, `response-000N.json`, and
`response-content-000N.*` files under that attempt directory and are linked from
the attempt `llm_calls` array. Prompt content is written inline in the prompt
artifact. The response metadata and body use the paired files described above;
the body is pretty-printed JSON when possible and raw text otherwise. Merge and
normalize retries use these stable paths:
```text
merge/<lane-id>/attempt-<NN>.json
merge/<lane-id>/attempt-<NN>/prompt-<NNNN>.json
merge/<lane-id>/attempt-<NN>/response-<NNNN>.json
merge/<lane-id>/attempt-<NN>/response-content-<NNNN>.<ext>
normalize/<lane-id>/attempt-<NN>.json
normalize/<lane-id>/attempt-<NN>/prompt-<NNNN>.json
normalize/<lane-id>/attempt-<NN>/response-<NNNN>.json
normalize/<lane-id>/attempt-<NN>/response-content-<NNNN>.<ext>
```
Checkpoint-reused merge and normalize work retains the stage-level input and
output artifacts but has no retry-attempt artifacts because no module attempt
executed. Debug artifacts may contain source material, reference material,
prompt inputs, model outputs, and other sensitive data. Typed artifact
envelopes include domain-neutral codec identity, redacted metadata and content,
and digests of the stable codec bytes. API keys are not written, and obvious
credential-shaped values and sensitive map keys are redacted, but debug
directories should still be protected as sensitive local state.
## Retention
Diagnostics retention is configured with `diagnostics.retention`,
`NOTARIUS_DIAGNOSTICS_RETENTION`, or the default `auto`.
Diagnostics retention uses the effective mode selected through configuration;
see [Configuration](config.md#diagnostics) for the fields, environment
overrides, precedence, and default.
- `auto`: keep failed runs and successful runs with warnings; remove successful
warning-free runs.
- `always`: keep every diagnostics run directory.
- `never`: remove successful run directories; failed runs are still retained.
Unknown retention values are rejected during config validation.
## Failures
Failures before diagnostics directory creation, such as a missing config file or
@@ -91,21 +175,19 @@ an unusable diagnostics work directory, are printed to stderr and may not have a
diagnostics run directory.
Failures after diagnostics directory creation are printed to stderr and written
to `error.log`. Depending on where the failure occurred, diagnostics may also
include invocation metadata, redacted effective config, resolved pipeline data,
the run manifest, warnings, and a run report.
to `error.log`. Depending on where the failure occurred, the directory may also
contain artifacts written before the failure.
If durable output writing fails after the pipeline completes, diagnostics are
retained for inspection and may include `run-manifest.json`, `warnings.json`,
`run-report.json`, and `error.log`.
retained for inspection.
## Warnings
A successful run with warnings exits with code `0`, prints a warning count to
stderr, and writes warnings to durable output and diagnostics when retained.
The run manifest `validation_status` indicates whether final artifacts were
approved or rejected after validation.
The [JSON output contract](integrations/json-output.md) defines durable warning
and validation-status fields.
## Cleanup
@@ -117,16 +199,29 @@ rm -rf /tmp/notarius/run-1234567890
rm -rf ./notarius-output/run-1234567890
```
Workspace checkpoint and debug directories can also be removed when no longer
needed. Remove exact identity or run directories, for example:
```sh
rm -rf /var/lib/notarius/checkpoints/dnd-session/seriatim-abcdef123456/7890abcd1234/identityabcd1234
rm -rf /var/lib/notarius/debug/run-1234567890
```
Use exact run-directory paths. Avoid broad cleanup commands against parent
directories unless they are part of your own operational policy.
## Operational Limits
There is no command to resume a failed run. Re-run `notarius run` after fixing
the cause.
Provider retries and timeouts are handled by Scriptorium according to the
selected execution profile. Pipeline module retry settings are defined in
[Configuration](config.md#module-bindings). There is no separate CLI retry
command.
Provider retries are limited to the OpenAI-compatible client retry behavior
configured by the effective LLM profile. There is no separate CLI retry command.
Extract worker concurrency and actual provider-call concurrency are separate
limits. Their configuration, defaults, and validation are defined in
[Configuration](config.md#concurrency). Cancellation stops undispatched extract
work; already started work is allowed to finish or observe cancellation before
the run reports failure.
Notarius writes local files only. Remote storage and archive management are not
part of the implemented CLI.

View File

@@ -1,210 +1,212 @@
# Architecture
This document defines Notarius development policy. It is inward-facing:
developers and LLM coding agents should use it to preserve the project's shape,
boundaries, and invariants as the code evolves.
This document defines the intended high-level architecture of Notarius and the
invariants that changes must preserve. Implemented component details belong in
[Internal Overview](../internal/overview.md) and its linked documents. The
reasoning behind significant architectural choices belongs in
[ADRs](../adr/).
Keep this document concise. It should describe durable architectural rules, not
CLI syntax, configuration reference material, module catalogs, or roadmap items.
## System Shape
## Project Shape
Notarius is a small, dependency-light Go application for extracting structured
artifacts from source material. It is a general extraction platform whose
source formats, extraction domains, validation policies, LLM providers, and
output formats are isolated behind explicit boundaries.
Notarius is a small, explicit, dependency-light Go application for extracting
structured artifacts from source material using modular pipeline stages.
The application is contract-first but not abstraction-heavy. Add interfaces and
extension points when they protect a real boundary:
- external source formats;
- pipeline stage modules;
- validators;
- LLM providers and runtime plumbing;
- output schemas and embedded assets.
Avoid abstractions that only anticipate hypothetical complexity. Prefer narrow
contracts that can be exercised by tests and real modules.
## Core Invariants
The framework must remain source-agnostic and domain-agnostic.
Source-format details belong in input modules. Transcript-specific concepts such
as segments, speakers, timestamps, and transcript schemas must not spread into
runner, extractor, validator, or LLM framework code.
Extraction-domain details belong in domain modules. D&D-specific concepts such
as spells, NPCs, items, combat turns, and encounters must not spread into core
source, runner, or LLM framework packages.
Extracted facts should be grounded with source references. Source references
should point to generic source units, not transcript-only structures. Framework
code should preserve source-reference ranges exactly and should not merge or
rewrite overlapping ranges unless a module explicitly owns that behavior.
The application workflow is fixed:
The application has one fixed pipeline shape:
```text
input -> chunk -> extract -> merge -> normalize -> output
```
These stages should remain explicit in the architecture. Chunking, merging, and
normalization must not be hidden inside domain extractors when they represent
general pipeline behavior.
Pipelines are configured compositions of this shape. They are not arbitrary
DAGs or a general workflow language. Every stage remains explicit; general
chunking, merging, or normalization behavior must not be hidden inside an
extractor.
Pipelines are fixed-shape templates for this workflow, not arbitrary DAGs or a
general workflow language. Module selection should be configuration- and
registry-driven, not scattered through conditionals.
Input and chunking are pipeline-wide. Each selected artifact lane owns its
extract, merge, and normalize stages, and the output stage aggregates the run's
lane outcomes.
## Package Boundaries
Notarius is contract-first without being abstraction-heavy. Interfaces and
extension points should protect demonstrated boundaries. New abstraction is not
itself an architectural goal.
Prefer fewer, larger framework packages until a boundary proves itself through
import direction, ownership, test seams, or substantial file size.
## Layers And Dependency Direction
Core packages should contain deterministic models and policy. Framework
packages should contain reusable orchestration and provider plumbing. Concrete
business logic should live under stage-oriented module packages:
The application boundary is the composition root and may depend on concrete
implementations. Domain-neutral model and framework layers provide reusable
policy, contracts, and orchestration. Concrete input, pipeline, output, and
validation extensions depend inward on those generic layers.
```text
internal/modules/input/...
internal/modules/chunk/...
internal/modules/extract/...
internal/modules/merge/...
internal/modules/normalize/...
internal/modules/output/...
```
Generic layers must not depend on production extensions. Concrete extensions
must not compose the application or take ownership of process behavior. The
current packages implementing these layers are inventoried in
[Internal Overview](../internal/overview.md).
Use short, lowercase, idiomatic Go package names. Avoid package names that repeat
parent-stage context.
The following dependency boundaries are mandatory:
Input modules translate external source formats into the core source model.
They may know about external schema details, source-specific metadata, and
format-specific validation rules. They should not own extraction-domain
decisions.
- extractors and validators do not depend on concrete input adapters;
- provider-specific types do not cross the LLM runtime boundary;
- external dependency types do not leak across internal package boundaries
unless that dependency is the package's explicit contract.
Extract modules own artifact semantics, prompt usage, structured response schema
selection, validator defaults, and domain-specific interpretation. They should
depend on framework contracts and core source/artifact types, not concrete input
module packages.
Shared helpers may support demonstrated common needs, but must not move
source-format or extraction-domain knowledge into generic framework packages.
External dependencies require a clear correctness, security, interoperability,
or complexity benefit.
Merge modules combine extracted candidates. Normalize modules reconcile merged
candidates for semantic consistency. Generic behavior may exist for simple
artifact types, but domain-specific behavior belongs in modules for the relevant
stage.
## Source And Domain Boundaries
Output modules serialize final artifacts and may report warnings out of band.
CLI, diagnostics, and reporting layers are responsible for surfacing those
warnings.
Input modules translate external source formats into the generic source model.
Format-specific schemas, fields, and validation remain with the input module
and its integration contract.
Framework stages operate on source documents, source units, and source
references rather than format-specific structures. A source reference identifies
an ordered range of generic source units. Framework code preserves those ranges
and does not merge or rewrite them unless a stage module explicitly owns that
behavior. Every source unit carries a validated self-reference to its containing
document and its own unit ID.
Extract modules own artifact semantics, prompt use, response schemas, and
domain interpretation. Domain-specific concepts remain in the relevant module,
validator, shared domain helper, and artifact contract.
Typed artifact registrations declare one stable artifact kind and exact Go
type from extraction through merge, normalization, and semantic validation.
Pipeline resolution requires a compatible codec and matching kind-specific
variants before a typed lane can be accepted. Framework-owned erasure remains
private and must report type incompatibility as an error rather than a panic.
Auxiliary references provide context or disambiguation. They are not source
evidence and must not be converted into source references.
## Pipeline Composition And Ownership
Module selection is configuration- and registry-driven. The framework resolves
named pipeline definitions, applies explicit defaults and runtime overrides,
and verifies module availability and capabilities before execution. Structural
pipeline choices must not be scattered through conditionals or hidden behind
ad hoc command flags.
Resolution validates every selected module and validator option set. A separate
preparation boundary then constructs the complete input, chunk, lane,
validation, and output implementation set in pipeline order. The runner accepts
only that prepared set, so construction and dependency failures occur before
source parsing or any other module operation.
Stage ownership is explicit:
- input modules convert external material into the generic source model;
- chunk modules partition source material for extraction;
- extract modules produce domain artifacts from chunks;
- merge modules combine accepted extraction outputs;
- normalize modules reconcile merged output;
- output modules encode accepted results and run outcomes into logical files.
The framework owns orchestration and handoff provenance. Modules return logical
results and warnings; they do not own CLI reporting, workspace paths, durable
file placement, checkpoints, or diagnostics.
After pipeline-wide chunking, extraction uses bounded framework concurrency.
One run-wide worker pool receives chunk-scoped lane jobs in deterministic
chunk-first, lane-second order. A lane may begin its merge and normalize
continuation only after all of its extract jobs are terminal; that continuation
remains serial within the lane, while bounded continuations for different lanes
may overlap. The framework must not create unbounded goroutines per lane or
chunk.
Completion timing does not choose public ordering or errors. The coordinator
orders accepted artifacts, warnings, rejections, checkpoint events, and
framework errors by stable pipeline scope. Rejections do not cancel unrelated
work. A framework error cancels derived work, prevents undispatched work from
starting, waits for started work, and prevents output encoding.
## Validation
Validators should be independently testable and composable.
Validation is a framework-managed boundary around outputs from chunk, extract,
merge, and normalize stages. Validators receive immutable stage output
and make an explicit whole-output decision: approve, approve with warnings, or
reject.
Deterministic validators should run before LLM-backed validators when both are
present. Validator decision semantics should be explicit: each candidate
artifact evaluated by a validator should receive exactly one decision from that
validator.
Typed artifact validators receive the domain value directly. Chunk validators
receive source-zone chunks, while serialized validators receive immutable
representation bytes and declared schema metadata. A validator registered for
one target or artifact kind cannot satisfy an incompatible selection.
LLM-backed review belongs in module-owned validator chains, not in an implicit
global review phase. Extract and normalize modules may both use deterministic
and LLM-backed validators.
Rejection is a recorded pipeline outcome, not a framework execution error.
Validator execution failures are framework errors. Rejected output does not
advance to the next stage.
Shared validator runtime mechanics belong in framework code. Concrete validator
behavior belongs in module or validator implementation packages.
Default validator chains are production composition policy and are registered
centrally by stage and module. Configuration may replace a stage-local default,
including with an explicitly empty chain. Configured validator order is
authoritative; the framework must not silently reorder it.
## LLM Runtime
## LLM Boundary
LLM provider details belong behind transport-neutral framework contracts.
Modules and validators use transport-neutral structured completion contracts.
Provider request and response types, authentication, transport behavior, and
provider error adaptation remain inside the LLM runtime.
Provider-specific HTTP request and response types should stay inside the LLM
runtime package. Prompt construction should stay in extractors, validators, or
shared prompt helpers; provider adapters should not own domain prompt logic.
The caller of the LLM owns prompt selection, prompt inputs, response schema,
and interpretation of structured output. Provider adapters do not own source-
or domain-specific prompt logic.
Errors, diagnostics, reports, manifests, and redacted configuration must not
expose secrets.
LLM calls and other external operations accept cancellation and respect
timeouts. Concurrency control belongs in shared runtime plumbing rather than in
individual modules.
## Configuration
The application-wide LLM scheduler bounds actual provider calls independently
of framework worker limits. Every LLM-backed module, retry, and validator uses
the single injected scheduled client, including work performed by overlapping
lanes.
Configuration should make pipeline composition explicit and discoverable.
## Configuration And Provenance
Centralize configuration loading, precedence, defaults, and validation. Structural
pipeline choices should come from named pipeline definitions, not ad hoc command
flags. Operational overrides may be handled separately when they do not obscure
the configured pipeline structure.
Configuration loading, precedence, defaults, environment overrides, redaction,
and validation are centralized. Named pipeline definitions make structural
composition explicit and discoverable. Operational overrides are permitted
when they do not obscure the configured pipeline structure.
Module registries should expose module metadata and capabilities without
requiring module construction. Configuration validation should fail fast when a
pipeline binds incompatible or unknown modules.
Run preparation fails before stage execution when statically discoverable
modules, capabilities, reference bindings, or explicitly selected profiles are
invalid or incompatible.
Run manifests should record enough resolved pipeline provenance to make a run
auditable after named configuration changes over time.
Run manifests record enough resolved pipeline, module, source, reference, and
LLM provenance to make a run auditable after configuration changes. Manifests
record identities and summaries rather than secret or large payload content.
## Dependencies
## State, Output, And Safety
Prefer the Go standard library where practical.
Durable output, diagnostics, checkpoints, and debug artifacts are separate
surfaces with separate ownership:
Use external dependencies only when justified by correctness, security,
interoperability, or substantial complexity reduction. Good reasons include
widely used file formats, complex validation behavior, or secure transport
handling.
- output modules define logical durable output; the application boundary owns
filesystem placement;
- diagnostics provide redacted run inspection and are not the durable output
contract;
- checkpoints support validated stage reuse and are not diagnostics;
- debug artifacts are opt-in inspection data and may contain sensitive source,
prompt, reference, and model-output content.
Avoid dependencies for small conveniences. Do not let external dependency types
leak across internal package boundaries unless the dependency is itself the
explicit contract of that package.
Writes of durable state are atomic where practical. Paths for writes, moves,
overwrites, and deletion must be narrow and explicit. Cleanup that can lose data
is opt-in.
## State, Files, and Safety
Secrets must not appear in errors, logs, diagnostics, manifests,
documentation, examples, or redacted configuration. Default logs and
diagnostics must not include large source, prompt, reference, or artifact
payloads.
If the application writes durable state, writes should be atomic where
practical. Multi-step workflows should preserve enough diagnostics to support
inspection after failure.
## Architectural Non-Goals
Code that deletes, moves, or overwrites files must use narrow, explicit paths.
Avoid broad parent-directory operations. Cleanup that can cause data loss must
be opt-in.
Notarius does not aim to provide:
## Errors and Logging
Errors should be actionable and preserve context. Wrap errors with operation and
path or resource context. CLI code should convert internal errors into concise
user-facing messages.
Errors and logs must not expose secrets. Logs should describe operations,
external calls, retries, and failure causes, but should not include large source
or artifact payloads by default.
Long-running operations should accept `context.Context`. External calls,
subprocesses, HTTP requests, storage operations, LLM calls, and multi-stage
workflows should respect cancellation and timeouts.
## Testing
Core logic should be testable without real external services. Use fakes,
fixtures, or local test doubles for input modules, extract modules, validators,
and LLM clients where practical.
Contract-first work should include fake implementations that prove interfaces
compose before real modules depend on them.
Maintain a fixture-driven walking skeleton that exercises the full pipeline with
fake modules and fake external clients. This protects stage composition as real
modules evolve.
Important CLI and configuration workflows should have tests. Adapter, extractor,
validator, and stage contracts should have focused tests that do not require
running the full application unless end-to-end coverage is intentional.
## Documentation
Documentation should follow the project documentation policy. Keep user docs
focused on implemented behavior. Put future, planned, or aspirational work only
under `docs/roadmap/`.
Core documentation should use generic terms such as source document, source
unit, source reference, input adapter, extractor, chunker, merger, normalizer,
artifact, validator, and run manifest.
Source-format details belong in input module or integration docs.
Domain-specific extraction details belong in extract module or artifact docs.
When changing architecture, config, CLI behavior, stage modules, extractor
contracts, validator contracts, LLM runtime behavior, or artifact schemas, update
the relevant docs and examples in the same change.
- an arbitrary workflow graph or general workflow language;
- source-format or extraction-domain behavior in generic framework packages;
- provider-specific contracts exposed to modules;
- structural pipeline composition through ad hoc CLI flags;
- implicit cross-stage behavior that bypasses the fixed pipeline;
- abstractions introduced solely for hypothetical future complexity.

View File

@@ -1,139 +0,0 @@
# Development
This document defines contributor workflow for Notarius. For architectural
invariants and package boundaries, read [Architecture](architecture.md) first.
## Required Reading
Before changing the repository, review:
- [Architecture](architecture.md)
- [Documentation Policy](documentation.md)
Keep current-behavior documentation limited to implemented behavior. Put planned
or deferred behavior under `docs/roadmap/`.
## Repository Layout
- `cmd/notarius`: executable entry point.
- `internal/cli`: CLI parsing, production catalog wiring, config loading, run
command orchestration, output writes, and user-facing errors.
- `internal/core`: deterministic models and policy for artifacts, source
documents, config, and diagnostics.
- `internal/framework`: reusable contracts, pipeline orchestration, prompt
helpers, validation helpers, and LLM runtime plumbing.
- `internal/modules`: concrete input, chunk, extract, merge, normalize, and
output modules.
- `docs`: policy, user/operator docs, internal docs, integration docs, and
roadmap files.
- `examples`: maintained, secret-free examples covered by tests where practical.
## Validation Commands
Run focused tests for the area changed, then run the broader checks when the
change affects shared contracts, CLI behavior, or documentation examples.
```sh
go test ./...
go vet ./...
go build ./cmd/notarius
```
Useful focused checks:
```sh
go test ./internal/cli
go test ./internal/core/config
go test ./internal/framework/pipeline
go test ./internal/framework/llm
go test ./internal/modules/input/seriatim
go test ./internal/modules/extract/dnd/spells
go test ./internal/modules/output/json
```
## Go Conventions
- Prefer the standard library unless a dependency is justified by correctness,
security, interoperability, or substantial complexity reduction.
- Keep package names short, lowercase, and idiomatic.
- Preserve import direction: framework and core code must not depend on concrete
production modules.
- Use `context.Context` for long-running operations and external calls.
- Return contextual errors that name the operation and relevant module, path, or
resource.
- Do not include secrets in errors, logs, diagnostics, manifests, or docs.
## Adding Config Fields
Config behavior is centralized under `internal/core/config`.
When adding a file config field:
1. Update file config structs and YAML parsing in `file_config.go`.
2. Apply the field over defaults in config application code.
3. Add validation in `validation.go` when the field has constraints.
4. Add environment override support in `env.go` only for operational overrides.
5. Update redaction if the field can contain secrets.
6. Add focused config tests.
7. Update [Configuration](../config.md) and maintained examples when behavior
changes.
Pipeline composition should remain config-driven. Do not add command flags that
silently replace structural pipeline definitions.
## Adding CLI Flags Or Commands
CLI behavior lives in `internal/cli`.
When adding CLI surface:
1. Keep syntax explicit and update usage text.
2. Validate arguments before running expensive work.
3. Convert internal errors into concise user-facing messages.
4. Add CLI tests for success, syntax errors, and failure modes.
5. Update [CLI Reference](../cli.md), and update
[Operations](../operations.md) or [Troubleshooting](../troubleshooting.md)
if run behavior changes.
## Adding Modules Or Adapters
Concrete modules live under `internal/modules/<kind>/...` and implement the
interfaces in `internal/framework/contracts`.
For a new production module:
1. Implement the relevant contract.
2. Expose a `ModuleSpec` with the correct module key, module kind, provided
capabilities, and required capabilities.
3. Expose a `Register` function that registers the module with its registry.
4. Add focused module tests for contract behavior, registration, options,
validation, and errors.
5. Register the module in `internal/cli/catalog.go` only when it is production
ready.
6. Update internal docs and user-facing docs only for implemented behavior.
Source-format behavior belongs in input modules and integration docs.
Extraction-domain behavior belongs in extract modules and artifact docs.
## Updating Examples
Examples must be valid, secret-free, and small.
- Prefer environment-based secret configuration.
- Keep `examples/dnd-spells.config.yml` loadable by CLI tests.
- Keep `examples/seriatim-minimal-transcript.json` compatible with the Seriatim
adapter.
- Do not add expected-output fixtures unless they are validated or have a clear
regeneration procedure.
## Documentation Updates
Update docs in the same change when behavior changes.
- CLI syntax: `docs/cli.md`
- Config fields and defaults: `docs/config.md`
- Output, diagnostics, retention, or recovery: `docs/operations.md`
- Common user-facing failures: `docs/troubleshooting.md`
- Internal architecture and contracts: `docs/internal/`
- External file formats and durable integration contracts: `docs/integrations/`
- Future or planned work only: `docs/roadmap/`

View File

@@ -1,446 +1,143 @@
# Go Project Documentation Policy
# Documentation Policy
## Purpose
Project documentation must help five audiences:
1. users who need to run the application;
2. administrators/operators who need to configure and operate it;
3. developers who need to understand and change it safely;
4. LLM coding agents that need clear scope, boundaries, and invariants;
5. developers and LLM coding agents integrating this project from another codebase.
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
This policy assigns each documentation topic to one canonical owner. Its goal is
to keep Notarius documentation accurate, concise, discoverable, and resistant
to drift for users, operators, developers, integrators, and LLM coding agents.
## Core Rules
### 1. Keep docs concise
### One Canonical Owner
Each document should cover a defined scope and only the essentials for that scope.
Each authoritative fact belongs in one document. A non-owning document may give
a short, stable summary for orientation, but it must link to the canonical owner
instead of repeating volatile details.
Avoid:
- long background explanations;
- repeated reference material;
- implementation detail in user-facing docs;
- aspirational language outside roadmap docs;
- verbose examples where one minimal example is clearer.
Volatile details include commands, flags, configuration fields and defaults,
module keys, schemas, file names, paths, status codes, retry behavior, and
runtime guarantees. If readers could reasonably treat a statement as a
contract, maintain it only in the owning document.
### 2. Document only implemented behavior outside roadmap files
### Current And Future Behavior
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
Outside `docs/roadmap/`, documentation describes implemented behavior only.
Partial features may be described only to their implemented boundary.
ADRs are the narrow exception: an ADR may record an accepted architectural
decision before implementation, but acceptance must not be presented as proof
that the behavior exists. The roadmap owns implementation status and sequencing
until the decision is implemented. Current architecture, user, operator,
integration, and internal documentation are updated when the behavior lands.
- `docs/roadmap/`
### Audience And Detail
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
Write for the document's stated audience and include only the detail needed for
its owned topic. User and operator docs should not expose implementation detail.
Developer docs should link to user-facing and external contracts rather than
restate them.
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
### Examples
### 3. Use canonical homes
Each type of information should have one canonical location.
Canonical homes:
- project purpose and quickstart: `README.md`
- development principles: `docs/policy/architecture.md`
- public HTTP API reference: `docs/api.md`
- configuration reference: `docs/config.md`
- CLI reference: `docs/cli.md`
- operations and recovery: `docs/operations.md`
- troubleshooting: `docs/troubleshooting.md`
- public API/package consumer guidance: `docs/consumers/`
- implemented internals: `docs/internal/`
- external protocol, service, and file-format contracts: `docs/integrations/`
- future work: `docs/roadmap/`
- contributor workflow: `docs/policy/development.md`
- copyable examples: `examples/`
Other files should summarize briefly and link to the canonical source.
### 4. Keep examples real
Examples should be valid, maintained, and free of secrets.
Where practical:
- example configs should load successfully;
- example commands should match real CLI syntax;
- important examples should be covered by tests.
## Documentation Profiles
All projects require:
- `README.md`
- `docs/policy/architecture.md`
Additional docs depend on the project.
### Small library
Recommended:
- `docs/policy/development.md`, if contributor conventions are non-obvious
### Simple CLI
Required:
- `docs/cli.md`
Recommended:
- `docs/policy/development.md`
### Config-driven CLI
Required:
- `docs/cli.md`
- `docs/config.md`
Recommended:
- `examples/`
- `docs/policy/development.md`
### Stateful or operator-facing application
Required:
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
Recommended:
- `docs/troubleshooting.md`
- `examples/`
- `docs/policy/development.md`
### Modular, service-oriented, or orchestration application
Required:
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
Recommended:
- `docs/troubleshooting.md`
- validated examples under `examples/`
### Public HTTP API service
Required:
- `docs/api.md`
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
Recommended:
- `docs/troubleshooting.md`
- `docs/consumers/`, for task-oriented client integration guides
- `docs/integrations/`, for upstream/downstream service contracts
- validated examples under `examples/`
### Project with public packages or consumer APIs
Required:
- `docs/consumers/api.md`
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
Recommended:
- copyable consumer examples under `examples/`, if practical
## Required Documents
### README.md
**Audience:** users, administrators, operators
The README is the outward-facing project orientation page.
It should include, in order:
1. concise description;
2. elevator pitch;
3. shortest useful command or usage example;
4. links to targeted docs.
The README should be short. It is not a manual.
The “shortest useful command” means the simplest command that performs the projects core use case. (It does not mean `app --help`.)
### docs/policy/architecture.md
**Audience:** developers, LLM coding agents
`docs/policy/architecture.md` is required for every project.
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
It should include:
- project shape;
- core design principles;
- package and boundary philosophy;
- state/persistence philosophy, if applicable;
- external integration philosophy, if applicable;
- error-handling and logging principles;
- testing expectations;
- documentation expectations;
- architectural invariants;
- explicit non-goals, if useful.
Notably, this file should prescribe a core development *policy* that should remain unchanged as the application evolves. It is not a place for details (e.g., CLI flags) that could change over time.
The contents of `architecture.md` should be trim and concise. LLMs may be directed to review it routinely via AGENTS.md, CLAUDE.md, or similar.
### docs/api.md
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
Required for projects whose primary public interface is HTTP.
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
It should include:
1. base URL conventions;
2. authentication and authorization behavior, if implemented;
3. response envelope;
4. supported media types and content negotiation behavior;
5. shared query parameters;
6. endpoint reference grouped by route family;
7. request parameters and validation rules;
8. response fields, units, nullability, and optionality;
9. error response shape and status codes;
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
11. compact request and response examples.
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
### docs/policy/development.md
**Audience:** developers, LLM coding agents
Required for projects maintained by humans and LLM coding agents.
It should include:
- repository layout;
- build/test commands;
- coding conventions;
- dependency policy;
- how to add config fields;
- how to add CLI flags;
- how to add modules or adapters, if applicable;
- how to update examples;
- documentation update expectations.
### docs/config.md
**Audience:** administrators, operators, advanced users
Required for applications with configuration files.
It should include, in order:
1. config file locations and discovery precedence;
2. minimal working config;
3. production-oriented config;
4. full configuration reference;
5. secrets handling, if applicable;
6. links to maintained examples.
The full configuration reference should be canonical.
### docs/cli.md
**Audience:** users, administrators, operators
Required for CLI applications.
It should include, in order:
1. shortest useful command;
2. command overview;
3. complete flag reference;
4. common workflows;
5. diagnostic or recovery commands, if applicable.
Explain when commands are useful, not just their syntax.
### docs/operations.md
**Audience:** administrators, operators
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
It should cover:
- normal workflow;
- filesystem layout;
- remote storage layout, if applicable;
- logs and manifests;
- resume/retry behavior;
- cleanup behavior;
- archive/backup behavior;
- safe recovery procedures;
- operational caveats.
### docs/troubleshooting.md
**Audience:** administrators, operators
Recommended once recurring failure modes exist.
Each entry should include:
- symptom;
- likely cause;
- diagnostic command or inspection step;
- safe fix;
- relevant links.
### docs/consumers/
**Audience:** developers and LLM coding agents integrating this project from another codebase
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
1. intended consumer audience and use cases;
2. required inputs supplied by operators or deployment configuration;
3. recommended public package or API workflow;
4. minimal copyable example;
5. consumer responsibilities and boundaries;
6. retry, idempotency, or status behavior, if applicable;
7. links to package-specific docs and canonical integration contracts.
Package-specific docs should be named `pkg-<name>.md` and should include:
1. import path;
2. intended use cases;
3. primary types and functions needed by consumers;
4. minimal examples;
5. validation, error, retry, and boundary behavior;
6. links to canonical file-format or wire-protocol contracts.
### docs/internal/
**Audience:** developers, LLM coding agents
Required for modular, service-oriented, or orchestration projects.
This directory describes implemented internal components. It is not the roadmap.
Use one file per major component where useful.
Each component doc should include:
1. purpose;
2. inputs and outputs;
3. boundaries;
4. config fields used;
5. external adapters used;
6. state or manifest behavior, if applicable;
7. skip/resume behavior, if applicable;
8. failure behavior;
9. tests to inspect before changing;
10. architectural invariants.
### docs/roadmap/
**Audience:** maintainers, developers, LLM coding agents
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
Roadmap docs should clearly distinguish:
- proposed work;
- accepted plans;
- deferred ideas;
- rejected ideas;
- implementation prompts or task breakdowns, if useful.
Roadmap docs should not be confused with current behavior.
### docs/integrations/
**Audience:** developers, LLM coding agents
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
Use one file per integration where useful.
## Examples Directory
Projects with non-trivial configuration or workflows should include `examples/`.
Useful examples include:
- minimal working config;
- production-oriented config;
- full annotated config;
- local development config;
- remote/object-storage config;
- minimal session/input file.
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
## Security and Privacy
Docs and examples must not include:
- real API keys;
- tokens;
- passwords;
- private keys;
- private environment dumps;
- sensitive user data;
- raw private transcripts;
- private infrastructure details unless intentionally public.
Document secret-handling mechanisms, not actual secret values.
## Maintenance Rules
When docs change, verify the affected behavior.
Where practical:
- load example config files in tests;
- test CLI examples or command parser behavior;
- validate documented flags against real flags;
- remove stale references;
- update links after renames;
- keep roadmap content out of non-roadmap docs.
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
Documentation is complete only when it matches the current code.
## Documentation Change Checklist
Before merging documentation changes, verify:
- README is concise and orientation-focused.
- `docs/policy/architecture.md` describes development principles.
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
- Future work appears only under `docs/roadmap/`.
- User-facing docs avoid unnecessary internals.
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
- Developer-facing docs preserve boundaries and invariants.
- Config examples match the schema.
- CLI examples match real commands and flags.
- Defaults appear in the canonical config reference.
- No secrets or private data are included.
- Links are accurate.
Complete copyable files belong in `examples/`. Documentation may use the
smallest illustrative snippet needed to explain its owned topic, but should link
to maintained examples instead of embedding a second complete copy.
Examples must be valid, secret-free, and tested where practical. Commands and
configuration used in documentation should match the application.
### Security And Privacy
Documentation and examples must not contain real credentials, private keys,
private environment dumps, sensitive source material, or private infrastructure
details unless intentionally public. Document secret-handling mechanisms, not
secret values.
## Canonical Ownership
| Topic | Canonical owner | Owned content | Content owned elsewhere |
| --- | --- | --- | --- |
| Product orientation and minimal end-to-end quickstart | `README.md` | What Notarius is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, diagnostics use, retention, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
| Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
Documents that do not exist are required only when the corresponding interface
or responsibility exists. Do not create placeholder API, consumer, integration,
or operations documents for behavior the application does not have.
## Boundary Rules
### Orientation
The README owns product orientation. The developer guide routes contributors.
Architecture owns normative structure. Internal overview owns the current
concrete component map. These documents may link to one another but should not
maintain parallel package or behavior descriptions.
### Commands, Configuration, And Operations
CLI documentation answers how to invoke the application. Configuration
documentation answers what settings mean. Operations answers what happens to
runtime state and how to operate or recover the application. When a workflow
crosses these topics, choose the document that owns the task and link to the
other contracts.
### Contracts And Implementation
Integration and API documents define externally observable shapes and
semantics. Internal documents explain how Notarius implements or consumes those
contracts. Internal docs may name a field, file, or protocol to identify a
dependency, but must link to its canonical contract for the definition.
### Security Topics
This policy owns what documentation and examples may contain. Architecture owns
application security invariants. Configuration owns credential-supply
mechanisms. Operations owns permissions and handling of sensitive runtime
artifacts. Internal docs own implementation mechanisms only.
## Architecture Decision Records
Use sequentially numbered ADR filenames such as
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
1. title;
2. status;
3. date;
4. context;
5. decision;
6. alternatives considered;
7. consequences.
Treat the decision content of an accepted ADR as immutable. When a decision
changes, create a new ADR and update the earlier ADR's status to superseded.
Rejected architectural alternatives belong in the ADR; rejected product ideas
belong in the roadmap.
## Maintenance
When behavior changes, update its canonical owner in the same change. If
ownership moves, remove the old definition and replace it with a link where
navigation remains useful.
Before completing documentation work:
- verify affected behavior and examples;
- check commands, flags, fields, defaults, schemas, and paths against their
implementation;
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
- remove stale references and validate links;
- confirm that non-owning documents summarize and link rather than redefine;
- confirm that no secrets or sensitive private data were added.

348
docs/roadmap/domain.md Normal file
View File

@@ -0,0 +1,348 @@
# Domain-Typed Pipeline Feature Roadmap
## Status
Implemented on 2026-07-17. This roadmap records the design delivered for
[ADR-0002](../adr/0002-linear-pipes-and-filters-pipeline.md),
[ADR-0003](../adr/0003-typed-interfaces-with-two-zone-data-model.md), and
[ADR-0004](../adr/0004-package-modules-by-domain.md). Its implementation history
is summarized in the [completion record](implementation.md).
This file is historical design context, not a current-behavior reference.
Implemented contracts and mechanics are defined by the architecture,
configuration, integration, operations, and internal documentation outside
`docs/roadmap/`.
## User Intent
Notarius should remain a small, explicit pipes-and-filters application while
making domain extensions safe to compose and straightforward to maintain. A
configured pipeline should fail before execution when its modules are
incompatible, should carry typed domain values rather than reparsed JSON between
artifact stages, and should preserve provenance and durable output contracts.
The application must also enforce one configurable, process-wide ceiling on
in-flight LLM calls. Pipeline scheduling may impose stricter limits, but no
stage, lane, retry, validator, or future LLM-backed extension may bypass that
global ceiling.
## Target Architecture
### Pipeline and outcome model
The topology remains:
```text
input -> chunk -> extract -> merge -> normalize -> output
```
Input and chunk are pipeline-wide. Extract, merge, normalize, and their
validators operate per artifact lane. Output aggregates the terminal artifacts
from all lanes. The resolved pipeline retains explicit fields for those roles;
it does not become a general DAG or a heterogeneous ordered-stage list.
Framework errors abort the run. Validator rejection is a recorded domain
outcome and does not abort unrelated work. Accepted extract results reach merge
in source-chunk order, regardless of execution completion order. Warnings,
rejections, artifacts, and reported errors are likewise ordered by stable
pipeline scope rather than goroutine completion time.
### Engine-owned source model
The engine owns `source.SourceDocument`, `source.SourceUnit`, `source.SourceRef`,
and `source.Chunk`. Domain modules may consume these types but must not redefine
their provenance semantics.
- Every source unit has a canonical self-reference identifying its source and
unit range.
- A chunk contains ordered source units and one canonical reference spanning
its first through last unit.
- Chunk and unit references are validated for source identity, order, and
containment.
- Auxiliary reference material remains distinct from source provenance.
- Cloning, canonicalization, checkpointing, debugging, and digest computation
preserve the source references exactly.
`source.Chunk.Ref` replaces duplicate start/end boundary fields. Because that
changes persisted workspace state, the workspace checkpoint schema advances to
`notarius.workspace.v2`. Existing v1 checkpoints are left intact but treated as
incompatible and recomputed; no in-place migration or deletion is required.
### Typed artifact lanes
Each lane has one canonical artifact type `T` from extraction through merge,
normalization, and typed validation. Module-facing Zone-B contracts are generic:
- `Extractor[T]` produces typed per-chunk values plus framework-owned
provenance and diagnostics;
- `Merger[T]` combines accepted values in source-chunk order;
- `Normalizer[T]` canonicalizes the merged value; and
- `TypedValidator[T]` applies semantic checks at its configured artifact stage.
The framework may use private erased adapters to keep heterogeneous lanes in
one resolved pipeline, but `any`, raw JSON, and a generic `Process(any)` API are
not module-facing handoffs. The extractor selected for a lane establishes its
artifact kind. Resolution uses that kind to select compatible merger,
normalizer, validator, and codec variants and rejects an incompatible lane
before any stage executes.
Generic strategies remain reusable without knowing concrete domains. In
particular, append-order merge is parameterized by a typed combine function
provided during domain registration, and no-op normalization is instantiated
for the lane's concrete type.
### Artifact identity and codecs
Every typed artifact kind has exactly one registered `ArtifactCodec[T]`. An
artifact kind is a stable logical identifier, separate from a Go type name. A
codec owns:
- artifact kind;
- schema identifier, name, and version;
- media type and JSON Schema bytes; and
- strict, deterministic encoding and decoding between `T` and the serialized
representation.
Equal canonical values must encode to equal bytes. Those bytes are the basis
for artifact digests. Codec decoding rejects malformed or schema-incompatible
content. Domain validators continue to own semantic validity; codecs do not
replace them.
`SerializedArtifact` is the Zone-C representation and includes the artifact
kind, schema metadata, media type, encoded content, and framework metadata.
Type erasure occurs through the codec after normalization for final output.
Intermediate checkpointing and opt-in debug recording may also use the codec,
but serialization for those side effects is not a stage handoff.
Checkpoint metadata records artifact kind, schema identifier, schema version,
and schema digest. Reuse requires an exact compatible registered codec;
otherwise the checkpoint is safely invalidated. Output remains domain-neutral
and consumes serialized artifacts.
Generic serialized validators remain supported for representation-level checks
such as valid JSON and JSON Schema validation. The framework encodes `T` through
its registered codec before invoking them. Domain validators receive `T`
directly. Chunk-stage validators remain in the source zone: semantic chunk
validators receive engine-owned chunks, while representation-level validators
receive the framework's canonical serialized chunk view. They do not force
source-zone values through a domain artifact codec.
### Registration and resolution
Registries expose typed registration helpers while privately retaining the Go
type identity needed to assemble erased lane executors.
- Codecs are keyed by artifact kind, with exactly one codec per kind.
- Extractors are keyed by their existing module key and declare an artifact
kind.
- Mergers, normalizers, and validators are keyed by `(module key, artifact
kind)`, allowing stable generic keys such as `appendorder` and `noop` to have
multiple typed specializations.
- Resolved pipeline identity and dependency fingerprints include artifact kind
and schema identity, version, and digest.
- Duplicate or incompatible registrations and selections fail deterministically
during composition or resolution.
The framework's public typed registration surface uses free generic functions,
because Go methods cannot declare their own type parameters. Private reflection
may verify and erase registered types, but it is not exposed to module authors.
### Preparation, options, and dependencies
Pipeline execution is split into resolution, preparation, and running.
Preparation constructs every selected module and validator before source input
begins and returns a prepared pipeline with explicit input, chunk, lane, and
output fields.
- Construction receives framework-owned dependencies, including the one shared
scheduled structured-LLM client.
- Raw configured options are decoded once into implementation-owned option
structs during preparation.
- Missing dependencies, malformed options, unknown options, and incompatible
typed selections fail before stage execution.
- Configuration validation uses the same option decoders without requiring live
provider dependencies.
- Per-run data such as sources, chunks, references, session identity, lane
identity, and metadata remains in operation requests.
- Constructed implementations that can be scheduled concurrently are immutable
after preparation or otherwise explicitly concurrency-safe.
Modules and validators must not construct provider clients, wrap their own
independent schedulers, or bypass the injected scheduled client.
### D&D artifact model
The D&D package root owns canonical `dnd.SpellList`, `dnd.SpellCast`, and
related evidence types, using engine-owned `source.SourceRef` values. The spell
extractor keeps its LLM response DTO and response schema private and maps the
canonicalized response to the domain model.
The D&D spell codec separately owns the existing durable spell artifact schema.
The LLM response schema and durable artifact schema remain distinct contracts
even if their current JSON shapes are similar. Shape, source-reference, and
source-relatedness validators operate on the canonical typed model. The
validator-only duplicate spell model and inter-stage JSON reparsing disappear.
The migration preserves the existing D&D spell payload, logical output bundle,
module and validator keys, prompt/schema identities, default validator chains,
warnings, rejection semantics, and manifest provenance unless a separate
compatibility decision explicitly changes one of those contracts.
## Package Ownership
The target production extension layout is:
```text
internal/modules/dnd/
types.go
codec/spells/
chunk/scenes/
extract/spells/
validate/spells/shape/
validate/spells/source_refs/
validate/spells/source_relatedness/
shared/
register/
internal/modules/generic/
chunk/units/
merge/appendorder/
normalize/noop/
validate/always_accept/
validate/always_reject/
validate/valid_json/
validate/valid_json_schema/
output/json/
register/
internal/modules/seriatim/
input/transcript/
register/
internal/framework/promptfs/
```
Shared domain types live at the domain root. Registration lives in a sibling
`register` package so that the root never imports child implementations. Each
registrar exposes one composition entry point accepting the pipeline registry
set and LLM asset registry. The CLI composition root creates those registries
and invokes the generic, Seriatim, and D&D registrars.
Concrete domain implementations do not import peer domains. Generic extensions
never import a concrete domain. A domain registrar may import generic packages
to register typed specializations for its domain. The application composition
root and designated black-box integration tests may compose multiple
registrars. Domain-neutral embedded prompt-asset filesystem support belongs to
the framework rather than a domain package.
## Concurrency Policy
### Configuration
The existing `concurrency.total_llm` setting remains the application-wide
ceiling on actual provider calls. Version-2 configuration gains an extensible
stage-worker map:
```yaml
concurrency:
total_llm: 4
stage_workers:
extract: 4
```
Initially, `extract` is the only recognized key. Unknown stage keys are rejected
so misspellings cannot silently alter scheduling. If omitted, the effective
extract worker count equals `total_llm`. Its valid range is
`1..concurrency.total_llm`. The environment override is
`NOTARIUS_STAGE_WORKERS_EXTRACT`; future stage overrides receive similarly
explicit names that map to the extensible file representation.
The worker setting bounds framework jobs, not provider calls. Only an actual
LLM call consumes a permit from the shared scheduled client. The global
scheduled client remains authoritative even if future stages gain worker limits.
### Scheduling
After pipeline-wide chunking, all lanes may run concurrently. A central
dispatcher submits `(lane, chunk)` jobs to one run-wide extract worker pool in
round-robin order: source chunk first, then resolved lane order. This avoids one
unbounded goroutine per job and prevents an early lane from monopolizing the
queue.
One job contains extraction, its stage-local retry behavior, and extract-stage
validation for that lane and chunk. Each lane begins its serial merge then
normalize continuation when all of its extract jobs reach a terminal state.
Different lanes' continuations may overlap, and any LLM-backed continuation or
validator still shares the global scheduled client.
Workers publish immutable task results to a coordinator. Only the coordinator
mutates aggregate results, manifests, checkpoint indexes, warnings, and
rejections. Debug artifacts use attempt-specific paths and do not rely on
concurrent writes to shared files.
Rejections do not cancel work. A framework error cancels the derived run
context, stops undispatched jobs, and waits for started jobs to finish or
observe cancellation. If the parent context was canceled, its error is
returned. Otherwise, internal cancellation errors are ignored when at least one
real framework error exists, and the primary returned error is selected from
all started-task framework errors by this stable ordering:
1. stage order: extract, merge, then normalize;
2. resolved lane order;
3. source chunk index for chunk-scoped work; and
4. configured validator or operation order within that scope.
The full per-task errors may be retained in opt-in diagnostics, but completion
timing never chooses the public error. Output runs only after every lane reaches
a successful or rejection-only terminal state and no framework error exists.
Extractors and any validator instance callable by multiple workers must be safe
for concurrent use. Production implementations should normally satisfy this by
being immutable after preparation.
## Compatibility and Safety
- Existing production module keys, validator keys, profiles, default chains,
and maintained configurations continue to resolve.
- Existing durable D&D JSON content and logical output paths remain unchanged.
- Framework errors, validator rejections, retries, checkpoints, diagnostics,
and debug behavior retain their current semantics except for the explicitly
documented workspace-v2 compatibility boundary and deterministic concurrent
ordering.
- All provider calls pass through the shared global scheduler, across lanes,
stages, retries, and validators.
- Source text and LLM payloads remain subject to the existing opt-in debug and
sensitive-data handling policies.
- Package moves do not create user-visible key changes or concrete cross-domain
dependencies.
## Non-Goals
This feature does not introduce:
- a general DAG or configurable stage topology;
- out-of-process plugins or an RPC extension protocol;
- cross-lane normalization;
- a new durable D&D spell schema merely to mirror internal Go types;
- per-stage LLM permit pools that could exceed or partition the global ceiling;
or
- concurrent work implemented through an unbounded goroutine per lane or chunk.
## Completion Criteria
The target state is reached when:
- all production lanes use one typed artifact from extract through normalize
and typed validation;
- codecs own schema-aware serialization at every type-erasure, checkpoint, and
debug boundary;
- incompatible lane composition and invalid options fail before source work;
- source units and chunks carry validated canonical provenance;
- production extensions follow the domain-first package and registrar rules;
- extract scheduling is bounded, deterministic, concurrent across lanes, and
race-free;
- instrumented tests prove actual concurrent LLM calls never exceed
`concurrency.total_llm` across all callers;
- the maintained D&D example and compatibility baselines retain their durable
contracts; and
- current-behavior documentation is updated as each implemented boundary lands.

50
docs/roadmap/future.md Normal file
View File

@@ -0,0 +1,50 @@
# Future Work
Current Notarius behavior is documented in the canonical README, CLI,
configuration, operations, internal, and integration docs. This roadmap records
future work only.
## Candidate Product Work
- Additional input adapters, such as Markdown or note-export formats.
- Additional D&D extractors beyond spell casts.
- Add non-file reference producers, such as prior-run artifacts, derived
summaries, or entity registries, without changing module-facing reference
item contracts.
- Add token budgeting and model context-window management for reference content.
- Add per-slot or per-chunk inclusion policies so modules can avoid repeating
large reference content in every prompt when that becomes important.
- Add structured or parsed references, such as typed roster schemas, when a
module has a clear need for more than opaque UTF-8 text.
- Add reference caching, preprocessing, summarization, embedding, or retrieval
if references become large enough to require preprocessing.
- Cross-lane entity normalization.
- Cross-chunk semantic deduplication.
- Additional validator packages and production default chains for future
modules.
- Production LLM-backed validators when there is a concrete review policy that
benefits from model judgment.
- Validator diagnostics and timing summaries if operators need more detail than
the current [durable output bundle](../integrations/json-output.md) provides.
- Media-type validators for non-JSON module outputs when such modules are
introduced.
- Validator compatibility metadata if real deployments need config-time
enforcement that a validator is suitable for a specific stage or module.
- Batching or context-window controls for LLM-backed validators if validator
inputs become large enough to require them.
- Additional output encoders.
## Candidate Operational Work
- Packaged release artifacts for alpha distribution.
- A documented versioning and release process.
- Optional generated example output fixtures with a regeneration procedure.
- Additional diagnostics or reporting views if operator workflows need them.
## Candidate Workspace Work
- Default-idempotent run behavior with an explicit force override.
- Remote workspace storage.
- Workspace garbage collection.
- Workspace archival policy.
- Cross-machine checkpoint reuse.

View File

@@ -0,0 +1,453 @@
# Domain-Typed Pipeline Implementation Roadmap
## Status
The domain-typed pipeline was implemented on 2026-07-17. A subsequent review
identified five follow-up issues. The stages below are the decision-complete
implementation plan for resolving them.
Implement the stages in order. Keep each stage independently reviewable and
leave the repository passing its full validation suite before beginning the
next stage. Unless a stage explicitly says otherwise, preserve public CLI,
configuration, module-key, validator-key, durable output, and checkpoint file
contracts.
Current architectural policy is defined by
[Architecture](../policy/architecture.md). The design context remains in the
[feature roadmap](domain.md) and
[ADR-0002](../adr/0002-linear-pipes-and-filters-pipeline.md),
[ADR-0003](../adr/0003-typed-interfaces-with-two-zone-data-model.md), and
[ADR-0004](../adr/0004-package-modules-by-domain.md).
## Delivered Baseline
The existing implementation already provides:
- domain-first production extensions and package-family registrars;
- engine-owned source provenance using `source.SourceRef` and workspace schema
v2;
- one domain-owned Go type through extraction, merge, normalization, and typed
validation;
- schema-aware artifact codecs at checkpoint, debug, and output boundaries;
- pre-execution typed resolution, option validation, and preparation;
- one shared scheduled LLM client; and
- bounded chunk-first, lane-second extraction with deterministic public
outcomes.
The following work corrects checkpoint identity, completes retry debugging,
removes kind-ambiguous registry lookup, strengthens architectural enforcement,
and removes the obsolete sequential extraction path.
## Implementation Rules
Apply these rules to every stage:
1. Read the task-specific documents listed in
[Development](../development.md) before changing the affected subsystem.
2. Add focused regression tests that fail against the pre-stage code and pass
after the change.
3. Preserve deterministic behavior. Do not base digests, public ordering,
reference resolution, or reported errors on Go map iteration or goroutine
completion order.
4. Keep framework-owned type erasure private. Do not reintroduce module-facing
`any`, raw JSON handoffs, or a generic processing interface.
5. Update current-behavior documentation in the same stage when externally
observable or documented internal behavior changes. Follow
[Documentation Policy](../policy/documentation.md); do not describe a later
stage as already implemented.
6. Run focused tests while iterating. At the end of every stage, run:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./cmd/notarius
git diff --check
```
## Stage 1: Include Resolved Validator Policy In Pipeline Identity
### Goal
Ensure that every effective validator-chain change alters the resolved
pipeline digest and therefore prevents reuse of checkpoints created under a
different validation policy.
### Required Changes
1. Extend `resolvedPipelineDigest` in
`internal/framework/pipeline/profile.go` to include the complete resolved
validator-chain collection. Include, for every chain:
- stage, lane ID, and owning module key;
- validator order;
- each complete resolved binding, including module key, LLM profile,
retries, options, and references;
- validator execution class;
- validator target; and
- artifact kind.
2. Hash the already canonical `ResolvedPipeline.ValidatorChains` order produced
by resolution. Do not independently sort validators or otherwise weaken
configured order. Continue excluding only the digest field itself.
3. Use the same deterministic JSON-and-SHA-256 mechanism as the existing
pipeline digest. Go map keys encoded inside bindings must retain the
deterministic ordering supplied by `encoding/json`.
4. Do not add validator fingerprints separately to individual stage
dependencies. Pipeline/workspace identity is the authoritative invalidation
boundary for validation-policy changes.
5. Do not advance the workspace schema version. The corrected digest naturally
selects a different checkpoint directory; existing checkpoints remain
intact and become reuse misses for the changed pipeline identity.
### Tests
Add focused tests under `internal/framework/pipeline` and, where useful,
`internal/core/workspace` proving that:
- registering a different default chain changes the resolved pipeline digest;
- adding, removing, or reordering a default validator changes the digest;
- changing a resolved validator binding option or LLM profile changes the
digest;
- changing execution class, target, or artifact kind changes the digest;
- resolving the same pipeline and catalog repeatedly produces the same digest;
- an explicit empty override produces an empty resolved chain and differs from
a non-empty inherited default; and
- a changed validator-chain digest produces a different checkpoint identity,
while an unchanged chain preserves it.
Retain the existing tests proving that the digest field itself is excluded and
that artifact schema identity participates in the digest.
### Documentation
Update the checkpoint invalidation description in `docs/operations.md` and the
identity description in `docs/internal/pipeline.md` to state concisely that the
effective resolved validator policy participates in pipeline identity.
### Completion Gate
A checkpoint accepted under one resolved default or explicit validator chain
must not be reusable after that chain changes.
## Stage 2: Complete Attempt-Scoped Debugging For Merge And Normalize
### Goal
Give merge and normalize retries the same attempt-level observability and
nested LLM-call association already provided for chunk and extract attempts.
### Required Changes
1. In `internal/framework/pipeline/runner_typed.go`, create an attempt-specific
debug context before invoking each merge or normalize module. Pass that
context to both the module operation and its validation chain.
2. Use these stable paths:
```text
merge/<lane-id>/attempt-<NN>.json
merge/<lane-id>/attempt-<NN>/prompt-<NNNN>.json
merge/<lane-id>/attempt-<NN>/response-<NNNN>.json
merge/<lane-id>/attempt-<NN>/response-content-<NNNN>.<ext>
normalize/<lane-id>/attempt-<NN>.json
normalize/<lane-id>/attempt-<NN>/prompt-<NNNN>.json
normalize/<lane-id>/attempt-<NN>/response-<NNNN>.json
normalize/<lane-id>/attempt-<NN>/response-content-<NNNN>.<ext>
```
Continue using two-digit retry attempt numbers and the existing debug path
sanitization and LLM call numbering behavior.
3. Write one module-attempt envelope for every attempted merge and normalize
operation:
- on success, record the codec-backed candidate artifact and warnings that
will be promoted if the attempt is accepted;
- on validator rejection, record the rejection without promoting discarded
warnings;
- on module, validation, serialization, or debug failure, record the error;
and
- in all cases, attach the LLM calls recorded in the module attempt scope.
4. Keep validator-specific debug scopes under the existing `validate/...`
hierarchy. A validator's LLM calls remain linked to its validator attempt;
the merge or normalize module envelope links only calls made by that module
attempt.
5. Preserve the existing stage-level `input.json` and `output.json` artifacts.
Checkpoint reuse should continue to emit stage-level artifacts but should
not synthesize retry attempts that did not execute.
6. Factor common attempt-envelope behavior into a small private helper where it
prevents chunk, extract, merge, and normalize instrumentation from drifting.
Do not introduce a new public runner abstraction solely for debugging.
7. Treat any debug write failure as a framework error, consistent with current
debug policy.
### Tests
Add tests with instrumented LLM-backed fake mergers and normalizers proving
that:
- first-attempt success writes the expected attempt and nested LLM artifacts;
- a failed first attempt followed by success writes two distinct attempt
envelopes and associates each call with the correct attempt;
- module errors, validator errors, and final rejection are represented in the
corresponding attempt envelope;
- validator LLM calls remain under validator paths rather than being attributed
to the module scope;
- discarded-attempt warnings are not promoted;
- checkpoint reuse produces no module-attempt files; and
- no merge or normalize call falls back to an unscoped stage-name debug path.
Extend the CLI debug integration test only as needed to verify the public debug
directory layout. Keep most behavioral coverage in the pipeline package.
### Documentation
Update `docs/operations.md` if necessary to show the stable merge and normalize
attempt paths. Update `docs/internal/pipeline.md` only where its implementation
description needs clarification; its existing attempt-scoping guarantee should
become fully true rather than be weakened.
### Completion Gate
Every executed merge and normalize retry must have a distinct debug envelope,
and every LLM call made by that module attempt must be nested under and linked
from that attempt.
## Stage 3: Make Typed Variant Spec Lookup Kind-Specific
### Goal
Ensure that CLI reference-target discovery and other behavior-sensitive lookup
select the merger or normalizer specification for the lane's actual artifact
kind, never an arbitrary map entry.
### Required Changes
1. Add explicit kind-specific lookup methods to `MergerRegistry` and
`NormalizerRegistry`, named:
```go
SpecForArtifact(key string, kind contracts.ArtifactKind) (ModuleSpec, bool)
```
Normalize the key and artifact kind in the same way as typed registration
and return a cloned spec.
2. Change CLI reference-target discovery in `internal/cli/run.go` to retain the
selected extractor's declared artifact kind and use it for merger and
normalizer spec lookup in that lane.
3. A missing typed variant must produce a deterministic error naming the
pipeline, lane, stage, module key, requested artifact kind, and sorted
registered kinds, matching the quality of full pipeline resolution errors.
4. Keep `Spec(key)` for kind-neutral catalog inspection and compatibility with
existing callers, but remove its map-order dependence. Select the first
registered artifact kind in sorted order before cloning its spec. Add a
comment making clear that behavior-sensitive code must use
`SpecForArtifact`.
5. Do not require typed variants under one reusable key to expose identical
reference slots or capabilities. Their kind-specific specs are allowed to
differ, and resolution must consistently choose the matching variant.
6. Audit all merger and normalizer `Spec` callers. Convert any caller making a
lane-specific decision to `SpecForArtifact`; leave only catalog or display
callers on the kind-neutral method.
### Tests
Register at least two artifact-kind variants under the same merger key and the
same normalizer key with intentionally different reference slots. Prove that:
- `SpecForArtifact` returns the correct cloned variant;
- lookup is stable regardless of registration order;
- CLI qualified and unqualified reference discovery uses the selected lane's
variant;
- a reference accepted by one variant is not incorrectly accepted for another;
- a missing variant reports sorted available kinds; and
- repeated `Spec(key)` calls return the same deterministic catalog result.
Retain coverage for the existing single-variant D&D catalog behavior.
### Documentation
Update `docs/internal/pipeline.md` or `docs/internal/modules.md` only if either
currently describes registry lookup mechanics. No CLI or configuration syntax
change is intended.
### Completion Gate
No behavior-sensitive merger or normalizer lookup may depend on map iteration,
and CLI reference resolution must use the artifact kind selected by the lane's
extractor.
## Stage 4: Enforce Domain Import Boundaries Generically
### Goal
Make the repository guard enforce ADR-0004 for present and future module
families without a hardcoded list of domain names or domain pairs.
### Required Changes
1. Refactor `internal/modules/import_boundaries_test.go` so that a module family
is derived from the first path segment under `internal/modules/`, rather than
recognized by a fixed `isDomain` list.
2. Treat `generic` as the domain-neutral reusable extension family. Treat every
other production family, including `dnd`, `seriatim`, and future families,
as concrete for import-boundary purposes.
3. Enforce these rules:
- a family root package must not import its own child implementation or
registrar packages;
- child packages within the same concrete family may import the family root,
shared helpers, or sibling implementations when needed;
- a concrete family must not import another concrete family;
- generic packages must not import concrete families;
- concrete implementation packages must not import generic implementation
packages directly;
- `internal/modules/<family>/register` may import its own family packages and
generic packages to compose typed strategies;
- the generic registrar may import generic child packages; and
- the application composition root and designated black-box integration
tests may compose multiple families.
4. Exclude `internal/modules/integration` from production-family discovery.
Preserve its exemption only for `_test.go` black-box composition files; do
not create a blanket exemption for production Go files.
5. Apply production import rules to white-box tests located in concrete module
packages. If an existing test composes concrete and generic implementations,
move that cross-family coverage to `internal/modules/integration` or replace
the foreign implementation with a package-local test double. Do not exempt
arbitrary `_test.go` files merely because they are tests.
6. Keep the test based on parsed Go imports. Do not add a new build tool or
external dependency for this guard.
### Tests
Expand the table-driven import tests to cover:
- a hypothetical future concrete family, demonstrating that no code change is
needed to enforce its boundaries;
- generic-to-concrete rejection for both current and hypothetical families;
- concrete-to-peer-concrete rejection;
- concrete implementation-to-generic rejection;
- concrete registrar-to-generic acceptance;
- family-root-to-child rejection;
- child-to-family-root and same-family sibling acceptance;
- application composition-root acceptance; and
- black-box integration-test acceptance without exempting non-test files.
Run the guard against the complete current repository and confirm that no
production package must be moved to satisfy it.
### Documentation
No ADR change is required. Update `docs/internal/modules.md` only if its package
boundary description needs to name the registrar-only generic composition
rule more clearly.
### Completion Gate
Adding a new directory under `internal/modules/<new-family>` must automatically
receive the same cross-family and registrar enforcement as current production
families.
## Stage 5: Remove The Obsolete Sequential Extraction Path
### Goal
Make the runner's structure match the implemented concurrent architecture:
the coordinator owns extraction, and a lane continuation begins from finalized
extract results and performs only merge and normalize work.
### Required Changes
1. Replace `runTypedLane` with a continuation-oriented private function whose
input explicitly contains the finalized extraction state needed by merge and
normalize:
- accepted typed extract artifacts;
- serialized checkpoint artifacts;
- accepted warnings;
- rejected outputs; and
- the original extract checkpoint decision.
Use a private struct if it keeps the call boundary clear and avoids a long
positional parameter list.
2. Change `continueLane` in `runner_concurrent.go` to pass the finalized state
directly. Do not make completed extraction look like checkpoint reuse.
3. Remove:
- `completedExtractLoader`;
- the `RunInput.extractDecision` private override;
- the sequential extraction branch formerly contained in `runTypedLane`;
and
- duplicate extraction retry, validation, checkpoint, and debug logic made
unreachable by concurrent coordination.
4. Keep extraction checkpoint loading and validation in `prepareLaneExtract`,
extract execution in `runExtractJob`, and deterministic final aggregation in
`finalizeLaneExtract`.
5. Preserve existing behavior exactly:
- extract checkpoint events report the real loader decision;
- reused and freshly computed extract results enter continuation through the
same typed state;
- accepted artifacts reach merge in chunk-index order;
- warnings and rejections retain stable ordering and promotion semantics;
- a lane with no accepted extracts follows the current merge behavior;
- stage-level extract input/output debug artifacts remain unchanged; and
- failure classification and cancellation retain their deterministic scope.
6. Keep merge and normalize serial within a lane and keep cross-lane
continuations bounded by the existing worker policy. Do not introduce a
goroutine per lane or chunk.
### Tests
Add or adjust focused runner tests proving parity for:
- fresh extraction, fully reused extraction, and a mixture of accepted and
rejected chunk results;
- deterministic extract ordering under reverse completion;
- warning promotion across retries;
- real checkpoint decision reporting for reused and recomputed extracts;
- extract input/output and attempt debug paths;
- framework cancellation and deterministic error selection; and
- unchanged bounded worker and global LLM concurrency behavior.
Use source search or a package-local compile-time assertion where practical to
confirm there is only one extraction execution path and no remaining
`completedExtractLoader` or `extractDecision` compatibility shim.
### Documentation
Update `docs/internal/pipeline.md` if its execution-flow description names the
old continuation mechanism. This stage is an internal refactor and must not
change operator or integration contracts.
### Completion Gate
The concurrent coordinator must be the only code path that executes extraction,
and lane continuation must consume finalized extraction state without routing
it through a synthetic checkpoint loader.
## Final Verification And Closeout
After all five stages:
1. Run the full validation commands from this document on a clean worktree.
2. Exercise the maintained D&D production example with both a fresh workspace
and checkpoint resume.
3. Verify that changing the registered default spell validator chain selects a
different checkpoint identity.
4. Verify merge and normalize attempt debug behavior with instrumented
LLM-backed test modules.
5. Confirm current production imports satisfy the generic boundary guard.
6. Re-read current-behavior documentation for statements made true or obsolete
by these stages.
7. Replace this roadmap's status with a concise completion record only after all
stages and documentation updates have landed.
## Open Questions
None. The implementation choices required for these five remediation stages
are specified above.

View File

@@ -1,29 +0,0 @@
# Future Work
Current Notarius behavior is documented in the canonical README, CLI,
configuration, operations, internal, and integration docs. This roadmap records
future work only.
## Candidate Product Work
- Additional input adapters, such as Markdown or note-export formats.
- Additional D&D extractors beyond spell casts.
- Cross-lane entity normalization.
- Cross-chunk semantic deduplication.
- Configurable validator chains with production validator modules.
- Multiple effective LLM profiles in one run.
- Parallel execution where it preserves deterministic manifests and diagnostics.
- Additional output encoders.
## Candidate Operational Work
- Packaged release artifacts for alpha distribution.
- A documented versioning and release process.
- Optional generated example output fixtures with a regeneration procedure.
- Additional diagnostics or reporting views if operator workflows need them.
## Non-Goals To Revisit Deliberately
- A general workflow language.
- Structural module selection through ad hoc run flags.
- Storing secrets in config files, diagnostics, manifests, or examples.

View File

@@ -1,215 +0,0 @@
# Troubleshooting
This guide maps common implemented failure modes to inspection steps and fixes.
For command syntax, see [CLI Reference](cli.md). For YAML fields and
environment overrides, see [Configuration](config.md). For output and
diagnostics layout, see [Operations](operations.md).
## Config File Not Found
Symptom:
```text
notarius: config file not found; pass --config or set NOTARIUS_CONFIG
```
Fix:
- Pass `--config path/to/config.yml`.
- Or set `NOTARIUS_CONFIG` to a readable file.
- Or install a config at `/usr/local/etc/notarius/config.yml`.
If the message says the config path is a directory or is not available, correct
the path or file permissions.
## Unsupported Or Invalid Config
Symptoms include:
- `unsupported config version`
- `config version is required`
- `field <name> not found`
- `total LLM concurrency must be greater than zero`
- `diagnostics retention "<value>" is not supported`
Fix:
- Use `version: 1`.
- Remove unknown YAML fields.
- Validate with:
```sh
go run ./cmd/notarius config validate --config path/to/config.yml
```
## Unknown Pipeline
Symptom:
```text
notarius: pipeline "..." is not configured
```
Fix:
- List configured pipeline IDs:
```sh
go run ./cmd/notarius pipelines list --config path/to/config.yml
```
- Use one of those IDs in `notarius run <pipeline-id>`.
- Check indentation under the top-level `pipelines` map.
## Unknown Or Incompatible Module
Symptoms mention a module key, pipeline slot, lane, capability, or `not
registered`.
Fix:
- Validate the pipeline against the production module catalog:
```sh
go run ./cmd/notarius config validate \
--config path/to/config.yml \
--pipeline dnd-session
```
- Use only implemented production module keys listed in
[Configuration](config.md#implemented-production-modules).
- Check that artifact lanes include an `extract` binding.
## Invalid `--only`
Symptoms include:
- `--only must contain comma-separated non-empty artifact lane IDs`
- `--only requires --pipeline`
- `selected artifact lane`
Fix:
- Use comma-separated lane IDs with no empty entries:
```sh
go run ./cmd/notarius run dnd-session \
--config path/to/config.yml \
--input path/to/input.json \
--only spells
```
- For `config validate`, include `--pipeline` when using `--only`.
- Confirm the lane ID exists under `pipelines.<id>.artifacts`.
## Seriatim Input Validation Failure
Symptoms include `seriatim input`, `parse JSON`, `segments must not be empty`,
or validation errors naming a segment field.
Fix:
- Compare the input to
[examples/seriatim-minimal-transcript.json](../examples/seriatim-minimal-transcript.json).
- Ensure the JSON has a `metadata` object and a non-empty `segments` array.
- Each segment needs a non-empty `id`, non-empty `speaker`, non-empty `text`,
non-negative numeric `start`, and non-negative numeric `end`.
- Segment IDs must be unique and must not contain leading or trailing
whitespace.
- `end` must be greater than or equal to `start`.
## Missing LLM Base URL Or Model
Symptoms include:
- `LLM profile "default" base URL must not be empty`
- `LLM profile "default" model must not be empty`
- `base URL must be valid`
Fix:
- Set `base_url` and `model` in `llm_profiles.default`.
- Or set `NOTARIUS_LLM_DEFAULT_BASE_URL` and
`NOTARIUS_LLM_DEFAULT_MODEL`.
- If a profile needs authentication, set `api_key_env` in YAML or set
`NOTARIUS_LLM_DEFAULT_API_KEY`.
## LLM Profile Override Failure
Symptom:
```text
notarius: LLM profile override "..." is not configured
```
Fix:
- Add the profile under `llm_profiles`.
- Or use an existing profile ID with `--llm-profile`.
Current runs require exactly one distinct effective LLM profile. If a pipeline
uses several profiles, run with `--llm-profile <id>` or align the bindings in
configuration.
## Provider HTTP Or Response Failure
Symptoms include:
- `provider request failed`
- `provider returned status 400`
- `provider returned status 403`
- `provider response missing choices`
- `provider response assistant message content is not valid JSON`
- `decode structured output`
Fix:
- Confirm the `base_url` points to an OpenAI-compatible endpoint root. Notarius
posts to `<base_url>/chat/completions`.
- Check `model` and provider credentials.
- Inspect the retained diagnostics `error.log`.
- For 400 and 403 responses, fix the request configuration or credentials.
- For 429 and 5xx responses, the client retries according to `max_retries`; if
the failure persists, inspect the provider response and adjust capacity,
credentials, or model settings.
- The assistant message content must decode as JSON matching the extractor's
structured response schema.
Provider error messages are redacted for configured API key values.
## Output Write Failure
Symptoms include:
- `create output directory`
- `write output file`
- `output file name must`
Fix:
- Ensure `--output-dir` points to a directory path or a path that can be
created.
- Check filesystem permissions and available disk space.
- If diagnostics were retained, inspect `run-report.json`, `run-manifest.json`,
and `error.log`.
The CLI rejects unsafe logical output paths before writing files.
## Diagnostics Directory Surprise
Symptom: the diagnostics directory is missing after a successful run.
Fix:
- Check `diagnostics.retention`.
- With `auto`, successful runs without warnings are removed.
- Use `diagnostics.retention: always` when every diagnostics run directory
should be kept.
- Use `--diagnostics-dir` to override the configured work directory for a run.
Symptom: diagnostics exist even with `retention: never`.
Explanation:
- Failed runs are retained so that `error.log` and available context can be
inspected.

View File

@@ -0,0 +1,2 @@
Cure Wounds: healing spell cast by touch.
Shield: defensive reaction spell.

View File

@@ -0,0 +1,27 @@
version: 2
concurrency:
total_llm: 1
stage_workers:
extract: 1
workspace:
directory: /var/lib/notarius
diagnostics:
enabled: true
retention: auto
resume:
enabled: false
debug:
enabled: false
pipelines:
dnd-session:
input: seriatim
references:
party: ./dnd-spells-roster.txt
glossary: ./dnd-spells-glossary.txt
chunk:
module: generic
options:
max_units: 50
artifacts:
spells:
extract: dnd/spells

View File

@@ -0,0 +1,3 @@
Aria: party cleric and recurring healer.
Borin: fighter ally.
Bandit mage: hostile spellcaster.

View File

@@ -1,16 +1,7 @@
version: 1
llm_profiles:
default:
provider: openai-compatible
base_url: http://127.0.0.1:1
model: fake-model
version: 2
pipelines:
dnd-session:
input: seriatim
chunk:
module: generic
options:
max_units: 50
artifacts:
spells:
extract: dnd/spells

View File

@@ -5,14 +5,14 @@
},
"segments": [
{
"id": "seg-001",
"id": 1,
"start": 0,
"end": 4,
"speaker": "Aria",
"text": "Aria raises her holy symbol and casts Cure Wounds."
},
{
"id": "seg-002",
"id": 2,
"start": 4,
"end": 8,
"speaker": "DM",

10
go.mod
View File

@@ -1,5 +1,11 @@
module gitea.maximumdirect.net/eric/notarius
go 1.24.0
go 1.25.5
require gopkg.in/yaml.v3 v3.0.1
require (
gitea.maximumdirect.net/eric/scriptorium v0.11.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1
)
require golang.org/x/text v0.14.0 // indirect

8
go.sum
View File

@@ -1,3 +1,11 @@
gitea.maximumdirect.net/eric/scriptorium v0.11.0 h1:rjvbt9FTaWHxYlHq7QlUzmMVUt3QdbTmeCkmH81N//o=
gitea.maximumdirect.net/eric/scriptorium v0.11.0/go.mod h1:FQ5lEuNxmrQyNgIomkpZdxvfTC0jWjbXYuq3tbJWF64=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@@ -3,50 +3,54 @@ package cli
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
"gitea.maximumdirect.net/eric/notarius/internal/modules/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/normalize/noop"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/output/json"
dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register"
genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register"
seriatimregister "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/register"
)
func productionRegistries() (pipeline.Registries, error) {
type productionComponents struct {
registries pipeline.Registries
assets *llm.AssetRegistry
}
func newProductionComponents() (productionComponents, error) {
registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(),
Chunkers: pipeline.NewChunkerRegistry(),
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(),
Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(),
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
}
if err := seriatim.Register(registries.Inputs); err != nil {
return pipeline.Registries{}, fmt.Errorf("register seriatim input: %w", err)
assets := llm.NewAssetRegistry()
registrars := []struct {
name string
register func(pipeline.Registries, *llm.AssetRegistry) error
}{
{name: "generic", register: genericregister.Register},
{name: "seriatim", register: seriatimregister.Register},
{name: "dnd", register: dndregister.Register},
}
if err := generic.Register(registries.Chunkers); err != nil {
return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err)
for _, registrar := range registrars {
if err := registrar.register(registries, assets); err != nil {
return productionComponents{}, fmt.Errorf("register %s module family: %w", registrar.name, err)
}
if err := spells.Register(registries.Extractors); err != nil {
return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err)
}
if err := appendorder.Register(registries.Mergers); err != nil {
return pipeline.Registries{}, fmt.Errorf("register appendorder merger: %w", err)
}
if err := noop.Register(registries.Normalizers); err != nil {
return pipeline.Registries{}, fmt.Errorf("register noop normalizer: %w", err)
}
if err := jsonoutput.Register(registries.Outputs); err != nil {
return pipeline.Registries{}, fmt.Errorf("register json output encoder: %w", err)
}
return registries, nil
return productionComponents{registries: registries, assets: assets}, nil
}
func productionRegistries() (pipeline.Registries, error) {
components, err := newProductionComponents()
return components.registries, err
}
func productionCatalog() (pipeline.ModuleCatalog, error) {
@@ -57,6 +61,11 @@ func productionCatalog() (pipeline.ModuleCatalog, error) {
return catalogFromRegistries(registries), nil
}
func productionPromptAssets() (*llm.AssetRegistry, error) {
components, err := newProductionComponents()
return components.assets, err
}
func effectiveCatalog(opts Options) (pipeline.ModuleCatalog, error) {
if !isEmptyCatalog(opts.Catalog) {
return opts.Catalog, nil
@@ -81,10 +90,12 @@ func catalogFromRegistries(registries pipeline.Registries) pipeline.ModuleCatalo
return pipeline.ModuleCatalog{
Inputs: registries.Inputs,
Chunkers: registries.Chunkers,
ArtifactCodecs: registries.ArtifactCodecs,
Extractors: registries.Extractors,
Mergers: registries.Mergers,
Normalizers: registries.Normalizers,
Validators: registries.Validators,
ValidatorChains: registries.ValidatorChains,
Outputs: registries.Outputs,
}
}
@@ -93,10 +104,12 @@ func registriesFromCatalog(catalog pipeline.ModuleCatalog) pipeline.Registries {
return pipeline.Registries{
Inputs: catalog.Inputs,
Chunkers: catalog.Chunkers,
ArtifactCodecs: catalog.ArtifactCodecs,
Extractors: catalog.Extractors,
Mergers: catalog.Mergers,
Normalizers: catalog.Normalizers,
Validators: catalog.Validators,
ValidatorChains: catalog.ValidatorChains,
Outputs: catalog.Outputs,
}
}
@@ -104,20 +117,24 @@ func registriesFromCatalog(catalog pipeline.ModuleCatalog) pipeline.Registries {
func isEmptyCatalog(catalog pipeline.ModuleCatalog) bool {
return catalog.Inputs == nil &&
catalog.Chunkers == nil &&
catalog.ArtifactCodecs == nil &&
catalog.Extractors == nil &&
catalog.Mergers == nil &&
catalog.Normalizers == nil &&
catalog.Validators == nil &&
catalog.ValidatorChains == nil &&
catalog.Outputs == nil
}
func isEmptyRegistries(registries pipeline.Registries) bool {
return registries.Inputs == nil &&
registries.Chunkers == nil &&
registries.ArtifactCodecs == nil &&
registries.Extractors == nil &&
registries.Mergers == nil &&
registries.Normalizers == nil &&
registries.Validators == nil &&
registries.ValidatorChains == nil &&
registries.Outputs == nil
}
@@ -125,48 +142,39 @@ func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileI
if err := ctx.Err(); err != nil {
return nil, nil, err
}
trimmedID := strings.TrimSpace(profileID)
if trimmedID == "" {
trimmedID = pipeline.DefaultLLMProfile
}
profile, ok := cfg.LLMProfile(trimmedID)
if !ok {
return nil, nil, fmt.Errorf("LLM profile %q is not configured", trimmedID)
}
clientCfg, err := cfg.OpenAICompatibleClientConfig(trimmedID)
assets, err := productionPromptAssets()
if err != nil {
return nil, nil, err
}
client, err := llm.NewOpenAICompatibleClient(clientCfg)
if err != nil {
return nil, nil, fmt.Errorf("create LLM client for profile %q: %w", trimmedID, err)
}
scheduler, err := llm.NewScheduler(effectiveLLMConcurrency(cfg, profile))
if err != nil {
return nil, nil, fmt.Errorf("create LLM scheduler for profile %q: %w", trimmedID, err)
}
provider := strings.TrimSpace(profile.Provider)
if provider == "" {
provider = "openai-compatible"
}
metadata := []artifacts.LLMProfileManifest{
{
ID: trimmedID,
Provider: provider,
Model: strings.TrimSpace(profile.Model),
},
}
return llm.NewScheduledClient(client, scheduler), metadata, nil
return buildProductionLLMClient(ctx, cfg, profileID, assets)
}
func effectiveLLMConcurrency(cfg config.Config, profile config.LLMProfile) int {
if profile.MaxConcurrency > 0 {
return profile.MaxConcurrency
func productionLLMClientFactoryWithAssets(assets *llm.AssetRegistry) LLMClientFactory {
return func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return buildProductionLLMClient(ctx, cfg, profileID, assets)
}
if cfg.Concurrency.TotalLLM > 0 {
return cfg.Concurrency.TotalLLM
}
return 1
}
func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID string, assets *llm.AssetRegistry) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
if assets == nil {
return nil, nil, fmt.Errorf("production asset registry must not be nil")
}
recorder := llm.NewLLMProfileRecorder()
client, err := llm.NewScriptoriumClient(llm.ScriptoriumClientConfig{
ProfileDir: cfg.Scriptorium.ProfileDir,
ProfileFile: cfg.Scriptorium.ProfileFile,
Assets: assets,
Recorder: recorder,
})
if err != nil {
return nil, nil, fmt.Errorf("create Scriptorium-backed LLM client: %w", err)
}
scheduler, err := llm.NewScheduler(cfg.Concurrency.TotalLLM)
if err != nil {
return nil, nil, fmt.Errorf("create LLM scheduler: %w", err)
}
return llm.NewScheduledClient(client, scheduler), nil, nil
}

View File

@@ -0,0 +1,600 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"io/fs"
"path/filepath"
"reflect"
"sort"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
spellrelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness"
validjson "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/valid_json"
validjsonschema "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/valid_json_schema"
)
func TestProductionCompatibilitySnapshot(t *testing.T) {
normalizedOptions, err := normalizeOptions(Options{})
if err != nil {
t.Fatalf("normalizeOptions() error = %v, want nil", err)
}
if normalizedOptions.Catalog.Inputs != normalizedOptions.Registries.Inputs ||
normalizedOptions.Catalog.Chunkers != normalizedOptions.Registries.Chunkers ||
normalizedOptions.Catalog.Extractors != normalizedOptions.Registries.Extractors ||
normalizedOptions.Catalog.Mergers != normalizedOptions.Registries.Mergers ||
normalizedOptions.Catalog.Normalizers != normalizedOptions.Registries.Normalizers ||
normalizedOptions.Catalog.Validators != normalizedOptions.Registries.Validators ||
normalizedOptions.Catalog.ValidatorChains != normalizedOptions.Registries.ValidatorChains ||
normalizedOptions.Catalog.Outputs != normalizedOptions.Registries.Outputs {
t.Fatal("production catalog and execution registries do not share one composition")
}
if normalizedOptions.LLMClientFactory == nil {
t.Fatal("production LLM client factory is nil")
}
registries, err := productionRegistries()
if err != nil {
t.Fatalf("productionRegistries() error = %v, want nil", err)
}
keySnapshots := []struct {
name string
got []string
want []string
}{
{name: "inputs", got: registries.Inputs.RegisteredKeys(), want: []string{"seriatim"}},
{name: "chunkers", got: registries.Chunkers.RegisteredKeys(), want: []string{"dnd/scenes", "generic"}},
{name: "extractors", got: registries.Extractors.RegisteredKeys(), want: []string{"dnd/spells"}},
{name: "mergers", got: registries.Mergers.RegisteredKeys(), want: []string{"appendorder"}},
{name: "normalizers", got: registries.Normalizers.RegisteredKeys(), want: []string{"noop"}},
{name: "validators", got: registries.Validators.RegisteredKeys(), want: []string{
"extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
"generic/always_accept", "generic/always_reject", "generic/valid_json", "generic/valid_json_schema",
}},
{name: "outputs", got: registries.Outputs.RegisteredKeys(), want: []string{"json"}},
}
for _, snapshot := range keySnapshots {
t.Run(snapshot.name, func(t *testing.T) {
if !reflect.DeepEqual(snapshot.got, snapshot.want) {
t.Fatalf("registered keys = %#v, want compatibility snapshot %#v", snapshot.got, snapshot.want)
}
})
}
wantChain := []pipeline.ModuleBinding{
pipeline.Binding(validjson.Key),
pipeline.Binding(validjsonschema.Key),
pipeline.Binding(spellshape.Key),
pipeline.Binding(spellsourcerefs.Key),
pipeline.Binding(spellrelatedness.Key),
}
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
t.Fatalf("spell validator chain = %#v, want compatibility snapshot %#v", got, wantChain)
}
assets, err := productionPromptAssets()
if err != nil {
t.Fatalf("productionPromptAssets() error = %v, want nil", err)
}
assertAssetNames(t, assets.PromptFS, []string{
"dnd.scenes/dnd.scenes.yaml",
"dnd.scenes/instructions.md",
"dnd.scenes/sharedassets/common-dnd-references.md",
"dnd.scenes/sharedassets/common-dnd-system.md",
"dnd.scenes/sharedassets/common-dnd-transcript.md",
"dnd.scenes/task.md",
"dnd.spells/dnd.spells.yaml",
"dnd.spells/instructions.md",
"dnd.spells/sharedassets/common-dnd-references.md",
"dnd.spells/sharedassets/common-dnd-system.md",
"dnd.spells/sharedassets/common-dnd-transcript.md",
"dnd.spells/task.md",
})
assertAssetNames(t, assets.SchemaFS, []string{
"dnd_scenes.v1.json",
"dnd_spells_llm.v1.json",
})
identitySnapshot := map[string]map[string]any{
"scenes": sceneManifestMetadata(t),
"spells": spellManifestMetadata(t),
}
for name, metadata := range identitySnapshot {
for _, key := range []string{"prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_id", "response_schema_name", "response_schema_version", "response_schema_sha256"} {
if value, ok := metadata[key].(string); !ok || value == "" {
t.Fatalf("%s metadata[%q] = %#v, want non-empty identity", name, key, metadata[key])
}
}
}
if got := []any{
identitySnapshot["scenes"]["prompt_id"], identitySnapshot["scenes"]["prompt_version"],
identitySnapshot["scenes"]["response_schema_key"], identitySnapshot["scenes"]["response_schema_id"], identitySnapshot["scenes"]["response_schema_name"], identitySnapshot["scenes"]["response_schema_version"],
}; !reflect.DeepEqual(got, []any{"dnd.scenes", "v1", "dnd_scenes", "notarius.dnd.scenes", "notarius_dnd_scenes_v1", "v1"}) {
t.Fatalf("scene identities = %#v, want compatibility snapshot", got)
}
if got := []any{
identitySnapshot["spells"]["prompt_id"], identitySnapshot["spells"]["prompt_version"],
identitySnapshot["spells"]["response_schema_key"], identitySnapshot["spells"]["response_schema_id"], identitySnapshot["spells"]["response_schema_name"], identitySnapshot["spells"]["response_schema_version"],
}; !reflect.DeepEqual(got, []any{"dnd.spells", "v1", "dnd_spells", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"}) {
t.Fatalf("spell identities = %#v, want compatibility snapshot", got)
}
fileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells.config.yml"))
if err != nil {
t.Fatalf("LoadFileConfig() error = %v, want nil", err)
}
cfg := config.Default()
if err := cfg.ApplyFileConfig(fileConfig); err != nil {
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
}
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
resolved := effective.ResolvedPipeline
if resolved.Input.Module != "seriatim" || resolved.Chunk.Module != "generic" || resolved.Output.Module != "json" || len(resolved.ArtifactLanes) != 1 {
t.Fatalf("resolved example = %#v, want maintained production topology", resolved)
}
lane := resolved.ArtifactLanes[0]
if lane.ID != "spells" || lane.Extract.Module != "dnd/spells" || lane.Merge.Module != "appendorder" || lane.Normalize.Module != "noop" {
t.Fatalf("resolved lane = %#v, want maintained spell lane", lane)
}
if got := resolvedValidatorKeys(resolved.ValidatorChains, pipeline.StageExtract, "spells", spells.Key); !reflect.DeepEqual(got, []string{
"generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
}) {
t.Fatalf("resolved validator keys = %#v, want compatibility snapshot", got)
}
productionFileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells-production.config.yml"))
if err != nil {
t.Fatalf("LoadFileConfig(production example) error = %v, want nil", err)
}
productionConfig := config.Default()
if err := productionConfig.ApplyFileConfig(productionFileConfig); err != nil {
t.Fatalf("ApplyFileConfig(production example) error = %v, want nil", err)
}
productionEffective, err := productionConfig.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)})
if err != nil {
t.Fatalf("Resolve(production example) error = %v, want nil", err)
}
productionResolved := productionEffective.ResolvedPipeline
if productionConfig.Concurrency.TotalLLM != 1 || productionConfig.Concurrency.StageWorkers["extract"] != 1 || !reflect.DeepEqual(productionResolved.Chunk.Options, map[string]any{"max_units": 50}) {
t.Fatalf("production example concurrency/options = %#v/%#v, want compatibility snapshot", productionConfig.Concurrency, productionResolved.Chunk.Options)
}
bindings := productionResolved.ArtifactLanes[0].ExtractReferences.Bindings
if len(bindings) != 2 || bindings[0].SlotName != "glossary" || bindings[0].Source != "./dnd-spells-glossary.txt" || bindings[1].SlotName != "party" || bindings[1].Source != "./dnd-spells-roster.txt" {
t.Fatalf("production example reference bindings = %#v, want maintained glossary and party bindings", bindings)
}
}
func TestMaintainedSeriatimToDNDCompatibilityBundle(t *testing.T) {
tests := []struct {
name string
client contracts.StructuredLLMClient
wantStatus string
wantLaneFile bool
wantRejectedCount int
}{
{name: "approved", client: newFakeRunLLMClient(false), wantStatus: "approved", wantLaneFile: true},
{name: "validator rejection is nonfatal", client: newFakeRunLLMClient(true), wantStatus: "rejected", wantRejectedCount: 1},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
outputDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "dnd-session",
"--config", fixturePath(t, "examples/dnd-spells.config.yml"),
"--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"),
"--output-dir", outputDir,
"--diagnostics-dir", t.TempDir(),
}, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(test.client, nil)})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
runDir := onlyChildDir(t, outputDir)
wantFiles := []string{"index.json", "manifest.json", "rejected.json", "warnings.json"}
if test.wantLaneFile {
wantFiles = append(wantFiles, "lanes/spells.json")
}
sort.Strings(wantFiles)
if got := relativeFileNames(t, runDir); !reflect.DeepEqual(got, wantFiles) {
t.Fatalf("durable files = %#v, want compatibility snapshot %#v", got, wantFiles)
}
var manifest artifacts.RunManifest
readJSONFile(t, filepath.Join(runDir, "manifest.json"), &manifest)
if manifest.PipelineID != "dnd-session" || manifest.InputModule != "seriatim" || manifest.Chunker != "generic" || manifest.OutputEncoder != "json" {
t.Fatalf("manifest module provenance = %#v, want maintained production modules", manifest)
}
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].ID != "spells" {
t.Fatalf("artifact lanes = %#v, want one spells lane", manifest.ArtifactLanes)
}
laneManifest := manifest.ArtifactLanes[0]
if laneManifest.Extractor != "dnd/spells" || laneManifest.Merger != "appendorder" || laneManifest.Normalizer != "noop" {
t.Fatalf("manifest lane module provenance = %#v, want maintained production modules", laneManifest)
}
if len(manifest.Extractors) != 0 || manifest.Merger != "" || manifest.Normalizer != "" {
t.Fatalf("legacy top-level lane summaries = %#v/%q/%q, want empty compatibility snapshot", manifest.Extractors, manifest.Merger, manifest.Normalizer)
}
if manifest.ValidationStatus != test.wantStatus || len(manifest.RejectedOutputs) != test.wantRejectedCount {
t.Fatalf("manifest outcome = status %q rejected %#v, want %q/%d", manifest.ValidationStatus, manifest.RejectedOutputs, test.wantStatus, test.wantRejectedCount)
}
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:1c98d94ae632fb10a2b56f684cd4fb1019cedb1a629e57dc0977cf4a54135be0"}) {
t.Fatalf("source digests = %#v, want maintained fixture provenance", manifest.SourceDigests)
}
if got := manifestValidatorKeys(manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)); !reflect.DeepEqual(got, []string{
"generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
}) {
t.Fatalf("manifest validator chain = %#v, want compatibility snapshot", got)
}
var index struct {
ManifestFile string `json:"manifest_file"`
OutputFiles []struct {
LaneID string `json:"lane_id"`
MediaType string `json:"media_type"`
File string `json:"file"`
ModuleKey string `json:"module_key"`
SchemaID string `json:"schema_id"`
SchemaName string `json:"schema_name"`
SchemaVersion string `json:"schema_version"`
} `json:"output_files"`
RejectedFile string `json:"rejected_file"`
WarningsFile string `json:"warnings_file"`
}
readJSONFile(t, filepath.Join(runDir, "index.json"), &index)
if index.ManifestFile != "manifest.json" || index.RejectedFile != "rejected.json" || index.WarningsFile != "warnings.json" {
t.Fatalf("output index fixed files = %#v, want compatibility snapshot", index)
}
if test.wantLaneFile {
if len(index.OutputFiles) != 1 {
t.Fatalf("output index entries = %#v, want one", index.OutputFiles)
}
wantOutput := struct {
LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string
}{"spells", "application/json", "lanes/spells.json", "noop", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"}
gotOutput := index.OutputFiles[0]
got := struct {
LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string
}{gotOutput.LaneID, gotOutput.MediaType, gotOutput.File, gotOutput.ModuleKey, gotOutput.SchemaID, gotOutput.SchemaName, gotOutput.SchemaVersion}
if got != wantOutput {
t.Fatalf("output index entries = %#v, want compatibility snapshot %#v", index.OutputFiles, wantOutput)
}
assertJSONEqual(t, readFile(t, filepath.Join(runDir, "lanes/spells.json")), []byte(`{
"spell_casts": [{
"caster": "Aria",
"spell": "Cure Wounds",
"effect": "Heals a wounded ally.",
"narrative_description": "Aria casts Cure Wounds.",
"source_refs": [{"source_id": "session-alpha", "start_unit_id": 1, "end_unit_id": 1}]
}]
}`))
} else if len(index.OutputFiles) != 0 {
t.Fatalf("output index entries = %#v, want none for rejected lane", index.OutputFiles)
}
var warnings struct {
Warnings []contracts.Warning `json:"warnings"`
}
readJSONFile(t, filepath.Join(runDir, "warnings.json"), &warnings)
if len(warnings.Warnings) != 0 {
t.Fatalf("warnings = %#v, want empty compatibility snapshot", warnings.Warnings)
}
var rejected struct {
Rejected []contracts.RejectedOutput `json:"rejected"`
}
readJSONFile(t, filepath.Join(runDir, "rejected.json"), &rejected)
if len(rejected.Rejected) != test.wantRejectedCount {
t.Fatalf("rejected outputs = %#v, want %d", rejected.Rejected, test.wantRejectedCount)
}
if test.wantRejectedCount == 1 {
got := rejected.Rejected[0]
if got.Stage != "extract" || got.LaneID != "spells" || got.ModuleKey != "dnd/spells" || got.ChunkID != "chunk-000001" || got.ChunkIndex != 0 || got.ValidatorName != "extract/dnd/spells/source_refs" || got.ReasonCode != "invalid_source_refs" || got.AttemptCount != 1 {
t.Fatalf("rejection = %#v, want maintained nonfatal validator outcome", got)
}
}
})
}
}
func TestProductionLLMCallersShareScheduledClient(t *testing.T) {
underlying := newBlockingProductionLLMClient()
scheduler, err := frameworkllm.NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler() error = %v, want nil", err)
}
client := frameworkllm.NewScheduledClient(underlying, scheduler)
doc := &source.SourceDocument{
ID: "session-alpha", Kind: "transcript", Format: "application/json", Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 1, Kind: "segment", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}},
{ID: 2, Kind: "segment", Text: "The spell takes effect.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}},
},
}
chunk := source.Chunk{
ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 2},
Content: []byte(`{"scene":"Aria casts Cure Wounds."}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), doc.Units...),
}
var started sync.WaitGroup
started.Add(2)
errs := make(chan error, 2)
go func() {
started.Done()
chunker, err := scenes.New(client, scenes.Options{})
if err == nil {
_, err = chunker.Chunk(context.Background(), contracts.ChunkRequest{Source: doc})
}
errs <- err
}()
go func() {
started.Done()
extractor, err := spells.New(client, spells.Options{})
if err == nil {
_, err = extractor.Extract(context.Background(), contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk})
}
errs <- err
}()
started.Wait()
for i := 0; i < 2; i++ {
<-underlying.entered
underlying.release <- struct{}{}
}
for i := 0; i < 2; i++ {
if err := <-errs; err != nil {
t.Fatalf("production LLM caller error = %v, want nil", err)
}
}
if underlying.maxActive != 1 {
t.Fatalf("maximum concurrent provider calls = %d, want total_llm limit 1", underlying.maxActive)
}
sort.Strings(underlying.stageNames)
if !reflect.DeepEqual(underlying.stageNames, []string{"dnd/scenes", "dnd/spells"}) {
t.Fatalf("scheduled stage names = %#v, want both production LLM callers", underlying.stageNames)
}
}
func TestProductionBundlePreservesLaneAndChunkOrder(t *testing.T) {
configPath := writeTestConfig(t, `version: 2
pipelines:
dnd-session:
input: seriatim
chunk:
module: generic
options:
max_units: 1
artifacts:
zeta:
extract: dnd/spells
alpha:
extract: dnd/spells
`)
outputDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "dnd-session",
"--config", configPath,
"--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"),
"--output-dir", outputDir,
"--diagnostics-dir", t.TempDir(),
}, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(orderingProductionLLMClient{}, nil)})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
runDir := onlyChildDir(t, outputDir)
var index struct {
OutputFiles []struct {
LaneID string `json:"lane_id"`
} `json:"output_files"`
}
readJSONFile(t, filepath.Join(runDir, "index.json"), &index)
if len(index.OutputFiles) != 2 {
t.Fatalf("output index entries = %#v, want two lanes", index.OutputFiles)
}
if got := []string{index.OutputFiles[0].LaneID, index.OutputFiles[1].LaneID}; !reflect.DeepEqual(got, []string{"alpha", "zeta"}) {
t.Fatalf("output lane order = %#v, want resolved lane order", got)
}
for _, laneID := range []string{"alpha", "zeta"} {
var payload struct {
SpellCasts []struct {
Spell string `json:"spell"`
SourceRefs []source.SourceRef `json:"source_refs"`
} `json:"spell_casts"`
}
readJSONFile(t, filepath.Join(runDir, "lanes", laneID+".json"), &payload)
if len(payload.SpellCasts) != 2 {
t.Fatalf("lane %q spell casts = %#v, want one per source chunk", laneID, payload.SpellCasts)
}
got := []any{
payload.SpellCasts[0].Spell, payload.SpellCasts[0].SourceRefs[0].StartUnitID,
payload.SpellCasts[1].Spell, payload.SpellCasts[1].SourceRefs[0].StartUnitID,
}
if !reflect.DeepEqual(got, []any{"Cure Wounds", 1, "Shield", 2}) {
t.Fatalf("lane %q chunk handoff order = %#v, want source chunk order", laneID, got)
}
}
}
type blockingProductionLLMClient struct {
mu sync.Mutex
active int
maxActive int
stageNames []string
entered chan struct{}
release chan struct{}
}
type orderingProductionLLMClient struct{}
func (orderingProductionLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
material := req.Inputs["transcript"]
unitID := 1
spellName := "Cure Wounds"
if strings.Contains(string(material.Content), "Shield") {
unitID = 2
spellName = "Shield"
}
payload, err := json.Marshal(map[string]any{
"spell_casts": []map[string]any{{
"caster": "Aria",
"spell": spellName,
"effect": "Fixture effect.",
"narrative_description": "Fixture spell cast.",
"source_refs": []map[string]any{{
"start_unit_id": unitID,
"end_unit_id": unitID,
}},
}},
})
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
if err := json.Unmarshal(payload, out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: payload}, nil
}
func newBlockingProductionLLMClient() *blockingProductionLLMClient {
return &blockingProductionLLMClient{entered: make(chan struct{}, 2), release: make(chan struct{}, 2)}
}
func (client *blockingProductionLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.mu.Lock()
client.active++
if client.active > client.maxActive {
client.maxActive = client.active
}
client.stageNames = append(client.stageNames, req.StageName)
client.mu.Unlock()
client.entered <- struct{}{}
select {
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
case <-client.release:
}
client.mu.Lock()
client.active--
client.mu.Unlock()
var payload []byte
switch req.StageName {
case scenes.Key:
payload = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Spell","primary_mode":"Narrative","main_participants":["Aria"],"summary":"Aria casts a spell.","boundary_note":"Complete source.","boundary_confidence":"High"}],"boundary_caveats":[]}`)
case spells.Key:
payload = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Healing","narrative_description":"Aria casts Cure Wounds.","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`)
}
if err := json.Unmarshal(payload, out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: payload}, nil
}
func sceneManifestMetadata(t *testing.T) map[string]any {
t.Helper()
chunker, err := scenes.New(orderingProductionLLMClient{}, scenes.Options{})
if err != nil {
t.Fatalf("construct scene chunker: %v", err)
}
return chunker.ManifestMetadata()
}
func spellManifestMetadata(t *testing.T) map[string]any {
t.Helper()
extractor, err := spells.New(orderingProductionLLMClient{}, spells.Options{})
if err != nil {
t.Fatalf("construct spell extractor: %v", err)
}
return extractor.ManifestMetadata()
}
func assertAssetNames(t *testing.T, getFS func() (fs.FS, error), want []string) {
t.Helper()
fSys, err := getFS()
if err != nil {
t.Fatalf("asset filesystem error = %v, want nil", err)
}
var got []string
if err := fs.WalkDir(fSys, ".", func(path string, entry fs.DirEntry, err error) error {
if err == nil && !entry.IsDir() {
got = append(got, path)
}
return err
}); err != nil {
t.Fatalf("walk assets: %v", err)
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("asset names = %#v, want compatibility snapshot %#v", got, want)
}
}
func resolvedValidatorKeys(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID, moduleKey string) []string {
for _, chain := range chains {
if chain.Stage == stage && chain.LaneID == laneID && chain.ModuleKey == moduleKey {
keys := make([]string, 0, len(chain.Validators))
for _, validator := range chain.Validators {
keys = append(keys, validator.Binding.Module)
}
return keys
}
}
return nil
}
func relativeFileNames(t *testing.T, root string) []string {
t.Helper()
var names []string
if err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
if err != nil || entry.IsDir() {
return err
}
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
names = append(names, filepath.ToSlash(rel))
return nil
}); err != nil {
t.Fatalf("walk durable output: %v", err)
}
sort.Strings(names)
return names
}
func assertJSONEqual(t *testing.T, got, want []byte) {
t.Helper()
var gotValue any
var wantValue any
if err := json.Unmarshal(got, &gotValue); err != nil {
t.Fatalf("unmarshal actual JSON: %v", err)
}
if err := json.Unmarshal(want, &wantValue); err != nil {
t.Fatalf("unmarshal expected JSON: %v", err)
}
if !reflect.DeepEqual(gotValue, wantValue) {
t.Fatalf("JSON = %#v, want compatibility snapshot %#v", gotValue, wantValue)
}
}

View File

@@ -2,6 +2,8 @@ package cli
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
@@ -16,7 +18,10 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -25,7 +30,7 @@ const defaultOutputRoot = "./notarius-output"
const usage = `Usage:
notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b]
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json]
`
@@ -46,7 +51,12 @@ func Run(args []string, stdout, stderr io.Writer) int {
}
func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
opts = normalizeOptions(opts)
var err error
opts, err = normalizeOptions(opts)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if len(args) == 0 {
writeUsage(stdout)
return 0
@@ -73,17 +83,28 @@ func writeUsage(w io.Writer) {
fmt.Fprint(w, usage)
}
func normalizeOptions(opts Options) Options {
func normalizeOptions(opts Options) (Options, error) {
if opts.LookupEnv == nil {
opts.LookupEnv = os.LookupEnv
}
if opts.Now == nil {
opts.Now = time.Now
}
if isEmptyCatalog(opts.Catalog) && isEmptyRegistries(opts.Registries) {
components, err := newProductionComponents()
if err != nil {
return Options{}, err
}
opts.Registries = components.registries
opts.Catalog = catalogFromRegistries(components.registries)
if opts.LLMClientFactory == nil {
opts.LLMClientFactory = productionLLMClientFactoryWithAssets(components.assets)
}
}
if opts.LLMClientFactory == nil {
opts.LLMClientFactory = productionLLMClientFactory
}
return opts
return opts, nil
}
func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) int {
@@ -95,6 +116,17 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
outputDir := fs.String("output-dir", "", "output directory")
diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory")
llmProfile := fs.String("llm-profile", "", "LLM profile override")
resume := fs.Bool("resume", false, "reuse valid workspace checkpoints")
sessionID := sessionIDFlag{}
referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{}
fs.Var(&sessionID, "session-id", "prompt session identifier")
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
if err := validateRunFlagValues(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
if err := fs.Parse(reorderRunArgs(args)); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
@@ -116,27 +148,48 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintln(stderr, "notarius: run requires --input")
return 2
}
if sessionID.set && strings.TrimSpace(sessionID.value) == "" {
fmt.Fprintln(stderr, "notarius: --session-id must not be empty")
return 2
}
only, err := parseOnly(*onlyRaw)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
referenceRequests, err := parseReferenceFlags(referenceFlags)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
referenceUnbindRequests, err := parseReferenceUnbindFlags(withoutReferenceFlags)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
cfg, loadedConfigPath, err := loadConfig(*configPath, opts)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
workspaceSettings := workspace.FromConfig(cfg)
if dir := strings.TrimSpace(*diagnosticsDir); dir != "" {
cfg.Diagnostics.WorkDir = dir
workspaceSettings.DiagnosticsRoot = dir
}
startedAt := opts.Now().UTC()
runDir, err := diagnostics.NewRunDirectory(cfg.Diagnostics.WorkDir, cfg.Diagnostics.Retention)
runID := fmt.Sprintf("run-%d", startedAt.UnixNano())
var runDir *diagnostics.RunDirectory
if workspaceSettings.DiagnosticsEnabled {
var err error
runDir, err = diagnostics.NewRunDirectory(workspaceSettings.DiagnosticsRoot, cfg.Diagnostics.Retention)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
runID = runDir.RunID()
}
invocation := diagnostics.InvocationMetadata{
Operation: "run",
PipelineID: pipelineID,
@@ -144,45 +197,71 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
ConfigPath: loadedConfigPath,
ConfigSource: configSource(*configPath),
OnlyLanes: append([]string(nil), only...),
RunID: runDir.RunID(),
Resume: *resume,
RunID: runID,
StartedAt: startedAt,
}
if err := runDir.WriteInvocationMetadata(invocation); err != nil {
if err := writeDiagnostics(runDir, func() error { return runDir.WriteInvocationMetadata(invocation) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
}
if *resume && !workspaceSettings.ResumeEnabled {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("--resume requires workspace.resume.enabled: true"))
}
debugRecorder, err := frameworkdebug.NewWorkspaceRecorder(workspaceSettings, runID)
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create debug recorder: %w", err))
}
debugRecorder = pipeline.SynchronizedDebugRecorder(debugRecorder)
catalog, err := effectiveCatalog(opts)
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests)
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
effective, err := cfg.Resolve(config.ResolveInput{
PipelineID: pipelineID,
Only: only,
Catalog: catalog,
LLMProfileOverride: *llmProfile,
ReferenceOverrides: referenceOverrides,
ReferenceUnbinds: referenceUnbinds,
})
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, profileIDs); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
workingDir, err := os.Getwd()
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("resolve working directory: %w", err))
}
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
ConfigPath: loadedConfigPath,
WorkingDir: workingDir,
})
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
effective.ResolvedPipeline = materialized
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
if err := runDir.WriteInvocationMetadata(invocation); err != nil {
if err := writeDiagnostics(runDir, func() error { return runDir.WriteInvocationMetadata(invocation) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
}
if err := runDir.WriteRedactedEffectiveConfig(effective); err != nil {
if err := writeDiagnostics(runDir, func() error { return runDir.WriteRedactedEffectiveConfig(effective) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics effective config: %w", err))
}
if err := runDir.WriteResolvedPipeline(effective.ResolvedPipeline); err != nil {
if err := writeDiagnostics(runDir, func() error { return runDir.WriteResolvedPipeline(effective.ResolvedPipeline) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved pipeline: %w", err))
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if len(profileIDs) != 1 {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("pipeline %q uses %d distinct LLM profiles; current runs require exactly one: %s", pipelineID, len(profileIDs), strings.Join(profileIDs, ", ")))
}
rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath))
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
if err := writeDiagnostics(runDir, func() error {
return runDir.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline))
}); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved references: %w", err))
}
registries, err := effectiveRegistries(opts)
@@ -191,59 +270,89 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
}
ctx := context.Background()
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, profileIDs[0])
factoryProfileID := ""
if len(profileIDs) == 1 {
factoryProfileID = profileIDs[0]
}
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID)
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", profileIDs[0], err))
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("prepare pipeline %q: %w", pipelineID, err))
}
rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath))
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
}
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(workspaceSettings, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
output, err := pipeline.New(registries).Run(ctx, pipeline.RunInput{
Pipeline: effective.ResolvedPipeline,
output, err := pipeline.New().Run(ctx, pipeline.RunInput{
Prepared: prepared,
Path: strings.TrimSpace(*inputPath),
RawInput: rawInput,
LLMClient: llmClient,
RunID: runDir.RunID(),
SessionID: strings.TrimSpace(sessionID.value),
RunID: runID,
StartedAt: startedAt,
LLMProfiles: llmProfiles,
Metadata: runMetadata(*outputDir, *diagnosticsDir),
Warnings: referenceWarnings,
Checkpoints: checkpointRecorder,
Checkpoint: checkpointLoader,
Debug: debugRecorder,
ExtractWorkers: cfg.Concurrency.StageWorkers["extract"],
})
if err != nil {
if output.Manifest.PipelineID != "" {
if output.Manifest.PipelineID != "" && runDir != nil {
_ = runDir.WriteRunManifest(output.Manifest)
_ = runDir.WriteCheckpointEvents(output.CheckpointEvents)
}
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("run pipeline %q: %w", pipelineID, err))
}
runOutputDir := filepath.Join(outputRoot(*outputDir), runDir.RunID())
if err := runDir.WriteRunManifest(output.Manifest); err != nil {
runOutputDir := filepath.Join(outputRoot(*outputDir), runID)
if err := writeDiagnostics(runDir, func() error { return runDir.WriteRunManifest(output.Manifest) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics run manifest: %w", err))
}
if err := runDir.WriteWarnings(output.Warnings); err != nil {
if err := writeDiagnostics(runDir, func() error { return runDir.WriteWarnings(output.Warnings) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics warnings: %w", err))
}
if err := runDir.WriteRunReport(runReport{
if err := writeDiagnostics(runDir, func() error { return runDir.WriteCheckpointEvents(output.CheckpointEvents) }); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics checkpoint events: %w", err))
}
if err := writeDiagnostics(runDir, func() error {
return runDir.WriteRunReport(runReport{
RunID: runDir.RunID(),
PipelineID: effective.PipelineID,
OutputPath: runOutputDir,
DiagnosticsPath: runDir.Path(),
ApprovedCount: len(output.Approved),
OutputCount: len(output.NormalizeOutputs),
RejectedCount: len(output.Rejected),
WarningCount: len(output.Warnings),
ValidationStatus: output.Manifest.ValidationStatus,
})
}); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics run report: %w", err))
}
if err := writeOutputFiles(runOutputDir, output.OutputFiles); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
if err := writeDiagnostics(runDir, func() error {
return runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
RetentionMode: cfg.Diagnostics.Retention,
RunSucceeded: true,
HasWarnings: len(output.Warnings) > 0,
})
}); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("apply diagnostics retention: %w", err))
}
fmt.Fprintf(stdout, "pipeline %q complete: approved=%d rejected=%d output=%s\n", effective.PipelineID, len(output.Approved), len(output.Rejected), runOutputDir)
fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir)
if len(output.Warnings) > 0 {
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings))
}
@@ -255,7 +364,7 @@ type runReport struct {
PipelineID string `json:"pipeline_id"`
OutputPath string `json:"output_path"`
DiagnosticsPath string `json:"diagnostics_path,omitempty"`
ApprovedCount int `json:"approved_count"`
OutputCount int `json:"output_count"`
RejectedCount int `json:"rejected_count"`
WarningCount int `json:"warning_count"`
ValidationStatus string `json:"validation_status,omitempty"`
@@ -277,6 +386,83 @@ func failPipelineCommand(stderr io.Writer, runDir *diagnostics.RunDirectory, ret
return 1
}
func writeDiagnostics(runDir *diagnostics.RunDirectory, write func() error) error {
if runDir == nil {
return nil
}
return write()
}
func checkpointHandlersForRun(
settings workspace.Settings,
resolved pipeline.ResolvedPipeline,
rawInput []byte,
only []string,
llmProfiles []artifacts.LLMProfileManifest,
llmProfileOverride string,
sessionID string,
resume bool,
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
identity, err := workspace.NewCheckpointIdentity(workspace.CheckpointIdentityInput{
Pipeline: resolved,
InputKey: resolved.Input.Module,
RawInputDigest: rawInputDigest(rawInput),
SelectedLanes: only,
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID),
References: pipeline.ReferenceProvenance(resolved),
ProvenanceFingerprints: llmProfileFingerprints(llmProfiles),
})
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
}
recorder, err := checkpoint.NewWorkspaceRecorder(settings, identity)
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err)
}
loader := pipeline.NoopCheckpointLoader()
if resume {
loader, err = checkpoint.NewWorkspaceLoader(settings, identity)
if err != nil {
return nil, nil, fmt.Errorf("create checkpoint loader: %w", err)
}
}
return recorder, loader, nil
}
func rawInputDigest(data []byte) string {
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])
}
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) []workspace.Fingerprint {
var values []workspace.Fingerprint
if strings.TrimSpace(llmProfileOverride) != "" {
values = append(values, workspace.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
}
if strings.TrimSpace(sessionID) != "" {
values = append(values, workspace.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
}
return values
}
func llmProfileFingerprints(profiles []artifacts.LLMProfileManifest) []workspace.Fingerprint {
if len(profiles) == 0 {
return nil
}
values := make([]workspace.Fingerprint, 0, len(profiles))
for _, profile := range profiles {
id := strings.TrimSpace(profile.ID)
if id == "" {
continue
}
values = append(values, workspace.Fingerprint{
Name: "llm_profile:" + id,
Value: strings.TrimSpace(profile.Provider) + ":" + strings.TrimSpace(profile.Model),
})
}
return values
}
func configSource(configPath string) string {
if strings.TrimSpace(configPath) != "" {
return "flag"
@@ -412,13 +598,25 @@ func reorderRunArgs(args []string) []string {
func runFlagTakesValue(arg string) bool {
switch arg {
case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile":
case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile", "--session-id", "--reference", "--without-reference":
return true
default:
return false
}
}
func validateRunFlagValues(args []string) error {
for i, arg := range args {
if arg != "--session-id" {
continue
}
if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") {
return fmt.Errorf("flag needs an argument: --session-id")
}
}
return nil
}
func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
seen := make(map[string]struct{})
add := func(binding pipeline.ModuleBinding) {
@@ -427,15 +625,17 @@ func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
seen[id] = struct{}{}
}
}
add(resolved.Input)
add(resolved.Chunk)
add(resolved.Output)
for _, lane := range resolved.ArtifactLanes {
add(lane.Extract)
add(lane.Merge)
add(lane.Normalize)
for _, validator := range lane.Validators {
add(validator)
}
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
add(validator.Binding)
}
}
}
ids := make([]string, 0, len(seen))
@@ -512,11 +712,16 @@ func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) in
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if _, err := cfg.Resolve(config.ResolveInput{
effective, err := cfg.Resolve(config.ResolveInput{
PipelineID: *pipelineID,
Only: only,
Catalog: catalog,
}); err != nil {
})
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline)); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
@@ -661,6 +866,510 @@ func parseOnly(raw string) ([]string, error) {
return result, nil
}
type stringListFlag []string
func (flag *stringListFlag) String() string {
if flag == nil {
return ""
}
return strings.Join(*flag, ",")
}
func (flag *stringListFlag) Set(value string) error {
*flag = append(*flag, value)
return nil
}
type sessionIDFlag struct {
value string
set bool
}
func (flag *sessionIDFlag) String() string {
if flag == nil {
return ""
}
return flag.value
}
func (flag *sessionIDFlag) Set(value string) error {
flag.value = value
flag.set = true
return nil
}
type cliReferenceRequest struct {
Selector cliReferenceSelector
Source string
}
type cliReferenceUnbindRequest struct {
Selector cliReferenceSelector
}
type cliReferenceSelector struct {
LaneID string
Stage pipeline.ModuleStage
SlotName string
}
func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
if len(values) == 0 {
return nil, nil
}
requests := make([]cliReferenceRequest, 0, len(values))
for _, raw := range values {
name, source, ok := strings.Cut(raw, "=")
if !ok {
return nil, fmt.Errorf("--reference must use slot=path or lane.slot=path")
}
if strings.TrimSpace(source) == "" {
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
}
selector, err := parseReferenceSelector(name, "--reference")
if err != nil {
return nil, err
}
requests = append(requests, cliReferenceRequest{
Selector: selector,
Source: strings.TrimSpace(source),
})
}
return requests, nil
}
func parseReferenceUnbindFlags(values []string) ([]cliReferenceUnbindRequest, error) {
if len(values) == 0 {
return nil, nil
}
requests := make([]cliReferenceUnbindRequest, 0, len(values))
for _, raw := range values {
if strings.Contains(raw, "=") {
return nil, fmt.Errorf("--without-reference must use a reference selector without =path")
}
selector, err := parseReferenceSelector(raw, "--without-reference")
if err != nil {
return nil, err
}
requests = append(requests, cliReferenceUnbindRequest{
Selector: selector,
})
}
return requests, nil
}
func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector, error) {
selector := strings.TrimSpace(raw)
if selector == "" {
return cliReferenceSelector{}, fmt.Errorf("%s reference slot must not be empty", flagName)
}
parts := strings.Split(selector, ".")
for _, part := range parts {
if strings.TrimSpace(part) == "" {
return cliReferenceSelector{}, fmt.Errorf("%s must use non-empty reference selector values", flagName)
}
}
switch len(parts) {
case 1:
return cliReferenceSelector{SlotName: strings.TrimSpace(parts[0])}, nil
case 2:
first := strings.TrimSpace(parts[0])
slotName := strings.TrimSpace(parts[1])
if first == string(pipeline.StageChunk) {
return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil
}
if first == string(pipeline.StageMerge) {
return cliReferenceSelector{Stage: pipeline.StageMerge, SlotName: slotName}, nil
}
return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
case 3:
laneID := strings.TrimSpace(parts[0])
stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
slotName := strings.TrimSpace(parts[2])
if stage != pipeline.StageExtract && stage != pipeline.StageMerge && stage != pipeline.StageNormalize {
return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
}
return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil
default:
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, merge.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
}
}
func resolveCLIReferenceRequests(
cfg config.Config,
pipelineID string,
only []string,
catalog pipeline.ModuleCatalog,
referenceRequests []cliReferenceRequest,
unbindRequests []cliReferenceUnbindRequest,
) ([]pipeline.ReferenceBinding, []pipeline.ReferenceUnbind, error) {
if len(referenceRequests) == 0 && len(unbindRequests) == 0 {
return nil, nil, nil
}
targets, err := selectedReferenceTargets(cfg, pipelineID, only, catalog)
if err != nil {
return nil, nil, err
}
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
for _, request := range referenceRequests {
target, err := resolveCLIReferenceTarget(targets, request.Selector)
if err != nil {
return nil, nil, err
}
overrides = append(overrides, pipeline.ReferenceBinding{
Stage: target.stage,
LaneID: target.laneID,
SlotName: request.Selector.SlotName,
Source: request.Source,
BindingSource: contracts.ReferenceBindingSourceCLI,
})
}
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
for _, request := range unbindRequests {
target, err := resolveCLIReferenceTarget(targets, request.Selector)
if err != nil {
return nil, nil, err
}
unbinds = append(unbinds, pipeline.ReferenceUnbind{
Stage: target.stage,
LaneID: target.laneID,
SlotName: request.Selector.SlotName,
})
}
return overrides, unbinds, nil
}
type selectedReferenceTarget struct {
laneID string
stage pipeline.ModuleStage
module string
slots map[string]struct{}
}
func selectedReferenceTargets(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceTarget, error) {
profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID)
if !ok {
return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID))
}
lanesByID := make(map[string]pipeline.ArtifactLaneProfile, len(profile.Artifacts))
for rawLaneID, lane := range profile.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
return nil, fmt.Errorf("pipeline %q artifact lane id must not be empty", strings.TrimSpace(pipelineID))
}
if _, ok := lanesByID[laneID]; ok {
return nil, fmt.Errorf("pipeline %q artifact lane %q is duplicated after trimming", strings.TrimSpace(pipelineID), laneID)
}
lanesByID[laneID] = lane
}
selectedIDs := make([]string, 0, len(lanesByID))
if len(only) == 0 {
for laneID := range lanesByID {
selectedIDs = append(selectedIDs, laneID)
}
} else {
seen := make(map[string]struct{}, len(only))
for _, rawLaneID := range only {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
return nil, fmt.Errorf("pipeline %q selected artifact lane id must not be empty", strings.TrimSpace(pipelineID))
}
if _, ok := lanesByID[laneID]; !ok {
return nil, fmt.Errorf("pipeline %q selected artifact lane %q is not declared", strings.TrimSpace(pipelineID), laneID)
}
if _, ok := seen[laneID]; !ok {
selectedIDs = append(selectedIDs, laneID)
seen[laneID] = struct{}{}
}
}
}
sort.Strings(selectedIDs)
targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*3)
chunk := pipeline.Binding(profile.Chunk.Module)
chunk.Module = strings.TrimSpace(profile.Chunk.Module)
if chunk.Module == "" {
chunk.Module = pipeline.DefaultChunkModule
}
chunkSpec, err := cliReferenceChunkerSpec(catalog, chunk.Module)
if err != nil {
return nil, fmt.Errorf("pipeline %q chunk module %q: %w", strings.TrimSpace(pipelineID), chunk.Module, err)
}
targets = append(targets, selectedReferenceTarget{
stage: pipeline.StageChunk,
module: chunk.Module,
slots: referenceSlotSet(chunkSpec.ReferenceSlots),
})
for _, laneID := range selectedIDs {
lane := lanesByID[laneID]
extractModule := strings.TrimSpace(lane.Extract.Module)
if extractModule == "" {
return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID)
}
extractSpec, err := cliReferenceExtractorSpec(catalog, extractModule)
if err != nil {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: %w", strings.TrimSpace(pipelineID), laneID, extractModule, err)
}
artifactKind := extractSpec.ArtifactKind
if artifactKind == "" {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q does not declare an artifact kind", strings.TrimSpace(pipelineID), laneID, extractModule)
}
targets = append(targets, selectedReferenceTarget{
laneID: laneID,
stage: pipeline.StageExtract,
module: extractModule,
slots: referenceSlotSet(extractSpec.ReferenceSlots),
})
mergeModule := strings.TrimSpace(lane.Merge.Module)
if mergeModule == "" {
mergeModule = pipeline.DefaultMergeModule
}
mergeSpec, err := cliReferenceMergerSpec(catalog, mergeModule, artifactKind)
if err != nil {
return nil, fmt.Errorf("pipeline %q lane %q merge module %q: %w", strings.TrimSpace(pipelineID), laneID, mergeModule, err)
}
targets = append(targets, selectedReferenceTarget{
laneID: laneID,
stage: pipeline.StageMerge,
module: mergeModule,
slots: referenceSlotSet(mergeSpec.ReferenceSlots),
})
normalizeModule := strings.TrimSpace(lane.Normalize.Module)
if normalizeModule == "" {
normalizeModule = pipeline.DefaultNormalizeModule
}
normalizeSpec, err := cliReferenceNormalizerSpec(catalog, normalizeModule, artifactKind)
if err != nil {
return nil, fmt.Errorf("pipeline %q lane %q normalize module %q: %w", strings.TrimSpace(pipelineID), laneID, normalizeModule, err)
}
targets = append(targets, selectedReferenceTarget{
laneID: laneID,
stage: pipeline.StageNormalize,
module: normalizeModule,
slots: referenceSlotSet(normalizeSpec.ReferenceSlots),
})
}
return targets, nil
}
func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
pipelineID = strings.TrimSpace(pipelineID)
for rawID, profile := range profiles {
if strings.TrimSpace(rawID) == pipelineID {
return profile, true
}
}
return pipeline.PipelineProfile{}, false
}
func cliReferenceChunkerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Chunkers == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Chunkers.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func cliReferenceExtractorSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Extractors == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Extractors.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func cliReferenceMergerSpec(catalog pipeline.ModuleCatalog, module string, kind contracts.ArtifactKind) (pipeline.ModuleSpec, error) {
if catalog.Mergers == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
registered := catalog.Mergers.RegisteredArtifactKinds(module)
if len(registered) == 0 {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Mergers.SpecForArtifact(module, kind)
if !ok {
return pipeline.ModuleSpec{}, cliReferenceArtifactVariantError("merger", module, kind, registered)
}
return spec, nil
}
func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string, kind contracts.ArtifactKind) (pipeline.ModuleSpec, error) {
if catalog.Normalizers == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
registered := catalog.Normalizers.RegisteredArtifactKinds(module)
if len(registered) == 0 {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Normalizers.SpecForArtifact(module, kind)
if !ok {
return pipeline.ModuleSpec{}, cliReferenceArtifactVariantError("normalizer", module, kind, registered)
}
return spec, nil
}
func cliReferenceArtifactVariantError(moduleType string, module string, kind contracts.ArtifactKind, registered []contracts.ArtifactKind) error {
values := make([]string, len(registered))
for i, value := range registered {
values[i] = string(value)
}
if len(values) == 0 {
return fmt.Errorf("%s %q has no typed variant for artifact kind %q", moduleType, module, kind)
}
return fmt.Errorf("%s %q has no typed variant for artifact kind %q; registered kinds: %s", moduleType, module, kind, strings.Join(values, ", "))
}
func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} {
slotSet := make(map[string]struct{}, len(slots))
for _, slot := range slots {
slotSet[slot.Name] = struct{}{}
}
return slotSet
}
func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliReferenceSelector) (selectedReferenceTarget, error) {
slotName := strings.TrimSpace(selector.SlotName)
if slotName == "" {
return selectedReferenceTarget{}, fmt.Errorf("reference slot must not be empty")
}
if selector.Stage == pipeline.StageChunk {
for _, target := range targets {
if target.stage != pipeline.StageChunk {
continue
}
if _, ok := target.slots[slotName]; !ok {
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
}
return target, nil
}
return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
}
if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageMerge || selector.Stage == pipeline.StageNormalize {
if selector.LaneID == "" && selector.Stage == pipeline.StageMerge {
return resolveCLIReferenceStageTarget(targets, selector.Stage, slotName)
}
for _, target := range targets {
if target.laneID == selector.LaneID && target.stage == selector.Stage {
if _, ok := target.slots[slotName]; !ok {
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
}
return target, nil
}
}
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
}
if strings.TrimSpace(selector.LaneID) != "" {
return resolveCLIReferenceLaneTarget(targets, strings.TrimSpace(selector.LaneID), slotName)
}
return resolveCLIReferenceFlatTarget(targets, slotName)
}
func resolveCLIReferenceStageTarget(targets []selectedReferenceTarget, stage pipeline.ModuleStage, slotName string) (selectedReferenceTarget, error) {
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if target.stage != stage {
continue
}
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
switch len(matches) {
case 0:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected %s target", slotName, stage)
case 1:
return matches[0], nil
default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected %s targets (%s); use a more specific selector such as %s", slotName, stage, targetList(matches), selectorSuggestions(matches, slotName))
}
}
func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
laneSelected := false
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if target.laneID != laneID {
continue
}
laneSelected = true
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
if !laneSelected {
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", laneID)
}
switch len(matches) {
case 0:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, laneID)
case 1:
return matches[0], nil
default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use a more specific selector such as %s", slotName, laneID, targetList(matches), selectorSuggestions(matches, slotName))
}
}
func resolveCLIReferenceFlatTarget(targets []selectedReferenceTarget, slotName string) (selectedReferenceTarget, error) {
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
switch len(matches) {
case 0:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected reference target", slotName)
case 1:
return matches[0], nil
default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets (%s); use a more specific selector such as %s", slotName, targetList(matches), selectorSuggestions(matches, slotName))
}
}
func targetList(targets []selectedReferenceTarget) string {
labels := make([]string, 0, len(targets))
for _, target := range targets {
labels = append(labels, targetLabel(target))
}
sort.Strings(labels)
return strings.Join(labels, ", ")
}
func targetLabel(target selectedReferenceTarget) string {
if target.stage == pipeline.StageChunk {
return "chunk"
}
return target.laneID + "." + string(target.stage)
}
func selectorSuggestions(targets []selectedReferenceTarget, slotName string) string {
suggestions := make([]string, 0, len(targets))
for _, target := range targets {
if target.stage == pipeline.StageChunk {
suggestions = append(suggestions, "chunk."+slotName)
continue
}
suggestions = append(suggestions, target.laneID+"."+string(target.stage)+"."+slotName)
}
sort.Strings(suggestions)
return strings.Join(suggestions, " or ")
}
func sortedPipelineIDs(cfg config.Config) []string {
ids := make([]string, 0, len(cfg.Pipelines))
for id := range cfg.Pipelines {

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,68 @@
package cli
import (
"context"
"errors"
"fmt"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/scriptorium"
)
const profileCheckPromptID = "notarius.profile.check"
var profileCheckPromptFS = fstest.MapFS{
"prompts/profile-check.yaml": &fstest.MapFile{Data: []byte(`id: notarius.profile.check
version: "1.0.0"
default_profile: mistral-small-3
inputs:
- name: transcript
required: true
messages:
- role: user
content: "{{input \"transcript\"}}"
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
}
func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
if len(profileIDs) == 0 {
return nil
}
engine, err := newProfileValidationEngine(cfg)
if err != nil {
return fmt.Errorf("load Scriptorium profiles: %w", err)
}
for _, profileID := range profileIDs {
if _, err := engine.Prepare(ctx, scriptorium.RunRequest{
PromptID: profileCheckPromptID,
ProfileID: profileID,
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("profile check"),
},
}); err != nil {
if errors.Is(err, scriptorium.ErrProfileNotFound) {
return fmt.Errorf("Scriptorium profile %q is not configured", profileID)
}
return fmt.Errorf("validate Scriptorium profile %q: %w", profileID, err)
}
}
return nil
}
func newProfileValidationEngine(cfg config.Config) (*scriptorium.Engine, error) {
opts := []scriptorium.Option{
scriptorium.WithPromptFS(profileCheckPromptFS, "prompts"),
}
if cfg.Scriptorium.ProfileFile != "" {
opts = append(opts, scriptorium.WithProfileFile(cfg.Scriptorium.ProfileFile))
}
return scriptorium.NewEngine(scriptorium.Config{
PromptDir: "unused",
ProfileDir: cfg.Scriptorium.ProfileDir,
}, opts...)
}

View File

@@ -1,53 +1,74 @@
package artifacts
import (
"encoding/json"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
type ArtifactCandidate struct {
Index int `json:"index"`
ExtractorKey string `json:"extractor_key"`
ArtifactType string `json:"artifact_type"`
SchemaVersion string `json:"schema_version"`
Payload json.RawMessage `json:"payload"`
SourceRefs []source.SourceRef `json:"source_refs,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type Artifact struct {
ExtractorKey string `json:"extractor_key"`
ArtifactType string `json:"artifact_type"`
SchemaVersion string `json:"schema_version"`
Payload json.RawMessage `json:"payload"`
SourceRefs []source.SourceRef `json:"source_refs,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type RejectedArtifact struct {
Candidate ArtifactCandidate `json:"candidate"`
ValidatorName string `json:"validator_name"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
}
type ArtifactLaneManifest struct {
ID string `json:"id"`
Extractor string `json:"extractor"`
Merger string `json:"merger"`
Normalizer string `json:"normalizer"`
Validators []string `json:"validators,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type ValidatorChainManifest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
Validators []ValidatorManifest `json:"validators"`
}
type ValidatorManifest struct {
Key string `json:"key"`
ExecutionClass string `json:"execution_class"`
}
type LLMProfileManifest struct {
ID string `json:"id"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
}
type ReferenceProvenance struct {
Stage string `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
OriginType string `json:"origin_type"`
OriginURI string `json:"origin_uri,omitempty"`
Digest string `json:"digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
SizeBytes int64 `json:"size_bytes,omitempty"`
BindingSource string `json:"binding_source,omitempty"`
}
type OutputSchemaProvenance struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
}
type NormalizedOutputManifest struct {
LaneID string `json:"lane_id"`
ModuleKey string `json:"module_key,omitempty"`
SourceID string `json:"source_id,omitempty"`
MediaType string `json:"media_type,omitempty"`
Schema OutputSchemaProvenance `json:"schema,omitempty"`
}
type RejectedOutputManifest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message,omitempty"`
AttemptCount int `json:"attempt_count,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type RunManifest struct {
RunID string `json:"run_id,omitempty"`
PipelineID string `json:"pipeline_id,omitempty"`
@@ -59,33 +80,16 @@ type RunManifest struct {
Merger string `json:"merger,omitempty"`
Normalizer string `json:"normalizer,omitempty"`
OutputEncoder string `json:"output_encoder,omitempty"`
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
ValidatorChains []ValidatorChainManifest `json:"validator_chains,omitempty"`
References []ReferenceProvenance `json:"references,omitempty"`
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
SchemaVersion string `json:"schema_version,omitempty"`
ValidationStatus string `json:"validation_status,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
}
func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact {
return Artifact{
ExtractorKey: candidate.ExtractorKey,
ArtifactType: candidate.ArtifactType,
SchemaVersion: candidate.SchemaVersion,
Payload: append(json.RawMessage(nil), candidate.Payload...),
SourceRefs: append([]source.SourceRef(nil), candidate.SourceRefs...),
Metadata: copyMetadata(candidate.Metadata),
}
}
func copyMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
copied := make(map[string]any, len(metadata))
for key, value := range metadata {
copied[key] = value
}
return copied
}

View File

@@ -2,116 +2,9 @@ package artifacts
import (
"encoding/json"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
func TestArtifactFromCandidatePreservesCandidateFields(t *testing.T) {
candidate := ArtifactCandidate{
Index: 7,
ExtractorKey: "generic-extractor",
ArtifactType: "generic-artifact",
SchemaVersion: "v1",
Payload: json.RawMessage(`{"name":"example"}`),
SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u2"},
},
Metadata: map[string]any{
"confidence": 0.75,
},
}
artifact := ArtifactFromCandidate(candidate)
if artifact.ExtractorKey != candidate.ExtractorKey {
t.Fatalf("ExtractorKey = %q, want %q", artifact.ExtractorKey, candidate.ExtractorKey)
}
if artifact.ArtifactType != candidate.ArtifactType {
t.Fatalf("ArtifactType = %q, want %q", artifact.ArtifactType, candidate.ArtifactType)
}
if artifact.SchemaVersion != candidate.SchemaVersion {
t.Fatalf("SchemaVersion = %q, want %q", artifact.SchemaVersion, candidate.SchemaVersion)
}
if string(artifact.Payload) != string(candidate.Payload) {
t.Fatalf("Payload = %s, want %s", artifact.Payload, candidate.Payload)
}
if !reflect.DeepEqual(artifact.SourceRefs, candidate.SourceRefs) {
t.Fatalf("SourceRefs = %#v, want %#v", artifact.SourceRefs, candidate.SourceRefs)
}
if !reflect.DeepEqual(artifact.Metadata, candidate.Metadata) {
t.Fatalf("Metadata = %#v, want %#v", artifact.Metadata, candidate.Metadata)
}
candidate.Payload[0] = '['
candidate.SourceRefs[0].StartUnitID = "changed"
candidate.Metadata["confidence"] = 0.5
if string(artifact.Payload) != `{"name":"example"}` {
t.Fatalf("Payload changed after candidate mutation: %s", artifact.Payload)
}
if artifact.SourceRefs[0].StartUnitID != "u1" {
t.Fatalf("SourceRefs changed after candidate mutation: %#v", artifact.SourceRefs)
}
if artifact.Metadata["confidence"] != 0.75 {
t.Fatalf("Metadata changed after candidate mutation: %#v", artifact.Metadata)
}
}
func TestJSONMarshalUsesExpectedFieldNames(t *testing.T) {
candidate := ArtifactCandidate{
Index: 1,
ExtractorKey: "generic-extractor",
ArtifactType: "generic-artifact",
SchemaVersion: "v1",
Payload: json.RawMessage(`{"value":true}`),
SourceRefs: []source.SourceRef{
{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u1"},
},
Metadata: map[string]any{
"reviewed": true,
},
}
rejected := RejectedArtifact{
Candidate: candidate,
ValidatorName: "generic-validator",
ReasonCode: "invalid",
Message: "candidate was not accepted",
}
gotJSON, err := json.Marshal(rejected)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var got map[string]any
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
assertHasKeys(t, got, "candidate", "validator_name", "reason_code", "message")
gotCandidate, ok := got["candidate"].(map[string]any)
if !ok {
t.Fatalf("candidate = %#v, want object", got["candidate"])
}
assertHasKeys(t, gotCandidate, "index", "extractor_key", "artifact_type", "schema_version", "payload", "source_refs", "metadata")
gotRefs, ok := gotCandidate["source_refs"].([]any)
if !ok {
t.Fatalf("source_refs = %#v, want array", gotCandidate["source_refs"])
}
if len(gotRefs) != 1 {
t.Fatalf("len(source_refs) = %d, want 1", len(gotRefs))
}
gotRef, ok := gotRefs[0].(map[string]any)
if !ok {
t.Fatalf("source_refs[0] = %#v, want object", gotRefs[0])
}
assertHasKeys(t, gotRef, "source_id", "start_unit_id", "end_unit_id")
}
func TestRunManifestOmitsEmptyOptionalFields(t *testing.T) {
gotJSON, err := json.Marshal(RunManifest{})
if err != nil {
@@ -128,7 +21,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
PipelineID: "pipeline-1",
PipelineDigest: "sha256:abc123",
LLMProfiles: []LLMProfileManifest{
{ID: "default", Provider: "openai-compatible", Model: "model-a"},
{ID: "default", Provider: "scriptorium", Model: "model-a"},
},
ArtifactLanes: []ArtifactLaneManifest{
{
@@ -136,12 +29,21 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
Extractor: "event-extractor",
Merger: "appendorder",
Normalizer: "noop",
Validators: []string{"grounded"},
Metadata: map[string]any{
"extractor": map[string]any{"prompt_id": "test.prompt"},
},
},
},
ValidatorChains: []ValidatorChainManifest{
{
Stage: "extract",
LaneID: "events",
ModuleKey: "event-extractor",
Validators: []ValidatorManifest{
{Key: "grounded", ExecutionClass: "deterministic"},
},
},
},
}
gotJSON, err := json.Marshal(manifest)
@@ -154,7 +56,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
t.Fatalf("json.Unmarshal() error = %v", err)
}
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "llm_profiles")
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "validator_chains", "llm_profiles")
profiles, ok := got["llm_profiles"].([]any)
if !ok {
@@ -180,7 +82,94 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
if !ok {
t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0])
}
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "metadata")
chains, ok := got["validator_chains"].([]any)
if !ok {
t.Fatalf("validator_chains = %#v, want array", got["validator_chains"])
}
if len(chains) != 1 {
t.Fatalf("len(validator_chains) = %d, want 1", len(chains))
}
chain, ok := chains[0].(map[string]any)
if !ok {
t.Fatalf("validator_chains[0] = %#v, want object", chains[0])
}
assertHasKeys(t, chain, "stage", "lane_id", "module_key", "validators")
}
func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
manifest := RunManifest{
References: []ReferenceProvenance{
{
Stage: "extract",
LaneID: "events",
SlotName: "roster",
OriginType: "file",
OriginURI: "file:///tmp/roster.txt",
Digest: "sha256:reference",
MediaType: "text/plain; charset=utf-8",
SizeBytes: 12,
BindingSource: "config",
},
},
}
gotJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var got RunManifest
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if len(got.References) != 1 {
t.Fatalf("len(References) = %d, want 1", len(got.References))
}
reference := got.References[0]
if reference.Stage != "extract" || reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference)
}
if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" {
t.Fatalf("reference provenance = %#v, want digest/media/size/source details", reference)
}
}
func TestRunManifestIncludesTopLevelModuleMetadata(t *testing.T) {
manifest := RunManifest{
ModuleMetadata: map[string]map[string]any{
"chunker": {
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:abc123",
"response_schema_key": "dnd_scenes",
"response_schema_name": "dnd_scenes",
},
},
}
gotJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var got map[string]any
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
moduleMetadata, ok := got["module_metadata"].(map[string]any)
if !ok {
t.Fatalf("module_metadata = %#v, want object", got["module_metadata"])
}
assertHasKeys(t, moduleMetadata, "chunker")
chunkerMetadata, ok := moduleMetadata["chunker"].(map[string]any)
if !ok {
t.Fatalf("module_metadata.chunker = %#v, want object", moduleMetadata["chunker"])
}
assertHasKeys(t, chunkerMetadata, "prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_name")
}
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {

View File

@@ -1,32 +1,34 @@
package config
import (
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const SupportedFileConfigVersion = 1
const SupportedFileConfigVersion = 2
type Config struct {
LLMProfiles map[string]LLMProfile `json:"llm_profiles"`
Scriptorium ScriptoriumConfig `json:"scriptorium,omitempty"`
Pipelines map[string]pipeline.PipelineProfile `json:"pipelines"`
Concurrency ConcurrencyConfig `json:"concurrency"`
Diagnostics DiagnosticsConfig `json:"diagnostics"`
Workspace WorkspaceConfig `json:"workspace"`
}
type LLMProfile struct {
Provider string `json:"provider,omitempty"`
BaseURL string `json:"base_url,omitempty"`
Model string `json:"model,omitempty"`
APIKey string `json:"api_key,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
MaxRetries int `json:"max_retries,omitempty"`
MaxConcurrency int `json:"max_concurrency,omitempty"`
type ScriptoriumConfig struct {
ProfileDir string `json:"profile_dir,omitempty"`
ProfileFile string `json:"profile_file,omitempty"`
}
type ConcurrencyConfig struct {
TotalLLM int `json:"total_llm"`
StageWorkers map[string]int `json:"stage_workers"`
extractWorkersConfigured bool
defaultedExtractWorkers int
}
type DiagnosticsConfig struct {
@@ -34,33 +36,78 @@ type DiagnosticsConfig struct {
Retention diagnostics.RetentionMode `json:"retention"`
}
type WorkspaceConfig struct {
Directory string `json:"directory,omitempty"`
Diagnostics WorkspaceDiagnosticsConfig `json:"diagnostics"`
Resume WorkspaceResumeConfig `json:"resume"`
Debug WorkspaceDebugConfig `json:"debug"`
}
type WorkspaceDiagnosticsConfig struct {
Enabled bool `json:"enabled"`
Retention diagnostics.RetentionMode `json:"retention,omitempty"`
enabledSet bool
retentionSet bool
}
type WorkspaceResumeConfig struct {
Enabled bool `json:"enabled"`
}
type WorkspaceDebugConfig struct {
Enabled bool `json:"enabled"`
}
func Default() Config {
return Config{
LLMProfiles: map[string]LLMProfile{
pipeline.DefaultLLMProfile: {
Provider: "openai-compatible",
TimeoutSeconds: 600,
MaxRetries: 3,
MaxConcurrency: 1,
},
},
Pipelines: map[string]pipeline.PipelineProfile{},
Concurrency: ConcurrencyConfig{
TotalLLM: 1,
StageWorkers: map[string]int{"extract": 1},
defaultedExtractWorkers: 1,
},
Diagnostics: DiagnosticsConfig{
WorkDir: "/tmp/notarius",
Retention: diagnostics.RetentionAuto,
},
Workspace: WorkspaceConfig{
Diagnostics: WorkspaceDiagnosticsConfig{
Enabled: true,
},
},
}
}
func (c *Config) RecomputeEffectiveDiagnostics() {
if c == nil {
return
}
if dir := c.workspaceDirectory(); dir != "" {
c.Diagnostics.WorkDir = filepath.Join(dir, "diagnostics")
}
if c.Workspace.Diagnostics.retentionSet {
c.Diagnostics.Retention = c.Workspace.Diagnostics.Retention
}
}
func (c Config) DiagnosticsEnabled() bool {
if !c.Workspace.Diagnostics.enabledSet {
return true
}
return c.Workspace.Diagnostics.Enabled
}
func (c Config) workspaceDirectory() string {
dir := strings.TrimSpace(c.Workspace.Directory)
if dir == "" {
return ""
}
return filepath.Clean(dir)
}
func cloneConfig(in Config) Config {
out := in
out.LLMProfiles = make(map[string]LLMProfile, len(in.LLMProfiles))
for key, profile := range in.LLMProfiles {
out.LLMProfiles[key] = profile
}
out.Concurrency.StageWorkers = cloneIntMap(in.Concurrency.StageWorkers)
out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines))
for key, profile := range in.Pipelines {
out.Pipelines[key] = clonePipelineProfile(profile)
@@ -68,11 +115,40 @@ func cloneConfig(in Config) Config {
return out
}
func cloneIntMap(in map[string]int) map[string]int {
if len(in) == 0 {
return nil
}
out := make(map[string]int, len(in))
for key, value := range in {
out[key] = value
}
return out
}
func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
if c == nil {
return
}
if c.StageWorkers == nil {
c.StageWorkers = make(map[string]int)
}
if !c.extractWorkersConfigured {
if value, ok := c.StageWorkers["extract"]; ok && (c.defaultedExtractWorkers == 0 || value != c.defaultedExtractWorkers) {
c.extractWorkersConfigured = true
return
}
c.StageWorkers["extract"] = c.TotalLLM
c.defaultedExtractWorkers = c.TotalLLM
}
}
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
out := in
out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk)
out.Output = cloneModuleBinding(in.Output)
out.References = cloneStringMap(in.References)
if len(in.Artifacts) > 0 {
out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts))
for key, lane := range in.Artifacts {
@@ -87,6 +163,7 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize)
out.References = cloneStringMap(in.References)
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators {
@@ -96,11 +173,35 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
return out
}
func cloneStringMap(in map[string]string) map[string]string {
if len(in) == 0 {
return nil
}
out := make(map[string]string, len(in))
for key, value := range in {
out[key] = value
}
return out
}
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out := in
if len(in.Options) > 0 {
out.Options = cloneOptions(in.Options)
}
out.References = cloneStringMap(in.References)
out.Validators = cloneValidatorOverride(in.Validators)
return out
}
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
out := pipeline.ValidatorOverride{Set: in.Set}
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators {
out.Validators[i] = cloneModuleBinding(binding)
}
}
return out
}

View File

@@ -4,24 +4,13 @@ import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestDefaultValues(t *testing.T) {
cfg := Default()
defaultProfile, ok := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
if !ok {
t.Fatalf("expected default LLM profile")
}
if defaultProfile.Provider != "openai-compatible" {
t.Fatalf("unexpected provider: %q", defaultProfile.Provider)
}
if defaultProfile.BaseURL != "" || defaultProfile.Model != "" {
t.Fatalf("default profile should not require base URL/model yet: %+v", defaultProfile)
}
if defaultProfile.TimeoutSeconds != 600 || defaultProfile.MaxRetries != 3 || defaultProfile.MaxConcurrency != 1 {
t.Fatalf("unexpected default LLM operational values: %+v", defaultProfile)
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
t.Fatalf("unexpected Scriptorium profile source defaults: %+v", cfg.Scriptorium)
}
if len(cfg.Pipelines) != 0 {
t.Fatalf("expected no built-in pipeline profiles, got %v", cfg.Pipelines)
@@ -29,20 +18,37 @@ func TestDefaultValues(t *testing.T) {
if cfg.Concurrency.TotalLLM != 1 {
t.Fatalf("unexpected total LLM concurrency: %d", cfg.Concurrency.TotalLLM)
}
if got := cfg.Concurrency.StageWorkers["extract"]; got != 1 {
t.Fatalf("unexpected extract workers: %d", got)
}
if cfg.Diagnostics.WorkDir != "/tmp/notarius" {
t.Fatalf("unexpected diagnostics work dir: %q", cfg.Diagnostics.WorkDir)
}
if cfg.Diagnostics.Retention != diagnostics.RetentionAuto {
t.Fatalf("unexpected diagnostics retention: %q", cfg.Diagnostics.Retention)
}
if cfg.Workspace.Directory != "" {
t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory)
}
if !cfg.Workspace.Diagnostics.Enabled || !cfg.DiagnosticsEnabled() {
t.Fatalf("expected workspace diagnostics enabled by default: %+v", cfg.Workspace.Diagnostics)
}
if cfg.Workspace.Diagnostics.Retention != "" {
t.Fatalf("unexpected workspace diagnostics retention: %q", cfg.Workspace.Diagnostics.Retention)
}
if cfg.Workspace.Resume.Enabled {
t.Fatalf("workspace resume should be disabled by default")
}
if cfg.Workspace.Debug.Enabled {
t.Fatalf("workspace debug should be disabled by default")
}
}
func TestApplyFileConfigMergesWithDefaults(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 1
llm_profiles:
default:
model: test-model
version: 2
scriptorium:
profile_dir: ./profiles
pipelines:
example:
input: fake/input
@@ -59,16 +65,15 @@ pipelines:
t.Fatalf("ApplyFileConfig: %v", err)
}
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
if profile.Model != "test-model" {
t.Fatalf("expected file model, got %+v", profile)
}
if profile.Provider != "openai-compatible" || profile.TimeoutSeconds != 600 || profile.MaxRetries != 3 {
t.Fatalf("expected default LLM fields to be preserved, got %+v", profile)
if cfg.Scriptorium.ProfileDir != "./profiles" {
t.Fatalf("expected Scriptorium profile dir, got %+v", cfg.Scriptorium)
}
if cfg.Concurrency.TotalLLM != 1 {
t.Fatalf("expected default concurrency preserved, got %d", cfg.Concurrency.TotalLLM)
}
if got := cfg.Concurrency.StageWorkers["extract"]; got != 1 {
t.Fatalf("expected default extract workers preserved, got %d", got)
}
if cfg.Diagnostics.Retention != diagnostics.RetentionAuto {
t.Fatalf("expected default diagnostics retention preserved, got %q", cfg.Diagnostics.Retention)
}

View File

@@ -3,9 +3,7 @@ package config
import (
"fmt"
"strings"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -14,16 +12,21 @@ type ResolveInput struct {
Only []string
Catalog pipeline.ModuleCatalog
LLMProfileOverride string
ReferenceOverrides []pipeline.ReferenceBinding
ReferenceUnbinds []pipeline.ReferenceUnbind
}
type EffectiveConfig struct {
Config Config
PipelineID string
Only []string
ReferenceOverrides []pipeline.ReferenceBinding
ReferenceUnbinds []pipeline.ReferenceUnbind
ResolvedPipeline pipeline.ResolvedPipeline
}
func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
c.Concurrency.recomputeStageWorkerDefaults()
if err := c.Validate(); err != nil {
return EffectiveConfig{}, err
}
@@ -40,13 +43,14 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
profile = clonePipelineProfile(profile)
profile.ID = pipelineID
if override := strings.TrimSpace(input.LLMProfileOverride); override != "" {
if !hasLLMProfile(c.LLMProfiles, override) {
return EffectiveConfig{}, fmt.Errorf("LLM profile override %q is not configured", override)
}
applyLLMProfileOverride(&profile, override)
}
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{Only: input.Only}, input.Catalog)
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{
Only: input.Only,
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
}, input.Catalog)
if err != nil {
return EffectiveConfig{}, fmt.Errorf("resolve pipeline %q: %w", pipelineID, err)
}
@@ -55,21 +59,18 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
Config: cloneConfig(c),
PipelineID: pipelineID,
Only: append([]string(nil), input.Only...),
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
ResolvedPipeline: resolved,
}, nil
}
func applyLLMProfileOverride(profile *pipeline.PipelineProfile, profileID string) {
profile.Input.LLMProfile = profileID
profile.Chunk.LLMProfile = profileID
profile.Output.LLMProfile = profileID
for laneID, lane := range profile.Artifacts {
lane.Extract.LLMProfile = profileID
lane.Merge.LLMProfile = profileID
lane.Normalize.LLMProfile = profileID
for i := range lane.Validators {
lane.Validators[i].LLMProfile = profileID
}
profile.Artifacts[laneID] = lane
}
}
@@ -83,36 +84,3 @@ func lookupPipelineProfile(profiles map[string]pipeline.PipelineProfile, pipelin
}
return pipeline.PipelineProfile{}, false
}
func (c Config) OpenAICompatibleClientConfig(profileID string) (llm.OpenAICompatibleClientConfig, error) {
trimmedID := strings.TrimSpace(profileID)
profile, ok := c.LLMProfile(trimmedID)
if !ok {
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q is not configured", trimmedID)
}
provider := strings.TrimSpace(profile.Provider)
if provider == "" {
provider = providerOpenAICompatible
}
if provider != providerOpenAICompatible {
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q provider %q is not supported", trimmedID, provider)
}
baseURL := strings.TrimSpace(profile.BaseURL)
if baseURL == "" {
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q base URL must not be empty", trimmedID)
}
model := strings.TrimSpace(profile.Model)
if model == "" {
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q model must not be empty", trimmedID)
}
return llm.OpenAICompatibleClientConfig{
BaseURL: baseURL,
Model: model,
APIKey: profile.APIKey,
MaxRetries: profile.MaxRetries,
RequestTimeout: time.Duration(profile.TimeoutSeconds) * time.Second,
}, nil
}

View File

@@ -3,7 +3,6 @@ package config
import (
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -28,6 +27,19 @@ func TestResolveRejectsEmptyAndUnknownPipelineID(t *testing.T) {
}
}
func TestResolveMaterializesDefaultExtractWorkersFromEffectiveTotal(t *testing.T) {
cfg := validConfig()
cfg.Concurrency.TotalLLM = 4
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
if got := effective.Config.Concurrency.StageWorkers["extract"]; got != 4 {
t.Fatalf("effective extract workers = %d, want total concurrency 4", got)
}
}
func TestResolveLaneFilteringSuccessAndFailure(t *testing.T) {
effective, err := validConfig().Resolve(ResolveInput{
PipelineID: " example ",
@@ -98,6 +110,43 @@ func TestResolveSurfacesMissingCapabilityThroughCatalog(t *testing.T) {
}
}
func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
profile.Chunk = pipeline.Binding("dnd/scenes")
lane := profile.Artifacts["events"]
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"events": lane}
cfg.Pipelines["example"] = profile
catalog := fakeCatalog(t,
pipeline.ModuleSpec{
Key: "fake/input",
Stage: pipeline.StageInput,
Provides: []string{"source.transcript"},
},
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"artifact"},
},
)
mustRegisterChunker(t, catalog.Chunkers, pipeline.ModuleSpec{
Key: "dnd/scenes",
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks", "chunks.scenes"},
})
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
if got := effective.ResolvedPipeline.Chunk.Module; got != "dnd/scenes" {
t.Fatalf("Chunk.Module = %q, want dnd/scenes", got)
}
}
func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
cfg := validConfig()
first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
@@ -120,7 +169,19 @@ func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
cfg := validConfig()
cfg.LLMProfiles["runtime"] = LLMProfile{Provider: "openai-compatible"}
profile := cfg.Pipelines["example"]
profile.Input.LLMProfile = "input-profile"
profile.Output.LLMProfile = "output-profile"
lane := profile.Artifacts["events"]
lane.Merge.LLMProfile = "merge-profile"
lane.Extract.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{
{Module: "fake/llm-validator", LLMProfile: "validator-profile"},
},
}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
base, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
if err != nil {
@@ -138,77 +199,43 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
if base.ResolvedPipeline.Digest == effective.ResolvedPipeline.Digest {
t.Fatalf("expected digest to change after LLM profile override")
}
for _, binding := range resolvedBindings(effective.ResolvedPipeline) {
for _, binding := range llmCapableBindings(effective.ResolvedPipeline) {
if binding.LLMProfile != "runtime" {
t.Fatalf("binding profile = %q, want runtime", binding.LLMProfile)
t.Fatalf("LLM-capable binding profile = %q, want runtime", binding.LLMProfile)
}
}
_, err = cfg.Resolve(ResolveInput{
PipelineID: "example",
Catalog: fakeCatalog(t),
LLMProfileOverride: "missing",
})
if err == nil || !strings.Contains(err.Error(), "LLM profile override") {
t.Fatalf("expected override profile error, got %v", err)
if effective.ResolvedPipeline.Input.LLMProfile != "input-profile" {
t.Fatalf("input profile = %q, want original input-profile", effective.ResolvedPipeline.Input.LLMProfile)
}
if effective.ResolvedPipeline.Output.LLMProfile != "output-profile" {
t.Fatalf("output profile = %q, want original output-profile", effective.ResolvedPipeline.Output.LLMProfile)
}
eventLane := effective.ResolvedPipeline.ArtifactLanes[0]
if eventLane.Merge.LLMProfile != "runtime" {
t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile)
}
validatorChain := findEffectiveValidatorChain(effective.ResolvedPipeline.ValidatorChains, pipeline.StageExtract, "events", "fake/extract")
if validatorChain == nil || len(validatorChain.Validators) != 1 {
t.Fatalf("validator chain = %#v, want one extract validator", effective.ResolvedPipeline.ValidatorChains)
}
if validatorChain.Validators[0].Binding.LLMProfile != "validator-profile" {
t.Fatalf("validator profile = %q, want original validator-profile", validatorChain.Validators[0].Binding.LLMProfile)
}
}
func resolvedBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
bindings := []pipeline.ModuleBinding{resolved.Input, resolved.Chunk, resolved.Output}
func findEffectiveValidatorChain(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID string, module string) *pipeline.ResolvedValidatorChain {
for i := range chains {
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
return &chains[i]
}
}
return nil
}
func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
bindings := []pipeline.ModuleBinding{resolved.Chunk}
for _, lane := range resolved.ArtifactLanes {
bindings = append(bindings, lane.Extract, lane.Merge, lane.Normalize)
bindings = append(bindings, lane.Validators...)
}
return bindings
}
func TestOpenAICompatibleClientConfigRejectsIncompleteDefaultProfile(t *testing.T) {
cfg := Default()
_, err := cfg.OpenAICompatibleClientConfig("default")
if err == nil || !strings.Contains(err.Error(), "base URL") {
t.Fatalf("expected incomplete profile error, got %v", err)
}
}
func TestOpenAICompatibleClientConfigSuccess(t *testing.T) {
cfg := validConfig()
profile := cfg.LLMProfiles["default"]
profile.APIKey = "secret"
profile.TimeoutSeconds = 45
profile.MaxRetries = 4
cfg.LLMProfiles["default"] = profile
llmCfg, err := cfg.OpenAICompatibleClientConfig(" default ")
if err != nil {
t.Fatalf("OpenAICompatibleClientConfig: %v", err)
}
if llmCfg.BaseURL != "https://example.invalid/v1" || llmCfg.Model != "test-model" || llmCfg.APIKey != "secret" {
t.Fatalf("unexpected client config strings: %+v", llmCfg)
}
if llmCfg.MaxRetries != 4 {
t.Fatalf("unexpected max retries: %d", llmCfg.MaxRetries)
}
if llmCfg.RequestTimeout != 45*time.Second {
t.Fatalf("unexpected timeout: %s", llmCfg.RequestTimeout)
}
}
func TestOpenAICompatibleClientConfigRejectsUnknownAndUnsupportedProfiles(t *testing.T) {
_, err := validConfig().OpenAICompatibleClientConfig("missing")
if err == nil || !strings.Contains(err.Error(), "not configured") {
t.Fatalf("expected unknown profile error, got %v", err)
}
cfg := validConfig()
profile := cfg.LLMProfiles["default"]
profile.Provider = "unsupported"
cfg.LLMProfiles["default"] = profile
_, err = cfg.OpenAICompatibleClientConfig("default")
if err == nil || !strings.Contains(err.Error(), "provider") {
t.Fatalf("expected unsupported provider error, got %v", err)
}
}

View File

@@ -7,7 +7,6 @@ import (
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func LoadFromEnv() (Config, error) {
@@ -30,43 +29,6 @@ func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool))
if c == nil {
return fmt.Errorf("config must not be nil")
}
if c.LLMProfiles == nil {
c.LLMProfiles = map[string]LLMProfile{}
}
defaultProfile := c.LLMProfiles[pipeline.DefaultLLMProfile]
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_API_KEY"); ok {
defaultProfile.APIKey = raw
}
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_BASE_URL"); ok {
defaultProfile.BaseURL = strings.TrimSpace(raw)
}
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MODEL"); ok {
defaultProfile.Model = strings.TrimSpace(raw)
}
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS"); ok {
value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS", raw)
if err != nil {
return err
}
defaultProfile.TimeoutSeconds = value
}
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MAX_RETRIES"); ok {
value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_MAX_RETRIES", raw)
if err != nil {
return err
}
defaultProfile.MaxRetries = value
}
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY"); ok {
value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY", raw)
if err != nil {
return err
}
defaultProfile.MaxConcurrency = value
}
c.LLMProfiles[pipeline.DefaultLLMProfile] = defaultProfile
if raw, ok := lookup("NOTARIUS_TOTAL_LLM_CONCURRENCY"); ok {
value, err := parseIntEnv("NOTARIUS_TOTAL_LLM_CONCURRENCY", raw)
if err != nil {
@@ -74,12 +36,54 @@ func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool))
}
c.Concurrency.TotalLLM = value
}
if raw, ok := lookup("NOTARIUS_STAGE_WORKERS_EXTRACT"); ok {
value, err := parseIntEnv("NOTARIUS_STAGE_WORKERS_EXTRACT", raw)
if err != nil {
return err
}
if c.Concurrency.StageWorkers == nil {
c.Concurrency.StageWorkers = make(map[string]int)
}
c.Concurrency.StageWorkers["extract"] = value
c.Concurrency.extractWorkersConfigured = true
}
c.Concurrency.recomputeStageWorkerDefaults()
if raw, ok := lookup("NOTARIUS_WORK_DIR"); ok {
c.Diagnostics.WorkDir = strings.TrimSpace(raw)
}
if raw, ok := lookup("NOTARIUS_DIAGNOSTICS_RETENTION"); ok {
c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(raw))
}
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIR"); ok {
c.Workspace.Directory = strings.TrimSpace(raw)
}
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED"); ok {
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED", raw)
if err != nil {
return err
}
c.Workspace.Diagnostics.Enabled = value
c.Workspace.Diagnostics.enabledSet = true
}
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION"); ok {
c.Workspace.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(raw))
c.Workspace.Diagnostics.retentionSet = true
}
if raw, ok := lookup("NOTARIUS_WORKSPACE_RESUME_ENABLED"); ok {
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_RESUME_ENABLED", raw)
if err != nil {
return err
}
c.Workspace.Resume.Enabled = value
}
if raw, ok := lookup("NOTARIUS_WORKSPACE_DEBUG_ENABLED"); ok {
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_DEBUG_ENABLED", raw)
if err != nil {
return err
}
c.Workspace.Debug.Enabled = value
}
c.RecomputeEffectiveDiagnostics()
return nil
}
@@ -90,3 +94,11 @@ func parseIntEnv(name string, raw string) (int, error) {
}
return value, nil
}
func parseBoolEnv(name string, raw string) (bool, error) {
value, err := strconv.ParseBool(strings.TrimSpace(raw))
if err != nil {
return false, fmt.Errorf("%s: must be a boolean", name)
}
return value, nil
}

View File

@@ -8,62 +8,176 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestApplyEnvOverridesOperationalAndLLMValues(t *testing.T) {
func TestApplyEnvOverridesOperationalValues(t *testing.T) {
cfg := Default()
cfg.Pipelines["example"] = pipeline.PipelineProfile{ID: "example", Input: pipeline.Binding("before")}
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
"NOTARIUS_LLM_DEFAULT_API_KEY": "secret",
"NOTARIUS_LLM_DEFAULT_BASE_URL": "https://example.invalid/v1",
"NOTARIUS_LLM_DEFAULT_MODEL": "test-model",
"NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS": "120",
"NOTARIUS_LLM_DEFAULT_MAX_RETRIES": "5",
"NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY": "2",
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "3",
"NOTARIUS_STAGE_WORKERS_EXTRACT": "2",
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
"NOTARIUS_WORKSPACE_DIR": "/var/lib/notarius-env",
"NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED": "false",
"NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION": "always",
"NOTARIUS_WORKSPACE_RESUME_ENABLED": "true",
"NOTARIUS_WORKSPACE_DEBUG_ENABLED": "true",
"NOTARIUS_PIPELINE_INPUT": "after",
}))
if err != nil {
t.Fatalf("ApplyEnvOverrides: %v", err)
}
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
if profile.APIKey != "secret" || profile.BaseURL != "https://example.invalid/v1" || profile.Model != "test-model" {
t.Fatalf("unexpected LLM profile strings: %+v", profile)
}
if profile.TimeoutSeconds != 120 || profile.MaxRetries != 5 || profile.MaxConcurrency != 2 {
t.Fatalf("unexpected LLM profile numeric values: %+v", profile)
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
t.Fatalf("LLM environment overrides must not change Scriptorium config: %+v", cfg.Scriptorium)
}
if cfg.Concurrency.TotalLLM != 3 {
t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM)
}
if cfg.Diagnostics.WorkDir != "/tmp/notarius-env" || cfg.Diagnostics.Retention != diagnostics.RetentionNever {
if got := cfg.Concurrency.StageWorkers["extract"]; got != 2 {
t.Fatalf("extract workers = %d, want 2", got)
}
if cfg.Workspace.Directory != "/var/lib/notarius-env" {
t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory)
}
if cfg.DiagnosticsEnabled() {
t.Fatalf("expected workspace diagnostics disabled")
}
if cfg.Diagnostics.WorkDir != "/var/lib/notarius-env/diagnostics" || cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
t.Fatalf("unexpected diagnostics config: %+v", cfg.Diagnostics)
}
if !cfg.Workspace.Resume.Enabled {
t.Fatalf("expected workspace resume enabled")
}
if !cfg.Workspace.Debug.Enabled {
t.Fatalf("expected workspace debug enabled")
}
if cfg.Pipelines["example"].Input.Module != "before" {
t.Fatalf("environment overrides must not change pipeline wiring: %+v", cfg.Pipelines["example"])
}
}
func TestApplyEnvOverridesRejectsInvalidIntegers(t *testing.T) {
for _, name := range []string{"NOTARIUS_TOTAL_LLM_CONCURRENCY", "NOTARIUS_STAGE_WORKERS_EXTRACT"} {
t.Run(name, func(t *testing.T) {
cfg := Default()
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "many",
}))
if err == nil || !strings.Contains(err.Error(), "NOTARIUS_TOTAL_LLM_CONCURRENCY") {
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{name: "many"}))
if err == nil || !strings.Contains(err.Error(), name) {
t.Fatalf("expected named integer error, got %v", err)
}
})
}
}
func TestStageWorkerEnvironmentPrecedenceAndDefaulting(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 2
concurrency:
total_llm: 4
stage_workers:
extract: 2
`))
if err != nil {
t.Fatalf("ParseFileConfigYAML() error = %v", err)
}
cfg := Default()
if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
t.Fatalf("ApplyFileConfig() error = %v", err)
}
if err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "5",
"NOTARIUS_STAGE_WORKERS_EXTRACT": "3",
})); err != nil {
t.Fatalf("ApplyEnvOverrides() error = %v", err)
}
if cfg.Concurrency.TotalLLM != 5 || cfg.Concurrency.StageWorkers["extract"] != 3 {
t.Fatalf("effective concurrency = %#v, want total 5 and extract 3", cfg.Concurrency)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate(overridden) error = %v, want nil", err)
}
defaulted := Default()
if err := defaulted.applyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6"})); err != nil {
t.Fatalf("ApplyEnvOverrides(defaulted) error = %v", err)
}
if got := defaulted.Concurrency.StageWorkers["extract"]; got != 6 {
t.Fatalf("defaulted extract workers = %d, want effective total 6", got)
}
}
func TestStageWorkerRangeValidationUsesFinalEnvironmentTotal(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 2
concurrency:
total_llm: 4
stage_workers:
extract: 5
`))
if err != nil {
t.Fatalf("ParseFileConfigYAML() error = %v", err)
}
cfg := Default()
if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
t.Fatalf("ApplyFileConfig() error = %v", err)
}
if err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6"})); err != nil {
t.Fatalf("ApplyEnvOverrides() error = %v", err)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v, want final total to make extract workers valid", err)
}
}
func TestApplyEnvOverridesRejectsInvalidBooleans(t *testing.T) {
for _, name := range []string{
"NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED",
"NOTARIUS_WORKSPACE_RESUME_ENABLED",
"NOTARIUS_WORKSPACE_DEBUG_ENABLED",
} {
t.Run(name, func(t *testing.T) {
cfg := Default()
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{name: "maybe"}))
if err == nil || !strings.Contains(err.Error(), name) {
t.Fatalf("expected named boolean error, got %v", err)
}
})
}
}
func TestApplyEnvOverridesLegacyDiagnosticsRemainCompatibleWithoutWorkspace(t *testing.T) {
cfg := Default()
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
}))
if err != nil {
t.Fatalf("ApplyEnvOverrides: %v", err)
}
if cfg.Diagnostics.WorkDir != "/tmp/notarius-env" {
t.Fatalf("diagnostics work dir = %q, want legacy env", cfg.Diagnostics.WorkDir)
}
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
t.Fatalf("diagnostics retention = %q, want legacy env", cfg.Diagnostics.Retention)
}
}
func TestLoadFromEnvUsesDefaultConfig(t *testing.T) {
t.Setenv("NOTARIUS_LLM_DEFAULT_MODEL", "env-model")
t.Setenv("NOTARIUS_TOTAL_LLM_CONCURRENCY", "2")
cfg, err := LoadFromEnv()
if err != nil {
t.Fatalf("LoadFromEnv: %v", err)
}
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].Model != "env-model" {
t.Fatalf("expected env model, got %+v", cfg.LLMProfiles[pipeline.DefaultLLMProfile])
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
t.Fatalf("unexpected Scriptorium config from env: %+v", cfg.Scriptorium)
}
if cfg.Concurrency.TotalLLM != 2 {
t.Fatalf("expected env concurrency override, got %+v", cfg.Concurrency)
}
if got := cfg.Concurrency.StageWorkers["extract"]; got != 2 {
t.Fatalf("expected extract workers to default to total, got %d", got)
}
}

View File

@@ -4,34 +4,26 @@ import (
"bytes"
"fmt"
"os"
"regexp"
"sort"
"strings"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gopkg.in/yaml.v3"
)
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
type FileConfig struct {
Version int `yaml:"version"`
LLMProfiles map[string]FileLLMProfile `yaml:"llm_profiles,omitempty"`
Scriptorium *FileScriptoriumConfig `yaml:"scriptorium,omitempty"`
Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"`
Diagnostics *FileDiagnosticsConfig `yaml:"diagnostics,omitempty"`
Workspace *FileWorkspaceConfig `yaml:"workspace,omitempty"`
}
type FileLLMProfile struct {
Provider *string `yaml:"provider,omitempty"`
BaseURL *string `yaml:"base_url,omitempty"`
Model *string `yaml:"model,omitempty"`
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
Timeout *fileDurationSeconds `yaml:"timeout,omitempty"`
MaxRetries *int `yaml:"max_retries,omitempty"`
MaxConcurrency *int `yaml:"max_concurrency,omitempty"`
type FileScriptoriumConfig struct {
ProfileDir *string `yaml:"profile_dir,omitempty"`
ProfileFile *string `yaml:"profile_file,omitempty"`
}
type FilePipelineProfile struct {
@@ -39,6 +31,7 @@ type FilePipelineProfile struct {
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]string `yaml:"references,omitempty"`
}
type FileArtifactLaneProfile struct {
@@ -46,10 +39,12 @@ type FileArtifactLaneProfile struct {
Merge *fileModuleBinding `yaml:"merge,omitempty"`
Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
Validators []fileModuleBinding `yaml:"validators,omitempty"`
References map[string]string `yaml:"references,omitempty"`
}
type FileConcurrencyConfig struct {
TotalLLM *int `yaml:"total_llm,omitempty"`
StageWorkers map[string]int `yaml:"stage_workers,omitempty"`
}
type FileDiagnosticsConfig struct {
@@ -57,46 +52,29 @@ type FileDiagnosticsConfig struct {
Retention *string `yaml:"retention,omitempty"`
}
type fileDurationSeconds struct {
seconds int
type FileWorkspaceConfig struct {
Directory *string `yaml:"directory,omitempty"`
Diagnostics *FileWorkspaceDiagnosticsConfig `yaml:"diagnostics,omitempty"`
Resume *FileWorkspaceEnabledConfig `yaml:"resume,omitempty"`
Debug *FileWorkspaceEnabledConfig `yaml:"debug,omitempty"`
}
func (d *fileDurationSeconds) UnmarshalYAML(node *yaml.Node) error {
if node.Kind != yaml.ScalarNode {
return fmt.Errorf("must be an integer seconds value or duration string")
}
if node.Tag == "!!int" {
var seconds int
if err := node.Decode(&seconds); err != nil {
return fmt.Errorf("must be an integer seconds value or duration string")
}
d.seconds = seconds
return nil
}
var raw string
if err := node.Decode(&raw); err != nil {
return fmt.Errorf("must be an integer seconds value or duration string")
}
duration, err := time.ParseDuration(strings.TrimSpace(raw))
if err != nil {
return fmt.Errorf("invalid duration %q", raw)
}
if duration%time.Second != 0 {
return fmt.Errorf("duration %q must resolve to whole seconds", raw)
}
d.seconds = int(duration / time.Second)
return nil
type FileWorkspaceDiagnosticsConfig struct {
Enabled *bool `yaml:"enabled,omitempty"`
Retention *string `yaml:"retention,omitempty"`
}
func (d fileDurationSeconds) Seconds() int {
return d.seconds
type FileWorkspaceEnabledConfig struct {
Enabled *bool `yaml:"enabled,omitempty"`
}
type fileModuleBinding struct {
Module string
LLMProfile string
Retries int
Options map[string]any
References map[string]string
Validators pipeline.ValidatorOverride
}
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
@@ -125,12 +103,34 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err
}
b.LLMProfile = strings.TrimSpace(llmProfile)
case "retries":
var retries int
if err := valueNode.Decode(&retries); err != nil {
return err
}
b.Retries = retries
case "options":
var options map[string]any
if err := valueNode.Decode(&options); err != nil {
return err
}
b.Options = normalizeOptions(options)
case "references":
var references map[string]string
if err := valueNode.Decode(&references); err != nil {
return err
}
b.References = references
case "validators":
b.Validators.Set = true
var validators []fileModuleBinding
if err := valueNode.Decode(&validators); err != nil {
return err
}
b.Validators.Validators = make([]pipeline.ModuleBinding, len(validators))
for i, validator := range validators {
b.Validators.Validators[i] = validator.toPipelineBinding()
}
default:
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
}
@@ -145,7 +145,10 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
return pipeline.ModuleBinding{
Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile),
Retries: b.Retries,
Options: cloneOptions(b.Options),
References: normalizedStringMap(b.References),
Validators: b.Validators,
}
}
@@ -186,23 +189,17 @@ func (c *Config) ApplyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
}
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
_ = lookup
if c == nil {
return fmt.Errorf("config must not be nil")
}
if fileCfg.Version != SupportedFileConfigVersion {
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
}
if c.LLMProfiles == nil {
c.LLMProfiles = map[string]LLMProfile{}
}
if c.Pipelines == nil {
c.Pipelines = map[string]pipeline.PipelineProfile{}
}
profileIDs, rawLLMProfileIDs, err := normalizedMapKeys(fileCfg.LLMProfiles, "llm profile id")
if err != nil {
return err
}
pipelineIDs, rawPipelineIDs, err := normalizedMapKeys(fileCfg.Pipelines, "pipeline id")
if err != nil {
return err
@@ -212,38 +209,66 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
return err
}
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Chunk != nil {
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
return err
}
}
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Output != nil {
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
return err
}
}
for rawLaneID, fileLane := range filePipeline.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
continue
}
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
return err
}
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
return err
}
if fileLane.Merge != nil {
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
return err
}
}
if fileLane.Normalize != nil {
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
return err
}
}
for i, validator := range fileLane.Validators {
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
return err
}
}
}
}
for _, profileID := range profileIDs {
fileProfile := fileCfg.LLMProfiles[rawLLMProfileIDs[profileID]]
profile := c.LLMProfiles[profileID]
if fileProfile.Provider != nil {
profile.Provider = strings.TrimSpace(*fileProfile.Provider)
if fileCfg.Scriptorium != nil {
if fileCfg.Scriptorium.ProfileDir != nil {
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileDir)
if value == "" {
return fmt.Errorf("scriptorium.profile_dir must not be empty when set")
}
if fileProfile.BaseURL != nil {
profile.BaseURL = strings.TrimSpace(*fileProfile.BaseURL)
c.Scriptorium.ProfileDir = value
}
if fileProfile.Model != nil {
profile.Model = strings.TrimSpace(*fileProfile.Model)
if fileCfg.Scriptorium.ProfileFile != nil {
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileFile)
if value == "" {
return fmt.Errorf("scriptorium.profile_file must not be empty when set")
}
if fileProfile.APIKeyEnv != nil {
apiKey, err := resolveAPIKeyEnv(*fileProfile.APIKeyEnv, lookup)
if err != nil {
return fmt.Errorf("llm_profiles.%s.api_key_env: %w", profileID, err)
c.Scriptorium.ProfileFile = value
}
profile.APIKeyEnv = strings.TrimSpace(*fileProfile.APIKeyEnv)
profile.APIKey = apiKey
}
if fileProfile.Timeout != nil {
profile.TimeoutSeconds = fileProfile.Timeout.Seconds()
}
if fileProfile.MaxRetries != nil {
profile.MaxRetries = *fileProfile.MaxRetries
}
if fileProfile.MaxConcurrency != nil {
profile.MaxConcurrency = *fileProfile.MaxConcurrency
}
c.LLMProfiles[profileID] = profile
}
for _, pipelineID := range pipelineIDs {
@@ -256,6 +281,7 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
ID: pipelineID,
Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: normalizedStringMap(filePipeline.References),
}
if filePipeline.Chunk != nil {
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
@@ -265,8 +291,11 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
}
for _, laneID := range laneIDs {
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
extract := fileLane.Extract.toPipelineBinding()
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
lane := pipeline.ArtifactLaneProfile{
Extract: fileLane.Extract.toPipelineBinding(),
Extract: extract,
References: normalizedStringMap(fileLane.References),
}
if fileLane.Merge != nil {
lane.Merge = fileLane.Merge.toPipelineBinding()
@@ -288,6 +317,15 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil {
c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM
}
if fileCfg.Concurrency != nil && fileCfg.Concurrency.StageWorkers != nil {
workers, configured, err := normalizeStageWorkers(fileCfg.Concurrency.StageWorkers)
if err != nil {
return err
}
c.Concurrency.StageWorkers = workers
c.Concurrency.extractWorkersConfigured = configured
}
c.Concurrency.recomputeStageWorkerDefaults()
if fileCfg.Diagnostics != nil {
if fileCfg.Diagnostics.WorkDir != nil {
c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir)
@@ -296,10 +334,52 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Diagnostics.Retention))
}
}
if fileCfg.Workspace != nil {
if fileCfg.Workspace.Directory != nil {
c.Workspace.Directory = strings.TrimSpace(*fileCfg.Workspace.Directory)
}
if fileCfg.Workspace.Diagnostics != nil {
if fileCfg.Workspace.Diagnostics.Enabled != nil {
c.Workspace.Diagnostics.Enabled = *fileCfg.Workspace.Diagnostics.Enabled
c.Workspace.Diagnostics.enabledSet = true
}
if fileCfg.Workspace.Diagnostics.Retention != nil {
c.Workspace.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Workspace.Diagnostics.Retention))
c.Workspace.Diagnostics.retentionSet = true
}
}
if fileCfg.Workspace.Resume != nil && fileCfg.Workspace.Resume.Enabled != nil {
c.Workspace.Resume.Enabled = *fileCfg.Workspace.Resume.Enabled
}
if fileCfg.Workspace.Debug != nil && fileCfg.Workspace.Debug.Enabled != nil {
c.Workspace.Debug.Enabled = *fileCfg.Workspace.Debug.Enabled
}
}
c.RecomputeEffectiveDiagnostics()
return nil
}
func normalizeStageWorkers(values map[string]int) (map[string]int, bool, error) {
workers := make(map[string]int, len(values))
configured := false
for rawKey, value := range values {
key := strings.TrimSpace(rawKey)
if key == "" {
return nil, false, fmt.Errorf("concurrency.stage_workers key must not be empty")
}
if key != "extract" {
return nil, false, fmt.Errorf("concurrency.stage_workers key %q is not supported", rawKey)
}
if _, exists := workers[key]; exists {
return nil, false, fmt.Errorf("concurrency.stage_workers key %q is duplicated after trimming", key)
}
workers[key] = value
configured = true
}
return workers, configured, nil
}
func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, map[string]string, error) {
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
@@ -318,19 +398,37 @@ func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, ma
return keys, rawByNormalized, nil
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName)
if name == "" {
return "", fmt.Errorf("must not be empty")
func normalizedStringMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
if !envVarNamePattern.MatchString(name) {
return "", fmt.Errorf("must be an environment variable name")
out := make(map[string]string, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
value, ok := lookup(name)
if !ok {
return "", fmt.Errorf("%s is not set", name)
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
}
return value, nil
return out
}
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
for key, value := range base {
out[key] = value
}
for key, value := range override {
out[key] = value
}
return out
}
func normalizeOptions(options map[string]any) map[string]any {

View File

@@ -12,7 +12,7 @@ import (
func TestParseMinimalValidConfig(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 1
version: 2
`))
if err != nil {
t.Fatalf("ParseFileConfigYAML: %v", err)
@@ -24,7 +24,7 @@ version: 1
func TestLoadFileConfig(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(path, []byte("version: 1\n"), 0o644); err != nil {
if err := os.WriteFile(path, []byte("version: 2\n"), 0o644); err != nil {
t.Fatalf("write config: %v", err)
}
@@ -39,7 +39,7 @@ func TestLoadFileConfig(t *testing.T) {
func TestParseFileConfigRejectsUnknownYAMLFields(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(`
version: 1
version: 2
unexpected: true
`))
if err == nil || !strings.Contains(err.Error(), "field unexpected not found") {
@@ -49,7 +49,7 @@ unexpected: true
func TestParseFileConfigRejectsUnknownModuleBindingFields(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(`
version: 1
version: 2
pipelines:
example:
input:
@@ -70,8 +70,8 @@ func TestParseFileConfigRejectsMissingAndUnsupportedVersion(t *testing.T) {
data string
want string
}{
{name: "missing", data: `llm_profiles: {}`, want: "version is required"},
{name: "unsupported", data: `version: 2`, want: "unsupported config version"},
{name: "missing", data: `scriptorium: {}`, want: "version is required"},
{name: "unsupported", data: `version: 1`, want: "unsupported config version"},
}
for _, tc := range tests {
@@ -84,14 +84,50 @@ func TestParseFileConfigRejectsMissingAndUnsupportedVersion(t *testing.T) {
}
}
func TestParseFileConfigRejectsStaleLLMProfiles(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(`
version: 2
llm_profiles:
default: {}
`))
if err == nil || !strings.Contains(err.Error(), "llm_profiles") {
t.Fatalf("expected stale llm_profiles error, got %v", err)
}
}
func TestParseFileConfigScriptoriumProfileSources(t *testing.T) {
t.Run("profile dir", func(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
scriptorium:
profile_dir: ./profiles
`)
if cfg.Scriptorium.ProfileDir != "./profiles" || cfg.Scriptorium.ProfileFile != "" {
t.Fatalf("Scriptorium = %+v, want profile_dir", cfg.Scriptorium)
}
})
t.Run("profile file", func(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
scriptorium:
profile_file: ./profiles.yml
`)
if cfg.Scriptorium.ProfileFile != "./profiles.yml" || cfg.Scriptorium.ProfileDir != "" {
t.Fatalf("Scriptorium = %+v, want profile_file", cfg.Scriptorium)
}
})
}
func TestParseFileConfigModuleBindingForms(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
version: 2
pipelines:
example:
input: fake/input
chunk:
module: generic
retries: 2
options:
size: 10
flags:
@@ -103,9 +139,12 @@ pipelines:
extract:
module: fake/extract
llm_profile: fast
retries: 3
options:
temperature: 0
merge: appendorder
merge:
module: appendorder
retries: 1
normalize:
module: noop
output: json
@@ -118,6 +157,9 @@ pipelines:
if profile.Chunk.Module != "generic" {
t.Fatalf("unexpected chunk binding: %+v", profile.Chunk)
}
if profile.Chunk.Retries != 2 {
t.Fatalf("chunk retries = %d, want 2", profile.Chunk.Retries)
}
if profile.Chunk.Options["size"] != 10 {
t.Fatalf("expected chunk options to preserve scalar, got %#v", profile.Chunk.Options)
}
@@ -133,6 +175,9 @@ pipelines:
if lane.Extract.Module != "fake/extract" || lane.Extract.LLMProfile != "fast" {
t.Fatalf("unexpected extract binding: %+v", lane.Extract)
}
if lane.Extract.Retries != 3 || lane.Merge.Retries != 1 {
t.Fatalf("unexpected retries: extract=%d merge=%d", lane.Extract.Retries, lane.Merge.Retries)
}
if lane.Extract.Options["temperature"] != 0 {
t.Fatalf("expected object options, got %#v", lane.Extract.Options)
}
@@ -144,9 +189,88 @@ pipelines:
}
}
func TestParseFileConfigReferenceMaps(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
pipelines:
example:
input: fake/input
references:
" roster ": " ./shared-roster.yml "
artifacts:
events:
extract: fake/extract
references:
" lore ": " ./lore.md "
`)
profile := cfg.Pipelines["example"]
if !reflect.DeepEqual(profile.References, map[string]string{"roster": "./shared-roster.yml"}) {
t.Fatalf("pipeline references = %#v, want trimmed map", profile.References)
}
gotLaneRefs := profile.Artifacts["events"].References
if !reflect.DeepEqual(gotLaneRefs, map[string]string{"lore": "./lore.md"}) {
t.Fatalf("lane references = %#v, want trimmed map", gotLaneRefs)
}
}
func TestParseFileConfigStageLocalReferenceMaps(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
pipelines:
example:
input: fake/input
chunk:
module: generic
references:
" scene_guide ": " ./scenes.md "
artifacts:
events:
extract:
module: fake/extract
references:
" glossary ": " ./glossary.md "
" roster ": " ./extract-roster.yml "
references:
roster: ./legacy-roster.yml
lore: ./lore.md
merge:
module: appendorder
references:
" merge_notes ": " ./merge.md "
normalize:
module: noop
references:
" normalization_notes ": " ./normalization.md "
`)
profile := cfg.Pipelines["example"]
if !reflect.DeepEqual(profile.Chunk.References, map[string]string{"scene_guide": "./scenes.md"}) {
t.Fatalf("chunk references = %#v, want trimmed map", profile.Chunk.References)
}
lane := profile.Artifacts["events"]
if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) {
t.Fatalf("lane references = %#v, want trimmed map", lane.References)
}
wantExtract := map[string]string{
"glossary": "./glossary.md",
"lore": "./lore.md",
"roster": "./extract-roster.yml",
}
if !reflect.DeepEqual(lane.Extract.References, wantExtract) {
t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract)
}
if !reflect.DeepEqual(lane.Merge.References, map[string]string{"merge_notes": "./merge.md"}) {
t.Fatalf("merge references = %#v, want trimmed map", lane.Merge.References)
}
if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) {
t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References)
}
}
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
version: 2
pipelines:
example:
input: fake/input
@@ -176,87 +300,64 @@ pipelines:
}
}
func TestParseFileConfigDurationParsing(t *testing.T) {
tests := []struct {
name string
raw string
want int
}{
{name: "integer seconds", raw: "600", want: 600},
{name: "duration string", raw: "10m", want: 600},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
func TestParseFileConfigStageLocalValidatorOverrides(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
llm_profiles:
default:
timeout: `+tc.raw+`
version: 2
pipelines:
example:
input: fake/input
chunk:
module: generic
validators: []
artifacts:
events:
extract:
module: fake/extract
validators:
- fake/validator
- module: fake/llm-validator
llm_profile: careful
options:
threshold: 0.7
merge:
module: appendorder
validators: []
normalize:
module: noop
`)
if got := cfg.LLMProfiles["default"].TimeoutSeconds; got != tc.want {
t.Fatalf("TimeoutSeconds = %d, want %d", got, tc.want)
}
})
}
}
func TestParseFileConfigRejectsSubsecondDuration(t *testing.T) {
_, err := ParseFileConfigYAML([]byte(`
version: 1
llm_profiles:
default:
timeout: 1500ms
`))
if err == nil || !strings.Contains(err.Error(), "whole seconds") {
t.Fatalf("expected whole-seconds duration error, got %v", err)
profile := cfg.Pipelines["example"]
if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 {
t.Fatalf("chunk validator override = %#v, want explicit empty", profile.Chunk.Validators)
}
}
func TestApplyFileConfigResolvesAPIKeyEnv(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 1
llm_profiles:
default:
api_key_env: NOTARIUS_TEST_API_KEY
`))
if err != nil {
t.Fatalf("ParseFileConfigYAML: %v", err)
lane := profile.Artifacts["events"]
if !lane.Extract.Validators.Set {
t.Fatalf("extract validator override Set = false, want true")
}
cfg := Default()
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{"NOTARIUS_TEST_API_KEY": "secret"})); err != nil {
t.Fatalf("ApplyFileConfig: %v", err)
validators := lane.Extract.Validators.Validators
if len(validators) != 2 {
t.Fatalf("extract validators = %#v, want two validators", validators)
}
profile := cfg.LLMProfiles["default"]
if profile.APIKeyEnv != "NOTARIUS_TEST_API_KEY" || profile.APIKey != "secret" {
t.Fatalf("unexpected resolved API key: %+v", profile)
if validators[0].Module != "fake/validator" {
t.Fatalf("first validator = %#v, want fake/validator", validators[0])
}
}
func TestApplyFileConfigRejectsDuplicateTrimmedLLMProfileIDs(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 1
llm_profiles:
default:
model: first
" default ":
model: second
`))
if err != nil {
t.Fatalf("ParseFileConfigYAML: %v", err)
if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" {
t.Fatalf("second validator = %#v, want LLM validator with profile", validators[1])
}
cfg := Default()
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
if err == nil || !strings.Contains(err.Error(), "llm profile id") || !strings.Contains(err.Error(), "duplicated") {
t.Fatalf("expected duplicate LLM profile ID error, got %v", err)
if validators[1].Options["threshold"] != 0.7 {
t.Fatalf("second validator options = %#v, want threshold", validators[1].Options)
}
if !lane.Merge.Validators.Set || len(lane.Merge.Validators.Validators) != 0 {
t.Fatalf("merge validator override = %#v, want explicit empty", lane.Merge.Validators)
}
if lane.Normalize.Validators.Set {
t.Fatalf("normalize validator override Set = true, want omitted")
}
}
func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 1
version: 2
pipelines:
example:
input: fake/input
@@ -276,7 +377,7 @@ pipelines:
func TestApplyFileConfigRejectsDuplicateTrimmedArtifactLaneIDs(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 1
version: 2
pipelines:
example:
input: fake/input
@@ -297,49 +398,134 @@ pipelines:
}
}
func TestApplyFileConfigAllowsRetryOnlyLLMProfile(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
llm_profiles:
retry-only:
max_retries: 3
`)
profile := cfg.LLMProfiles["retry-only"]
if profile.MaxRetries != 3 {
t.Fatalf("unexpected max retries: %d", profile.MaxRetries)
}
if profile.TimeoutSeconds != 0 {
t.Fatalf("expected unset timeout, got %d", profile.TimeoutSeconds)
}
if profile.MaxConcurrency != 0 {
t.Fatalf("expected unset max concurrency, got %d", profile.MaxConcurrency)
}
}
func TestApplyFileConfigRejectsInvalidAPIKeyEnv(t *testing.T) {
func TestApplyFileConfigRejectsDuplicateTrimmedReferenceSlots(t *testing.T) {
tests := []struct {
name string
env string
raw string
want string
}{
{name: "invalid name", env: "NOTARIUS-KEY", want: "environment variable name"},
{name: "not set", env: "NOTARIUS_TEST_API_KEY", want: "is not set"},
{
name: "pipeline",
raw: `
version: 2
pipelines:
example:
input: fake/input
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" reference slot`,
},
{
name: "lane",
raw: `
version: 2
pipelines:
example:
input: fake/input
artifacts:
events:
extract: fake/extract
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" reference slot`,
},
{
name: "chunk",
raw: `
version: 2
pipelines:
example:
input: fake/input
chunk:
module: generic
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" chunk reference slot`,
},
{
name: "extract",
raw: `
version: 2
pipelines:
example:
input: fake/input
artifacts:
events:
extract:
module: fake/extract
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" extract reference slot`,
},
{
name: "normalize",
raw: `
version: 2
pipelines:
example:
input: fake/input
artifacts:
events:
extract: fake/extract
normalize:
module: noop
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" normalize reference slot`,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 1
llm_profiles:
default:
api_key_env: ` + tc.env + `
`))
fileCfg, err := ParseFileConfigYAML([]byte(tc.raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML: %v", err)
}
cfg := Default()
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "duplicated") {
t.Fatalf("expected duplicate reference slot error, got %v", err)
}
})
}
}
func TestApplyFileConfigRejectsInvalidScriptoriumSources(t *testing.T) {
tests := []struct {
name string
raw string
want string
}{
{name: "empty profile dir", raw: "profile_dir: ' '", want: "profile_dir"},
{name: "empty profile file", raw: "profile_file: ' '", want: "profile_file"},
{name: "both sources", raw: "profile_dir: ./profiles\n profile_file: ./profiles.yml", want: "mutually exclusive"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
cfg := Default()
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 2
scriptorium:
` + tc.raw + `
`))
if err != nil {
t.Fatalf("ParseFileConfigYAML: %v", err)
}
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
if err == nil {
err = cfg.Validate()
}
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("expected error containing %q, got %v", tc.want, err)
}
@@ -349,7 +535,7 @@ llm_profiles:
func TestApplyFileConfigOperationalSections(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
version: 2
concurrency:
total_llm: 4
diagnostics:
@@ -360,6 +546,9 @@ diagnostics:
if cfg.Concurrency.TotalLLM != 4 {
t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM)
}
if got := cfg.Concurrency.StageWorkers["extract"]; got != 4 {
t.Fatalf("default extract workers = %d, want total concurrency", got)
}
if cfg.Diagnostics.WorkDir != "/tmp/notarius-test" {
t.Fatalf("unexpected work dir: %q", cfg.Diagnostics.WorkDir)
}
@@ -368,6 +557,147 @@ diagnostics:
}
}
func TestApplyFileConfigStageWorkers(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
concurrency:
total_llm: 4
stage_workers:
extract: 3
`)
if got := cfg.Concurrency.StageWorkers["extract"]; got != 3 {
t.Fatalf("extract workers = %d, want 3", got)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
}
func TestApplyFileConfigEmptyStageWorkersDefaultsExtractToTotal(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
concurrency:
total_llm: 4
stage_workers: {}
`)
if got := cfg.Concurrency.StageWorkers["extract"]; got != 4 {
t.Fatalf("extract workers = %d, want total concurrency 4", got)
}
}
func TestApplyFileConfigRejectsUnsupportedStageWorkerKeys(t *testing.T) {
for _, test := range []struct {
name string
key string
want string
}{
{name: "empty", key: "' '", want: "must not be empty"},
{name: "unknown", key: "merge", want: "not supported"},
} {
t.Run(test.name, func(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte("version: 2\nconcurrency:\n stage_workers:\n " + test.key + ": 1\n"))
if err != nil {
t.Fatalf("ParseFileConfigYAML() error = %v", err)
}
cfg := Default()
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("ApplyFileConfig() error = %v, want %q", err, test.want)
}
})
}
}
func TestApplyFileConfigWorkspaceSection(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
workspace:
directory: /var/lib/notarius
diagnostics:
enabled: false
retention: never
resume:
enabled: true
debug:
enabled: true
diagnostics:
work_dir: /tmp/legacy
retention: always
`)
if cfg.Workspace.Directory != "/var/lib/notarius" {
t.Fatalf("workspace directory = %q, want /var/lib/notarius", cfg.Workspace.Directory)
}
if cfg.DiagnosticsEnabled() {
t.Fatalf("expected diagnostics disabled")
}
if cfg.Diagnostics.WorkDir != "/var/lib/notarius/diagnostics" {
t.Fatalf("effective diagnostics work dir = %q, want workspace diagnostics root", cfg.Diagnostics.WorkDir)
}
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
t.Fatalf("effective diagnostics retention = %q, want workspace override", cfg.Diagnostics.Retention)
}
if !cfg.Workspace.Resume.Enabled {
t.Fatalf("expected resume enabled")
}
if !cfg.Workspace.Debug.Enabled {
t.Fatalf("expected debug enabled")
}
}
func TestApplyFileConfigLegacyDiagnosticsRemainCompatible(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
diagnostics:
work_dir: /tmp/legacy
retention: always
`)
if cfg.Workspace.Directory != "" {
t.Fatalf("workspace directory = %q, want unset", cfg.Workspace.Directory)
}
if !cfg.DiagnosticsEnabled() {
t.Fatalf("expected diagnostics enabled")
}
if cfg.Diagnostics.WorkDir != "/tmp/legacy" {
t.Fatalf("effective diagnostics work dir = %q, want legacy", cfg.Diagnostics.WorkDir)
}
if cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
t.Fatalf("effective diagnostics retention = %q, want legacy", cfg.Diagnostics.Retention)
}
}
func TestApplyFileConfigWorkspaceRetentionOverridesLegacyRetentionOnlyWhenSet(t *testing.T) {
t.Run("legacy retained", func(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
workspace:
directory: /var/lib/notarius
diagnostics:
retention: never
`)
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
t.Fatalf("effective diagnostics retention = %q, want legacy", cfg.Diagnostics.Retention)
}
})
t.Run("workspace overrides", func(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
workspace:
directory: /var/lib/notarius
diagnostics:
retention: always
diagnostics:
retention: never
`)
if cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
t.Fatalf("effective diagnostics retention = %q, want workspace", cfg.Diagnostics.Retention)
}
})
}
func parseAndApplyConfig(t *testing.T, raw string) Config {
t.Helper()
fileCfg, err := ParseFileConfigYAML([]byte(raw))

View File

@@ -2,17 +2,8 @@ package config
import "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
const redactedSecret = "[REDACTED]"
func (c Config) Redacted() Config {
redacted := cloneConfig(c)
for id, profile := range redacted.LLMProfiles {
if profile.APIKey != "" {
profile.APIKey = redactedSecret
}
redacted.LLMProfiles[id] = profile
}
return redacted
return cloneConfig(c)
}
func (c Config) RedactedDiagnosticsPayload() any {
@@ -24,6 +15,8 @@ func (e EffectiveConfig) RedactedDiagnosticsPayload() any {
Config: e.Config.Redacted(),
PipelineID: e.PipelineID,
Only: append([]string(nil), e.Only...),
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), e.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), e.ReferenceUnbinds...),
ResolvedPipeline: cloneResolvedPipeline(e.ResolvedPipeline),
}
}
@@ -32,7 +25,14 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
out := in
out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk)
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
out.Output = cloneModuleBinding(in.Output)
if len(in.ValidatorChains) > 0 {
out.ValidatorChains = make([]pipeline.ResolvedValidatorChain, len(in.ValidatorChains))
for i, chain := range in.ValidatorChains {
out.ValidatorChains[i] = cloneResolvedValidatorChain(chain)
}
}
if len(in.ArtifactLanes) > 0 {
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
for i, lane := range in.ArtifactLanes {
@@ -42,11 +42,30 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
return out
}
func cloneResolvedValidatorChain(in pipeline.ResolvedValidatorChain) pipeline.ResolvedValidatorChain {
out := in
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ResolvedValidator, len(in.Validators))
for i, validator := range in.Validators {
out.Validators[i] = pipeline.ResolvedValidator{
Binding: cloneModuleBinding(validator.Binding),
ExecutionClass: validator.ExecutionClass,
Target: validator.Target,
ArtifactKind: validator.ArtifactKind,
}
}
}
return out
}
func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.ResolvedArtifactLane {
out := in
out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize)
out.ExtractReferences = pipeline.CloneReferenceTarget(in.ExtractReferences)
out.MergeReferences = pipeline.CloneReferenceTarget(in.MergeReferences)
out.NormalizeReferences = pipeline.CloneReferenceTarget(in.NormalizeReferences)
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators {

View File

@@ -3,74 +3,99 @@ package config
import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestRedactedConfigRemovesAPIKeyValues(t *testing.T) {
func TestRedactedConfigCopiesScriptoriumConfig(t *testing.T) {
cfg := Default()
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = LLMProfile{
Provider: "openai-compatible",
BaseURL: "https://example.invalid/v1",
Model: "test-model",
APIKey: "secret",
APIKeyEnv: "NOTARIUS_TEST_API_KEY",
TimeoutSeconds: 600,
MaxRetries: 3,
MaxConcurrency: 1,
}
cfg.LLMProfiles["other"] = LLMProfile{APIKey: "other-secret", Model: "other-model"}
cfg.Scriptorium.ProfileDir = "./profiles"
cfg.Workspace.Directory = "/var/lib/notarius"
cfg.Workspace.Resume.Enabled = true
cfg.Concurrency.StageWorkers["extract"] = 1
redacted := cfg.Redacted()
if redacted.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != redactedSecret {
t.Fatalf("expected default API key redacted, got %+v", redacted.LLMProfiles[pipeline.DefaultLLMProfile])
if redacted.Scriptorium.ProfileDir != "./profiles" {
t.Fatalf("expected Scriptorium profile source preserved, got %+v", redacted.Scriptorium)
}
if redacted.LLMProfiles["other"].APIKey != redactedSecret {
t.Fatalf("expected other API key redacted, got %+v", redacted.LLMProfiles["other"])
}
if redacted.LLMProfiles[pipeline.DefaultLLMProfile].Model != "test-model" {
t.Fatalf("expected non-secret fields preserved, got %+v", redacted.LLMProfiles[pipeline.DefaultLLMProfile])
}
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != "secret" {
redacted.Scriptorium.ProfileDir = "./changed"
if cfg.Scriptorium.ProfileDir != "./profiles" {
t.Fatalf("redaction mutated original config")
}
if redacted.Workspace.Directory != "/var/lib/notarius" || !redacted.Workspace.Resume.Enabled {
t.Fatalf("expected workspace config preserved, got %+v", redacted.Workspace)
}
redacted.Workspace.Directory = "/changed"
if cfg.Workspace.Directory != "/var/lib/notarius" {
t.Fatalf("redaction mutated original workspace config")
}
redacted.Concurrency.StageWorkers["extract"] = 9
if cfg.Concurrency.StageWorkers["extract"] != 1 {
t.Fatalf("redaction aliased stage worker map")
}
}
func TestConfigRedactedDiagnosticsPayloadRedactsAPIKeys(t *testing.T) {
func TestConfigRedactedDiagnosticsPayloadCopiesConfig(t *testing.T) {
cfg := Default()
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
profile.APIKey = "secret"
profile.Model = "test-model"
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile
cfg.Scriptorium.ProfileFile = "./profiles.yml"
payload, ok := cfg.RedactedDiagnosticsPayload().(Config)
if !ok {
t.Fatalf("expected Config payload, got %T", cfg.RedactedDiagnosticsPayload())
}
if payload.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != redactedSecret {
t.Fatalf("expected API key redacted, got %+v", payload.LLMProfiles[pipeline.DefaultLLMProfile])
}
if payload.LLMProfiles[pipeline.DefaultLLMProfile].Model != "test-model" {
t.Fatalf("expected non-secret fields preserved, got %+v", payload.LLMProfiles[pipeline.DefaultLLMProfile])
}
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != "secret" {
t.Fatalf("redacted diagnostics payload mutated original config")
if payload.Scriptorium.ProfileFile != "./profiles.yml" {
t.Fatalf("expected Scriptorium profile file preserved, got %+v", payload.Scriptorium)
}
}
func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T) {
func TestEffectiveConfigRedactedDiagnosticsPayloadCopies(t *testing.T) {
cfg := validConfig()
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
profile.APIKey = "secret"
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile
cfg.Concurrency.TotalLLM = 4
cfg.Concurrency.StageWorkers["extract"] = 2
lane := cfg.Pipelines["example"].Artifacts["events"]
lane.Extract.Options = map[string]any{"temperature": 0.2}
lane.References = map[string]string{"roster": "./roster.yml"}
lane.Extract.References = map[string]string{"glossary": "./glossary.md"}
lane.Normalize.References = map[string]string{"notes": "./normalize.md"}
cfg.Pipelines["example"].Artifacts["events"] = lane
pipelineProfile := cfg.Pipelines["example"]
pipelineProfile.Chunk.References = map[string]string{"scene_guide": "./scene.md"}
cfg.Pipelines["example"] = pipelineProfile
effective, err := cfg.Resolve(ResolveInput{
PipelineID: "example",
Only: []string{"events"},
Catalog: fakeCatalog(t),
Catalog: fakeCatalog(t,
pipeline.ModuleSpec{
Key: "generic",
Stage: pipeline.StageChunk,
Requires: []string{"source"},
Provides: []string{"chunks"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "scene_guide"},
},
},
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "glossary"},
{Name: "roster"},
},
},
pipeline.ModuleSpec{
Key: "noop",
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes"},
},
},
),
})
if err != nil {
t.Fatalf("Resolve: %v", err)
@@ -80,15 +105,13 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
if !ok {
t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload())
}
if payload.Config.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != redactedSecret {
t.Fatalf("expected nested API key redacted, got %+v", payload.Config.LLMProfiles[pipeline.DefaultLLMProfile])
}
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != "secret" {
t.Fatalf("redacted diagnostics payload mutated source config")
}
if payload.PipelineID != effective.PipelineID || payload.ResolvedPipeline.Digest != effective.ResolvedPipeline.Digest {
t.Fatalf("expected pipeline metadata preserved, got %+v", payload)
}
payload.Config.Concurrency.StageWorkers["extract"] = 4
if effective.Config.Concurrency.StageWorkers["extract"] != 2 {
t.Fatalf("expected effective stage worker map to be copied")
}
payload.Only[0] = "changed"
if effective.Only[0] != "events" {
@@ -98,4 +121,52 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 {
t.Fatalf("expected resolved pipeline options to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml"
if referenceBindingSource(effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings, "roster") != "./roster.yml" {
t.Fatalf("expected resolved pipeline references to be copied")
}
payload.ResolvedPipeline.Chunk.References["scene_guide"] = "./changed-scene.md"
if effective.ResolvedPipeline.Chunk.References["scene_guide"] != "./scene.md" {
t.Fatalf("expected chunk references to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] = "./changed-glossary.md"
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] != "./glossary.md" {
t.Fatalf("expected extract references to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] = "./changed-normalize.md"
if effective.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] != "./normalize.md" {
t.Fatalf("expected normalize references to be copied")
}
effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
Content: []byte("reference content"),
},
},
},
},
}
payload, ok = effective.RedactedDiagnosticsPayload().(EffectiveConfig)
if !ok {
t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload())
}
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content[0] = 'X'
got := effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content
if string(got) != "reference content" {
t.Fatalf("expected materialized reference content to be copied, got %q", got)
}
}
func referenceBindingSource(bindings []pipeline.ReferenceBinding, slotName string) string {
for _, binding := range bindings {
if binding.SlotName == slotName {
return binding.Source
}
}
return ""
}

View File

@@ -2,16 +2,19 @@ package config
import (
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const providerOpenAICompatible = "openai-compatible"
func (c Config) Validate() error {
if err := validateLLMProfiles(c.LLMProfiles); err != nil {
c.Concurrency.recomputeStageWorkerDefaults()
if err := validateScriptorium(c.Scriptorium); err != nil {
return err
}
if err := validateWorkspace(c.Workspace); err != nil {
return err
}
if err := validateDiagnostics(c.Diagnostics); err != nil {
@@ -20,43 +23,49 @@ func (c Config) Validate() error {
if c.Concurrency.TotalLLM <= 0 {
return fmt.Errorf("total LLM concurrency must be greater than zero")
}
return validatePipelineProfiles(c.Pipelines, c.LLMProfiles)
if err := validateStageWorkers(c.Concurrency); err != nil {
return err
}
return validatePipelineProfiles(c.Pipelines)
}
func (c Config) LLMProfile(id string) (LLMProfile, bool) {
trimmedID := strings.TrimSpace(id)
for rawID, profile := range c.LLMProfiles {
if strings.TrimSpace(rawID) == trimmedID {
return profile, true
func validateStageWorkers(cfg ConcurrencyConfig) error {
keys := make([]string, 0, len(cfg.StageWorkers))
for key := range cfg.StageWorkers {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if strings.TrimSpace(key) == "" {
return fmt.Errorf("concurrency.stage_workers key must not be empty")
}
if key != "extract" {
return fmt.Errorf("concurrency.stage_workers key %q is not supported", key)
}
}
return LLMProfile{}, false
extractWorkers, ok := cfg.StageWorkers["extract"]
if !ok {
extractWorkers = cfg.TotalLLM
}
if extractWorkers < 1 || extractWorkers > cfg.TotalLLM {
return fmt.Errorf("concurrency.stage_workers.extract must be between 1 and concurrency.total_llm (%d)", cfg.TotalLLM)
}
return nil
}
func validateLLMProfiles(profiles map[string]LLMProfile) error {
seen := make(map[string]struct{}, len(profiles))
for rawID, profile := range profiles {
id := strings.TrimSpace(rawID)
if id == "" {
return fmt.Errorf("LLM profile id must not be empty")
func validateScriptorium(cfg ScriptoriumConfig) error {
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
return fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")
}
if _, ok := seen[id]; ok {
return fmt.Errorf("LLM profile id %q is duplicated after trimming", id)
}
seen[id] = struct{}{}
return nil
}
provider := strings.TrimSpace(profile.Provider)
if provider != "" && provider != providerOpenAICompatible {
return fmt.Errorf("LLM profile %q provider %q is not supported", id, provider)
}
if profile.TimeoutSeconds < 0 {
return fmt.Errorf("LLM profile %q timeout seconds must not be negative", id)
}
if profile.MaxRetries < 0 {
return fmt.Errorf("LLM profile %q max retries must not be negative", id)
}
if profile.MaxConcurrency < 0 {
return fmt.Errorf("LLM profile %q max concurrency must not be negative", id)
func validateWorkspace(cfg WorkspaceConfig) error {
if cfg.Diagnostics.retentionSet {
switch cfg.Diagnostics.Retention {
case "", diagnostics.RetentionAuto, diagnostics.RetentionAlways, diagnostics.RetentionNever:
default:
return fmt.Errorf("workspace diagnostics retention %q is not supported", cfg.Diagnostics.Retention)
}
}
return nil
@@ -74,7 +83,7 @@ func validateDiagnostics(cfg DiagnosticsConfig) error {
}
}
func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmProfiles map[string]LLMProfile) error {
func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) error {
seen := make(map[string]struct{}, len(profiles))
for rawID, profile := range profiles {
id := strings.TrimSpace(rawID)
@@ -89,13 +98,16 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
}
if err := validateBindingLLMProfile(id, "", "input", profile.Input, llmProfiles); err != nil {
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
return err
}
if err := validateBindingLLMProfile(id, "", "chunk", profile.Chunk, llmProfiles); err != nil {
if err := validateBinding(id, "", "chunk", profile.Chunk, true); err != nil {
return err
}
if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil {
if err := validateBinding(id, "", "output", profile.Output, false); err != nil {
return err
}
if err := validateReferenceMap(id, "", profile.References); err != nil {
return err
}
for rawLaneID, lane := range profile.Artifacts {
@@ -103,51 +115,133 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if laneID == "" {
return fmt.Errorf("pipeline %q artifact lane id must not be empty", id)
}
if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil {
if err := validateReferenceMap(id, laneID, lane.References); err != nil {
return err
}
if err := validateBindingLLMProfile(id, laneID, "merge", lane.Merge, llmProfiles); err != nil {
if err := validateBinding(id, laneID, "extract", lane.Extract, true); err != nil {
return err
}
if err := validateBindingLLMProfile(id, laneID, "normalize", lane.Normalize, llmProfiles); err != nil {
if err := validateBinding(id, laneID, "merge", lane.Merge, true); err != nil {
return err
}
for i, validator := range lane.Validators {
if err := validateBindingLLMProfile(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles); err != nil {
if err := validateBinding(id, laneID, "normalize", lane.Normalize, true); err != nil {
return err
}
if len(lane.Validators) > 0 {
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", id, laneID)
}
}
}
return nil
}
func validateBinding(
pipelineID string,
laneID string,
slot string,
binding pipeline.ModuleBinding,
referencesAllowed bool,
) error {
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
return err
}
if binding.Retries < 0 {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q %s retries must be greater than or equal to zero", pipelineID, laneID, slot)
}
return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot)
}
if err := validateValidatorOverride(pipelineID, laneID, slot, binding.Validators); err != nil {
return err
}
if len(binding.References) == 0 {
return nil
}
if !referencesAllowed {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q %s references are not supported", pipelineID, laneID, slot)
}
return fmt.Errorf("pipeline %q %s references are not supported", pipelineID, slot)
}
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
}
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
if !override.Set {
return nil
}
switch slot {
case "chunk", "extract", "merge", "normalize":
default:
return fmt.Errorf("%s validators are not supported", referenceContext(pipelineID, laneID, slot))
}
for i, validator := range override.Validators {
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
if strings.TrimSpace(validator.Module) == "" {
return fmt.Errorf("%s module must not be empty", context)
}
if len(validator.References) > 0 {
return fmt.Errorf("%s references are not supported", context)
}
if validator.Validators.Set {
return fmt.Errorf("%s nested validators are not supported", context)
}
if validator.Retries != 0 {
return fmt.Errorf("%s retries are not supported", context)
}
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
return fmt.Errorf("%s llm_profile must not be empty when set", context)
}
}
return nil
}
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
return validateReferenceMapForContext(pipelineID, laneID, "", references)
}
func validateReferenceMapForContext(pipelineID string, laneID string, slot string, references map[string]string) error {
seen := make(map[string]struct{}, len(references))
for rawSlotName, rawSource := range references {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
return fmt.Errorf("%s reference slot name must not be empty", referenceContext(pipelineID, laneID, slot))
}
if _, ok := seen[slotName]; ok {
return fmt.Errorf("%s reference slot %q is duplicated after trimming", referenceContext(pipelineID, laneID, slot), slotName)
}
seen[slotName] = struct{}{}
if strings.TrimSpace(rawSource) == "" {
return fmt.Errorf("%s reference slot %q source must not be empty", referenceContext(pipelineID, laneID, slot), slotName)
}
}
return nil
}
func referenceContext(pipelineID string, laneID string, slot string) string {
if laneID != "" && slot != "" {
return fmt.Sprintf("pipeline %q lane %q %s", pipelineID, laneID, slot)
}
if laneID != "" {
return fmt.Sprintf("pipeline %q lane %q", pipelineID, laneID)
}
if slot != "" {
return fmt.Sprintf("pipeline %q %s", pipelineID, slot)
}
return fmt.Sprintf("pipeline %q", pipelineID)
}
func validateBindingLLMProfile(
pipelineID string,
laneID string,
slot string,
binding pipeline.ModuleBinding,
profiles map[string]LLMProfile,
) error {
profileID := strings.TrimSpace(binding.LLMProfile)
if profileID == "" {
profileID = pipeline.DefaultLLMProfile
}
if hasLLMProfile(profiles, profileID) {
return nil
}
if binding.LLMProfile != "" && strings.TrimSpace(binding.LLMProfile) == "" {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q %s references unknown LLM profile %q", pipelineID, laneID, slot, profileID)
return fmt.Errorf("pipeline %q lane %q %s llm_profile must not be empty when set", pipelineID, laneID, slot)
}
return fmt.Errorf("pipeline %q %s references unknown LLM profile %q", pipelineID, slot, profileID)
}
func hasLLMProfile(profiles map[string]LLMProfile, profileID string) bool {
profileID = strings.TrimSpace(profileID)
for rawID := range profiles {
if strings.TrimSpace(rawID) == profileID {
return true
}
}
return false
return fmt.Errorf("pipeline %q %s llm_profile must not be empty when set", pipelineID, slot)
}
return nil
}

View File

@@ -1,6 +1,7 @@
package config
import (
"fmt"
"strings"
"testing"
@@ -17,27 +18,26 @@ func TestValidateSuccessForValidConfig(t *testing.T) {
}
}
func TestValidateRejectsUnknownLLMProfileReferencedByBinding(t *testing.T) {
func TestValidateAllowsExplicitScriptoriumProfileIDOnBinding(t *testing.T) {
cfg := validConfig()
lane := cfg.Pipelines["example"].Artifacts["events"]
lane.Extract.LLMProfile = "missing"
lane.Extract.LLMProfile = "scriptorium-profile"
cfg.Pipelines["example"].Artifacts["events"] = lane
err := cfg.Validate()
if err == nil || !strings.Contains(err.Error(), "unknown LLM profile") || !strings.Contains(err.Error(), "events") {
t.Fatalf("expected unknown LLM profile error with lane context, got %v", err)
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
}
func TestValidateRejectsInvalidProvider(t *testing.T) {
func TestValidateRejectsWhitespaceOnlyExplicitLLMProfile(t *testing.T) {
cfg := validConfig()
profile := cfg.LLMProfiles["default"]
profile.Provider = "unsupported"
cfg.LLMProfiles["default"] = profile
lane := cfg.Pipelines["example"].Artifacts["events"]
lane.Extract.LLMProfile = " "
cfg.Pipelines["example"].Artifacts["events"] = lane
err := cfg.Validate()
if err == nil || !strings.Contains(err.Error(), "provider") {
t.Fatalf("expected provider error, got %v", err)
if err == nil || !strings.Contains(err.Error(), "llm_profile") || !strings.Contains(err.Error(), "events") {
t.Fatalf("expected llm_profile error with lane context, got %v", err)
}
}
@@ -56,34 +56,16 @@ func TestValidateRejectsInvalidNumericFields(t *testing.T) {
want: "total LLM concurrency",
},
{
name: "timeout",
name: "negative retries",
mutate: func(cfg Config) Config {
profile := cfg.LLMProfiles["default"]
profile.TimeoutSeconds = -1
cfg.LLMProfiles["default"] = profile
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Merge.Retries = -1
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: "timeout",
},
{
name: "max retries",
mutate: func(cfg Config) Config {
profile := cfg.LLMProfiles["default"]
profile.MaxRetries = -1
cfg.LLMProfiles["default"] = profile
return cfg
},
want: "max retries",
},
{
name: "max concurrency",
mutate: func(cfg Config) Config {
profile := cfg.LLMProfiles["default"]
profile.MaxConcurrency = -1
cfg.LLMProfiles["default"] = profile
return cfg
},
want: "max concurrency",
want: "retries",
},
}
@@ -97,12 +79,49 @@ func TestValidateRejectsInvalidNumericFields(t *testing.T) {
}
}
func TestValidateAllowsPartialLLMProfileNumericConfig(t *testing.T) {
func TestValidateStageWorkerBoundaries(t *testing.T) {
for _, test := range []struct {
name string
workers int
wantErr bool
}{
{name: "below minimum", workers: 0, wantErr: true},
{name: "minimum", workers: 1},
{name: "maximum", workers: 4},
{name: "above maximum", workers: 5, wantErr: true},
} {
t.Run(test.name, func(t *testing.T) {
cfg := validConfig()
cfg.LLMProfiles["retry-only"] = LLMProfile{MaxRetries: 3}
cfg.Concurrency.TotalLLM = 4
cfg.Concurrency.StageWorkers["extract"] = test.workers
err := cfg.Validate()
if test.wantErr && (err == nil || !strings.Contains(err.Error(), "stage_workers.extract")) {
t.Fatalf("Validate() error = %v, want extract worker range error", err)
}
if !test.wantErr && err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
})
}
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate: %v", err)
func TestValidateRejectsUnknownEffectiveStageWorkerKey(t *testing.T) {
cfg := validConfig()
cfg.Concurrency.StageWorkers["merge"] = 1
err := cfg.Validate()
if err == nil || !strings.Contains(err.Error(), "stage_workers key") || !strings.Contains(err.Error(), "merge") {
t.Fatalf("Validate() error = %v, want unknown stage worker key", err)
}
}
func TestValidateRejectsMutuallyExclusiveScriptoriumProfileSources(t *testing.T) {
cfg := validConfig()
cfg.Scriptorium.ProfileDir = "./profiles"
cfg.Scriptorium.ProfileFile = "./profiles.yml"
err := cfg.Validate()
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
t.Fatalf("expected Scriptorium source conflict, got %v", err)
}
}
@@ -116,20 +135,201 @@ func TestValidateRejectsInvalidDiagnosticsRetention(t *testing.T) {
}
}
func TestValidateRejectsInvalidWorkspaceDiagnosticsRetention(t *testing.T) {
cfg := validConfig()
cfg.Workspace.Diagnostics.Retention = diagnostics.RetentionMode("sometimes")
cfg.Workspace.Diagnostics.retentionSet = true
err := cfg.Validate()
if err == nil || !strings.Contains(err.Error(), "workspace diagnostics retention") {
t.Fatalf("expected workspace retention error, got %v", err)
}
}
func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
tests := []struct {
name string
mutate func(Config) Config
want []string
}{
{
name: "empty chunk slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Chunk.References = map[string]string{" ": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "chunk", "reference slot", "empty"},
},
{
name: "empty chunk source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Chunk.References = map[string]string{"roster": " "}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "chunk", "roster", "source", "empty"},
},
{
name: "empty extract slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "extract", "reference slot", "empty"},
},
{
name: "empty extract source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "extract", "roster", "source", "empty"},
},
{
name: "empty normalize slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "normalize", "reference slot", "empty"},
},
{
name: "empty normalize source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "normalize", "roster", "source", "empty"},
},
{
name: "empty pipeline slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.References = map[string]string{" ": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "reference slot", "empty"},
},
{
name: "empty pipeline source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.References = map[string]string{"roster": " "}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "roster", "source", "empty"},
},
{
name: "empty lane slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "reference slot", "empty"},
},
{
name: "empty lane source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "roster", "source", "empty"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := tc.mutate(validConfig()).Validate()
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
})
}
}
func TestValidateRejectsReferencesOnUnsupportedBindings(t *testing.T) {
tests := []struct {
name string
mutate func(Config) Config
want []string
}{
{
name: "input",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Input.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "input", "references", "not supported"},
},
{
name: "output",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Output.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "output", "references", "not supported"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := tc.mutate(validConfig()).Validate()
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
})
}
}
func TestValidateRejectsEmptyIDs(t *testing.T) {
tests := []struct {
name string
mutate func(Config) Config
want string
}{
{
name: "LLM profile",
mutate: func(cfg Config) Config {
cfg.LLMProfiles[" "] = LLMProfile{}
return cfg
},
want: "LLM profile id",
},
{
name: "pipeline",
mutate: func(cfg Config) Config {
@@ -156,14 +356,6 @@ func TestValidateRejectsIDsDuplicatedAfterTrimming(t *testing.T) {
mutate func(Config) Config
want string
}{
{
name: "LLM profile",
mutate: func(cfg Config) Config {
cfg.LLMProfiles[" default "] = cfg.LLMProfiles["default"]
return cfg
},
want: "duplicated",
},
{
name: "pipeline",
mutate: func(cfg Config) Config {
@@ -184,31 +376,101 @@ func TestValidateRejectsIDsDuplicatedAfterTrimming(t *testing.T) {
}
}
func TestValidateUsesTrimmedLLMProfileIDs(t *testing.T) {
func TestValidateRejectsConfiguredValidators(t *testing.T) {
cfg := validConfig()
cfg.LLMProfiles[" default "] = cfg.LLMProfiles["default"]
delete(cfg.LLMProfiles, "default")
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Validators = []pipeline.ModuleBinding{pipeline.Binding("fake/validator")}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
err := cfg.Validate()
if err == nil {
t.Fatal("Validate() error = nil, want configured validators error")
}
for _, want := range []string{"example", "events", "validators", "extract.validators", "merge.validators", "normalize.validators"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
}
func TestValidateAcceptsStageLocalValidatorOverrides(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
profile.Chunk.Validators = pipeline.ValidatorOverride{Set: true}
lane := profile.Artifacts["events"]
lane.Extract.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{
pipeline.Binding("fake/validator"),
{Module: "fake/llm-validator", LLMProfile: "careful", Options: map[string]any{"threshold": 0.7}},
},
}
lane.Merge.Validators = pipeline.ValidatorOverride{Set: true}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate: %v", err)
t.Fatalf("Validate() error = %v, want nil", err)
}
if _, ok := cfg.LLMProfile("default"); !ok {
t.Fatalf("expected trimmed LLM profile lookup to succeed")
}
func TestValidateRejectsInvalidValidatorBindings(t *testing.T) {
tests := []struct {
name string
binding pipeline.ModuleBinding
want string
}{
{
name: "empty module",
binding: pipeline.ModuleBinding{},
want: "module must not be empty",
},
{
name: "references",
binding: pipeline.ModuleBinding{Module: "fake/validator", References: map[string]string{"roster": "./roster.txt"}},
want: "references are not supported",
},
{
name: "nested validators",
binding: pipeline.ModuleBinding{Module: "fake/validator", Validators: pipeline.ValidatorOverride{Set: true}},
want: "nested validators are not supported",
},
{
name: "retries",
binding: pipeline.ModuleBinding{Module: "fake/validator", Retries: 1},
want: "retries are not supported",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{test.binding},
}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
err := cfg.Validate()
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Validate() error = %v, want %q", err, test.want)
}
})
}
}
func validConfig() Config {
cfg := Default()
profile := cfg.LLMProfiles["default"]
profile.BaseURL = "https://example.invalid/v1"
profile.Model = "test-model"
cfg.LLMProfiles["default"] = profile
cfg.Pipelines["example"] = pipeline.PipelineProfile{
Input: pipeline.Binding("fake/input"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"events": {
Extract: pipeline.Binding("fake/extract"),
Validators: []pipeline.ModuleBinding{pipeline.Binding("fake/validator")},
},
"notes": {
Extract: pipeline.Binding("fake/extract"),
@@ -256,6 +518,12 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
Requires: []string{"normalized"},
Provides: []string{"validated"},
},
"fake/llm-validator": {
Key: "fake/llm-validator",
Stage: pipeline.StageValidate,
Requires: []string{"normalized"},
Provides: []string{"validated"},
},
"json": {
Key: "json",
Stage: pipeline.StageOutput,
@@ -265,6 +533,11 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
for _, override := range overrides {
specs[override.Key] = override
}
for _, key := range []string{"fake/extract", "appendorder", "noop"} {
spec := specs[key]
spec.ArtifactKind = fakeArtifactKind
specs[key] = spec
}
inputs := pipeline.NewInputAdapterRegistry()
chunkers := pipeline.NewChunkerRegistry()
@@ -280,15 +553,23 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
mustRegisterMerger(t, mergers, specs["appendorder"])
mustRegisterNormalizer(t, normalizers, specs["noop"])
mustRegisterValidator(t, validators, specs["fake/validator"])
mustRegisterValidator(t, validators, specs["fake/llm-validator"])
mustRegisterOutput(t, outputs, specs["json"])
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, fakeArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
ArtifactCodecs: codecs,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Validators: validators,
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
}
}
@@ -309,32 +590,69 @@ func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.Extractor, error) { return nil, nil }); err != nil {
validateOptions := func(options map[string]any) error {
if err := pipeline.RejectUnknownOptions(options, "temperature"); err != nil {
return err
}
if value, ok := options["temperature"]; ok {
if _, ok := value.(float64); !ok {
return fmt.Errorf("temperature must be a number")
}
}
return nil
}
if err := pipeline.RegisterExtractorBuilder[fakeArtifact](registry, spec, validateOptions, func(pipeline.BuildRequest) (contracts.Extractor[fakeArtifact], error) { return nil, nil }); err != nil {
t.Fatalf("register extractor: %v", err)
}
}
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.Merger, error) { return nil, nil }); err != nil {
if err := pipeline.RegisterMerger[fakeArtifact](registry, spec, func() (contracts.Merger[fakeArtifact], error) { return nil, nil }); err != nil {
t.Fatalf("register merger: %v", err)
}
}
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.Normalizer, error) { return nil, nil }); err != nil {
if err := pipeline.RegisterNormalizer[fakeArtifact](registry, spec, func() (contracts.Normalizer[fakeArtifact], error) { return nil, nil }); err != nil {
t.Fatalf("register normalizer: %v", err)
}
}
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
executionClass := contracts.ExecutionClassDeterministic
if spec.Key == "fake/llm-validator" {
executionClass = contracts.ExecutionClassLLMBacked
}
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass}
if err := pipeline.RegisterTypedValidator[fakeArtifact](registry, fakeArtifactKind, validatorSpec, func() (contracts.TypedValidator[fakeArtifact], error) { return nil, nil }); err != nil {
t.Fatalf("register validator: %v", err)
}
}
const fakeArtifactKind contracts.ArtifactKind = "test/artifact"
type fakeArtifact string
type fakeArtifactCodec struct{}
func (fakeArtifactCodec) Kind() contracts.ArtifactKind { return fakeArtifactKind }
func (fakeArtifactCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "urn:notarius:test:artifact", Name: "Test artifact", Version: "1", JSONSchema: []byte(`{"type":"string"}`)}
}
func (fakeArtifactCodec) MediaType() string { return "application/json" }
func (fakeArtifactCodec) Encode(value fakeArtifact) ([]byte, error) {
return []byte(fmt.Sprintf("%q", value)), nil
}
func (fakeArtifactCodec) Decode(content []byte) (fakeArtifact, error) {
if len(content) < 2 {
return "", fmt.Errorf("invalid test artifact")
}
return fakeArtifact(content[1 : len(content)-1]), nil
}
func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { return nil, nil }); err != nil {

View File

@@ -4,6 +4,8 @@ const (
ArtifactInvocationMetadata = "invocation.json"
ArtifactEffectiveConfig = "effective-config.json"
ArtifactResolvedPipeline = "resolved-pipeline.json"
ArtifactResolvedReferences = "resolved-references.json"
ArtifactCheckpointEvents = "checkpoint-events.json"
ArtifactSourceDocument = "source-document.json"
ArtifactRunManifest = "run-manifest.json"
ArtifactRunReport = "run-report.json"

View File

@@ -7,6 +7,7 @@ func TestArtifactNamesUseExtractionOrientedNames(t *testing.T) {
ArtifactInvocationMetadata,
ArtifactEffectiveConfig,
ArtifactResolvedPipeline,
ArtifactResolvedReferences,
ArtifactSourceDocument,
ArtifactRunManifest,
ArtifactRunReport,

View File

@@ -51,6 +51,7 @@ type InvocationMetadata struct {
Operation string `json:"operation"`
PipelineID string `json:"pipeline_id,omitempty"`
PipelineDigest string `json:"pipeline_digest,omitempty"`
Resume bool `json:"resume,omitempty"`
InputPath string `json:"input_path,omitempty"`
ConfigPath string `json:"config_path,omitempty"`
ConfigSource string `json:"config_source,omitempty"`
@@ -149,6 +150,14 @@ func (r *RunDirectory) WriteResolvedPipeline(payload any) error {
return r.WriteJSONArtifact(ArtifactResolvedPipeline, payload)
}
func (r *RunDirectory) WriteResolvedReferences(payload any) error {
return r.WriteJSONArtifact(ArtifactResolvedReferences, payload)
}
func (r *RunDirectory) WriteCheckpointEvents(payload any) error {
return r.WriteJSONArtifact(ArtifactCheckpointEvents, payload)
}
func (r *RunDirectory) WriteSourceDocument(payload any) error {
return r.WriteJSONArtifact(ArtifactSourceDocument, payload)
}

View File

@@ -191,6 +191,9 @@ func TestWriteTypedArtifacts(t *testing.T) {
if err := runDir.WriteResolvedPipeline(map[string]any{"pipeline": "test"}); err != nil {
t.Fatalf("WriteResolvedPipeline: %v", err)
}
if err := runDir.WriteResolvedReferences([]artifacts.ReferenceProvenance{{LaneID: "events", SlotName: "roster"}}); err != nil {
t.Fatalf("WriteResolvedReferences: %v", err)
}
if err := runDir.WriteSourceDocument(map[string]any{"source_id": "source-1"}); err != nil {
t.Fatalf("WriteSourceDocument: %v", err)
}
@@ -207,6 +210,7 @@ func TestWriteTypedArtifacts(t *testing.T) {
for _, name := range []string{
ArtifactEffectiveConfig,
ArtifactResolvedPipeline,
ArtifactResolvedReferences,
ArtifactSourceDocument,
ArtifactRunManifest,
ArtifactRunReport,

View File

@@ -0,0 +1,65 @@
package source
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
)
// DigestDocument returns a deterministic digest of the canonical source
// document content. The existing Digest field is excluded from its own digest.
func DigestDocument(doc *SourceDocument) (string, error) {
if doc == nil {
return "", fmt.Errorf("source document must not be nil")
}
payload := struct {
ID string `json:"id"`
Kind string `json:"kind"`
Format string `json:"format"`
Units []SourceUnit `json:"units"`
Metadata map[string]any `json:"metadata,omitempty"`
}{
ID: doc.ID,
Kind: doc.Kind,
Format: doc.Format,
Units: doc.Units,
Metadata: doc.Metadata,
}
encoded, err := json.Marshal(payload)
if err != nil {
return "", fmt.Errorf("encode source document for digest: %w", err)
}
sum := sha256.Sum256(encoded)
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
// DigestChunk returns a deterministic digest of a chunk, including its source
// provenance, content, units, and metadata.
func DigestChunk(chunk Chunk) (string, error) {
payload := struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
Index int `json:"index"`
Ref SourceRef `json:"ref"`
Content []byte `json:"content"`
MediaType string `json:"media_type"`
Units []SourceUnit `json:"units"`
Metadata map[string]any `json:"metadata,omitempty"`
}{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Ref: chunk.Ref,
Content: chunk.Content,
MediaType: chunk.MediaType,
Units: chunk.Units,
Metadata: chunk.Metadata,
}
encoded, err := json.Marshal(payload)
if err != nil {
return "", fmt.Errorf("encode source chunk for digest: %w", err)
}
sum := sha256.Sum256(encoded)
return "sha256:" + hex.EncodeToString(sum[:]), nil
}

View File

@@ -10,14 +10,26 @@ type SourceDocument struct {
}
type SourceUnit struct {
ID string `json:"id"`
ID int `json:"id"`
Kind string `json:"kind"`
Text string `json:"text"`
Ref SourceRef `json:"ref"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type SourceRef struct {
SourceID string `json:"source_id"`
StartUnitID string `json:"start_unit_id"`
EndUnitID string `json:"end_unit_id"`
StartUnitID int `json:"start_unit_id"`
EndUnitID int `json:"end_unit_id"`
}
type Chunk struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
Index int `json:"index"`
Ref SourceRef `json:"ref"`
Content []byte `json:"-"`
MediaType string `json:"media_type"`
Units []SourceUnit `json:"units"`
Metadata map[string]any `json:"metadata,omitempty"`
}

View File

@@ -96,13 +96,8 @@ func TestValidateDocumentMissingUnitFields(t *testing.T) {
}{
{
name: "id",
mutate: func(doc *SourceDocument) { doc.Units[1].ID = "" },
wantErr: "source unit[1].id must not be empty",
},
{
name: "id surrounding whitespace",
mutate: func(doc *SourceDocument) { doc.Units[1].ID = " u2 " },
wantErr: "source unit[1].id \" u2 \" must not contain leading or trailing whitespace",
mutate: func(doc *SourceDocument) { doc.Units[1].ID = 0 },
wantErr: "source unit[1].id must be positive",
},
{
name: "kind",
@@ -135,24 +130,140 @@ func TestValidateDocumentMissingUnitFields(t *testing.T) {
func TestValidateDocumentDuplicateUnitIDs(t *testing.T) {
doc := validDocument()
doc.Units[1].ID = "u1"
doc.Units[1].ID = 1
err := ValidateDocument(doc)
if err == nil {
t.Fatal("ValidateDocument() error = nil, want error")
}
if err.Error() != "source unit id \"u1\" is duplicated" {
if err.Error() != "source unit id 1 is duplicated" {
t.Fatalf("ValidateDocument() error = %q", err.Error())
}
}
func TestValidateDocumentUnitReferences(t *testing.T) {
tests := []struct {
name string
mutate func(*SourceDocument)
wantErr string
}{
{
name: "missing",
mutate: func(doc *SourceDocument) { doc.Units[0].Ref = SourceRef{} },
wantErr: "source unit[0].ref: source ref source_id must not be empty",
},
{
name: "foreign source",
mutate: func(doc *SourceDocument) { doc.Units[0].Ref.SourceID = "source-2" },
wantErr: "source unit[0].ref: source ref source_id \"source-2\" does not match document id \"source-1\"",
},
{
name: "non-self range",
mutate: func(doc *SourceDocument) {
doc.Units[0].Ref.StartUnitID = 2
doc.Units[0].Ref.EndUnitID = 2
},
wantErr: "source unit[0].ref must identify source unit id 1",
},
{
name: "reversed range",
mutate: func(doc *SourceDocument) {
doc.Units[0].Ref.StartUnitID = 2
doc.Units[0].Ref.EndUnitID = 1
},
wantErr: "source unit[0].ref: source ref start_unit_id 2 appears after end_unit_id 1",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
doc := validDocument()
tt.mutate(doc)
err := ValidateDocument(doc)
if err == nil {
t.Fatal("ValidateDocument() error = nil, want unit reference error")
}
if err.Error() != tt.wantErr {
t.Fatalf("ValidateDocument() error = %q, want %q", err.Error(), tt.wantErr)
}
})
}
}
func TestDigestDocumentIsDeterministicAndIncludesUnitReference(t *testing.T) {
doc := validDocument()
doc.Metadata = map[string]any{"second": "value", "first": true}
first, err := DigestDocument(doc)
if err != nil {
t.Fatalf("DigestDocument() error = %v, want nil", err)
}
reordered := validDocument()
reordered.Metadata = map[string]any{"first": true, "second": "value"}
second, err := DigestDocument(reordered)
if err != nil {
t.Fatalf("DigestDocument(reordered) error = %v, want nil", err)
}
if first != second {
t.Fatalf("digests = %q and %q, want deterministic map ordering", first, second)
}
changed := validDocument()
changed.Metadata = map[string]any{"first": true, "second": "value"}
changed.Units[0].Ref.SourceID = "different-source"
changedDigest, err := DigestDocument(changed)
if err != nil {
t.Fatalf("DigestDocument(changed) error = %v, want nil", err)
}
if first == changedDigest {
t.Fatalf("digest = %q after reference change, want different digest", changedDigest)
}
}
func TestDigestChunkIsDeterministicAndIncludesReference(t *testing.T) {
doc := validDocument()
chunk := Chunk{
ID: "chunk-1",
SourceID: doc.ID,
Index: 0,
Ref: SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 2},
Content: []byte("chunk content"),
MediaType: "text/plain",
Units: doc.Units,
Metadata: map[string]any{"second": "value", "first": true},
}
first, err := DigestChunk(chunk)
if err != nil {
t.Fatalf("DigestChunk() error = %v, want nil", err)
}
chunk.Metadata = map[string]any{"first": true, "second": "value"}
second, err := DigestChunk(chunk)
if err != nil {
t.Fatalf("DigestChunk(reordered metadata) error = %v, want nil", err)
}
if first != second {
t.Fatalf("digests = %q and %q, want deterministic map ordering", first, second)
}
chunk.Ref.EndUnitID = 1
changed, err := DigestChunk(chunk)
if err != nil {
t.Fatalf("DigestChunk(changed ref) error = %v, want nil", err)
}
if first == changed {
t.Fatalf("digest = %q after reference change, want different digest", changed)
}
}
func TestValidateRefValid(t *testing.T) {
doc := validDocument()
ref := SourceRef{
SourceID: "source-1",
StartUnitID: "u1",
EndUnitID: "u2",
StartUnitID: 1,
EndUnitID: 2,
}
if err := ValidateRef(doc, ref); err != nil {
@@ -164,8 +275,8 @@ func TestValidateRefSourceIDMismatch(t *testing.T) {
doc := validDocument()
ref := SourceRef{
SourceID: "source-2",
StartUnitID: "u1",
EndUnitID: "u2",
StartUnitID: 1,
EndUnitID: 2,
}
err := ValidateRef(doc, ref)
@@ -186,43 +297,33 @@ func TestValidateRefMissingUnitIDs(t *testing.T) {
}{
{
name: "missing source id",
ref: SourceRef{StartUnitID: "u1", EndUnitID: "u2"},
ref: SourceRef{StartUnitID: 1, EndUnitID: 2},
wantErr: "source ref source_id must not be empty",
},
{
name: "source id surrounding whitespace",
ref: SourceRef{SourceID: " source-1 ", StartUnitID: "u1", EndUnitID: "u2"},
ref: SourceRef{SourceID: " source-1 ", StartUnitID: 1, EndUnitID: 2},
wantErr: "source ref source_id \" source-1 \" must not contain leading or trailing whitespace",
},
{
name: "missing start id",
ref: SourceRef{SourceID: "source-1", EndUnitID: "u2"},
wantErr: "source ref start_unit_id must not be empty",
},
{
name: "start id surrounding whitespace",
ref: SourceRef{SourceID: "source-1", StartUnitID: " u1 ", EndUnitID: "u2"},
wantErr: "source ref start_unit_id \" u1 \" must not contain leading or trailing whitespace",
ref: SourceRef{SourceID: "source-1", EndUnitID: 2},
wantErr: "source ref start_unit_id must be positive",
},
{
name: "missing end id",
ref: SourceRef{SourceID: "source-1", StartUnitID: "u1"},
wantErr: "source ref end_unit_id must not be empty",
},
{
name: "end id surrounding whitespace",
ref: SourceRef{SourceID: "source-1", StartUnitID: "u1", EndUnitID: " u2 "},
wantErr: "source ref end_unit_id \" u2 \" must not contain leading or trailing whitespace",
ref: SourceRef{SourceID: "source-1", StartUnitID: 1},
wantErr: "source ref end_unit_id must be positive",
},
{
name: "unknown start id",
ref: SourceRef{SourceID: "source-1", StartUnitID: "u9", EndUnitID: "u2"},
wantErr: "source ref start_unit_id \"u9\" was not found",
ref: SourceRef{SourceID: "source-1", StartUnitID: 9, EndUnitID: 2},
wantErr: "source ref start_unit_id 9 was not found",
},
{
name: "unknown end id",
ref: SourceRef{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u9"},
wantErr: "source ref end_unit_id \"u9\" was not found",
ref: SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 9},
wantErr: "source ref end_unit_id 9 was not found",
},
}
@@ -244,8 +345,8 @@ func TestValidateRefReversedUnitOrder(t *testing.T) {
doc := validDocument()
ref := SourceRef{
SourceID: "source-1",
StartUnitID: "u2",
EndUnitID: "u1",
StartUnitID: 2,
EndUnitID: 1,
}
err := ValidateRef(doc, ref)
@@ -261,7 +362,7 @@ func TestValidateRefReversedUnitOrder(t *testing.T) {
func TestUnitIndex(t *testing.T) {
doc := validDocument()
index, ok := UnitIndex(doc, "u2")
index, ok := UnitIndex(doc, 2)
if !ok {
t.Fatal("UnitIndex() ok = false, want true")
}
@@ -269,7 +370,7 @@ func TestUnitIndex(t *testing.T) {
t.Fatalf("UnitIndex() index = %d, want 1", index)
}
index, ok = UnitIndex(doc, "u9")
index, ok = UnitIndex(doc, 9)
if ok {
t.Fatal("UnitIndex() ok = true, want false")
}
@@ -286,14 +387,16 @@ func validDocument() *SourceDocument {
Digest: "sha256:abc123",
Units: []SourceUnit{
{
ID: "u1",
ID: 1,
Kind: "paragraph",
Text: "First unit.",
Ref: SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
},
{
ID: "u2",
ID: 2,
Kind: "paragraph",
Text: "Second unit.",
Ref: SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2},
},
},
}

View File

@@ -28,13 +28,10 @@ func ValidateDocument(doc *SourceDocument) error {
return fmt.Errorf("source document units must not be empty")
}
seenUnitIDs := make(map[string]struct{}, len(doc.Units))
seenUnitIDs := make(map[int]struct{}, len(doc.Units))
for i, unit := range doc.Units {
if isBlank(unit.ID) {
return fmt.Errorf("source unit[%d].id must not be empty", i)
}
if hasSurroundingWhitespace(unit.ID) {
return fmt.Errorf("source unit[%d].id %q must not contain leading or trailing whitespace", i, unit.ID)
if unit.ID <= 0 {
return fmt.Errorf("source unit[%d].id must be positive", i)
}
if isBlank(unit.Kind) {
return fmt.Errorf("source unit[%d].kind must not be empty", i)
@@ -43,10 +40,18 @@ func ValidateDocument(doc *SourceDocument) error {
return fmt.Errorf("source unit[%d].text must not be empty", i)
}
if _, ok := seenUnitIDs[unit.ID]; ok {
return fmt.Errorf("source unit id %q is duplicated", unit.ID)
return fmt.Errorf("source unit id %d is duplicated", unit.ID)
}
seenUnitIDs[unit.ID] = struct{}{}
}
for i, unit := range doc.Units {
if err := ValidateRef(doc, unit.Ref); err != nil {
return fmt.Errorf("source unit[%d].ref: %w", i, err)
}
if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
return fmt.Errorf("source unit[%d].ref must identify source unit id %d", i, unit.ID)
}
}
return nil
}
@@ -61,17 +66,11 @@ func ValidateRef(doc *SourceDocument, ref SourceRef) error {
if hasSurroundingWhitespace(ref.SourceID) {
return fmt.Errorf("source ref source_id %q must not contain leading or trailing whitespace", ref.SourceID)
}
if isBlank(ref.StartUnitID) {
return fmt.Errorf("source ref start_unit_id must not be empty")
if ref.StartUnitID <= 0 {
return fmt.Errorf("source ref start_unit_id must be positive")
}
if hasSurroundingWhitespace(ref.StartUnitID) {
return fmt.Errorf("source ref start_unit_id %q must not contain leading or trailing whitespace", ref.StartUnitID)
}
if isBlank(ref.EndUnitID) {
return fmt.Errorf("source ref end_unit_id must not be empty")
}
if hasSurroundingWhitespace(ref.EndUnitID) {
return fmt.Errorf("source ref end_unit_id %q must not contain leading or trailing whitespace", ref.EndUnitID)
if ref.EndUnitID <= 0 {
return fmt.Errorf("source ref end_unit_id must be positive")
}
if ref.SourceID != doc.ID {
return fmt.Errorf("source ref source_id %q does not match document id %q", ref.SourceID, doc.ID)
@@ -79,20 +78,20 @@ func ValidateRef(doc *SourceDocument, ref SourceRef) error {
startIndex, ok := UnitIndex(doc, ref.StartUnitID)
if !ok {
return fmt.Errorf("source ref start_unit_id %q was not found", ref.StartUnitID)
return fmt.Errorf("source ref start_unit_id %d was not found", ref.StartUnitID)
}
endIndex, ok := UnitIndex(doc, ref.EndUnitID)
if !ok {
return fmt.Errorf("source ref end_unit_id %q was not found", ref.EndUnitID)
return fmt.Errorf("source ref end_unit_id %d was not found", ref.EndUnitID)
}
if startIndex > endIndex {
return fmt.Errorf("source ref start_unit_id %q appears after end_unit_id %q", ref.StartUnitID, ref.EndUnitID)
return fmt.Errorf("source ref start_unit_id %d appears after end_unit_id %d", ref.StartUnitID, ref.EndUnitID)
}
return nil
}
func UnitIndex(doc *SourceDocument, unitID string) (int, bool) {
func UnitIndex(doc *SourceDocument, unitID int) (int, bool) {
if doc == nil {
return 0, false
}

View File

@@ -0,0 +1,114 @@
package workspace
import (
"encoding/json"
"fmt"
"os"
"path"
"path/filepath"
"strings"
)
func SafePath(root string, name string) (string, error) {
root = strings.TrimSpace(root)
if root == "" {
return "", fmt.Errorf("workspace root must not be empty")
}
name = strings.TrimSpace(name)
if name == "" {
return "", fmt.Errorf("workspace artifact name must not be empty")
}
if strings.Contains(name, `\`) {
return "", fmt.Errorf("workspace artifact name %q must use slash-separated relative paths", name)
}
if path.IsAbs(name) || filepath.IsAbs(name) {
return "", fmt.Errorf("workspace artifact name %q must be relative", name)
}
if name == "." || strings.Contains(name, "..") {
return "", fmt.Errorf("workspace artifact name %q must not contain ..", name)
}
cleaned := path.Clean(name)
if cleaned != name {
return "", fmt.Errorf("workspace artifact name %q must be clean", name)
}
absRoot, err := filepath.Abs(root)
if err != nil {
return "", fmt.Errorf("resolve workspace root %q: %w", root, err)
}
target, err := filepath.Abs(filepath.Join(absRoot, filepath.FromSlash(cleaned)))
if err != nil {
return "", fmt.Errorf("resolve workspace artifact %q: %w", name, err)
}
rel, err := filepath.Rel(absRoot, target)
if err != nil {
return "", fmt.Errorf("resolve workspace artifact %q: %w", name, err)
}
if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("workspace artifact name %q resolves outside workspace root", name)
}
return target, nil
}
func WriteJSON(root string, name string, payload any) error {
target, err := SafePath(root, name)
if err != nil {
return err
}
data, err := json.MarshalIndent(payload, "", " ")
if err != nil {
return fmt.Errorf("marshal workspace artifact %q: %w", name, err)
}
data = append(data, '\n')
if err := writeFileAtomic(target, data, 0o644); err != nil {
return fmt.Errorf("write workspace artifact %q: %w", name, err)
}
return nil
}
func WriteBytes(root string, name string, data []byte) error {
target, err := SafePath(root, name)
if err != nil {
return err
}
if err := writeFileAtomic(target, data, 0o644); err != nil {
return fmt.Errorf("write workspace artifact %q: %w", name, err)
}
return nil
}
func writeFileAtomic(target string, data []byte, perm os.FileMode) error {
dir := filepath.Dir(target)
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
temp, err := os.CreateTemp(dir, "."+filepath.Base(target)+".tmp-*")
if err != nil {
return err
}
tempPath := temp.Name()
removeTemp := true
defer func() {
if removeTemp {
_ = os.Remove(tempPath)
}
}()
if _, err := temp.Write(data); err != nil {
_ = temp.Close()
return err
}
if err := temp.Chmod(perm); err != nil {
_ = temp.Close()
return err
}
if err := temp.Close(); err != nil {
return err
}
if err := os.Rename(tempPath, target); err != nil {
return err
}
removeTemp = false
return nil
}

View File

@@ -0,0 +1,148 @@
package workspace
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestSafePathAcceptsCleanRelativePaths(t *testing.T) {
root := t.TempDir()
got, err := SafePath(root, "source/manifest.json")
if err != nil {
t.Fatalf("SafePath: %v", err)
}
want := filepath.Join(root, "source", "manifest.json")
if got != want {
t.Fatalf("SafePath = %q, want %q", got, want)
}
}
func TestSafePathRejectsUnsafeNames(t *testing.T) {
root := t.TempDir()
tests := []struct {
name string
path string
want string
}{
{name: "empty", path: " ", want: "empty"},
{name: "absolute", path: filepath.Join(root, "artifact.json"), want: "relative"},
{name: "parent segment", path: "../artifact.json", want: ".."},
{name: "embedded parent", path: "source/../artifact.json", want: ".."},
{name: "backslash", path: `source\artifact.json`, want: "slash-separated"},
{name: "unclean", path: "source//artifact.json", want: "clean"},
{name: "dot", path: ".", want: ".."},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, err := SafePath(root, tc.path)
if err == nil {
t.Fatalf("SafePath returned %q, want error", got)
}
if !strings.Contains(err.Error(), tc.want) {
t.Fatalf("SafePath error = %v, want containing %q", err, tc.want)
}
})
}
}
func TestSafePathRejectsEmptyRoot(t *testing.T) {
got, err := SafePath(" ", "artifact.json")
if err == nil {
t.Fatalf("SafePath returned %q, want error", got)
}
if !strings.Contains(err.Error(), "root") {
t.Fatalf("SafePath error = %v, want root error", err)
}
}
func TestSafePathDoesNotPermitEscapingRoot(t *testing.T) {
root := t.TempDir()
for _, name := range []string{
"..",
"../outside.json",
"nested/../../outside.json",
} {
t.Run(name, func(t *testing.T) {
got, err := SafePath(root, name)
if err == nil {
t.Fatalf("SafePath returned %q, want error", got)
}
})
}
}
func TestWriteJSONWritesIndentedAtomicArtifact(t *testing.T) {
root := t.TempDir()
err := WriteJSON(root, "source/manifest.json", map[string]any{
"status": "succeeded",
"count": 2,
})
if err != nil {
t.Fatalf("WriteJSON: %v", err)
}
got := string(readFile(t, filepath.Join(root, "source", "manifest.json")))
if !strings.HasSuffix(got, "\n") {
t.Fatalf("expected trailing newline, got %q", got)
}
if !strings.Contains(got, `"status": "succeeded"`) || !strings.Contains(got, `"count": 2`) {
t.Fatalf("unexpected JSON: %s", got)
}
assertNoTempFiles(t, filepath.Join(root, "source"))
}
func TestWriteBytesWritesNestedArtifact(t *testing.T) {
root := t.TempDir()
if err := WriteBytes(root, "chunk/chunks.json", []byte("payload")); err != nil {
t.Fatalf("WriteBytes: %v", err)
}
got := string(readFile(t, filepath.Join(root, "chunk", "chunks.json")))
if got != "payload" {
t.Fatalf("bytes = %q, want payload", got)
}
assertNoTempFiles(t, filepath.Join(root, "chunk"))
}
func TestWritersRejectUnsafePaths(t *testing.T) {
root := t.TempDir()
if err := WriteBytes(root, "../outside.json", []byte("payload")); err == nil {
t.Fatalf("WriteBytes accepted unsafe path")
}
if err := WriteJSON(root, `debug\trace.json`, map[string]string{"x": "y"}); err == nil {
t.Fatalf("WriteJSON accepted unsafe path")
}
if _, err := os.Stat(filepath.Join(root, "..", "outside.json")); !os.IsNotExist(err) {
t.Fatalf("outside path stat err = %v, want not exist", err)
}
}
func readFile(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %q: %v", path, err)
}
return data
}
func assertNoTempFiles(t *testing.T, dir string) {
t.Helper()
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatalf("read dir %q: %v", dir, err)
}
for _, entry := range entries {
if strings.Contains(entry.Name(), ".tmp-") {
t.Fatalf("temporary file was not cleaned up: %s", entry.Name())
}
}
}

View File

@@ -0,0 +1,276 @@
package workspace
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"path/filepath"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const digestPrefixLength = 16
type Fingerprint struct {
Name string `json:"name"`
Value string `json:"value"`
}
type CheckpointIdentityInput struct {
Pipeline pipeline.ResolvedPipeline
InputKey string
RawInputDigest string
SourceDigest string
SelectedLanes []string
RuntimeOverrides []Fingerprint
References []artifacts.ReferenceProvenance
ProvenanceFingerprints []Fingerprint
}
type CheckpointIdentity struct {
Digest string `json:"digest"`
PipelineID string `json:"pipeline_id"`
PipelineDigest string `json:"pipeline_digest"`
InputKey string `json:"input_key"`
RawInputDigest string `json:"raw_input_digest,omitempty"`
SourceDigest string `json:"source_digest,omitempty"`
SelectedLanes []string `json:"selected_lanes,omitempty"`
RuntimeOverrides []Fingerprint `json:"runtime_overrides,omitempty"`
ReferenceDigests []Fingerprint `json:"reference_digests,omitempty"`
ProvenanceFingerprints []Fingerprint `json:"provenance_fingerprints,omitempty"`
}
func NewCheckpointIdentity(input CheckpointIdentityInput) (CheckpointIdentity, error) {
pipelineID := strings.TrimSpace(input.Pipeline.ID)
if pipelineID == "" {
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity pipeline id must not be empty")
}
pipelineDigest := strings.TrimSpace(input.Pipeline.Digest)
if pipelineDigest == "" {
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity pipeline digest must not be empty")
}
inputKey := strings.TrimSpace(input.InputKey)
if inputKey == "" {
inputKey = strings.TrimSpace(input.Pipeline.Input.Module)
}
if inputKey == "" {
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity input key must not be empty")
}
rawInputDigest := strings.TrimSpace(input.RawInputDigest)
sourceDigest := strings.TrimSpace(input.SourceDigest)
if rawInputDigest == "" && sourceDigest == "" {
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity raw input digest or source digest must be set")
}
identity := CheckpointIdentity{
PipelineID: pipelineID,
PipelineDigest: pipelineDigest,
InputKey: inputKey,
RawInputDigest: rawInputDigest,
SourceDigest: sourceDigest,
SelectedLanes: normalizedLanes(input.SelectedLanes, input.Pipeline.ArtifactLanes),
RuntimeOverrides: normalizeFingerprints(input.RuntimeOverrides),
ReferenceDigests: referenceFingerprints(input.References),
ProvenanceFingerprints: normalizeFingerprints(input.ProvenanceFingerprints),
}
digest, err := identityDigest(identity)
if err != nil {
return CheckpointIdentity{}, err
}
identity.Digest = digest
return identity, nil
}
func (s Settings) CheckpointDirectory(identity CheckpointIdentity) (string, error) {
if !s.ResumeEnabled || strings.TrimSpace(s.CheckpointsRoot) == "" {
return "", nil
}
relative, err := identity.RelativePath()
if err != nil {
return "", err
}
return SafePath(s.CheckpointsRoot, relative)
}
func (i CheckpointIdentity) RelativePath() (string, error) {
pipelineID, err := safePathComponent(i.PipelineID)
if err != nil {
return "", fmt.Errorf("checkpoint identity pipeline id: %w", err)
}
inputKey, err := safePathComponent(i.InputKey)
if err != nil {
return "", fmt.Errorf("checkpoint identity input key: %w", err)
}
sourceDigest := digestPrefix(i.SourceDigest)
if sourceDigest == "" {
sourceDigest = digestPrefix(i.RawInputDigest)
}
if sourceDigest == "" {
return "", fmt.Errorf("checkpoint identity source digest prefix must not be empty")
}
pipelineDigest := digestPrefix(i.PipelineDigest)
if pipelineDigest == "" {
return "", fmt.Errorf("checkpoint identity pipeline digest prefix must not be empty")
}
sourceComponent, err := safePathComponent(sourceDigest)
if err != nil {
return "", fmt.Errorf("checkpoint identity source digest: %w", err)
}
pipelineComponent, err := safePathComponent(pipelineDigest)
if err != nil {
return "", fmt.Errorf("checkpoint identity pipeline digest: %w", err)
}
identityDigest := digestPrefix(i.Digest)
if identityDigest == "" {
return "", fmt.Errorf("checkpoint identity digest prefix must not be empty")
}
identityComponent, err := safePathComponent(identityDigest)
if err != nil {
return "", fmt.Errorf("checkpoint identity digest: %w", err)
}
return filepath.ToSlash(filepath.Join(pipelineID, inputKey+"-"+sourceComponent, pipelineComponent, identityComponent)), nil
}
func identityDigest(identity CheckpointIdentity) (string, error) {
payload := identity
payload.Digest = ""
data, err := json.Marshal(payload)
if err != nil {
return "", fmt.Errorf("marshal checkpoint identity: %w", err)
}
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
func normalizedLanes(selected []string, resolved []pipeline.ResolvedArtifactLane) []string {
if len(selected) > 0 {
return normalizeStrings(selected)
}
lanes := make([]string, 0, len(resolved))
for _, lane := range resolved {
lanes = append(lanes, lane.ID)
}
return normalizeStrings(lanes)
}
func normalizeFingerprints(values []Fingerprint) []Fingerprint {
if len(values) == 0 {
return nil
}
byName := make(map[string]string, len(values))
for _, value := range values {
name := strings.TrimSpace(value.Name)
fingerprint := strings.TrimSpace(value.Value)
if name == "" || fingerprint == "" {
continue
}
byName[name] = fingerprint
}
if len(byName) == 0 {
return nil
}
names := make([]string, 0, len(byName))
for name := range byName {
names = append(names, name)
}
sort.Strings(names)
out := make([]Fingerprint, 0, len(names))
for _, name := range names {
out = append(out, Fingerprint{Name: name, Value: byName[name]})
}
return out
}
func referenceFingerprints(references []artifacts.ReferenceProvenance) []Fingerprint {
if len(references) == 0 {
return nil
}
values := make([]Fingerprint, 0, len(references))
for _, reference := range references {
digest := strings.TrimSpace(reference.Digest)
if digest == "" {
continue
}
parts := []string{
strings.TrimSpace(reference.Stage),
strings.TrimSpace(reference.LaneID),
strings.TrimSpace(reference.SlotName),
strings.TrimSpace(reference.OriginURI),
}
values = append(values, Fingerprint{
Name: strings.Join(parts, ":"),
Value: digest,
})
}
return normalizeFingerprints(values)
}
func normalizeStrings(values []string) []string {
if len(values) == 0 {
return nil
}
seen := make(map[string]struct{}, len(values))
for _, value := range values {
value = strings.TrimSpace(value)
if value == "" {
continue
}
seen[value] = struct{}{}
}
if len(seen) == 0 {
return nil
}
out := make([]string, 0, len(seen))
for value := range seen {
out = append(out, value)
}
sort.Strings(out)
return out
}
func digestPrefix(digest string) string {
digest = strings.TrimSpace(digest)
if digest == "" {
return ""
}
if idx := strings.Index(digest, ":"); idx >= 0 {
digest = digest[idx+1:]
}
digest = strings.TrimSpace(digest)
if len(digest) > digestPrefixLength {
return digest[:digestPrefixLength]
}
return digest
}
func safePathComponent(value string) (string, error) {
value = strings.TrimSpace(value)
if value == "" {
return "", fmt.Errorf("must not be empty")
}
var b strings.Builder
for _, r := range value {
switch {
case r >= 'a' && r <= 'z':
b.WriteRune(r)
case r >= 'A' && r <= 'Z':
b.WriteRune(r)
case r >= '0' && r <= '9':
b.WriteRune(r)
case r == '-' || r == '_' || r == '.':
b.WriteRune(r)
default:
b.WriteString(fmt.Sprintf("~%x", r))
}
}
encoded := b.String()
if encoded == "." || encoded == ".." || strings.Contains(encoded, "..") || strings.ContainsAny(encoded, `/\`) {
return "", fmt.Errorf("%q is not filesystem safe", value)
}
return encoded, nil
}

View File

@@ -0,0 +1,266 @@
package workspace
import (
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestCheckpointIdentityIsDeterministic(t *testing.T) {
first := mustIdentity(t, identityInput())
second := mustIdentity(t, identityInput())
if first.Digest != second.Digest {
t.Fatalf("digest changed for same input: %q != %q", first.Digest, second.Digest)
}
if !strings.HasPrefix(first.Digest, "sha256:") {
t.Fatalf("digest = %q, want sha256 prefix", first.Digest)
}
}
func TestCheckpointIdentityChangesWhenInputsChange(t *testing.T) {
base := mustIdentity(t, identityInput())
tests := []struct {
name string
mutate func(CheckpointIdentityInput) CheckpointIdentityInput
}{
{
name: "pipeline digest",
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
input.Pipeline.Digest = "sha256:pipeline-b"
return input
},
},
{
name: "raw input digest",
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
input.RawInputDigest = "sha256:raw-b"
return input
},
},
{
name: "selected lanes",
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
input.SelectedLanes = []string{"items"}
return input
},
},
{
name: "reference digest",
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
input.References[0].Digest = "sha256:reference-b"
return input
},
},
{
name: "runtime override",
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
input.RuntimeOverrides = []Fingerprint{{Name: "llm_profile", Value: "careful"}}
return input
},
},
{
name: "provenance fingerprint",
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
input.ProvenanceFingerprints = []Fingerprint{{Name: "prompt:dnd.spells", Value: "sha256:prompt-b"}}
return input
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
changed := mustIdentity(t, tc.mutate(identityInput()))
if changed.Digest == base.Digest {
t.Fatalf("digest did not change after %s mutation: %q", tc.name, changed.Digest)
}
})
}
}
func TestCheckpointIdentityNormalizesOrder(t *testing.T) {
input := identityInput()
input.SelectedLanes = []string{"spells", "items", "spells"}
input.RuntimeOverrides = []Fingerprint{
{Name: "z", Value: "2"},
{Name: "a", Value: "1"},
}
input.ProvenanceFingerprints = []Fingerprint{
{Name: "schema", Value: "sha256:schema"},
{Name: "prompt", Value: "sha256:prompt"},
}
identity := mustIdentity(t, input)
if got := strings.Join(identity.SelectedLanes, ","); got != "items,spells" {
t.Fatalf("selected lanes = %q, want sorted unique values", got)
}
if identity.RuntimeOverrides[0].Name != "a" || identity.ProvenanceFingerprints[0].Name != "prompt" {
t.Fatalf("fingerprints not sorted: runtime=%+v provenance=%+v", identity.RuntimeOverrides, identity.ProvenanceFingerprints)
}
}
func TestCheckpointIdentityPathIsFilesystemSafe(t *testing.T) {
input := identityInput()
input.Pipeline.ID = "campaign/main"
input.InputKey = "seriatim/input"
input.SourceDigest = "sha256:abcdef0123456789ffffffff"
input.Pipeline.Digest = "sha256:1234567890abcdefeeeeeeee"
identity := mustIdentity(t, input)
relative, err := identity.RelativePath()
if err != nil {
t.Fatalf("RelativePath: %v", err)
}
if strings.Contains(relative, `\`) || strings.Contains(relative, "..") {
t.Fatalf("relative path is not filesystem safe: %q", relative)
}
identityDigest := digestPrefix(identity.Digest)
wantRelative := "campaign~2fmain/seriatim~2finput-abcdef0123456789/1234567890abcdef/" + identityDigest
if relative != wantRelative {
t.Fatalf("relative path = %q", relative)
}
root := t.TempDir()
settings := Settings{
CheckpointsRoot: filepath.Join(root, "checkpoints"),
ResumeEnabled: true,
}
got, err := settings.CheckpointDirectory(identity)
if err != nil {
t.Fatalf("CheckpointDirectory: %v", err)
}
want := filepath.Join(root, "checkpoints", "campaign~2fmain", "seriatim~2finput-abcdef0123456789", "1234567890abcdef", identityDigest)
if got != want {
t.Fatalf("checkpoint directory = %q, want %q", got, want)
}
}
func TestCheckpointIdentityPathIncludesInvocationIdentity(t *testing.T) {
base := mustIdentity(t, identityInput())
changedInput := identityInput()
changedInput.References[0].Digest = "sha256:reference-b"
changed := mustIdentity(t, changedInput)
if base.Digest == changed.Digest {
t.Fatalf("test setup produced same identity digest: %q", base.Digest)
}
basePath, err := base.RelativePath()
if err != nil {
t.Fatalf("base RelativePath: %v", err)
}
changedPath, err := changed.RelativePath()
if err != nil {
t.Fatalf("changed RelativePath: %v", err)
}
if basePath == changedPath {
t.Fatalf("relative path did not change with invocation identity: %q", basePath)
}
}
func TestCheckpointDirectoryDisabledReturnsEmptyPath(t *testing.T) {
settings := Settings{CheckpointsRoot: filepath.Join(t.TempDir(), "checkpoints")}
got, err := settings.CheckpointDirectory(mustIdentity(t, identityInput()))
if err != nil {
t.Fatalf("CheckpointDirectory: %v", err)
}
if got != "" {
t.Fatalf("CheckpointDirectory = %q, want empty path", got)
}
}
func TestNewCheckpointIdentityRequiresCoreInputs(t *testing.T) {
tests := []struct {
name string
mutate func(CheckpointIdentityInput) CheckpointIdentityInput
want string
}{
{
name: "pipeline id",
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
input.Pipeline.ID = ""
return input
},
want: "pipeline id",
},
{
name: "pipeline digest",
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
input.Pipeline.Digest = ""
return input
},
want: "pipeline digest",
},
{
name: "input key",
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
input.InputKey = ""
input.Pipeline.Input.Module = ""
return input
},
want: "input key",
},
{
name: "input digest",
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
input.RawInputDigest = ""
input.SourceDigest = ""
return input
},
want: "raw input digest or source digest",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := NewCheckpointIdentity(tc.mutate(identityInput()))
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("expected error containing %q, got %v", tc.want, err)
}
})
}
}
func identityInput() CheckpointIdentityInput {
return CheckpointIdentityInput{
Pipeline: pipeline.ResolvedPipeline{
ID: "dnd-session",
Digest: "sha256:pipeline-a",
Input: pipeline.Binding("seriatim"),
ArtifactLanes: []pipeline.ResolvedArtifactLane{
{ID: "spells"},
{ID: "items"},
},
},
InputKey: "seriatim",
RawInputDigest: "sha256:raw-a",
SelectedLanes: []string{"spells"},
RuntimeOverrides: []Fingerprint{
{Name: "llm_profile", Value: "fast"},
},
References: []artifacts.ReferenceProvenance{
{
Stage: "extract",
LaneID: "spells",
SlotName: "party",
OriginURI: "file:///party.yml",
Digest: "sha256:reference-a",
},
},
ProvenanceFingerprints: []Fingerprint{
{Name: "prompt:dnd.spells", Value: "sha256:prompt-a"},
},
}
}
func mustIdentity(t *testing.T, input CheckpointIdentityInput) CheckpointIdentity {
t.Helper()
identity, err := NewCheckpointIdentity(input)
if err != nil {
t.Fatalf("NewCheckpointIdentity: %v", err)
}
return identity
}

View File

@@ -0,0 +1,85 @@
package workspace
import "time"
const (
WorkspaceSchemaVersion = "notarius.workspace.v2"
WorkspaceSchemaVersionV1 = "notarius.workspace.v1"
)
type StageName string
const (
StageSource StageName = "source"
StageChunk StageName = "chunk"
StageExtract StageName = "extract"
StageMerge StageName = "merge"
StageNormalize StageName = "normalize"
)
type StageStatus string
const (
StatusPending StageStatus = "pending"
StatusRunning StageStatus = "running"
StatusSucceeded StageStatus = "succeeded"
StatusSucceededWithRejections StageStatus = "succeeded_with_rejections"
StatusFailed StageStatus = "failed"
StatusInvalidated StageStatus = "invalidated"
)
type StageManifest struct {
WorkspaceSchemaVersion string `json:"workspace_schema_version"`
Stage StageName `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
DependencyFingerprints []Fingerprint `json:"dependency_fingerprints,omitempty"`
Status StageStatus `json:"status"`
OutputDigests []Fingerprint `json:"output_digests,omitempty"`
ValidationStatus string `json:"validation_status,omitempty"`
Rejections []RejectionSummary `json:"rejections,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
}
type RejectionSummary struct {
ValidatorName string `json:"validator_name,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message,omitempty"`
Count int `json:"count,omitempty"`
}
type SourceManifest struct {
StageManifest
SourceID string `json:"source_id,omitempty"`
}
type ChunkManifest struct {
StageManifest
ChunkCount int `json:"chunk_count,omitempty"`
}
type ExtractLaneManifest struct {
StageManifest
ChunkCount int `json:"chunk_count,omitempty"`
OutputCount int `json:"output_count,omitempty"`
}
type MergeLaneManifest struct {
StageManifest
InputCount int `json:"input_count,omitempty"`
}
type NormalizeLaneManifest struct {
StageManifest
InputCount int `json:"input_count,omitempty"`
}
func NewStageManifest(stage StageName, status StageStatus) StageManifest {
return StageManifest{
WorkspaceSchemaVersion: WorkspaceSchemaVersion,
Stage: stage,
Status: status,
}
}

View File

@@ -0,0 +1,149 @@
package workspace
import (
"encoding/json"
"testing"
"time"
)
func TestStageManifestDefaults(t *testing.T) {
if WorkspaceSchemaVersion != "notarius.workspace.v2" {
t.Fatalf("current schema version = %q, want notarius.workspace.v2", WorkspaceSchemaVersion)
}
if WorkspaceSchemaVersionV1 != "notarius.workspace.v1" {
t.Fatalf("legacy schema version = %q, want notarius.workspace.v1", WorkspaceSchemaVersionV1)
}
manifest := NewStageManifest(StageExtract, StatusRunning)
if manifest.WorkspaceSchemaVersion != WorkspaceSchemaVersion {
t.Fatalf("schema version = %q, want %q", manifest.WorkspaceSchemaVersion, WorkspaceSchemaVersion)
}
if manifest.Stage != StageExtract {
t.Fatalf("stage = %q, want extract", manifest.Stage)
}
if manifest.Status != StatusRunning {
t.Fatalf("status = %q, want running", manifest.Status)
}
}
func TestManifestJSONRoundTrips(t *testing.T) {
started := time.Unix(100, 0).UTC()
completed := time.Unix(200, 0).UTC()
t.Run("source", func(t *testing.T) {
manifest := SourceManifest{
StageManifest: populatedManifest(StageSource, "", "seriatim", started, completed),
SourceID: "source-1",
}
var got SourceManifest
roundTripManifest(t, manifest, &got)
if got.SourceID != manifest.SourceID || got.Stage != StageSource {
t.Fatalf("round trip source manifest = %+v", got)
}
})
t.Run("chunk", func(t *testing.T) {
manifest := ChunkManifest{
StageManifest: populatedManifest(StageChunk, "", "generic", started, completed),
ChunkCount: 3,
}
var got ChunkManifest
roundTripManifest(t, manifest, &got)
if got.ChunkCount != manifest.ChunkCount || got.Stage != StageChunk {
t.Fatalf("round trip chunk manifest = %+v", got)
}
})
t.Run("extract", func(t *testing.T) {
manifest := ExtractLaneManifest{
StageManifest: populatedManifest(StageExtract, "spells", "dnd/spells", started, completed),
ChunkCount: 3,
OutputCount: 2,
}
var got ExtractLaneManifest
roundTripManifest(t, manifest, &got)
if got.LaneID != "spells" || got.OutputCount != manifest.OutputCount || got.Stage != StageExtract {
t.Fatalf("round trip extract manifest = %+v", got)
}
})
t.Run("merge", func(t *testing.T) {
manifest := MergeLaneManifest{
StageManifest: populatedManifest(StageMerge, "spells", "appendorder", started, completed),
InputCount: 2,
}
var got MergeLaneManifest
roundTripManifest(t, manifest, &got)
if got.InputCount != manifest.InputCount || got.Stage != StageMerge {
t.Fatalf("round trip merge manifest = %+v", got)
}
})
t.Run("normalize", func(t *testing.T) {
manifest := NormalizeLaneManifest{
StageManifest: populatedManifest(StageNormalize, "spells", "noop", started, completed),
InputCount: 1,
}
var got NormalizeLaneManifest
roundTripManifest(t, manifest, &got)
if got.InputCount != manifest.InputCount || got.Stage != StageNormalize {
t.Fatalf("round trip normalize manifest = %+v", got)
}
})
}
func TestStatusValues(t *testing.T) {
values := []StageStatus{
StatusPending,
StatusRunning,
StatusSucceeded,
StatusSucceededWithRejections,
StatusFailed,
StatusInvalidated,
}
want := []string{
"pending",
"running",
"succeeded",
"succeeded_with_rejections",
"failed",
"invalidated",
}
for i, value := range values {
if string(value) != want[i] {
t.Fatalf("status[%d] = %q, want %q", i, value, want[i])
}
}
}
func populatedManifest(stage StageName, laneID string, moduleKey string, started time.Time, completed time.Time) StageManifest {
manifest := NewStageManifest(stage, StatusSucceededWithRejections)
manifest.LaneID = laneID
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = []Fingerprint{{Name: "source", Value: "sha256:source"}}
manifest.OutputDigests = []Fingerprint{{Name: "output", Value: "sha256:output"}}
manifest.ValidationStatus = "approved_with_warnings"
manifest.Rejections = []RejectionSummary{
{
ValidatorName: "shape",
ReasonCode: "invalid_shape",
Message: "invalid output shape",
Count: 1,
},
}
manifest.StartedAt = &started
manifest.CompletedAt = &completed
manifest.Metadata = map[string]string{"attempt": "1"}
return manifest
}
func roundTripManifest(t *testing.T, in any, out any) {
t.Helper()
data, err := json.Marshal(in)
if err != nil {
t.Fatalf("marshal manifest: %v", err)
}
if err := json.Unmarshal(data, out); err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
}

View File

@@ -0,0 +1,76 @@
package workspace
import (
"fmt"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
)
type Settings struct {
RootDir string
DiagnosticsRoot string
CheckpointsRoot string
DebugRoot string
DiagnosticsEnabled bool
ResumeEnabled bool
DebugEnabled bool
}
func FromConfig(cfg config.Config) Settings {
root := cleanPath(cfg.Workspace.Directory)
settings := Settings{
RootDir: root,
DiagnosticsEnabled: cfg.DiagnosticsEnabled(),
}
if settings.DiagnosticsEnabled {
settings.DiagnosticsRoot = cleanPath(cfg.Diagnostics.WorkDir)
}
if root == "" {
return settings
}
settings.CheckpointsRoot = filepath.Join(root, "checkpoints")
settings.DebugRoot = filepath.Join(root, "debug")
settings.ResumeEnabled = cfg.Workspace.Resume.Enabled
settings.DebugEnabled = cfg.Workspace.Debug.Enabled
return settings
}
func (s Settings) DiagnosticsRunDirectory(runID string) (string, error) {
if !s.DiagnosticsEnabled || strings.TrimSpace(s.DiagnosticsRoot) == "" {
return "", nil
}
return safeSingleDirectory(s.DiagnosticsRoot, runID, "diagnostics run ID")
}
func (s Settings) CheckpointIdentityDirectory(identity string) (string, error) {
if !s.ResumeEnabled || strings.TrimSpace(s.CheckpointsRoot) == "" {
return "", nil
}
return SafePath(s.CheckpointsRoot, identity)
}
func (s Settings) DebugRunDirectory(runID string) (string, error) {
if !s.DebugEnabled || strings.TrimSpace(s.DebugRoot) == "" {
return "", nil
}
return safeSingleDirectory(s.DebugRoot, runID, "debug run ID")
}
func cleanPath(path string) string {
path = strings.TrimSpace(path)
if path == "" {
return ""
}
return filepath.Clean(path)
}
func safeSingleDirectory(root string, name string, label string) (string, error) {
name = strings.TrimSpace(name)
if strings.Contains(name, "/") || strings.Contains(name, `\`) {
return "", fmt.Errorf("%s %q must be a single directory name", label, name)
}
return SafePath(root, name)
}

View File

@@ -0,0 +1,127 @@
package workspace
import (
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
)
func TestFromConfigBuildsWorkspaceRoots(t *testing.T) {
cfg := config.Default()
cfg.Workspace.Directory = "/var/lib/notarius"
cfg.Workspace.Resume.Enabled = true
cfg.Workspace.Debug.Enabled = true
cfg.RecomputeEffectiveDiagnostics()
settings := FromConfig(cfg)
if settings.RootDir != "/var/lib/notarius" {
t.Fatalf("RootDir = %q, want /var/lib/notarius", settings.RootDir)
}
if settings.DiagnosticsRoot != "/var/lib/notarius/diagnostics" || !settings.DiagnosticsEnabled {
t.Fatalf("diagnostics settings = %+v, want workspace diagnostics root enabled", settings)
}
if settings.CheckpointsRoot != "/var/lib/notarius/checkpoints" || !settings.ResumeEnabled {
t.Fatalf("checkpoint settings = %+v, want workspace checkpoints root enabled", settings)
}
if settings.DebugRoot != "/var/lib/notarius/debug" || !settings.DebugEnabled {
t.Fatalf("debug settings = %+v, want workspace debug root enabled", settings)
}
}
func TestFromConfigKeepsLegacyDiagnosticsRootWithoutWorkspaceRoot(t *testing.T) {
cfg := config.Default()
cfg.Diagnostics.WorkDir = "/tmp/notarius-legacy"
cfg.Workspace.Resume.Enabled = true
cfg.Workspace.Debug.Enabled = true
settings := FromConfig(cfg)
if settings.RootDir != "" {
t.Fatalf("RootDir = %q, want empty", settings.RootDir)
}
if settings.DiagnosticsRoot != "/tmp/notarius-legacy" || !settings.DiagnosticsEnabled {
t.Fatalf("diagnostics settings = %+v, want legacy diagnostics root enabled", settings)
}
if settings.CheckpointsRoot != "" || settings.ResumeEnabled {
t.Fatalf("checkpoint settings = %+v, want disabled empty root", settings)
}
if settings.DebugRoot != "" || settings.DebugEnabled {
t.Fatalf("debug settings = %+v, want disabled empty root", settings)
}
}
func TestPathConstructors(t *testing.T) {
root := t.TempDir()
settings := Settings{
RootDir: root,
DiagnosticsRoot: filepath.Join(root, "diagnostics"),
CheckpointsRoot: filepath.Join(root, "checkpoints"),
DebugRoot: filepath.Join(root, "debug"),
DiagnosticsEnabled: true,
ResumeEnabled: true,
DebugEnabled: true,
}
diagnosticsDir, err := settings.DiagnosticsRunDirectory("run-123")
if err != nil {
t.Fatalf("DiagnosticsRunDirectory: %v", err)
}
if diagnosticsDir != filepath.Join(root, "diagnostics", "run-123") {
t.Fatalf("diagnostics dir = %q", diagnosticsDir)
}
checkpointDir, err := settings.CheckpointIdentityDirectory("pipeline/input-digest/pipeline-digest")
if err != nil {
t.Fatalf("CheckpointIdentityDirectory: %v", err)
}
if checkpointDir != filepath.Join(root, "checkpoints", "pipeline", "input-digest", "pipeline-digest") {
t.Fatalf("checkpoint dir = %q", checkpointDir)
}
debugDir, err := settings.DebugRunDirectory("run-456")
if err != nil {
t.Fatalf("DebugRunDirectory: %v", err)
}
if debugDir != filepath.Join(root, "debug", "run-456") {
t.Fatalf("debug dir = %q", debugDir)
}
}
func TestDisabledPathConstructorsReturnEmptyPaths(t *testing.T) {
settings := Settings{}
for name, call := range map[string]func() (string, error){
"diagnostics": func() (string, error) { return settings.DiagnosticsRunDirectory("run-1") },
"checkpoint": func() (string, error) { return settings.CheckpointIdentityDirectory("identity") },
"debug": func() (string, error) { return settings.DebugRunDirectory("run-1") },
} {
t.Run(name, func(t *testing.T) {
got, err := call()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got != "" {
t.Fatalf("path = %q, want empty", got)
}
})
}
}
func TestRunDirectoryConstructorsRejectNestedNames(t *testing.T) {
root := t.TempDir()
settings := Settings{
DiagnosticsRoot: filepath.Join(root, "diagnostics"),
DebugRoot: filepath.Join(root, "debug"),
DiagnosticsEnabled: true,
DebugEnabled: true,
}
if got, err := settings.DiagnosticsRunDirectory("run-1/nested"); err == nil {
t.Fatalf("DiagnosticsRunDirectory returned %q, want error", got)
}
if got, err := settings.DebugRunDirectory("run-1/nested"); err == nil {
t.Fatalf("DebugRunDirectory returned %q, want error", got)
}
}

View File

@@ -0,0 +1,301 @@
package checkpoint
import (
"encoding/base64"
"encoding/json"
"fmt"
"os"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
type WorkspaceLoader struct {
root string
identityDigest string
}
func NewWorkspaceLoader(settings coreworkspace.Settings, identity coreworkspace.CheckpointIdentity) (pipeline.CheckpointLoader, error) {
root, err := settings.CheckpointDirectory(identity)
if err != nil {
return nil, err
}
if strings.TrimSpace(root) == "" {
return pipeline.NoopCheckpointLoader(), nil
}
return &WorkspaceLoader{root: root, identityDigest: identity.Digest}, nil
}
func (l *WorkspaceLoader) Enabled() bool {
return l != nil && strings.TrimSpace(l.root) != ""
}
func (l *WorkspaceLoader) Source(moduleKey string) (pipeline.SourceCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.SourceManifest
if decision := l.readJSON("source/manifest.json", &manifest); !decision.Reused {
return pipeline.SourceCheckpoint{}, decision
}
if decision := l.validateManifest(manifest.StageManifest, coreworkspace.StageSource, "", moduleKey, coreworkspace.StatusSucceeded, nil); !decision.Reused {
return pipeline.SourceCheckpoint{}, decision
}
var payload sourceDocumentEnvelope
if decision := l.readJSON("source/source-document.json", &payload); !decision.Reused {
return pipeline.SourceCheckpoint{}, decision
}
doc := cloneSourceDocument(payload.Document)
if err := source.ValidateDocument(&doc); err != nil {
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint document is invalid: %v", err)
}
if strings.TrimSpace(manifest.SourceID) != "" && manifest.SourceID != doc.ID {
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint source id does not match payload")
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), digestFingerprints("source_document", doc.Digest)) {
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint output digest does not match payload")
}
return pipeline.SourceCheckpoint{Document: &doc}, reusedDecision()
}
func (l *WorkspaceLoader) Chunk(moduleKey string, sourceDigest string) (pipeline.ChunkCheckpoint, pipeline.CheckpointDecision) {
expectedDependencies := digestFingerprints("source_document", sourceDigest)
var manifest coreworkspace.ChunkManifest
if decision := l.readJSON("chunk/manifest.json", &manifest); !decision.Reused {
return pipeline.ChunkCheckpoint{}, decision
}
if decision := l.validateManifest(manifest.StageManifest, coreworkspace.StageChunk, "", moduleKey, coreworkspace.StatusSucceeded, expectedDependencies); !decision.Reused {
return pipeline.ChunkCheckpoint{}, decision
}
var payload chunksEnvelope
if decision := l.readJSON("chunk/chunks.json", &payload); !decision.Reused {
return pipeline.ChunkCheckpoint{}, decision
}
chunks, err := sourceChunksFromEnvelope(payload.Chunks)
if err != nil {
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint payload is invalid: %v", err)
}
if len(chunks) == 0 {
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint payload has no chunks")
}
outputDigests, err := chunkOutputDigests(chunks)
if err != nil {
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint output cannot be digested: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), outputDigests) {
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint output digests do not match payload")
}
return pipeline.ChunkCheckpoint{Chunks: chunks, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) Extract(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.ExtractLaneManifest
if d := l.readJSON(laneManifestPath("extract", laneID), &manifest); !d.Reused {
return pipeline.ExtractCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageExtract, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded, coreworkspace.StatusSucceededWithRejections); !d.Reused {
return pipeline.ExtractCheckpoint{}, d
}
var payload artifactExtractEnvelope
if d := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !d.Reused {
return pipeline.ExtractCheckpoint{}, d
}
outputs, err := artifactCheckpointOutputs(payload.Outputs)
if err != nil {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(outputs)) {
return pipeline.ExtractCheckpoint{}, invalidDecision("extract artifact checkpoint output digests do not match payload")
}
return pipeline.ExtractCheckpoint{Outputs: outputs, Rejected: cloneRejectedOutputs(payload.Rejected), Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) Merge(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.MergeLaneManifest
if d := l.readJSON(laneManifestPath("merge", laneID), &manifest); !d.Reused {
return pipeline.MergeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageMerge, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused {
return pipeline.MergeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !d.Reused {
return pipeline.MergeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.MergeCheckpoint{}, invalidDecision("merge artifact checkpoint output digest does not match payload")
}
return pipeline.MergeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func (l *WorkspaceLoader) Normalize(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
var manifest coreworkspace.NormalizeLaneManifest
if d := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !d.Reused {
return pipeline.NormalizeCheckpoint{}, d
}
if d := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageNormalize, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !d.Reused {
return pipeline.NormalizeCheckpoint{}, d
}
var payload artifactSingleEnvelope
if d := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !d.Reused {
return pipeline.NormalizeCheckpoint{}, d
}
values, err := artifactCheckpointOutputs([]artifactCheckpointEnvelope{payload.Output})
if err != nil {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint payload is invalid: %v", err)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), artifactOutputDigests(values)) {
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize artifact checkpoint output digest does not match payload")
}
return pipeline.NormalizeCheckpoint{Output: values[0], Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
}
func artifactCheckpointOutputs(values []artifactCheckpointEnvelope) ([]pipeline.CheckpointArtifact, error) {
if len(values) == 0 {
return nil, nil
}
out := make([]pipeline.CheckpointArtifact, 0, len(values))
for _, v := range values {
content, err := contentFromEnvelope(v.Content)
if err != nil {
return nil, err
}
if strings.TrimSpace(string(v.Kind)) == "" || strings.TrimSpace(v.Schema.ID) == "" || strings.TrimSpace(v.Schema.Version) == "" || strings.TrimSpace(v.SchemaDigest) == "" {
return nil, fmt.Errorf("artifact codec identity is incomplete")
}
out = append(out, pipeline.CheckpointArtifact{LaneID: v.LaneID, ModuleKey: v.ModuleKey, SourceID: v.SourceID, ChunkID: v.ChunkID, ChunkIndex: v.ChunkIndex, ChunkRef: v.ChunkRef, SchemaDigest: v.SchemaDigest, Artifact: contracts.SerializedArtifact{Kind: v.Kind, Schema: v.Schema, MediaType: v.Content.MediaType, Content: content, Metadata: cloneMetadata(v.Content.Metadata)}})
}
return out, nil
}
func (l *WorkspaceLoader) readJSON(name string, out any) pipeline.CheckpointDecision {
if !l.Enabled() {
return pipeline.CheckpointDecision{Reason: "checkpoint loading disabled"}
}
target, err := coreworkspace.SafePath(l.root, name)
if err != nil {
return invalidDecision("checkpoint path is invalid: %v", err)
}
data, err := os.ReadFile(target)
if err != nil {
if os.IsNotExist(err) {
return pipeline.CheckpointDecision{Reason: "checkpoint artifact is missing"}
}
return invalidDecision("read checkpoint artifact: %v", err)
}
if err := json.Unmarshal(data, out); err != nil {
return invalidDecision("decode checkpoint artifact: %v", err)
}
return reusedDecision()
}
func (l *WorkspaceLoader) validateManifest(manifest coreworkspace.StageManifest, stage coreworkspace.StageName, laneID string, moduleKey string, status coreworkspace.StageStatus, dependencies []pipeline.CheckpointFingerprint) pipeline.CheckpointDecision {
return l.validateLaneManifest(manifest, stage, laneID, moduleKey, dependencies, status)
}
func (l *WorkspaceLoader) validateLaneManifest(manifest coreworkspace.StageManifest, stage coreworkspace.StageName, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, statuses ...coreworkspace.StageStatus) pipeline.CheckpointDecision {
if manifest.WorkspaceSchemaVersion == coreworkspace.WorkspaceSchemaVersionV1 {
return invalidDecision("checkpoint workspace schema version %q is incompatible with %q and must be recomputed", manifest.WorkspaceSchemaVersion, coreworkspace.WorkspaceSchemaVersion)
}
if manifest.WorkspaceSchemaVersion != coreworkspace.WorkspaceSchemaVersion {
return invalidDecision("checkpoint workspace schema version %q is not supported", manifest.WorkspaceSchemaVersion)
}
if strings.TrimSpace(l.identityDigest) != "" && manifest.Metadata["checkpoint_identity_digest"] != l.identityDigest {
return invalidDecision("checkpoint identity digest does not match current invocation")
}
if manifest.Stage != stage {
return invalidDecision("checkpoint stage %q does not match %q", manifest.Stage, stage)
}
if strings.TrimSpace(laneID) != "" && manifest.LaneID != laneID {
return invalidDecision("checkpoint lane %q does not match %q", manifest.LaneID, laneID)
}
if strings.TrimSpace(moduleKey) != "" && manifest.ModuleKey != moduleKey {
return invalidDecision("checkpoint module %q does not match %q", manifest.ModuleKey, moduleKey)
}
statusOK := false
for _, status := range statuses {
if manifest.Status == status {
statusOK = true
break
}
}
if !statusOK {
return invalidDecision("checkpoint status %q cannot be reused", manifest.Status)
}
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.DependencyFingerprints), dependencies) {
return invalidDecision("checkpoint dependency fingerprints do not match")
}
return reusedDecision()
}
func sourceChunksFromEnvelope(values []chunkEnvelope) ([]source.Chunk, error) {
if len(values) == 0 {
return nil, nil
}
out := make([]source.Chunk, 0, len(values))
for _, value := range values {
content, err := contentFromEnvelope(value.Content)
if err != nil {
return nil, err
}
out = append(out, source.Chunk{
ID: value.ID,
SourceID: value.SourceID,
Index: value.Index,
Ref: value.Ref,
Content: content,
MediaType: value.Content.MediaType,
Units: cloneSourceUnits(value.Units),
Metadata: cloneMetadata(value.Metadata),
})
}
return out, nil
}
func contentFromEnvelope(value binaryEnvelope) ([]byte, error) {
content, err := base64.StdEncoding.DecodeString(value.ContentBase64)
if err != nil {
return nil, fmt.Errorf("decode content_base64: %w", err)
}
if digest := strings.TrimSpace(value.ContentDigest); digest != "" && digest != contentDigest(content) {
return nil, fmt.Errorf("content digest mismatch")
}
return content, nil
}
func coreworkspaceToPipelineFingerprints(values []coreworkspace.Fingerprint) []pipeline.CheckpointFingerprint {
if len(values) == 0 {
return nil
}
out := make([]pipeline.CheckpointFingerprint, 0, len(values))
for _, value := range values {
out = append(out, pipeline.CheckpointFingerprint{Name: value.Name, Value: value.Value})
}
return normalizeFingerprints(out)
}
func fingerprintsEqual(a []pipeline.CheckpointFingerprint, b []pipeline.CheckpointFingerprint) bool {
a = normalizeFingerprints(a)
b = normalizeFingerprints(b)
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func reusedDecision() pipeline.CheckpointDecision {
return pipeline.CheckpointDecision{Reused: true, Reason: "checkpoint is valid"}
}
func invalidDecision(format string, args ...any) pipeline.CheckpointDecision {
return pipeline.CheckpointDecision{Reason: fmt.Sprintf(format, args...)}
}

View File

@@ -0,0 +1,561 @@
package checkpoint
import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"path"
"sort"
"strings"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
type WorkspaceRecorder struct {
root string
identityDigest string
now func() time.Time
}
func NewWorkspaceRecorder(settings coreworkspace.Settings, identity coreworkspace.CheckpointIdentity) (pipeline.CheckpointRecorder, error) {
root, err := settings.CheckpointDirectory(identity)
if err != nil {
return nil, err
}
if strings.TrimSpace(root) == "" {
return pipeline.NoopCheckpointRecorder(), nil
}
return &WorkspaceRecorder{root: root, identityDigest: identity.Digest, now: time.Now}, nil
}
func (r *WorkspaceRecorder) SourceRunning(moduleKey string) error {
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusRunning)
manifest.ModuleKey = moduleKey
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) SourceSucceeded(moduleKey string, doc *source.SourceDocument) error {
if doc == nil {
return fmt.Errorf("checkpoint source document must not be nil")
}
if err := r.writePayload("source/source-document.json", sourceDocumentEnvelope{Document: cloneSourceDocument(*doc)}); err != nil {
return err
}
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusSucceeded)
manifest.ModuleKey = moduleKey
manifest.OutputDigests = workspaceFingerprints(digestFingerprints("source_document", doc.Digest))
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{
StageManifest: manifest,
SourceID: doc.ID,
})
}
func (r *WorkspaceRecorder) SourceFailed(moduleKey string, err error) error {
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusFailed)
manifest.ModuleKey = moduleKey
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) ChunkRunning(moduleKey string, sourceDigest string) error {
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusRunning)
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) ChunkSucceeded(moduleKey string, sourceDigest string, chunks []source.Chunk, warnings []contracts.Warning) error {
outputDigests, err := chunkOutputDigests(chunks)
if err != nil {
return fmt.Errorf("digest chunk checkpoint output: %w", err)
}
payload := chunksEnvelope{Chunks: chunkEnvelopes(chunks), Warnings: cloneWarnings(warnings)}
if err := r.writePayload("chunk/chunks.json", payload); err != nil {
return err
}
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceeded)
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
manifest.OutputDigests = workspaceFingerprints(outputDigests)
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{
StageManifest: manifest,
ChunkCount: len(chunks),
})
}
func (r *WorkspaceRecorder) ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error {
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceededWithRejections)
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
manifest.ValidationStatus = "rejected"
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) ChunkFailed(moduleKey string, sourceDigest string, err error) error {
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusFailed)
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) ExtractRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) ExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected), Warnings: cloneWarnings(warnings)}
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests(outputs))
manifest.ValidationStatus = validationStatusString(warnings, rejected)
manifest.Rejections = rejectionSummaries(rejected)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest, ChunkCount: len(outputs) + len(rejected), OutputCount: len(outputs)})
}
func (r *WorkspaceRecorder) ExtractFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) MergeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) MergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *WorkspaceRecorder) MergeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
manifest.ValidationStatus = "rejected"
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *WorkspaceRecorder) MergeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) NormalizeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
manifest.StartedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) NormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
return err
}
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
manifest.ValidationStatus = validationStatusString(warnings, nil)
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *WorkspaceRecorder) NormalizeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
manifest.ValidationStatus = "rejected"
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
manifest.CompletedAt = timePtr(r.timestamp())
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
}
func (r *WorkspaceRecorder) NormalizeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
manifest.CompletedAt = timePtr(r.timestamp())
manifest.Metadata = errorMetadata(err)
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
}
func (r *WorkspaceRecorder) writeManifest(name string, payload any) error {
return r.writeJSON(name, payload)
}
func (r *WorkspaceRecorder) writePayload(name string, payload any) error {
return r.writeJSON(name, payload)
}
func (r *WorkspaceRecorder) writeJSON(name string, payload any) error {
if r == nil || strings.TrimSpace(r.root) == "" {
return nil
}
return coreworkspace.WriteJSON(r.root, name, payload)
}
func (r *WorkspaceRecorder) timestamp() time.Time {
if r == nil || r.now == nil {
return time.Now().UTC()
}
return r.now().UTC()
}
func (r *WorkspaceRecorder) newStageManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus) coreworkspace.StageManifest {
manifest := coreworkspace.NewStageManifest(stage, status)
if strings.TrimSpace(r.identityDigest) != "" {
manifest.Metadata = map[string]string{"checkpoint_identity_digest": r.identityDigest}
}
return manifest
}
func (r *WorkspaceRecorder) laneManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) coreworkspace.StageManifest {
manifest := r.newStageManifest(stage, status)
manifest.LaneID = laneID
manifest.ModuleKey = moduleKey
manifest.DependencyFingerprints = workspaceFingerprints(dependencies)
return manifest
}
type sourceDocumentEnvelope struct {
Document source.SourceDocument `json:"document"`
}
type chunksEnvelope struct {
Chunks []chunkEnvelope `json:"chunks"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type chunkEnvelope struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
Index int `json:"index"`
Ref source.SourceRef `json:"ref"`
Content binaryEnvelope `json:"content"`
Units []source.SourceUnit `json:"units,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type binaryEnvelope struct {
ContentBase64 string `json:"content_base64,omitempty"`
ContentDigest string `json:"content_digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type artifactCheckpointEnvelope struct {
LaneID string `json:"lane_id"`
ModuleKey string `json:"module_key"`
SourceID string `json:"source_id,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
ChunkRef source.SourceRef `json:"chunk_ref,omitempty"`
Kind contracts.ArtifactKind `json:"artifact_kind"`
Schema contracts.ArtifactSchema `json:"schema"`
SchemaDigest string `json:"schema_digest"`
Content binaryEnvelope `json:"content"`
}
type artifactExtractEnvelope struct {
Outputs []artifactCheckpointEnvelope `json:"outputs"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type artifactSingleEnvelope struct {
Output artifactCheckpointEnvelope `json:"output"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
func artifactCheckpointEnvelopeFromOutput(output pipeline.CheckpointArtifact) artifactCheckpointEnvelope {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
schema.JSONSchema = nil
return artifactCheckpointEnvelope{LaneID: output.LaneID, ModuleKey: output.ModuleKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: output.SchemaDigest, Content: binaryEnvelopeFromContent(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)}
}
func artifactCheckpointEnvelopes(outputs []pipeline.CheckpointArtifact) []artifactCheckpointEnvelope {
if len(outputs) == 0 {
return nil
}
out := make([]artifactCheckpointEnvelope, 0, len(outputs))
for _, v := range outputs {
out = append(out, artifactCheckpointEnvelopeFromOutput(v))
}
return out
}
func artifactOutputDigests(outputs []pipeline.CheckpointArtifact) []pipeline.CheckpointFingerprint {
values := make([]pipeline.CheckpointFingerprint, 0, len(outputs))
for i, v := range outputs {
values = append(values, pipeline.CheckpointFingerprint{Name: fmt.Sprintf("artifact[%d]", i), Value: contentDigest(v.Artifact.Content)})
}
return normalizeFingerprints(values)
}
func chunkEnvelopes(chunks []source.Chunk) []chunkEnvelope {
if len(chunks) == 0 {
return nil
}
out := make([]chunkEnvelope, 0, len(chunks))
for _, chunk := range chunks {
out = append(out, chunkEnvelope{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Ref: chunk.Ref,
Content: binaryEnvelopeFromContent(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
Units: cloneSourceUnits(chunk.Units),
Metadata: cloneMetadata(chunk.Metadata),
})
}
return out
}
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
return binaryEnvelope{
ContentBase64: base64.StdEncoding.EncodeToString(content),
ContentDigest: contentDigest(content),
MediaType: mediaType,
Metadata: cloneMetadata(metadata),
Warnings: cloneWarnings(warnings),
}
}
func cloneSourceDocument(doc source.SourceDocument) source.SourceDocument {
doc.Units = cloneSourceUnits(doc.Units)
doc.Metadata = cloneMetadata(doc.Metadata)
return doc
}
func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
if len(units) == 0 {
return nil
}
out := make([]source.SourceUnit, 0, len(units))
for _, unit := range units {
out = append(out, source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
})
}
return out
}
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
if len(warnings) == 0 {
return nil
}
return append([]contracts.Warning(nil), warnings...)
}
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 {
return nil
}
return append([]contracts.RejectedOutput(nil), rejected...)
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
func chunkOutputDigests(chunks []source.Chunk) ([]pipeline.CheckpointFingerprint, error) {
values := make([]pipeline.CheckpointFingerprint, 0, len(chunks))
for _, chunk := range chunks {
digest, err := source.DigestChunk(chunk)
if err != nil {
return nil, fmt.Errorf("chunk %q: %w", chunk.ID, err)
}
values = append(values, pipeline.CheckpointFingerprint{
Name: chunk.ID,
Value: digest,
})
}
return normalizeFingerprints(values), nil
}
func digestFingerprints(name string, digest string) []pipeline.CheckpointFingerprint {
digest = strings.TrimSpace(digest)
if digest == "" {
return nil
}
return []pipeline.CheckpointFingerprint{{Name: name, Value: digest}}
}
func workspaceFingerprints(values []pipeline.CheckpointFingerprint) []coreworkspace.Fingerprint {
normalized := normalizeFingerprints(values)
if len(normalized) == 0 {
return nil
}
out := make([]coreworkspace.Fingerprint, 0, len(normalized))
for _, value := range normalized {
out = append(out, coreworkspace.Fingerprint{Name: value.Name, Value: value.Value})
}
return out
}
func normalizeFingerprints(values []pipeline.CheckpointFingerprint) []pipeline.CheckpointFingerprint {
if len(values) == 0 {
return nil
}
byName := make(map[string]string, len(values))
for _, value := range values {
name := strings.TrimSpace(value.Name)
fingerprint := strings.TrimSpace(value.Value)
if name == "" || fingerprint == "" {
continue
}
byName[name] = fingerprint
}
if len(byName) == 0 {
return nil
}
names := make([]string, 0, len(byName))
for name := range byName {
names = append(names, name)
}
sort.Strings(names)
out := make([]pipeline.CheckpointFingerprint, 0, len(names))
for _, name := range names {
out = append(out, pipeline.CheckpointFingerprint{Name: name, Value: byName[name]})
}
return out
}
func rejectionSummaries(rejected []contracts.RejectedOutput) []coreworkspace.RejectionSummary {
if len(rejected) == 0 {
return nil
}
type key struct {
validatorName string
reasonCode string
message string
}
counts := make(map[key]int, len(rejected))
for _, item := range rejected {
k := key{validatorName: item.ValidatorName, reasonCode: item.ReasonCode, message: item.Message}
counts[k]++
}
keys := make([]key, 0, len(counts))
for k := range counts {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
if keys[i].validatorName != keys[j].validatorName {
return keys[i].validatorName < keys[j].validatorName
}
if keys[i].reasonCode != keys[j].reasonCode {
return keys[i].reasonCode < keys[j].reasonCode
}
return keys[i].message < keys[j].message
})
out := make([]coreworkspace.RejectionSummary, 0, len(keys))
for _, k := range keys {
out = append(out, coreworkspace.RejectionSummary{
ValidatorName: k.validatorName,
ReasonCode: k.reasonCode,
Message: k.message,
Count: counts[k],
})
}
return out
}
func statusForRejected(rejected []contracts.RejectedOutput) coreworkspace.StageStatus {
if len(rejected) > 0 {
return coreworkspace.StatusSucceededWithRejections
}
return coreworkspace.StatusSucceeded
}
func validationStatusString(warnings []contracts.Warning, rejected []contracts.RejectedOutput) string {
if len(rejected) > 0 {
return "rejected"
}
if len(warnings) > 0 {
return "approved_with_warnings"
}
return "approved"
}
func errorMetadata(err error) map[string]string {
if err == nil {
return nil
}
return map[string]string{"error": err.Error()}
}
func laneManifestPath(stage string, laneID string) string {
return lanePayloadPath(stage, laneID, "manifest.json")
}
func lanePayloadPath(stage string, laneID string, file string) string {
return path.Join(stage, checkpointPathComponent(laneID), file)
}
func checkpointPathComponent(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return "_"
}
var b strings.Builder
for _, r := range value {
switch {
case r >= 'a' && r <= 'z':
b.WriteRune(r)
case r >= 'A' && r <= 'Z':
b.WriteRune(r)
case r >= '0' && r <= '9':
b.WriteRune(r)
case r == '-' || r == '_' || r == '.':
b.WriteRune(r)
default:
b.WriteString(fmt.Sprintf("~%x", r))
}
}
out := b.String()
if out == "." || out == ".." || strings.Contains(out, "..") {
return "_"
}
return out
}
func contentDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
func timePtr(t time.Time) *time.Time {
return &t
}

View File

@@ -0,0 +1,318 @@
package checkpoint
import (
"encoding/base64"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
}
chunks := []source.Chunk{
{
ID: "chunk-1",
SourceID: "source-1",
Index: 0,
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
Content: []byte("chunk content"),
MediaType: "text/plain",
Units: doc.Units,
},
}
if err := recorder.SourceRunning("seriatim"); err != nil {
t.Fatalf("SourceRunning: %v", err)
}
assertManifestStatus(t, filepath.Join(root, "source", "manifest.json"), coreworkspace.StatusRunning)
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
t.Fatalf("SourceSucceeded: %v", err)
}
assertManifestStatus(t, filepath.Join(root, "source", "manifest.json"), coreworkspace.StatusSucceeded)
if _, err := os.Stat(filepath.Join(root, "source", "source-document.json")); err != nil {
t.Fatalf("expected source checkpoint payload: %v", err)
}
if err := recorder.ChunkRunning("generic", doc.Digest); err != nil {
t.Fatalf("ChunkRunning: %v", err)
}
if err := recorder.ChunkSucceeded("generic", doc.Digest, chunks, nil); err != nil {
t.Fatalf("ChunkSucceeded: %v", err)
}
assertManifestStatus(t, filepath.Join(root, "chunk", "manifest.json"), coreworkspace.StatusSucceeded)
var chunkPayload struct {
Chunks []struct {
Content struct {
ContentBase64 string `json:"content_base64"`
ContentDigest string `json:"content_digest"`
} `json:"content"`
} `json:"chunks"`
}
readJSON(t, filepath.Join(root, "chunk", "chunks.json"), &chunkPayload)
if len(chunkPayload.Chunks) != 1 {
t.Fatalf("checkpoint chunks = %#v, want one", chunkPayload.Chunks)
}
decoded, err := base64.StdEncoding.DecodeString(chunkPayload.Chunks[0].Content.ContentBase64)
if err != nil {
t.Fatalf("decode chunk content: %v", err)
}
if string(decoded) != "chunk content" {
t.Fatalf("chunk content = %q, want original content", decoded)
}
if got, want := chunkPayload.Chunks[0].Content.ContentDigest, contentDigest([]byte("chunk content")); got != want {
t.Fatalf("content digest = %q, want %q", got, want)
}
}
func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
loader := &WorkspaceLoader{root: root}
schema := contracts.ArtifactSchema{ID: "dnd.spell_response", Name: "spell response", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
artifact := contracts.SerializedArtifact{Kind: "dnd.spells", Schema: schema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`), Metadata: map[string]any{"spell_cast_count": float64(0)}}
stored := pipeline.CheckpointArtifact{LaneID: "spells", ModuleKey: "dnd/spells", SourceID: "source-1", ChunkID: "chunk-1", ChunkIndex: 2, ChunkRef: source.SourceRef{SourceID: "source-1", StartUnitID: 4, EndUnitID: 8}, Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(schema)}
extractDeps := []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}
if err := recorder.ExtractSucceeded("spells", "dnd/spells", extractDeps, []pipeline.CheckpointArtifact{stored}, nil, nil); err != nil {
t.Fatalf("ExtractSucceeded: %v", err)
}
extracted, decision := loader.Extract("spells", "dnd/spells", extractDeps)
if !decision.Reused || len(extracted.Outputs) != 1 {
t.Fatalf("extract decision=%#v checkpoint=%#v, want reused", decision, extracted)
}
got := extracted.Outputs[0]
if got.Artifact.Kind != artifact.Kind || got.Artifact.Schema.ID != schema.ID || got.Artifact.Schema.Version != schema.Version || got.SchemaDigest != stored.SchemaDigest || string(got.Artifact.Content) != string(artifact.Content) || got.ChunkRef != stored.ChunkRef {
t.Fatalf("artifact checkpoint = %#v, want codec identity, bytes, and provenance", got)
}
mergeDeps := artifactOutputDigests([]pipeline.CheckpointArtifact{stored})
if err := recorder.MergeSucceeded("spells", "merge", mergeDeps, stored, nil); err != nil {
t.Fatalf("MergeSucceeded: %v", err)
}
merged, decision := loader.Merge("spells", "merge", mergeDeps)
if !decision.Reused || string(merged.Output.Artifact.Content) != string(artifact.Content) {
t.Fatalf("merge decision=%#v checkpoint=%#v, want reused", decision, merged)
}
normalizeDeps := artifactOutputDigests([]pipeline.CheckpointArtifact{stored})
if err := recorder.NormalizeSucceeded("spells", "normalize", normalizeDeps, stored, nil); err != nil {
t.Fatalf("NormalizeSucceeded: %v", err)
}
normalized, decision := loader.Normalize("spells", "normalize", normalizeDeps)
if !decision.Reused || normalized.Output.SchemaDigest != stored.SchemaDigest {
t.Fatalf("normalize decision=%#v checkpoint=%#v, want reused", decision, normalized)
}
}
func TestWorkspaceLoaderInvalidatesMissingCorruptAndMismatchedCheckpoints(t *testing.T) {
t.Run("missing", func(t *testing.T) {
loader := &WorkspaceLoader{root: t.TempDir()}
if _, decision := loader.Source("seriatim"); decision.Reused || !strings.Contains(decision.Reason, "missing") {
t.Fatalf("decision = %#v, want missing invalidation", decision)
}
})
t.Run("dependency mismatch", func(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
chunks := []source.Chunk{{
ID: "chunk-1",
SourceID: "source-1",
Content: []byte("chunk content"),
MediaType: "text/plain",
}}
if err := recorder.ChunkSucceeded("generic", "sha256:source-a", chunks, nil); err != nil {
t.Fatalf("ChunkSucceeded: %v", err)
}
loader := &WorkspaceLoader{root: root}
if _, decision := loader.Chunk("generic", "sha256:source-b"); decision.Reused || !strings.Contains(decision.Reason, "dependency") {
t.Fatalf("decision = %#v, want dependency invalidation", decision)
}
})
t.Run("incompatible workspace schema remains untouched", func(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
doc := &source.SourceDocument{
ID: "source-1", Kind: "document", Format: "text/plain", Digest: "sha256:source",
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
}
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
t.Fatalf("SourceSucceeded: %v", err)
}
manifestPath := filepath.Join(root, "source", "manifest.json")
manifest := strings.Replace(string(readFile(t, manifestPath)), coreworkspace.WorkspaceSchemaVersion, coreworkspace.WorkspaceSchemaVersionV1, 1)
if err := os.WriteFile(manifestPath, []byte(manifest), 0o644); err != nil {
t.Fatalf("write legacy manifest: %v", err)
}
beforeManifest := readFile(t, manifestPath)
payloadPath := filepath.Join(root, "source", "source-document.json")
beforePayload := readFile(t, payloadPath)
loader := &WorkspaceLoader{root: root}
if _, decision := loader.Source("seriatim"); decision.Reused || !strings.Contains(decision.Reason, "incompatible") || !strings.Contains(decision.Reason, coreworkspace.WorkspaceSchemaVersionV1) {
t.Fatalf("decision = %#v, want incompatible legacy schema invalidation", decision)
}
if got := readFile(t, manifestPath); string(got) != string(beforeManifest) {
t.Fatal("legacy manifest changed during reuse decision")
}
if got := readFile(t, payloadPath); string(got) != string(beforePayload) {
t.Fatal("legacy payload changed during reuse decision")
}
})
t.Run("corrupt payload", func(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
chunks := []source.Chunk{{
ID: "chunk-1",
SourceID: "source-1",
Content: []byte("chunk content"),
MediaType: "text/plain",
}}
if err := recorder.ChunkSucceeded("generic", "sha256:source", chunks, nil); err != nil {
t.Fatalf("ChunkSucceeded: %v", err)
}
payloadPath := filepath.Join(root, "chunk", "chunks.json")
data := strings.ReplaceAll(string(readFile(t, payloadPath)), contentDigest([]byte("chunk content")), "sha256:bad")
if err := os.WriteFile(payloadPath, []byte(data), 0o644); err != nil {
t.Fatalf("corrupt chunk payload: %v", err)
}
loader := &WorkspaceLoader{root: root}
if _, decision := loader.Chunk("generic", "sha256:source"); decision.Reused || !strings.Contains(decision.Reason, "invalid") {
t.Fatalf("decision = %#v, want corrupt payload invalidation", decision)
}
})
}
func TestWorkspaceRecorderRecordsRejectedExtractOutputs(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
rejected := []contracts.RejectedOutput{
{
Stage: string(pipeline.StageExtract),
LaneID: "spells",
ModuleKey: "dnd/spells",
ChunkID: "chunk-1",
ValidatorName: "shape",
ReasonCode: "invalid_shape",
Message: "bad shape",
},
}
if err := recorder.ExtractRunning("spells", "dnd/spells", []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}); err != nil {
t.Fatalf("ExtractRunning: %v", err)
}
if err := recorder.ExtractSucceeded("spells", "dnd/spells", nil, nil, rejected, nil); err != nil {
t.Fatalf("ExtractSucceeded: %v", err)
}
var manifest coreworkspace.ExtractLaneManifest
readJSON(t, filepath.Join(root, "extract", "spells", "manifest.json"), &manifest)
if manifest.Status != coreworkspace.StatusSucceededWithRejections || manifest.ValidationStatus != "rejected" {
t.Fatalf("extract manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus)
}
if len(manifest.Rejections) != 1 || manifest.Rejections[0].Count != 1 || manifest.Rejections[0].ReasonCode != "invalid_shape" {
t.Fatalf("rejections = %#v", manifest.Rejections)
}
var payload struct {
Rejected []contracts.RejectedOutput `json:"rejected"`
}
readJSON(t, filepath.Join(root, "extract", "spells", "outputs.json"), &payload)
if len(payload.Rejected) != 1 || payload.Rejected[0].ChunkID != "chunk-1" {
t.Fatalf("checkpoint rejected payload = %#v", payload.Rejected)
}
}
func TestWorkspaceRecorderRecordsFailedStages(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
if err := recorder.MergeRunning("spells", "appendorder", nil); err != nil {
t.Fatalf("MergeRunning: %v", err)
}
if err := recorder.MergeFailed("spells", "appendorder", nil, assertErr("merge failed")); err != nil {
t.Fatalf("MergeFailed: %v", err)
}
var manifest coreworkspace.MergeLaneManifest
readJSON(t, filepath.Join(root, "merge", "spells", "manifest.json"), &manifest)
if manifest.Status != coreworkspace.StatusFailed {
t.Fatalf("status = %q, want failed", manifest.Status)
}
if !strings.Contains(manifest.Metadata["error"], "merge failed") {
t.Fatalf("metadata = %#v, want error", manifest.Metadata)
}
}
func TestWorkspaceRecorderRecordsWarningOnlyValidation(t *testing.T) {
root := t.TempDir()
recorder := newTestRecorder(t, root)
schema := contracts.ArtifactSchema{ID: "test.artifact", Name: "test_artifact", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
output := pipeline.CheckpointArtifact{
LaneID: "events", ModuleKey: "noop", SourceID: "source-1",
Artifact: contracts.SerializedArtifact{Kind: "test/artifact", Schema: schema, MediaType: "application/json", Content: []byte(`{"ok":true}`)},
SchemaDigest: contracts.DigestArtifactSchema(schema),
}
warnings := []contracts.Warning{{ReasonCode: "note", Message: "warning"}}
if err := recorder.NormalizeSucceeded("events", "noop", nil, output, warnings); err != nil {
t.Fatalf("NormalizeSucceeded: %v", err)
}
var manifest coreworkspace.NormalizeLaneManifest
readJSON(t, filepath.Join(root, "normalize", "events", "manifest.json"), &manifest)
if manifest.Status != coreworkspace.StatusSucceeded || manifest.ValidationStatus != "approved_with_warnings" {
t.Fatalf("normalize manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus)
}
}
func newTestRecorder(t *testing.T, root string) *WorkspaceRecorder {
t.Helper()
return &WorkspaceRecorder{root: root}
}
func assertManifestStatus(t *testing.T, path string, want coreworkspace.StageStatus) {
t.Helper()
var manifest coreworkspace.StageManifest
readJSON(t, path, &manifest)
if manifest.Status != want {
t.Fatalf("%s status = %q, want %q", path, manifest.Status, want)
}
}
func readJSON(t *testing.T, path string, out any) {
t.Helper()
data := readFile(t, path)
if err := json.Unmarshal(data, out); err != nil {
t.Fatalf("decode %q: %v", path, err)
}
}
func readFile(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %q: %v", path, err)
}
return data
}
type assertErr string
func (e assertErr) Error() string { return string(e) }

View File

@@ -0,0 +1,81 @@
package contracts
import (
"crypto/sha256"
"encoding/hex"
)
// ArtifactKind is the stable logical identity of a domain artifact.
type ArtifactKind string
// ArtifactSchema describes the durable representation owned by an artifact
// codec. JSONSchema is cloned whenever framework ownership changes.
type ArtifactSchema struct {
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
JSONSchema []byte `json:"-"`
}
// SerializedArtifact is the domain-neutral representation of a typed
// artifact at an explicit serialization boundary.
type SerializedArtifact struct {
Kind ArtifactKind `json:"kind"`
Schema ArtifactSchema `json:"schema"`
MediaType string `json:"media_type"`
Content []byte `json:"-"`
Metadata map[string]any `json:"metadata,omitempty"`
}
// SerializedOutput associates a domain-neutral artifact with the pipeline
// operation that produced it. Provenance remains outside codec-owned bytes.
type SerializedOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Artifact SerializedArtifact `json:"artifact"`
}
// ArtifactCodec owns the stable encoding for one concrete artifact type.
type ArtifactCodec[T any] interface {
Kind() ArtifactKind
Schema() ArtifactSchema
MediaType() string
Encode(T) ([]byte, error)
Decode([]byte) (T, error)
}
// DigestArtifactSchema returns the SHA-256 digest of the exact JSON Schema
// bytes. Schema formatting is therefore part of the registered identity.
func DigestArtifactSchema(schema ArtifactSchema) string {
sum := sha256.Sum256(schema.JSONSchema)
return "sha256:" + hex.EncodeToString(sum[:])
}
func CloneArtifactSchema(schema ArtifactSchema) ArtifactSchema {
schema.JSONSchema = append([]byte(nil), schema.JSONSchema...)
return schema
}
func CloneSerializedArtifact(artifact SerializedArtifact) SerializedArtifact {
artifact.Schema = CloneArtifactSchema(artifact.Schema)
artifact.Content = append([]byte(nil), artifact.Content...)
artifact.Metadata = cloneArtifactMetadata(artifact.Metadata)
return artifact
}
func CloneSerializedOutput(output SerializedOutput) SerializedOutput {
output.Artifact = CloneSerializedArtifact(output.Artifact)
return output
}
func cloneArtifactMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}

View File

@@ -0,0 +1,45 @@
package contracts
import "testing"
func TestDigestArtifactSchemaUsesExactBytes(t *testing.T) {
first := ArtifactSchema{JSONSchema: []byte(`{"type":"object"}`)}
second := ArtifactSchema{JSONSchema: []byte("{\n \"type\": \"object\"\n}")}
if got := DigestArtifactSchema(first); got != "sha256:a2c799262a3ce3c19ef5cdd983bf3d12b43ab3c426227091b909dcb7054738c0" {
t.Fatalf("DigestArtifactSchema() = %q, want stable SHA-256", got)
}
if DigestArtifactSchema(first) == DigestArtifactSchema(second) {
t.Fatal("schema digests match for different exact bytes")
}
}
func TestArtifactCloneHelpersOwnSlicesAndMaps(t *testing.T) {
schema := ArtifactSchema{ID: "notes.v1", Name: "notes", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
artifact := SerializedArtifact{
Kind: "test/notes",
Schema: schema,
MediaType: "application/json",
Content: []byte(`{"items":["one"]}`),
Metadata: map[string]any{"origin": "test"},
}
clonedSchema := CloneArtifactSchema(schema)
cloned := CloneSerializedArtifact(artifact)
schema.JSONSchema[0] = '['
artifact.Content[0] = '['
artifact.Metadata["origin"] = "changed"
if string(clonedSchema.JSONSchema) != `{"type":"object"}` {
t.Fatalf("cloned schema = %q, want original bytes", clonedSchema.JSONSchema)
}
if string(cloned.Schema.JSONSchema) != `{"type":"object"}` {
t.Fatalf("serialized artifact schema = %q, want original bytes", cloned.Schema.JSONSchema)
}
if string(cloned.Content) != `{"items":["one"]}` {
t.Fatalf("cloned content = %q, want original bytes", cloned.Content)
}
if cloned.Metadata["origin"] != "test" {
t.Fatalf("cloned metadata = %#v, want independent map", cloned.Metadata)
}
}

View File

@@ -1,307 +0,0 @@
package contracts_test
import (
"context"
"encoding/json"
"errors"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
var _ contracts.InputAdapter = compositionAdapter{}
var _ contracts.Chunker = compositionChunker{}
var _ contracts.Extractor = compositionExtractor{}
var _ contracts.Merger = compositionMerger{}
var _ contracts.Normalizer = compositionNormalizer{}
var _ contracts.Validator = compositionValidator{}
var _ contracts.OutputEncoder = compositionOutputEncoder{}
func TestContractsComposeAcrossPackages(t *testing.T) {
ctx := context.Background()
adapter := compositionAdapter{}
chunker := compositionChunker{}
extractor := compositionExtractor{}
merger := compositionMerger{}
normalizer := compositionNormalizer{}
validator := compositionValidator{}
encoder := compositionOutputEncoder{}
doc, err := adapter.Parse(ctx, contracts.ParseRequest{SourceID: "source-1"})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
if err := source.ValidateDocument(doc); err != nil {
t.Fatalf("ValidateDocument() error = %v, want nil", err)
}
chunking, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc,
Metadata: map[string]any{"max_units": 2},
})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
if len(chunking.Chunks) != 1 {
t.Fatalf("len(Chunks) = %d, want 1", len(chunking.Chunks))
}
extraction, err := extractor.Extract(ctx, contracts.ExtractionRequest{
Source: doc,
Chunk: &chunking.Chunks[0],
AmbientContext: map[string]any{"synopsis": "example synopsis"},
})
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(extraction.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(extraction.Candidates))
}
candidate := extraction.Candidates[0]
for _, ref := range candidate.SourceRefs {
if err := source.ValidateRef(doc, ref); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
}
merge, err := merger.Merge(ctx, contracts.MergeRequest{
Source: doc,
LaneID: candidate.ArtifactType,
ChunkArtifacts: []contracts.ChunkArtifacts{
{
Chunk: chunking.Chunks[0],
Candidates: extraction.Candidates,
},
},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if len(merge.Candidates) != 1 {
t.Fatalf("len(merge.Candidates) = %d, want 1", len(merge.Candidates))
}
normalize, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
Source: doc,
LaneID: candidate.ArtifactType,
Candidates: merge.Candidates,
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if len(normalize.Candidates) != 1 {
t.Fatalf("len(normalize.Candidates) = %d, want 1", len(normalize.Candidates))
}
validation, err := validator.Validate(ctx, contracts.ValidationRequest{
Source: doc,
Candidates: normalize.Candidates,
})
if err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
if len(validation.Decisions) != 1 {
t.Fatalf("len(Decisions) = %d, want 1", len(validation.Decisions))
}
decision := validation.Decisions[0]
if !decision.Approved {
t.Fatal("Approved = false, want true")
}
if decision.CandidateIndex != candidate.Index {
t.Fatalf("CandidateIndex = %d, want %d", decision.CandidateIndex, candidate.Index)
}
output, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
Approved: []artifacts.Artifact{
artifacts.ArtifactFromCandidate(normalize.Candidates[0]),
},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if len(output.Files) != 1 {
t.Fatalf("len(Files) = %d, want 1", len(output.Files))
}
if output.Files[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.Files[0].ContentType)
}
if len(output.Files[0].Bytes) == 0 {
t.Fatal("len(Bytes) = 0, want encoded bytes")
}
}
type compositionAdapter struct{}
func (adapter compositionAdapter) Key() string {
return "generic-input"
}
func (adapter compositionAdapter) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
return &source.SourceDocument{
ID: req.SourceID,
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: "u1", Kind: "unit", Text: "First source unit."},
{ID: "u2", Kind: "unit", Text: "Second source unit."},
},
}, nil
}
type compositionChunker struct{}
func (chunker compositionChunker) Key() string {
return "generic-chunker"
}
func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if req.Source == nil {
return contracts.ChunkResult{}, errors.New("source document is required")
}
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{
{
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
Units: append([]source.SourceUnit(nil), req.Source.Units...),
Metadata: map[string]any{"strategy": "whole-document"},
},
},
}, nil
}
type compositionExtractor struct{}
func (extractor compositionExtractor) Key() string {
return "generic-extractor"
}
func (extractor compositionExtractor) ArtifactType() string {
return "generic-artifact"
}
func (extractor compositionExtractor) SchemaVersion() string {
return "v1"
}
func (extractor compositionExtractor) Validators() []contracts.Validator {
return []contracts.Validator{compositionValidator{}}
}
func (extractor compositionExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
if req.Source == nil {
return contracts.ExtractionResult{}, errors.New("source document is required")
}
units := req.Source.Units
if req.Chunk != nil {
units = req.Chunk.Units
}
if req.AmbientContext["synopsis"] == "" {
return contracts.ExtractionResult{}, errors.New("ambient synopsis is required")
}
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{
Index: 0,
ExtractorKey: extractor.Key(),
ArtifactType: extractor.ArtifactType(),
SchemaVersion: extractor.SchemaVersion(),
Payload: json.RawMessage(`{"value":"example"}`),
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: units[0].ID,
EndUnitID: units[len(units)-1].ID,
},
},
},
},
}, nil
}
type compositionMerger struct{}
func (merger compositionMerger) Key() string {
return "generic-merger"
}
func (merger compositionMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
}
return contracts.MergeResult{Candidates: candidates}, nil
}
type compositionNormalizer struct{}
func (normalizer compositionNormalizer) Key() string {
return "generic-normalizer"
}
func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
}
type compositionValidator struct{}
func (validator compositionValidator) Name() string {
return "generic-validator"
}
func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
for _, candidate := range req.Candidates {
decisions = append(decisions, contracts.ValidationDecision{
CandidateIndex: candidate.Index,
Approved: true,
ReasonCode: "accepted",
Message: "candidate accepted",
})
}
return contracts.ValidationResult{
ValidatorName: validator.Name(),
Decisions: decisions,
}, nil
}
type compositionOutputEncoder struct{}
func (encoder compositionOutputEncoder) Key() string {
return "generic-output"
}
func (encoder compositionOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
payload := struct {
RunID string `json:"run_id"`
ApprovedCount int `json:"approved_count"`
}{
RunID: req.Manifest.RunID,
ApprovedCount: len(req.Approved),
}
encoded, err := json.Marshal(payload)
if err != nil {
return contracts.OutputResult{}, err
}
return contracts.OutputResult{
Files: []contracts.OutputFile{
{
Name: "artifacts/generic.json",
ContentType: "application/json",
Bytes: encoded,
},
},
}, nil
}

View File

@@ -8,38 +8,127 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
type LLMMessage struct {
Role string `json:"role"`
Content string `json:"content"`
}
type StructuredCompletionRequest struct {
StageName string `json:"stage_name"`
Messages []LLMMessage `json:"messages"`
Model string `json:"model,omitempty"`
ResponseSchemaName string `json:"response_schema_name,omitempty"`
ResponseSchema json.RawMessage `json:"response_schema,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
Inputs LLMInputSet `json:"inputs,omitempty"`
Vars map[string]any `json:"vars,omitempty"`
}
type StructuredCompletionResponse struct {
Content json.RawMessage `json:"content"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Debug *LLMDebugMaterial `json:"debug,omitempty"`
}
type StructuredLLMClient interface {
CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error)
}
type LLMDebugMaterial struct {
Prompt *LLMDebugPrompt `json:"prompt,omitempty"`
Response *LLMDebugResponse `json:"response,omitempty"`
}
type LLMDebugPrompt struct {
PromptID string `json:"prompt_id,omitempty"`
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
Messages []LLMDebugMessage `json:"messages,omitempty"`
EffectiveModelParams map[string]any `json:"effective_model_params,omitempty"`
OutputContract map[string]any `json:"output_contract,omitempty"`
StructuredOutput map[string]any `json:"structured_output,omitempty"`
InputHashes map[string]string `json:"input_hashes,omitempty"`
}
type LLMDebugMessage struct {
Role string `json:"role"`
Content string `json:"content"`
CacheControl map[string]any `json:"cache_control,omitempty"`
}
type LLMDebugResponse struct {
Content string `json:"content,omitempty"`
RunID string `json:"run_id,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id,omitempty"`
ModelName string `json:"model_name,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
EffectiveModelParams map[string]any `json:"effective_model_params,omitempty"`
InputHashes map[string]string `json:"input_hashes,omitempty"`
Validation map[string]any `json:"validation,omitempty"`
Usage LLMDebugUsage `json:"usage,omitempty"`
}
type LLMDebugUsage struct {
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
CachedTokens int `json:"cached_tokens,omitempty"`
CacheWriteTokens int `json:"cache_write_tokens,omitempty"`
}
type LLMProfileManifestProvider interface {
LLMProfileManifests() []artifacts.LLMProfileManifest
}
type LLMInputMaterial struct {
Name string `json:"name"`
MediaType string `json:"media_type,omitempty"`
Content []byte `json:"-"`
Digest string `json:"digest,omitempty"`
OriginURI string `json:"origin_uri,omitempty"`
SizeBytes int64 `json:"size_bytes,omitempty"`
}
func NewLLMInputMaterial(name string, mediaType string, content []byte, digest string, originURI string) LLMInputMaterial {
return LLMInputMaterial{
Name: name,
MediaType: mediaType,
Content: append([]byte(nil), content...),
Digest: digest,
OriginURI: originURI,
SizeBytes: int64(len(content)),
}
}
func (material LLMInputMaterial) Clone() LLMInputMaterial {
material.Content = append([]byte(nil), material.Content...)
return material
}
type LLMInputSet map[string]LLMInputMaterial
func (set LLMInputSet) Clone() LLMInputSet {
if len(set) == 0 {
return nil
}
out := make(LLMInputSet, len(set))
for key, material := range set {
out[key] = material.Clone()
}
return out
}
type ParseRequest struct {
SourceID string `json:"source_id,omitempty"`
Path string `json:"path,omitempty"`
Raw []byte `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
@@ -48,124 +137,91 @@ type InputAdapter interface {
Parse(ctx context.Context, req ParseRequest) (*source.SourceDocument, error)
}
type SourceChunk struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
Index int `json:"index"`
Units []source.SourceUnit `json:"units"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type ChunkRequest struct {
Source *source.SourceDocument `json:"-"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type ChunkResult struct {
Chunks []SourceChunk `json:"chunks"`
Chunks []source.Chunk `json:"chunks"`
Warnings []Warning `json:"warnings,omitempty"`
}
type Chunker interface {
Key() string
ReferenceSlots() []ReferenceSlot
Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
}
type ExtractionRequest struct {
Source *source.SourceDocument `json:"-"`
Chunk *SourceChunk `json:"chunk,omitempty"`
AmbientContext map[string]any `json:"ambient_context,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
const (
ReferenceBindingSourceConfig = "config"
ReferenceBindingSourceCLI = "cli"
)
type ReferenceSlot struct {
Name string `json:"name"`
Description string `json:"description,omitempty"`
Required bool `json:"required,omitempty"`
AcceptedMediaTypes []string `json:"accepted_media_types,omitempty"`
Multiple bool `json:"multiple,omitempty"`
MaxBytes int64 `json:"max_bytes,omitempty"`
}
type ExtractionResult struct {
Candidates []artifacts.ArtifactCandidate `json:"candidates,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
func CloneReferenceSlots(slots []ReferenceSlot) []ReferenceSlot {
if len(slots) == 0 {
return nil
}
out := make([]ReferenceSlot, len(slots))
for i, slot := range slots {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
out[i] = slot
}
return out
}
type Extractor interface {
Key() string
ArtifactType() string
SchemaVersion() string
Validators() []Validator
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
type ReferenceOrigin struct {
Type string `json:"type"`
URI string `json:"uri,omitempty"`
}
type ChunkArtifacts struct {
Chunk SourceChunk `json:"chunk"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
type ReferenceItem struct {
SlotName string `json:"slot_name"`
MediaType string `json:"media_type,omitempty"`
Content []byte `json:"-"`
Digest string `json:"digest,omitempty"`
Origin ReferenceOrigin `json:"origin"`
SizeBytes int64 `json:"size_bytes,omitempty"`
BindingSource string `json:"binding_source,omitempty"`
}
type MergeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
ChunkArtifacts []ChunkArtifacts `json:"chunk_artifacts"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
type ResolvedReferenceSlot struct {
Slot ReferenceSlot `json:"slot"`
Items []ReferenceItem `json:"items,omitempty"`
}
type MergeResult struct {
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
Warnings []Warning `json:"warnings,omitempty"`
type ReferenceSet struct {
Slots map[string]ResolvedReferenceSlot `json:"slots,omitempty"`
}
type Merger interface {
Key() string
Merge(ctx context.Context, req MergeRequest) (MergeResult, error)
}
type ExecutionClass string
type NormalizeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type NormalizeResult struct {
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
Warnings []Warning `json:"warnings,omitempty"`
}
type Normalizer interface {
Key() string
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
}
type ValidationRequest struct {
Source *source.SourceDocument `json:"-"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type ValidationDecision struct {
CandidateIndex int `json:"candidate_index"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
const (
ExecutionClassDeterministic ExecutionClass = "deterministic"
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
)
type ValidationResult struct {
ValidatorName string `json:"validator_name"`
Decisions []ValidationDecision `json:"decisions"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
}
type Validator interface {
Name() string
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
}
type Warning struct {
Scope string `json:"scope,omitempty"`
ReasonCode string `json:"reason_code"`
@@ -174,11 +230,10 @@ type Warning struct {
type OutputRequest struct {
Manifest artifacts.RunManifest `json:"manifest"`
Approved []artifacts.Artifact `json:"approved,omitempty"`
Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"`
NormalizeOutputs []SerializedOutput `json:"normalize_outputs,omitempty"`
Rejected []RejectedOutput `json:"rejected,omitempty"`
Warnings []Warning `json:"warnings,omitempty"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
@@ -198,6 +253,19 @@ type OutputEncoder interface {
Encode(ctx context.Context, req OutputRequest) (OutputResult, error)
}
type RejectedOutput struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message"`
AttemptCount int `json:"attempt_count,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type ManifestMetadataProvider interface {
ManifestMetadata() map[string]any
}

View File

@@ -3,6 +3,7 @@ package contracts
import (
"context"
"encoding/json"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
@@ -11,20 +12,15 @@ import (
var _ InputAdapter = fakeAdapter{}
var _ Chunker = fakeChunker{}
var _ Extractor = fakeExtractor{}
var _ Merger = fakeMerger{}
var _ Normalizer = fakeNormalizer{}
var _ Validator = fakeValidator{}
var _ Extractor[fakeArtifact] = fakeExtractor{}
var _ Merger[fakeArtifact] = fakeMerger{}
var _ Normalizer[fakeArtifact] = fakeNormalizer{}
var _ StructuredLLMClient = fakeLLMClient{}
var _ OutputEncoder = fakeOutputEncoder{}
func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
validator := fakeValidator{name: "generic-validator"}
func TestFakeExtractorReturnsTypedOutput(t *testing.T) {
extractor := fakeExtractor{
key: "generic-extractor",
artifactType: "generic-artifact",
schemaVersion: "v1",
validators: []Validator{validator},
}
doc := &source.SourceDocument{
ID: "source-1",
@@ -32,11 +28,11 @@ func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "Source text."},
{ID: 1, Kind: "section", Text: "Source text."},
},
}
result, err := extractor.Extract(context.Background(), ExtractionRequest{Source: doc})
result, err := extractor.Extract(context.Background(), TypedExtractionRequest{Source: doc})
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
@@ -44,37 +40,8 @@ func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
if extractor.Key() != "generic-extractor" {
t.Fatalf("Key() = %q, want generic-extractor", extractor.Key())
}
if extractor.ArtifactType() != "generic-artifact" {
t.Fatalf("ArtifactType() = %q, want generic-artifact", extractor.ArtifactType())
}
if extractor.SchemaVersion() != "v1" {
t.Fatalf("SchemaVersion() = %q, want v1", extractor.SchemaVersion())
}
if len(extractor.Validators()) != 1 {
t.Fatalf("len(Validators()) = %d, want 1", len(extractor.Validators()))
}
if extractor.Validators()[0].Name() != "generic-validator" {
t.Fatalf("Validators()[0].Name() = %q, want generic-validator", extractor.Validators()[0].Name())
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
}
candidate := result.Candidates[0]
if candidate.Index != 0 {
t.Fatalf("ArtifactCandidate.Index = %d, want 0", candidate.Index)
}
if candidate.ExtractorKey != extractor.Key() {
t.Fatalf("ArtifactCandidate.ExtractorKey = %q, want %q", candidate.ExtractorKey, extractor.Key())
}
if candidate.ArtifactType != extractor.ArtifactType() {
t.Fatalf("ArtifactCandidate.ArtifactType = %q, want %q", candidate.ArtifactType, extractor.ArtifactType())
}
if candidate.SchemaVersion != extractor.SchemaVersion() {
t.Fatalf("ArtifactCandidate.SchemaVersion = %q, want %q", candidate.SchemaVersion, extractor.SchemaVersion())
}
if string(candidate.Payload) != `{"value":"example"}` {
t.Fatalf("ArtifactCandidate.Payload = %s, want example payload", candidate.Payload)
if result.Value.Value != "example" {
t.Fatalf("Value = %q, want example", result.Value.Value)
}
}
@@ -85,7 +52,7 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "Source text."},
{ID: 1, Kind: "section", Text: "Source text."},
},
}
chunker := fakeChunker{key: "generic-chunker"}
@@ -104,43 +71,68 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
chunk := result.Chunks[0]
if chunk.ID != "source-1:chunk:0" {
t.Fatalf("SourceChunk.ID = %q, want source-1:chunk:0", chunk.ID)
t.Fatalf("source.Chunk.ID = %q, want source-1:chunk:0", chunk.ID)
}
if chunk.SourceID != doc.ID {
t.Fatalf("SourceChunk.SourceID = %q, want %q", chunk.SourceID, doc.ID)
t.Fatalf("source.Chunk.SourceID = %q, want %q", chunk.SourceID, doc.ID)
}
if chunk.Index != 0 {
t.Fatalf("SourceChunk.Index = %d, want 0", chunk.Index)
t.Fatalf("source.Chunk.Index = %d, want 0", chunk.Index)
}
if chunk.Ref.StartUnitID != 1 || chunk.Ref.EndUnitID != 1 {
t.Fatalf("source.Chunk.Ref = %#v, want source-1:1-1", chunk.Ref)
}
if chunk.MediaType != "application/json" || string(chunk.Content) != `{"units":[{"id":1,"kind":"section","text":"Source text."}]}` {
t.Fatalf("source.Chunk payload = %q %s, want JSON units", chunk.MediaType, chunk.Content)
}
if len(chunk.Units) != 1 {
t.Fatalf("len(SourceChunk.Units) = %d, want 1", len(chunk.Units))
t.Fatalf("len(source.Chunk.Units) = %d, want 1", len(chunk.Units))
}
}
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
extractor := fakeExtractor{
key: "generic-extractor",
artifactType: "generic-artifact",
schemaVersion: "v1",
}
func TestFakeChunkerReceivesPerRunContext(t *testing.T) {
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "First source text."},
{ID: "u2", Kind: "section", Text: "Second source text."},
{ID: 1, Kind: "section", Text: "Source text."},
},
}
chunk := SourceChunk{
chunker := &recordingChunker{key: "llm-chunker"}
if _, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc, SessionID: "session", LLMProfile: "profile"}); err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
if chunker.request.SessionID != "session" || chunker.request.LLMProfile != "profile" {
t.Fatalf("ChunkRequest = %#v, want per-run session and profile", chunker.request)
}
}
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
extractor := fakeExtractor{key: "generic-extractor"}
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "section", Text: "First source text."},
{ID: 2, Kind: "section", Text: "Second source text."},
},
}
chunk := source.Chunk{
ID: "source-1:chunk:1",
SourceID: doc.ID,
Index: 1,
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 2, EndUnitID: 2},
Content: []byte(`{"units":[{"id":2,"kind":"section","text":"Second source text."}]}`),
MediaType: "application/json",
Units: []source.SourceUnit{doc.Units[1]},
}
result, err := extractor.Extract(context.Background(), ExtractionRequest{
result, err := extractor.Extract(context.Background(), TypedExtractionRequest{
Source: doc,
Chunk: &chunk,
AmbientContext: map[string]any{"mode": "chunked"},
@@ -148,51 +140,211 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(result.Candidates) != 1 {
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
if result.Value.Value != "chunked" {
t.Fatalf("Value = %q, want chunked", result.Value.Value)
}
}
func TestReferenceSetDataTypes(t *testing.T) {
references := ReferenceSet{
Slots: map[string]ResolvedReferenceSlot{
"roster": {
Slot: ReferenceSlot{
Name: "roster",
Description: "Known characters",
Required: true,
AcceptedMediaTypes: []string{"text/plain"},
Multiple: true,
MaxBytes: 4096,
},
Items: []ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain",
Content: []byte("Aria\nBryn\n"),
Digest: "sha256:reference",
Origin: ReferenceOrigin{
Type: "file",
URI: "file:///tmp/roster.txt",
},
SizeBytes: 10,
BindingSource: ReferenceBindingSourceConfig,
},
},
},
},
}
candidate := result.Candidates[0]
if string(candidate.Payload) != `{"value":"chunked"}` {
t.Fatalf("ArtifactCandidate.Payload = %s, want chunked payload", candidate.Payload)
item := references.Slots["roster"].Items[0]
if item.SlotName != "roster" || item.MediaType != "text/plain" || string(item.Content) != "Aria\nBryn\n" {
t.Fatalf("reference item = %#v, want constructed item fields", item)
}
if len(candidate.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(candidate.SourceRefs))
if item.BindingSource != ReferenceBindingSourceConfig {
t.Fatalf("BindingSource = %q, want %q", item.BindingSource, ReferenceBindingSourceConfig)
}
ref := candidate.SourceRefs[0]
if ref.StartUnitID != "u2" || ref.EndUnitID != "u2" {
t.Fatalf("SourceRef = %+v, want u2 range", ref)
}
func TestCloneReferenceSlotsEmptyInputReturnsNil(t *testing.T) {
if got := CloneReferenceSlots(nil); got != nil {
t.Fatalf("CloneReferenceSlots(nil) = %#v, want nil", got)
}
if got := CloneReferenceSlots([]ReferenceSlot{}); got != nil {
t.Fatalf("CloneReferenceSlots(empty) = %#v, want nil", got)
}
}
func TestCloneReferenceSlotsPreservesFields(t *testing.T) {
slots := []ReferenceSlot{
{
Name: "roster",
Description: "Known characters",
Required: true,
AcceptedMediaTypes: []string{"text/plain", "text/markdown"},
Multiple: true,
MaxBytes: 4096,
},
{
Name: "glossary",
Description: "Campaign terms",
MaxBytes: 2048,
},
}
got := CloneReferenceSlots(slots)
if !reflect.DeepEqual(got, slots) {
t.Fatalf("CloneReferenceSlots() = %#v, want %#v", got, slots)
}
}
func TestCloneReferenceSlotsCopiesAcceptedMediaTypes(t *testing.T) {
slots := []ReferenceSlot{
{
Name: "party",
AcceptedMediaTypes: []string{"application/json", "text/plain"},
},
}
got := CloneReferenceSlots(slots)
got[0].Name = "changed"
got[0].AcceptedMediaTypes[0] = "text/markdown"
if slots[0].Name != "party" {
t.Fatalf("source slot name = %q, want unchanged", slots[0].Name)
}
if slots[0].AcceptedMediaTypes[0] != "application/json" {
t.Fatalf("source AcceptedMediaTypes aliased clone: %#v", slots[0].AcceptedMediaTypes)
}
}
func TestReferenceItemJSONOmitsContent(t *testing.T) {
item := ReferenceItem{
SlotName: "roster",
MediaType: "text/plain",
Content: []byte("reference content"),
Digest: "sha256:reference",
Origin: ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
}
encoded, err := json.Marshal(item)
if err != nil {
t.Fatalf("json.Marshal() error = %v, want nil", err)
}
var got map[string]any
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
}
if _, ok := got["content"]; ok {
t.Fatalf("encoded reference item leaked content: %s", encoded)
}
if _, ok := got["Content"]; ok {
t.Fatalf("encoded reference item leaked Content: %s", encoded)
}
}
func TestLLMInputMaterialCopiesContentAndOmitsContentFromJSON(t *testing.T) {
content := []byte("raw source bytes")
material := NewLLMInputMaterial("transcript", "application/json", content, "sha256:source", "file:///tmp/source.json")
content[0] = 'R'
if got := string(material.Content); got != "raw source bytes" {
t.Fatalf("material content = %q, want defensive copy", got)
}
if material.SizeBytes != int64(len("raw source bytes")) {
t.Fatalf("SizeBytes = %d, want content length", material.SizeBytes)
}
clone := material.Clone()
clone.Content[0] = 'X'
if got := string(material.Content); got != "raw source bytes" {
t.Fatalf("cloned material content aliased original: %q", got)
}
encoded, err := json.Marshal(material)
if err != nil {
t.Fatalf("json.Marshal() error = %v, want nil", err)
}
var got map[string]any
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
}
if _, ok := got["content"]; ok {
t.Fatalf("encoded material leaked content: %s", encoded)
}
if _, ok := got["Content"]; ok {
t.Fatalf("encoded material leaked Content: %s", encoded)
}
if got["digest"] != "sha256:source" || got["origin_uri"] != "file:///tmp/source.json" {
t.Fatalf("encoded material = %#v, want non-secret provenance", got)
}
}
func TestLLMInputSetCloneCopiesContent(t *testing.T) {
set := LLMInputSet{
"transcript": NewLLMInputMaterial("transcript", "application/json", []byte("source"), "sha256:source", "file:///tmp/source.json"),
}
clone := set.Clone()
clone["transcript"].Content[0] = 'S'
if got := string(set["transcript"].Content); got != "source" {
t.Fatalf("input set clone aliased content: %q", got)
}
}
func TestArtifactSchemaJSONOmitsSchemaContent(t *testing.T) {
schema := ArtifactSchema{
ID: "schema-id",
Name: "schema-name",
Version: "v1",
JSONSchema: []byte(`{"type":"object"}`),
}
encoded, err := json.Marshal(schema)
if err != nil {
t.Fatalf("json.Marshal() error = %v, want nil", err)
}
var got map[string]any
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
}
if got["id"] != "schema-id" || got["name"] != "schema-name" || got["version"] != "v1" {
t.Fatalf("encoded schema = %#v, want schema provenance", got)
}
if _, ok := got["json_schema"]; ok {
t.Fatalf("encoded schema leaked raw schema content: %s", encoded)
}
if _, ok := got["JSONSchema"]; ok {
t.Fatalf("encoded schema leaked raw schema content: %s", encoded)
}
}
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
candidate := artifacts.ArtifactCandidate{
Index: 0,
ExtractorKey: "generic-extractor",
ArtifactType: "generic-artifact",
SchemaVersion: "v1",
Payload: json.RawMessage(`{"value":"example"}`),
}
chunk := SourceChunk{
ID: "source-1:chunk:0",
SourceID: "source-1",
Index: 0,
Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "Source text."},
},
}
extractOutput := ExtractArtifact[fakeArtifact]{LaneID: "generic-lane", ExtractorKey: "generic-extractor", SourceID: "source-1", ChunkID: "source-1:chunk:0", ChunkIndex: 0, Value: fakeArtifact{Value: "example"}}
merger := fakeMerger{key: "generic-merger"}
normalizer := fakeNormalizer{key: "generic-normalizer"}
encoder := fakeOutputEncoder{key: "generic-output"}
merged, err := merger.Merge(context.Background(), MergeRequest{
LaneID: "generic-artifact",
ChunkArtifacts: []ChunkArtifacts{
{
Chunk: chunk,
Candidates: []artifacts.ArtifactCandidate{candidate},
},
},
merged, err := merger.Merge(context.Background(), TypedMergeRequest[fakeArtifact]{
LaneID: "generic-lane",
ExtractOutputs: []ExtractArtifact[fakeArtifact]{extractOutput},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
@@ -200,13 +352,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if merger.Key() != "generic-merger" {
t.Fatalf("Merger.Key() = %q, want generic-merger", merger.Key())
}
if len(merged.Candidates) != 1 {
t.Fatalf("len(merged.Candidates) = %d, want 1", len(merged.Candidates))
if merged.Value.Value != "example" {
t.Fatalf("merged value = %q, want example", merged.Value.Value)
}
normalized, err := normalizer.Normalize(context.Background(), NormalizeRequest{
LaneID: "generic-artifact",
Candidates: merged.Candidates,
normalized, err := normalizer.Normalize(context.Background(), TypedNormalizeRequest[fakeArtifact]{
LaneID: "generic-lane",
MergeOutput: MergeArtifact[fakeArtifact]{LaneID: "generic-lane", MergerKey: merger.Key(), SourceID: "source-1", Value: merged.Value},
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
@@ -214,15 +366,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if normalizer.Key() != "generic-normalizer" {
t.Fatalf("Normalizer.Key() = %q, want generic-normalizer", normalizer.Key())
}
if len(normalized.Candidates) != 1 {
t.Fatalf("len(normalized.Candidates) = %d, want 1", len(normalized.Candidates))
if normalized.Value.Value != "example" {
t.Fatalf("normalized value = %q, want example", normalized.Value.Value)
}
encoded, err := encoder.Encode(context.Background(), OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
Approved: []artifacts.Artifact{
artifacts.ArtifactFromCandidate(normalized.Candidates[0]),
},
NormalizeOutputs: []SerializedOutput{{LaneID: "generic-lane", NormalizerKey: normalizer.Key(), SourceID: "source-1", Artifact: SerializedArtifact{Kind: "test/artifact", Schema: ArtifactSchema{ID: "schema-id", Name: "schema-name", Version: "v1"}, MediaType: "application/json", Content: []byte(`{"value":"example"}`)}}},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
@@ -236,7 +386,7 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
if encoded.Files[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", encoded.Files[0].ContentType)
}
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","approved_count":1}` {
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","output_count":1}` {
t.Fatalf("Bytes = %s, want encoded output", encoded.Files[0].Bytes)
}
}
@@ -292,70 +442,68 @@ func (chunker fakeChunker) Key() string {
return chunker.key
}
func (chunker fakeChunker) ReferenceSlots() []ReferenceSlot {
return nil
}
func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
return ChunkResult{
Chunks: []SourceChunk{
Chunks: []source.Chunk{
{
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
Ref: source.SourceRef{
SourceID: req.Source.ID,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
},
Content: []byte(`{"units":[{"id":1,"kind":"section","text":"Source text."}]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units...),
},
},
}, nil
}
type recordingChunker struct {
key string
request ChunkRequest
}
func (chunker *recordingChunker) Key() string {
return chunker.key
}
func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
return nil
}
func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
chunker.request = req
return fakeChunker{key: chunker.key}.Chunk(ctx, req)
}
type fakeExtractor struct {
key string
artifactType string
schemaVersion string
validators []Validator
}
type fakeArtifact struct{ Value string }
func (extractor fakeExtractor) Key() string {
return extractor.key
}
func (extractor fakeExtractor) ArtifactType() string {
return extractor.artifactType
func (extractor fakeExtractor) ReferenceSlots() []ReferenceSlot {
return nil
}
func (extractor fakeExtractor) SchemaVersion() string {
return extractor.schemaVersion
}
func (extractor fakeExtractor) Validators() []Validator {
return extractor.validators
}
func (extractor fakeExtractor) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) {
units := req.Source.Units
if req.Chunk != nil {
units = req.Chunk.Units
}
payload := json.RawMessage(`{"value":"example"}`)
func (extractor fakeExtractor) Extract(ctx context.Context, req TypedExtractionRequest) (TypedExtractionResult[fakeArtifact], error) {
value := "example"
if req.AmbientContext["mode"] == "chunked" {
payload = json.RawMessage(`{"value":"chunked"}`)
value = "chunked"
}
return ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{
Index: 0,
ExtractorKey: extractor.key,
ArtifactType: extractor.artifactType,
SchemaVersion: extractor.schemaVersion,
Payload: payload,
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: units[0].ID,
EndUnitID: units[len(units)-1].ID,
},
},
},
},
}, nil
return TypedExtractionResult[fakeArtifact]{Value: fakeArtifact{Value: value}}, nil
}
type fakeMerger struct {
@@ -366,13 +514,8 @@ func (merger fakeMerger) Key() string {
return merger.key
}
func (merger fakeMerger) Merge(ctx context.Context, req MergeRequest) (MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
}
return MergeResult{Candidates: candidates}, nil
func (merger fakeMerger) Merge(ctx context.Context, req TypedMergeRequest[fakeArtifact]) (TypedMergeResult[fakeArtifact], error) {
return TypedMergeResult[fakeArtifact]{Value: req.ExtractOutputs[0].Value}, nil
}
type fakeNormalizer struct {
@@ -383,33 +526,12 @@ func (normalizer fakeNormalizer) Key() string {
return normalizer.key
}
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) {
return NormalizeResult{Candidates: req.Candidates}, nil
func (normalizer fakeNormalizer) ReferenceSlots() []ReferenceSlot {
return nil
}
type fakeValidator struct {
name string
}
func (validator fakeValidator) Name() string {
return validator.name
}
func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) {
decisions := make([]ValidationDecision, 0, len(req.Candidates))
for _, candidate := range req.Candidates {
decisions = append(decisions, ValidationDecision{
CandidateIndex: candidate.Index,
Approved: true,
ReasonCode: "accepted",
Message: "candidate accepted",
})
}
return ValidationResult{
ValidatorName: validator.name,
Decisions: decisions,
}, nil
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req TypedNormalizeRequest[fakeArtifact]) (TypedNormalizeResult[fakeArtifact], error) {
return TypedNormalizeResult[fakeArtifact]{Value: req.MergeOutput.Value}, nil
}
type fakeLLMClient struct{}
@@ -434,7 +556,7 @@ func (encoder fakeOutputEncoder) Encode(ctx context.Context, req OutputRequest)
{
Name: "artifacts/generic.json",
ContentType: "application/json",
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","approved_count":1}`),
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","output_count":1}`),
},
},
}, nil

View File

@@ -0,0 +1,164 @@
package contracts
import (
"context"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
// ExtractArtifact carries a typed per-chunk value with framework provenance.
type ExtractArtifact[T any] struct {
LaneID string
ExtractorKey string
SourceID string
ChunkID string
ChunkIndex int
ChunkRef source.SourceRef
Value T
}
// MergeArtifact carries a typed merged value with framework provenance.
type MergeArtifact[T any] struct {
LaneID string
MergerKey string
SourceID string
Value T
}
// NormalizeArtifact carries a typed normalized value with framework provenance.
type NormalizeArtifact[T any] struct {
LaneID string
NormalizerKey string
SourceID string
Value T
}
type TypedExtractionRequest struct {
Source *source.SourceDocument
Chunk *source.Chunk
AmbientContext map[string]any
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
}
type TypedExtractionResult[T any] struct {
Value T
Warnings []Warning
}
type Extractor[T any] interface {
Key() string
ReferenceSlots() []ReferenceSlot
Extract(context.Context, TypedExtractionRequest) (TypedExtractionResult[T], error)
}
type TypedMergeRequest[T any] struct {
Source *source.SourceDocument
LaneID string
ExtractOutputs []ExtractArtifact[T]
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
}
type TypedMergeResult[T any] struct {
Value T
Warnings []Warning
}
type Merger[T any] interface {
Key() string
Merge(context.Context, TypedMergeRequest[T]) (TypedMergeResult[T], error)
}
type TypedNormalizeRequest[T any] struct {
Source *source.SourceDocument
LaneID string
MergeOutput MergeArtifact[T]
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
}
type TypedNormalizeResult[T any] struct {
Value T
Warnings []Warning
}
type Normalizer[T any] interface {
Key() string
ReferenceSlots() []ReferenceSlot
Normalize(context.Context, TypedNormalizeRequest[T]) (TypedNormalizeResult[T], error)
}
type TypedValidationRequest[T any] struct {
Stage string
LaneID string
ModuleKey string
Source *source.SourceDocument
SourceID string
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
Chunk *source.Chunk
Chunks []source.Chunk
Ref source.SourceRef
Value T
}
type TypedValidator[T any] interface {
Name() string
ExecutionClass() ExecutionClass
Validate(context.Context, TypedValidationRequest[T]) (ValidationResult, error)
}
type ChunkValidationRequest struct {
ModuleKey string
Source *source.SourceDocument
SourceID string
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
Chunks []source.Chunk
}
type ChunkValidator interface {
Name() string
ExecutionClass() ExecutionClass
Validate(context.Context, ChunkValidationRequest) (ValidationResult, error)
}
type SerializedValidationRequest struct {
Stage string
LaneID string
ModuleKey string
Source *source.SourceDocument
SourceID string
SourceInput LLMInputMaterial
SessionID string
References ReferenceSet
LLMProfile string
Metadata map[string]any
Chunk *source.Chunk
Chunks []source.Chunk
Schema ArtifactSchema
MediaType string
Content []byte
}
type SerializedValidator interface {
Name() string
ExecutionClass() ExecutionClass
Validate(context.Context, SerializedValidationRequest) (ValidationResult, error)
}

View File

@@ -0,0 +1,41 @@
package debug
import (
"strings"
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
type WorkspaceRecorder struct {
root string
}
func NewWorkspaceRecorder(settings coreworkspace.Settings, runID string) (pipeline.DebugRecorder, error) {
root, err := settings.DebugRunDirectory(runID)
if err != nil {
return nil, err
}
if strings.TrimSpace(root) == "" {
return pipeline.NoopDebugRecorder(), nil
}
return &WorkspaceRecorder{root: root}, nil
}
func (r *WorkspaceRecorder) Enabled() bool {
return r != nil && strings.TrimSpace(r.root) != ""
}
func (r *WorkspaceRecorder) WriteJSON(name string, payload any) error {
if !r.Enabled() {
return nil
}
return coreworkspace.WriteJSON(r.root, name, payload)
}
func (r *WorkspaceRecorder) WriteBytes(name string, data []byte) error {
if !r.Enabled() {
return nil
}
return coreworkspace.WriteBytes(r.root, name, data)
}

View File

@@ -0,0 +1,333 @@
package llm
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"io/fs"
"path"
"sort"
"strings"
"time"
"gitea.maximumdirect.net/eric/scriptorium"
)
type AssetSource struct {
FS fs.FS
Root string
}
type AssetRegistry struct {
prompts []AssetSource
schemas []AssetSource
}
type AssetHashPart struct {
FS fs.FS
Path string
}
func NewAssetRegistry() *AssetRegistry {
return &AssetRegistry{}
}
func (r *AssetRegistry) RegisterPromptFS(fsys fs.FS, root string) error {
if r == nil {
return fmt.Errorf("asset registry must not be nil")
}
source, err := newAssetSource(fsys, root)
if err != nil {
return fmt.Errorf("register prompt assets: %w", err)
}
r.prompts = append(r.prompts, source)
return nil
}
func (r *AssetRegistry) RegisterSchemaFS(fsys fs.FS, root string) error {
if r == nil {
return fmt.Errorf("asset registry must not be nil")
}
source, err := newAssetSource(fsys, root)
if err != nil {
return fmt.Errorf("register schema assets: %w", err)
}
r.schemas = append(r.schemas, source)
return nil
}
func (r *AssetRegistry) PromptFS() (fs.FS, error) {
if r == nil {
return nil, fmt.Errorf("asset registry must not be nil")
}
return flattenAssetSources(r.prompts)
}
func (r *AssetRegistry) SchemaFS() (fs.FS, error) {
if r == nil {
return nil, fmt.Errorf("asset registry must not be nil")
}
return flattenAssetSources(r.schemas)
}
func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
promptFS, err := r.PromptFS()
if err != nil {
return nil, fmt.Errorf("prepare prompt assets: %w", err)
}
schemaFS, err := r.SchemaFS()
if err != nil {
return nil, fmt.Errorf("prepare schema assets: %w", err)
}
return []scriptorium.Option{
scriptorium.WithPromptFS(promptFS, "."),
scriptorium.WithSchemaFS(schemaFS, "."),
}, nil
}
func HashAssets(parts []AssetHashPart) (string, error) {
if len(parts) == 0 {
return "", fmt.Errorf("asset hash requires at least one part")
}
hash := sha256.New()
for _, part := range parts {
cleanPath, err := cleanAssetPath(part.Path)
if err != nil {
return "", fmt.Errorf("hash asset %q: %w", part.Path, err)
}
data, err := fs.ReadFile(part.FS, cleanPath)
if err != nil {
return "", fmt.Errorf("read hash asset %s: %w", cleanPath, err)
}
if _, err := io.WriteString(hash, cleanPath); err != nil {
return "", err
}
if _, err := hash.Write([]byte{0}); err != nil {
return "", err
}
if _, err := hash.Write(data); err != nil {
return "", err
}
if _, err := hash.Write([]byte{0}); err != nil {
return "", err
}
}
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil
}
func newAssetSource(fsys fs.FS, root string) (AssetSource, error) {
if fsys == nil {
return AssetSource{}, fmt.Errorf("filesystem must not be nil")
}
cleanRoot, err := cleanAssetRoot(root)
if err != nil {
return AssetSource{}, err
}
return AssetSource{FS: fsys, Root: cleanRoot}, nil
}
func flattenAssetSources(sources []AssetSource) (fs.FS, error) {
out := assetMapFS{}
for _, source := range sources {
if err := fs.WalkDir(source.FS, source.Root, func(name string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
return nil
}
rel := name
if source.Root != "." {
rel = strings.TrimPrefix(name, source.Root+"/")
}
rel, err := cleanAssetPath(rel)
if err != nil {
return err
}
if _, exists := out[rel]; exists {
return fmt.Errorf("duplicate asset path %q", rel)
}
data, err := fs.ReadFile(source.FS, name)
if err != nil {
return err
}
out[rel] = append([]byte(nil), data...)
return nil
}); err != nil {
return nil, fmt.Errorf("walk asset root %s: %w", source.Root, err)
}
}
return out, nil
}
func cleanAssetRoot(root string) (string, error) {
trimmed := strings.TrimSpace(root)
if trimmed == "" || trimmed == "." {
return ".", nil
}
return cleanAssetPath(trimmed)
}
func cleanAssetPath(name string) (string, error) {
trimmed := strings.TrimSpace(name)
if trimmed == "" {
return "", fmt.Errorf("path must not be empty")
}
cleaned := path.Clean(strings.TrimPrefix(trimmed, "/"))
if cleaned == "." || !fs.ValidPath(cleaned) {
return "", fmt.Errorf("invalid path %q", name)
}
return cleaned, nil
}
type assetMapFS map[string][]byte
func (m assetMapFS) Open(name string) (fs.File, error) {
cleaned, err := cleanOpenPath(name)
if err != nil {
return nil, &fs.PathError{Op: "open", Path: name, Err: err}
}
if data, ok := m[cleaned]; ok {
return &assetFile{
reader: bytes.NewReader(data),
info: assetFileInfo{name: path.Base(cleaned), size: int64(len(data))},
}, nil
}
entries := m.dirEntries(cleaned)
if entries != nil {
return &assetDir{name: path.Base(cleaned), entries: entries}, nil
}
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
}
func (m assetMapFS) ReadFile(name string) ([]byte, error) {
cleaned, err := cleanOpenPath(name)
if err != nil {
return nil, &fs.PathError{Op: "readfile", Path: name, Err: err}
}
data, ok := m[cleaned]
if !ok {
return nil, &fs.PathError{Op: "readfile", Path: name, Err: fs.ErrNotExist}
}
return append([]byte(nil), data...), nil
}
func (m assetMapFS) ReadDir(name string) ([]fs.DirEntry, error) {
cleaned, err := cleanOpenPath(name)
if err != nil {
return nil, &fs.PathError{Op: "readdir", Path: name, Err: err}
}
entries := m.dirEntries(cleaned)
if entries == nil {
return nil, &fs.PathError{Op: "readdir", Path: name, Err: fs.ErrNotExist}
}
return entries, nil
}
func (m assetMapFS) dirEntries(dir string) []fs.DirEntry {
children := map[string]assetDirEntry{}
prefix := ""
if dir != "." {
prefix = dir + "/"
}
for name, data := range m {
if !strings.HasPrefix(name, prefix) {
continue
}
rest := strings.TrimPrefix(name, prefix)
if rest == "" {
continue
}
childName, _, hasSlash := strings.Cut(rest, "/")
entry := assetDirEntry{name: childName, dir: hasSlash}
if !hasSlash {
entry.size = int64(len(data))
}
children[childName] = entry
}
if len(children) == 0 {
return nil
}
names := make([]string, 0, len(children))
for name := range children {
names = append(names, name)
}
sort.Strings(names)
entries := make([]fs.DirEntry, 0, len(names))
for _, name := range names {
entries = append(entries, children[name])
}
return entries
}
func cleanOpenPath(name string) (string, error) {
if name == "." {
return ".", nil
}
return cleanAssetPath(name)
}
type assetFile struct {
reader *bytes.Reader
info assetFileInfo
}
func (f *assetFile) Stat() (fs.FileInfo, error) { return f.info, nil }
func (f *assetFile) Read(p []byte) (int, error) { return f.reader.Read(p) }
func (f *assetFile) Close() error { return nil }
type assetDir struct {
name string
offset int
entries []fs.DirEntry
}
func (d *assetDir) Stat() (fs.FileInfo, error) { return assetFileInfo{name: d.name, dir: true}, nil }
func (d *assetDir) Read([]byte) (int, error) { return 0, fmt.Errorf("cannot read directory") }
func (d *assetDir) Close() error { return nil }
func (d *assetDir) ReadDir(n int) ([]fs.DirEntry, error) {
if d.offset >= len(d.entries) {
return nil, io.EOF
}
end := len(d.entries)
if n > 0 && d.offset+n < end {
end = d.offset + n
}
out := append([]fs.DirEntry(nil), d.entries[d.offset:end]...)
d.offset = end
return out, nil
}
type assetDirEntry struct {
name string
dir bool
size int64
}
func (e assetDirEntry) Name() string { return e.name }
func (e assetDirEntry) IsDir() bool { return e.dir }
func (e assetDirEntry) Type() fs.FileMode { return e.InfoMode().Type() }
func (e assetDirEntry) Info() (fs.FileInfo, error) {
return assetFileInfo{name: e.name, dir: e.dir, size: e.size}, nil
}
func (e assetDirEntry) InfoMode() fs.FileMode {
if e.dir {
return fs.ModeDir | 0o555
}
return 0o444
}
type assetFileInfo struct {
name string
dir bool
size int64
}
func (i assetFileInfo) Name() string { return i.name }
func (i assetFileInfo) Size() int64 { return i.size }
func (i assetFileInfo) Mode() fs.FileMode { return assetDirEntry{dir: i.dir}.InfoMode() }
func (i assetFileInfo) ModTime() time.Time { return time.Time{} }
func (i assetFileInfo) IsDir() bool { return i.dir }
func (i assetFileInfo) Sys() any { return nil }

View File

@@ -0,0 +1,198 @@
package llm
import (
"context"
"strings"
"testing"
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/scriptorium"
)
func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
registry := NewAssetRegistry()
mustRegisterPromptFS(t, registry, fstest.MapFS{
"prompts/test.yaml": {Data: []byte(validPromptYAML("schemas/out.json"))},
"prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
"prompts/messages/task.tmpl": {Data: []byte("Return JSON.")},
"schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)},
}, "prompts")
mustRegisterSchemaFS(t, registry, fstest.MapFS{
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
}, "root")
engine := newAssetTestEngine(t, registry)
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
},
})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
if got := len(prepared.Messages); got != 2 {
t.Fatalf("message count = %d, want 2", got)
}
if prepared.OutputContract.SchemaPath != "schemas/out.json" {
t.Fatalf("schema path = %q, want schemas/out.json", prepared.OutputContract.SchemaPath)
}
}
func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
registry := NewAssetRegistry()
mustRegisterPromptFS(t, registry, fstest.MapFS{
"test.yaml": {Data: []byte(validPromptYAML("out.json"))},
}, ".")
mustRegisterSchemaFS(t, registry, fstest.MapFS{
"out.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
},
})
if err == nil || !strings.Contains(err.Error(), "content_file") {
t.Fatalf("Prepare() error = %v, want missing content_file error", err)
}
}
func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
registry := NewAssetRegistry()
mustRegisterPromptFS(t, registry, fstest.MapFS{
"test.yaml": {Data: []byte(validPromptYAML("missing.json"))},
"messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
"messages/task.tmpl": {Data: []byte("Return JSON.")},
}, ".")
mustRegisterSchemaFS(t, registry, fstest.MapFS{
"present.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline(`{"ok":true}`),
},
})
if err == nil || !strings.Contains(err.Error(), "missing.json") {
t.Fatalf("Prepare() error = %v, want missing schema error", err)
}
}
func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
registry := NewAssetRegistry()
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("id: one")}}, "one")
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("id: two")}}, "two")
_, err := registry.PromptFS()
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
t.Fatalf("PromptFS() error = %v, want duplicate path error", err)
}
}
func TestAssetRegistryCombinesNamespacedPromptSources(t *testing.T) {
registry := NewAssetRegistry()
mustRegisterPromptFS(t, registry, fstest.MapFS{
"dnd.spells/dnd.spells.yaml": {Data: []byte(validPromptYAML("schema.json"))},
"dnd.spells/task.md": {Data: []byte("spell task")},
"dnd.spells/instructions.md": {Data: []byte("spell instructions")},
}, ".")
mustRegisterPromptFS(t, registry, fstest.MapFS{
"dnd.scenes/dnd.scenes.yaml": {Data: []byte(validPromptYAML("schema.json"))},
"dnd.scenes/task.md": {Data: []byte("scene task")},
"dnd.scenes/instructions.md": {Data: []byte("scene instructions")},
}, ".")
fsys, err := registry.PromptFS()
if err != nil {
t.Fatalf("PromptFS() error = %v, want nil", err)
}
for _, name := range []string{
"dnd.spells/dnd.spells.yaml",
"dnd.spells/task.md",
"dnd.spells/instructions.md",
"dnd.scenes/dnd.scenes.yaml",
"dnd.scenes/task.md",
"dnd.scenes/instructions.md",
} {
if _, err := fsys.Open(name); err != nil {
t.Fatalf("PromptFS().Open(%q) error = %v, want nil", name, err)
}
}
}
func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
hash, err := HashAssets([]AssetHashPart{{
FS: fstest.MapFS{"prompt.md": {Data: []byte("secret prompt text")}},
Path: "prompt.md",
}})
if err != nil {
t.Fatalf("HashAssets() error = %v, want nil", err)
}
if !strings.HasPrefix(hash, "sha256:") {
t.Fatalf("hash = %q, want sha256-prefixed value", hash)
}
if strings.Contains(hash, "secret prompt text") {
t.Fatalf("hash leaked asset content")
}
}
func newAssetTestEngine(t *testing.T, registry *AssetRegistry) *scriptorium.Engine {
t.Helper()
options, err := registry.ScriptoriumOptions()
if err != nil {
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
}
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "asset-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "asset-test-model",
})))
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
return engine
}
func mustRegisterPromptFS(t *testing.T, registry *AssetRegistry, fsys fstest.MapFS, root string) {
t.Helper()
if err := registry.RegisterPromptFS(fsys, root); err != nil {
t.Fatalf("RegisterPromptFS() error = %v, want nil", err)
}
}
func mustRegisterSchemaFS(t *testing.T, registry *AssetRegistry, fsys fstest.MapFS, root string) {
t.Helper()
if err := registry.RegisterSchemaFS(fsys, root); err != nil {
t.Fatalf("RegisterSchemaFS() error = %v, want nil", err)
}
}
func validPromptYAML(schemaPath string) string {
return `id: asset.test
version: "v1"
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content_file: ./messages/user.tmpl
- role: user
content_file: ./messages/task.tmpl
output:
format: json
validation_mode: json_schema
schema_path: ` + schemaPath + `
repair_attempts: 0
`
}

View File

@@ -1,364 +0,0 @@
package llm
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
const openAICompatibleProviderName = "openai-compatible"
// OpenAICompatibleClientConfig configures the direct HTTP structured-output adapter.
type OpenAICompatibleClientConfig struct {
BaseURL string
Model string
APIKey string
MaxRetries int
HTTPClient *http.Client
RequestTimeout time.Duration
}
// OpenAICompatibleClient sends OpenAI-compatible chat-completion requests with
// response_format.type=json_schema.
type OpenAICompatibleClient struct {
baseURL string
model string
apiKey string
maxRetries int
httpClient *http.Client
requestTimeout time.Duration
}
var _ contracts.StructuredLLMClient = (*OpenAICompatibleClient)(nil)
func NewOpenAICompatibleClient(cfg OpenAICompatibleClientConfig) (*OpenAICompatibleClient, error) {
normalized, err := normalizeOpenAICompatibleConfig(cfg)
if err != nil {
return nil, err
}
client := normalized.HTTPClient
if client == nil {
client = http.DefaultClient
}
return &OpenAICompatibleClient{
baseURL: normalized.BaseURL,
model: normalized.Model,
apiKey: normalized.APIKey,
maxRetries: normalized.MaxRetries,
httpClient: client,
requestTimeout: normalized.RequestTimeout,
}, nil
}
func (c *OpenAICompatibleClient) CompleteStructured(
ctx context.Context,
req contracts.StructuredCompletionRequest,
out any,
) (contracts.StructuredCompletionResponse, error) {
if c == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("openai-compatible client must not be nil")
}
if err := validateOutputTarget(out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
model := strings.TrimSpace(req.Model)
if model == "" {
model = c.model
}
if model == "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion model must not be empty")
}
schemaName := strings.TrimSpace(req.ResponseSchemaName)
if schemaName == "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema name must not be empty")
}
if len(bytes.TrimSpace(req.ResponseSchema)) == 0 || !json.Valid(req.ResponseSchema) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema JSON must be valid")
}
messages, err := toOpenAICompatibleMessages(req.Messages)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
endpoint := buildChatCompletionsURL(c.baseURL)
var lastErr error
for attempt := 0; attempt <= c.maxRetries; attempt++ {
content, metadata, callErr := c.completeStructuredOnce(ctx, endpoint, model, messages, schemaName, req.ResponseSchema)
if callErr == nil {
if decodeErr := json.Unmarshal(content, out); decodeErr != nil {
callErr = retryableError{err: fmt.Errorf("decode structured output: %w", decodeErr)}
} else {
return contracts.StructuredCompletionResponse{
Content: content,
Provider: openAICompatibleProviderName,
Model: firstNonEmpty(metadata.Model, model),
PromptTokens: metadata.PromptTokens,
CompletionTokens: metadata.CompletionTokens,
TotalTokens: metadata.TotalTokens,
}, nil
}
}
if ctx.Err() != nil {
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
lastErr = c.redactError(callErr)
if !canRetry(ctx, attempt, c.maxRetries, callErr) {
return contracts.StructuredCompletionResponse{}, lastErr
}
}
if lastErr == nil {
lastErr = fmt.Errorf("structured completion failed")
}
return contracts.StructuredCompletionResponse{}, lastErr
}
type openAICompatibleMessage struct {
Role string `json:"role"`
Content string `json:"content"`
}
type openAICompatibleRequest struct {
Model string `json:"model"`
Messages []openAICompatibleMessage `json:"messages"`
ResponseFormat openAICompatibleStructuredOutputShape `json:"response_format"`
}
type openAICompatibleStructuredOutputShape struct {
Type string `json:"type"`
JSONSchema openAICompatibleSchemaEnvelope `json:"json_schema"`
}
type openAICompatibleSchemaEnvelope struct {
Name string `json:"name"`
Strict bool `json:"strict"`
Schema json.RawMessage `json:"schema"`
}
type openAICompatibleChatCompletionsResponse struct {
Model string `json:"model"`
Choices []struct {
Message struct {
Content json.RawMessage `json:"content"`
} `json:"message"`
} `json:"choices"`
Usage *openAICompatibleUsage `json:"usage,omitempty"`
}
type openAICompatibleUsage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
}
type openAICompatibleResponseMetadata struct {
Model string
PromptTokens int
CompletionTokens int
TotalTokens int
}
func normalizeOpenAICompatibleConfig(cfg OpenAICompatibleClientConfig) (OpenAICompatibleClientConfig, error) {
cfg.BaseURL = strings.TrimSpace(cfg.BaseURL)
cfg.Model = strings.TrimSpace(cfg.Model)
cfg.APIKey = strings.TrimSpace(cfg.APIKey)
if cfg.MaxRetries < 0 {
return OpenAICompatibleClientConfig{}, fmt.Errorf("max retries must be zero or greater")
}
if cfg.BaseURL == "" {
return OpenAICompatibleClientConfig{}, fmt.Errorf("base URL must not be empty")
}
if _, err := url.ParseRequestURI(cfg.BaseURL); err != nil {
return OpenAICompatibleClientConfig{}, fmt.Errorf("base URL must be valid: %w", err)
}
if cfg.Model == "" {
return OpenAICompatibleClientConfig{}, fmt.Errorf("model must not be empty")
}
cfg.BaseURL = strings.TrimRight(cfg.BaseURL, "/")
return cfg, nil
}
func (c *OpenAICompatibleClient) completeStructuredOnce(
ctx context.Context,
endpoint string,
model string,
messages []openAICompatibleMessage,
responseSchemaName string,
responseSchemaJSON json.RawMessage,
) (json.RawMessage, openAICompatibleResponseMetadata, error) {
requestCtx := ctx
var cancel context.CancelFunc
if c.requestTimeout > 0 {
requestCtx, cancel = context.WithTimeout(ctx, c.requestTimeout)
defer cancel()
}
requestBody := openAICompatibleRequest{
Model: model,
Messages: messages,
ResponseFormat: openAICompatibleStructuredOutputShape{
Type: "json_schema",
JSONSchema: openAICompatibleSchemaEnvelope{
Name: responseSchemaName,
Strict: true,
Schema: responseSchemaJSON,
},
},
}
payload, err := json.Marshal(requestBody)
if err != nil {
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("marshal provider request: %w", err)
}
httpReq, err := http.NewRequestWithContext(requestCtx, http.MethodPost, endpoint, bytes.NewReader(payload))
if err != nil {
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("build provider request: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")
if c.apiKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)
}
httpResp, err := c.httpClient.Do(httpReq)
if err != nil {
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("provider request failed: %w", err)}
}
defer func() {
_ = httpResp.Body.Close()
}()
rawResp, err := io.ReadAll(httpResp.Body)
if err != nil {
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("read provider response: %w", err)}
}
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
statusErr := parseProviderErrorBody(httpResp.StatusCode, rawResp)
if httpResp.StatusCode == http.StatusTooManyRequests || httpResp.StatusCode >= 500 {
return nil, openAICompatibleResponseMetadata{}, retryableError{err: statusErr}
}
return nil, openAICompatibleResponseMetadata{}, statusErr
}
return decodeChatCompletionsResponse(rawResp)
}
func toOpenAICompatibleMessages(messages []contracts.LLMMessage) ([]openAICompatibleMessage, error) {
if len(messages) == 0 {
return nil, fmt.Errorf("structured completion messages must not be empty")
}
result := make([]openAICompatibleMessage, len(messages))
for i, message := range messages {
role := strings.TrimSpace(message.Role)
content := strings.TrimSpace(message.Content)
if role == "" {
return nil, fmt.Errorf("message[%d] role must not be empty", i)
}
if content == "" {
return nil, fmt.Errorf("message[%d] content must not be empty", i)
}
result[i] = openAICompatibleMessage{
Role: role,
Content: content,
}
}
return result, nil
}
func buildChatCompletionsURL(baseURL string) string {
return strings.TrimRight(baseURL, "/") + "/chat/completions"
}
func decodeChatCompletionsResponse(raw []byte) (json.RawMessage, openAICompatibleResponseMetadata, error) {
var parsed openAICompatibleChatCompletionsResponse
if err := json.Unmarshal(raw, &parsed); err != nil {
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("decode provider response envelope: %w", err)}
}
if len(parsed.Choices) == 0 {
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("provider response missing choices")}
}
content, err := extractAssistantContentJSON(parsed.Choices[0].Message.Content)
if err != nil {
return nil, openAICompatibleResponseMetadata{}, retryableError{err: err}
}
metadata := openAICompatibleResponseMetadata{
Model: parsed.Model,
}
if parsed.Usage != nil {
metadata.PromptTokens = parsed.Usage.PromptTokens
metadata.CompletionTokens = parsed.Usage.CompletionTokens
metadata.TotalTokens = parsed.Usage.TotalTokens
}
return content, metadata, nil
}
func extractAssistantContentJSON(raw json.RawMessage) (json.RawMessage, error) {
trimmedRaw := bytes.TrimSpace(raw)
if len(trimmedRaw) == 0 || bytes.Equal(trimmedRaw, []byte("null")) {
return nil, fmt.Errorf("provider response missing assistant message content")
}
var textContent string
if err := json.Unmarshal(trimmedRaw, &textContent); err == nil {
textContent = strings.TrimSpace(textContent)
if textContent == "" {
return nil, fmt.Errorf("provider response assistant message content is empty")
}
if !json.Valid([]byte(textContent)) {
return nil, fmt.Errorf("provider response assistant message content is not valid JSON")
}
return json.RawMessage(textContent), nil
}
if json.Valid(trimmedRaw) {
return append(json.RawMessage(nil), trimmedRaw...), nil
}
return nil, fmt.Errorf("provider response assistant message content is not valid JSON")
}
func parseProviderErrorBody(status int, body []byte) error {
trimmed := strings.TrimSpace(string(body))
if trimmed == "" {
return fmt.Errorf("provider returned status %d", status)
}
var payload map[string]any
if err := json.Unmarshal(body, &payload); err == nil {
if nested, ok := payload["error"].(map[string]any); ok {
if msg, ok := nested["message"].(string); ok && strings.TrimSpace(msg) != "" {
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
}
}
if msg, ok := payload["message"].(string); ok && strings.TrimSpace(msg) != "" {
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
}
}
return fmt.Errorf("provider returned status %d: %s", status, trimmed)
}
func (c *OpenAICompatibleClient) redactError(err error) error {
secrets := []string{c.apiKey}
if c.apiKey != "" {
secrets = append(secrets, "Bearer "+c.apiKey)
}
return ErrorWithSecretsRedacted(err, secrets)
}

View File

@@ -1,494 +0,0 @@
package llm
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type testArtifact struct {
Value string `json:"value"`
}
func TestNewOpenAICompatibleClientValidation(t *testing.T) {
tests := []struct {
name string
cfg OpenAICompatibleClientConfig
want string
}{
{
name: "empty base URL",
cfg: OpenAICompatibleClientConfig{
BaseURL: " ",
Model: "model",
},
want: "base URL",
},
{
name: "invalid base URL",
cfg: OpenAICompatibleClientConfig{
BaseURL: "://bad",
Model: "model",
},
want: "base URL",
},
{
name: "empty model",
cfg: OpenAICompatibleClientConfig{
BaseURL: "https://example.test/v1",
Model: " ",
},
want: "model",
},
{
name: "negative retries",
cfg: OpenAICompatibleClientConfig{
BaseURL: "https://example.test/v1",
Model: "model",
MaxRetries: -1,
},
want: "max retries",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := NewOpenAICompatibleClient(tc.cfg)
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("expected error containing %q, got %v", tc.want, err)
}
})
}
}
func TestOpenAICompatibleClientSuccessfulStructuredCompletion(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{
"model":"provider-model",
"choices":[{"message":{"content":"{\"value\":\"ok\"}"}}],
"usage":{"prompt_tokens":11,"completion_tokens":7,"total_tokens":18}
}`)
}))
defer server.Close()
client := newTestClient(t, server.URL, "default-model", 0)
var out testArtifact
resp, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &out)
if err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if out.Value != "ok" {
t.Fatalf("unexpected decoded output: %+v", out)
}
if string(resp.Content) != `{"value":"ok"}` {
t.Fatalf("unexpected raw content: %s", resp.Content)
}
if resp.Provider != openAICompatibleProviderName {
t.Fatalf("unexpected provider: %q", resp.Provider)
}
if resp.Model != "provider-model" {
t.Fatalf("unexpected model: %q", resp.Model)
}
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
t.Fatalf("unexpected token metadata: %+v", resp)
}
}
func TestOpenAICompatibleClientRequestBodyIncludesStructuredOutputShape(t *testing.T) {
var seenPath string
var seenAuthorization string
var seenReq map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
seenPath = r.URL.Path
seenAuthorization = r.Header.Get("Authorization")
if err := json.NewDecoder(r.Body).Decode(&seenReq); err != nil {
t.Fatalf("decode request: %v", err)
}
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"value\":\"ok\"}"}}]}`)
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL + "/v1",
Model: "default-model",
APIKey: "secret-key",
MaxRetries: 0,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
var out testArtifact
_, err = client.CompleteStructured(context.Background(), validStructuredRequest("request-model"), &out)
if err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if seenPath != "/v1/chat/completions" {
t.Fatalf("unexpected request path: %q", seenPath)
}
if seenAuthorization != "Bearer secret-key" {
t.Fatalf("unexpected authorization header: %q", seenAuthorization)
}
if seenReq["model"] != "request-model" {
t.Fatalf("unexpected model: %v", seenReq["model"])
}
messages, ok := seenReq["messages"].([]any)
if !ok || len(messages) != 1 {
t.Fatalf("unexpected messages: %#v", seenReq["messages"])
}
message, ok := messages[0].(map[string]any)
if !ok {
t.Fatalf("unexpected message shape: %#v", messages[0])
}
if message["role"] != "user" || message["content"] != "extract this" {
t.Fatalf("unexpected message: %#v", message)
}
responseFormat, ok := seenReq["response_format"].(map[string]any)
if !ok {
t.Fatalf("expected response_format object, got %T", seenReq["response_format"])
}
if responseFormat["type"] != "json_schema" {
t.Fatalf("unexpected response_format.type: %v", responseFormat["type"])
}
jsonSchema, ok := responseFormat["json_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_format.json_schema object, got %T", responseFormat["json_schema"])
}
if jsonSchema["name"] != "test_artifact" {
t.Fatalf("unexpected schema name: %v", jsonSchema["name"])
}
if jsonSchema["strict"] != true {
t.Fatalf("expected strict=true, got %v", jsonSchema["strict"])
}
schema, ok := jsonSchema["schema"].(map[string]any)
if !ok {
t.Fatalf("expected schema object, got %T", jsonSchema["schema"])
}
if schema["type"] != "object" {
t.Fatalf("unexpected schema: %#v", schema)
}
}
func TestOpenAICompatibleClientDefaultModelFallbackAndOverride(t *testing.T) {
var seenModels []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var req map[string]any
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatalf("decode request: %v", err)
}
seenModels = append(seenModels, fmt.Sprint(req["model"]))
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"value\":\"ok\"}"}}]}`)
}))
defer server.Close()
client := newTestClient(t, server.URL, "default-model", 0)
var first testArtifact
if _, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &first); err != nil {
t.Fatalf("first CompleteStructured: %v", err)
}
var second testArtifact
if _, err := client.CompleteStructured(context.Background(), validStructuredRequest("override-model"), &second); err != nil {
t.Fatalf("second CompleteStructured: %v", err)
}
if len(seenModels) != 2 || seenModels[0] != "default-model" || seenModels[1] != "override-model" {
t.Fatalf("unexpected models: %v", seenModels)
}
}
func TestOpenAICompatibleClientInvalidOutputTarget(t *testing.T) {
client := newTestClient(t, "https://example.test/v1", "default-model", 0)
tests := []struct {
name string
out any
}{
{name: "nil", out: nil},
{name: "non-pointer", out: testArtifact{}},
{name: "nil pointer", out: (*testArtifact)(nil)},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), tc.out)
if err == nil || !strings.Contains(err.Error(), "output target") {
t.Fatalf("expected output target error, got %v", err)
}
})
}
}
func TestOpenAICompatibleClientMissingAndInvalidSchema(t *testing.T) {
client := newTestClient(t, "https://example.test/v1", "default-model", 0)
tests := []struct {
name string
mutate func(*contracts.StructuredCompletionRequest)
want string
}{
{
name: "missing schema name",
mutate: func(req *contracts.StructuredCompletionRequest) {
req.ResponseSchemaName = " "
},
want: "schema name",
},
{
name: "missing schema JSON",
mutate: func(req *contracts.StructuredCompletionRequest) {
req.ResponseSchema = nil
},
want: "schema JSON",
},
{
name: "invalid schema JSON",
mutate: func(req *contracts.StructuredCompletionRequest) {
req.ResponseSchema = json.RawMessage(`{"type":`)
},
want: "schema JSON",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
req := validStructuredRequest("")
tc.mutate(&req)
var out testArtifact
_, err := client.CompleteStructured(context.Background(), req, &out)
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("expected error containing %q, got %v", tc.want, err)
}
})
}
}
func TestOpenAICompatibleClientRejectsEmptyMessages(t *testing.T) {
client := newTestClient(t, "https://example.test/v1", "default-model", 0)
tests := []struct {
name string
mutate func(*contracts.StructuredCompletionRequest)
want string
}{
{
name: "no messages",
mutate: func(req *contracts.StructuredCompletionRequest) {
req.Messages = nil
},
want: "messages",
},
{
name: "empty role",
mutate: func(req *contracts.StructuredCompletionRequest) {
req.Messages[0].Role = " "
},
want: "role",
},
{
name: "empty content",
mutate: func(req *contracts.StructuredCompletionRequest) {
req.Messages[0].Content = " "
},
want: "content",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
req := validStructuredRequest("")
tc.mutate(&req)
var out testArtifact
_, err := client.CompleteStructured(context.Background(), req, &out)
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("expected error containing %q, got %v", tc.want, err)
}
})
}
}
func TestOpenAICompatibleClientProviderNon2xxBehavior(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest)
_, _ = io.WriteString(w, `{"error":{"message":"bad request"}}`)
}))
defer server.Close()
client := newTestClient(t, server.URL, "default-model", 0)
var out testArtifact
_, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &out)
if err == nil || !strings.Contains(err.Error(), "status 400: bad request") {
t.Fatalf("expected provider status error, got %v", err)
}
}
func TestOpenAICompatibleClientRetries429And5xx(t *testing.T) {
var attempts atomic.Int32
statuses := []int{http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusOK}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempt := int(attempts.Add(1)) - 1
if statuses[attempt] != http.StatusOK {
w.WriteHeader(statuses[attempt])
_, _ = io.WriteString(w, `{"error":{"message":"try again"}}`)
return
}
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"value\":\"ok\"}"}}]}`)
}))
defer server.Close()
client := newTestClient(t, server.URL, "default-model", 2)
var out testArtifact
if _, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &out); err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if attempts.Load() != 3 {
t.Fatalf("expected 3 attempts, got %d", attempts.Load())
}
}
func TestOpenAICompatibleClientRetriesMalformedResponses(t *testing.T) {
tests := []struct {
name string
firstBody string
}{
{
name: "malformed provider envelope",
firstBody: `{"choices":[]}`,
},
{
name: "malformed assistant JSON",
firstBody: `{"choices":[{"message":{"content":"{"}}]}`,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var attempts atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if attempts.Add(1) == 1 {
_, _ = io.WriteString(w, tc.firstBody)
return
}
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"value\":\"ok\"}"}}]}`)
}))
defer server.Close()
client := newTestClient(t, server.URL, "default-model", 1)
var out testArtifact
if _, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &out); err != nil {
t.Fatalf("CompleteStructured: %v", err)
}
if attempts.Load() != 2 {
t.Fatalf("expected 2 attempts, got %d", attempts.Load())
}
})
}
}
func TestOpenAICompatibleClientNoRetryForNonRetryable4xx(t *testing.T) {
var attempts atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts.Add(1)
w.WriteHeader(http.StatusForbidden)
_, _ = io.WriteString(w, `{"error":{"message":"forbidden"}}`)
}))
defer server.Close()
client := newTestClient(t, server.URL, "default-model", 3)
var out testArtifact
_, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &out)
if err == nil || !strings.Contains(err.Error(), "status 403") {
t.Fatalf("expected forbidden error, got %v", err)
}
if attempts.Load() != 1 {
t.Fatalf("expected 1 attempt, got %d", attempts.Load())
}
}
func TestOpenAICompatibleClientProviderErrorRedactsAPIKey(t *testing.T) {
const apiKey = "secret-api-key"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
_, _ = io.WriteString(w, `{"error":{"message":"Bearer secret-api-key failed for secret-api-key"}}`)
}))
defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: server.URL,
Model: "default-model",
APIKey: apiKey,
MaxRetries: 0,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
var out testArtifact
_, err = client.CompleteStructured(context.Background(), validStructuredRequest(""), &out)
if err == nil {
t.Fatalf("expected provider error")
}
if strings.Contains(err.Error(), apiKey) || strings.Contains(err.Error(), "Bearer "+apiKey) {
t.Fatalf("expected API key to be redacted, got %q", err.Error())
}
}
func TestOpenAICompatibleClientRespectsContextCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
client := newTestClient(t, "https://example.test/v1", "default-model", 1)
var out testArtifact
_, err := client.CompleteStructured(ctx, validStructuredRequest(""), &out)
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context canceled, got %v", err)
}
}
func newTestClient(t *testing.T, baseURL string, model string, maxRetries int) *OpenAICompatibleClient {
t.Helper()
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
BaseURL: baseURL,
Model: model,
MaxRetries: maxRetries,
})
if err != nil {
t.Fatalf("NewOpenAICompatibleClient: %v", err)
}
return client
}
func validStructuredRequest(model string) contracts.StructuredCompletionRequest {
return contracts.StructuredCompletionRequest{
Messages: []contracts.LLMMessage{
{Role: " user ", Content: " extract this "},
},
Model: model,
ResponseSchemaName: " test_artifact ",
ResponseSchema: testResponseSchema(),
}
}
func testResponseSchema() json.RawMessage {
return json.RawMessage(`{
"type": "object",
"properties": {
"value": {"type": "string"}
},
"required": ["value"],
"additionalProperties": false
}`)
}

View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
@@ -41,3 +42,14 @@ func (c *scheduledClient) CompleteStructured(ctx context.Context, req contracts.
}
return response, nil
}
func (c *scheduledClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
if c == nil || c.client == nil {
return nil
}
provider, ok := c.client.(contracts.LLMProfileManifestProvider)
if !ok {
return nil
}
return provider.LLMProfileManifests()
}

View File

@@ -0,0 +1,175 @@
package llm
import (
"context"
"errors"
"testing"
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/scriptorium"
)
func TestScriptoriumPublicAPIGrounding(t *testing.T) {
// Keep this compile-time grounding close to the future Notarius adapter so
// dependency upgrades reveal API drift before the runtime cutover.
engine, err := scriptorium.NewEngine(
scriptorium.Config{
PromptDir: "unused-when-prompt-option-is-set",
ProfileDir: "",
SchemaDir: "",
Timeout: time.Second,
},
scriptorium.WithPromptFS(fstest.MapFS{}, "."),
scriptorium.WithProfileFS(fstest.MapFS{}, "."),
scriptorium.WithSchemaFS(fstest.MapFS{}, "."),
scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "test-model",
APIKeyRequired: true,
ExtraParams: map[string]any{"mode": "test"},
})),
scriptorium.WithLLMClient(scriptoriumGroundingLLMClient{}),
)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
if engine == nil {
t.Fatalf("NewEngine() = nil, want engine")
}
var (
_ func(string) scriptorium.Option = scriptorium.WithPromptFile
_ func(string) scriptorium.Option = scriptorium.WithProfileFile
_ func(string) scriptorium.Option = scriptorium.WithSchemaFile
)
req := scriptorium.RunRequest{
PromptID: "dnd.spells",
PromptVersion: "v1",
ProfileID: "test-profile",
APIKey: "request-scoped-secret",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///tmp/transcript.json", `{"segments":[]}`),
"glossary": scriptorium.Inline(""),
"roster": scriptorium.File("/tmp/roster.txt"),
},
Vars: map[string]string{
"session_id": "session-1",
},
Execution: &scriptorium.ExecutionTargetOverride{
Model: "override-model",
Temperature: ptr(0.2),
MaxTokens: ptr(100),
TopP: ptr(0.9),
TimeoutSeconds: ptr(30),
ServiceTier: "standard",
ReasoningEffort: "low",
APIKeyEnv: "SCRIPTORIUM_API_KEY",
ExtraParams: map[string]any{"provider_option": "value"},
},
Validation: &scriptorium.OutputContract{
Format: scriptorium.FormatJSON,
ValidationMode: scriptorium.ValidationJSONSchema,
SchemaPath: "schemas/dnd_spells.v1.json",
RepairAttempts: 1,
},
Metadata: map[string]string{
"artifact_kind": "dnd_spell",
},
}
if req.Inputs["transcript"].Type != scriptorium.ArtifactRefInline {
t.Fatalf("inline input type = %q, want %q", req.Inputs["transcript"].Type, scriptorium.ArtifactRefInline)
}
if req.Inputs["roster"].Type != scriptorium.ArtifactRefFile {
t.Fatalf("file input type = %q, want %q", req.Inputs["roster"].Type, scriptorium.ArtifactRefFile)
}
result := scriptorium.RunResult{
RunID: "run-1",
Artifact: scriptorium.Artifact{
Name: "output",
ContentType: "application/json",
Body: []byte(`{"ok":true}`),
URI: "inline://output",
Size: int64(len(`{"ok":true}`)),
Hash: "sha256:abc",
},
RawOutput: `{"ok":true}`,
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-prompt-hash",
SelectedProfileID: req.ProfileID,
ModelName: "test-model",
Endpoint: "http://127.0.0.1:1/v1",
EffectiveModelParams: scriptorium.ExecutionTarget{
Model: "test-model",
APIKeyEnv: "SCRIPTORIUM_API_KEY",
ExtraParams: map[string]any{"provider_option": "value"},
ReasoningEffort: "low",
},
InputHashes: map[string]string{
"transcript": "sha256:def",
},
Validation: scriptorium.ValidationResult{
Status: scriptorium.ValidationPassed,
Mode: scriptorium.ValidationJSONSchema,
SchemaPath: req.Validation.SchemaPath,
RepairAttempts: 1,
IsValid: true,
},
Usage: scriptorium.TokenUsage{
PromptTokens: 10,
CompletionTokens: 5,
TotalTokens: 15,
CachedTokens: 3,
CacheWriteTokens: 2,
},
StartTime: time.Unix(1, 0),
EndTime: time.Unix(2, 0),
Duration: time.Second,
}
if result.Validation.Status != scriptorium.ValidationPassed {
t.Fatalf("validation status = %q, want %q", result.Validation.Status, scriptorium.ValidationPassed)
}
if result.Usage.TotalTokens != 15 {
t.Fatalf("total tokens = %d, want 15", result.Usage.TotalTokens)
}
publicErrors := []error{
scriptorium.ErrInvalidConfig,
scriptorium.ErrInvalidRequest,
scriptorium.ErrPromptNotFound,
scriptorium.ErrProfileNotFound,
scriptorium.ErrPromptLoad,
scriptorium.ErrProfileLoad,
scriptorium.ErrArtifactLoad,
scriptorium.ErrPromptRender,
scriptorium.ErrLLMGenerate,
scriptorium.ErrValidation,
}
for _, publicErr := range publicErrors {
if !errors.Is(publicErr, publicErr) {
t.Fatalf("sentinel error does not match itself: %v", publicErr)
}
}
}
type scriptoriumGroundingLLMClient struct{}
func (scriptoriumGroundingLLMClient) Generate(context.Context, scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
return &scriptorium.GenerateResponse{
Content: `{"ok":true}`,
Usage: scriptorium.TokenUsage{
PromptTokens: 1,
CompletionTokens: 1,
TotalTokens: 2,
},
}, nil
}
func ptr[T any](v T) *T {
return &v
}

View File

@@ -0,0 +1,379 @@
package llm
import (
"context"
"encoding/json"
"fmt"
"net/http"
"regexp"
"sort"
"strings"
"sync"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/scriptorium"
)
const scriptoriumProviderName = "scriptorium"
type ScriptoriumClientConfig struct {
ProfileDir string
ProfileFile string
Assets *AssetRegistry
Timeout time.Duration
HTTPClient *http.Client
EngineOptions []scriptorium.Option
Recorder *LLMProfileRecorder
}
type ScriptoriumClient struct {
engine *scriptorium.Engine
recorder *LLMProfileRecorder
}
type LLMProfileRecorder struct {
mu sync.Mutex
profiles map[string]artifacts.LLMProfileManifest
}
var _ contracts.StructuredLLMClient = (*ScriptoriumClient)(nil)
var _ contracts.LLMProfileManifestProvider = (*ScriptoriumClient)(nil)
func NewScriptoriumClient(cfg ScriptoriumClientConfig) (*ScriptoriumClient, error) {
if cfg.Assets == nil {
return nil, fmt.Errorf("scriptorium client assets must not be nil")
}
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
return nil, fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")
}
options, err := cfg.Assets.ScriptoriumOptions()
if err != nil {
return nil, err
}
if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" {
options = append(options, scriptorium.WithProfileFile(profileFile))
}
options = append(options, cfg.EngineOptions...)
engine, err := scriptorium.NewEngine(scriptorium.Config{
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
Timeout: cfg.Timeout,
HTTPClient: cfg.HTTPClient,
}, options...)
if err != nil {
return nil, fmt.Errorf("create Scriptorium engine: %w", err)
}
recorder := cfg.Recorder
if recorder == nil {
recorder = NewLLMProfileRecorder()
}
return &ScriptoriumClient{
engine: engine,
recorder: recorder,
}, nil
}
func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
if c == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client must not be nil")
}
if c.engine == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client engine must not be nil")
}
if err := validateOutputTarget(out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
promptID := strings.TrimSpace(req.PromptID)
if promptID == "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
}
runReq := scriptorium.RunRequest{
PromptID: promptID,
PromptVersion: strings.TrimSpace(req.PromptVersion),
ProfileID: strings.TrimSpace(req.ProfileID),
Inputs: scriptoriumInputs(req.Inputs),
Vars: scriptoriumVars(req),
Metadata: scriptoriumMetadata(req),
}
prepared, err := c.engine.Prepare(ctx, runReq)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
}
result, err := c.engine.Run(ctx, runReq)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
}
if result == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: empty result", promptID)
}
response := c.responseFromResult(result, prepared)
if result.Validation.Status == scriptorium.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run Scriptorium prompt %q: validation failed: %s", promptID, strings.Join(result.Validation.Errors, "; "))
}
if len(strings.TrimSpace(string(response.Content))) == 0 {
return response, fmt.Errorf("run Scriptorium prompt %q: empty structured output", promptID)
}
if err := json.Unmarshal(response.Content, out); err != nil {
return response, fmt.Errorf("decode Scriptorium structured output for prompt %q: %w", promptID, err)
}
return response, nil
}
func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, prepared *scriptorium.PreparedRun) contracts.StructuredCompletionResponse {
content := result.Artifact.Body
if len(content) == 0 {
content = []byte(result.RawOutput)
}
profile := artifacts.LLMProfileManifest{
ID: strings.TrimSpace(result.SelectedProfileID),
Provider: scriptoriumProviderName,
Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model),
}
if c.recorder != nil {
c.recorder.Record(profile)
}
return contracts.StructuredCompletionResponse{
Content: append(json.RawMessage(nil), content...),
Provider: profile.Provider,
Model: profile.Model,
ProfileID: profile.ID,
PromptTokens: result.Usage.PromptTokens,
CompletionTokens: result.Usage.CompletionTokens,
TotalTokens: result.Usage.TotalTokens,
Debug: scriptoriumDebugMaterial(prepared, result),
}
}
func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scriptorium.RunResult) *contracts.LLMDebugMaterial {
material := &contracts.LLMDebugMaterial{}
if prepared != nil {
material.Prompt = scriptoriumDebugPrompt(prepared)
}
if result != nil {
material.Response = scriptoriumDebugResponse(result)
}
if material.Prompt == nil && material.Response == nil {
return nil
}
return material
}
func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDebugPrompt {
if prepared == nil {
return nil
}
messages := make([]contracts.LLMDebugMessage, 0, len(prepared.Messages))
for _, message := range prepared.Messages {
messages = append(messages, contracts.LLMDebugMessage{
Role: message.Role,
Content: message.Content,
CacheControl: jsonObject(message.CacheControl),
})
}
return &contracts.LLMDebugPrompt{
PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SessionID: prepared.SessionID,
RenderedPromptHash: prepared.RenderedPromptHash,
Messages: messages,
EffectiveModelParams: jsonObject(prepared.EffectiveModelParams),
OutputContract: jsonObject(prepared.OutputContract),
StructuredOutput: jsonObject(prepared.StructuredOutput),
InputHashes: cloneStringMap(prepared.InputHashes),
}
}
func scriptoriumDebugResponse(result *scriptorium.RunResult) *contracts.LLMDebugResponse {
if result == nil {
return nil
}
content := result.RawOutput
if content == "" {
content = string(result.Artifact.Body)
}
return &contracts.LLMDebugResponse{
Content: content,
RunID: result.RunID,
PromptID: result.PromptID,
PromptVersion: result.PromptVersion,
PromptHash: result.PromptHash,
RenderedPromptHash: result.RenderedPromptHash,
SelectedProfileID: result.SelectedProfileID,
ModelName: result.ModelName,
Endpoint: result.Endpoint,
EffectiveModelParams: jsonObject(result.EffectiveModelParams),
InputHashes: cloneStringMap(result.InputHashes),
Validation: jsonObject(result.Validation),
Usage: contracts.LLMDebugUsage{
PromptTokens: result.Usage.PromptTokens,
CompletionTokens: result.Usage.CompletionTokens,
TotalTokens: result.Usage.TotalTokens,
CachedTokens: result.Usage.CachedTokens,
CacheWriteTokens: result.Usage.CacheWriteTokens,
},
}
}
func jsonObject(value any) map[string]any {
if value == nil {
return nil
}
data, err := json.Marshal(value)
if err != nil || string(data) == "null" {
return nil
}
var out map[string]any
if err := json.Unmarshal(data, &out); err != nil {
return nil
}
if len(out) == 0 {
return nil
}
return out
}
func cloneStringMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
for key, value := range values {
out[key] = value
}
return out
}
func (c *ScriptoriumClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
if c == nil || c.recorder == nil {
return nil
}
return c.recorder.Manifests()
}
func NewLLMProfileRecorder() *LLMProfileRecorder {
return &LLMProfileRecorder{profiles: map[string]artifacts.LLMProfileManifest{}}
}
func (r *LLMProfileRecorder) Record(profile artifacts.LLMProfileManifest) {
if r == nil {
return
}
profile.ID = strings.TrimSpace(profile.ID)
profile.Provider = strings.TrimSpace(profile.Provider)
profile.Model = strings.TrimSpace(profile.Model)
key := profile.ID + "\x00" + profile.Provider + "\x00" + profile.Model
r.mu.Lock()
defer r.mu.Unlock()
if r.profiles == nil {
r.profiles = map[string]artifacts.LLMProfileManifest{}
}
r.profiles[key] = profile
}
func (r *LLMProfileRecorder) Manifests() []artifacts.LLMProfileManifest {
if r == nil {
return nil
}
r.mu.Lock()
defer r.mu.Unlock()
if len(r.profiles) == 0 {
return nil
}
keys := make([]string, 0, len(r.profiles))
for key := range r.profiles {
keys = append(keys, key)
}
sort.Strings(keys)
out := make([]artifacts.LLMProfileManifest, 0, len(keys))
for _, key := range keys {
out = append(out, r.profiles[key])
}
return out
}
func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.ArtifactRef {
if len(inputs) == 0 {
return nil
}
out := make(map[string]scriptorium.ArtifactRef, len(inputs))
for key, material := range inputs {
name := strings.TrimSpace(key)
if name == "" {
name = strings.TrimSpace(material.Name)
}
if name == "" {
continue
}
body := string(material.Content)
if body == "" {
body = " "
}
if origin := strings.TrimSpace(material.OriginURI); origin != "" {
out[name] = scriptorium.InlineWithURI(origin, body)
} else {
out[name] = scriptorium.Inline(body)
}
}
return out
}
func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]string {
vars := make(map[string]string, len(req.Vars)+1)
for key, value := range req.Vars {
name := strings.TrimSpace(key)
if name == "" || value == nil {
continue
}
vars[name] = fmt.Sprint(value)
}
if sessionID := strings.TrimSpace(req.SessionID); sessionID != "" {
vars["session_id"] = sessionID
}
if len(vars) == 0 {
return nil
}
return vars
}
func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]string {
metadata := map[string]string{}
if stageName := strings.TrimSpace(req.StageName); stageName != "" {
metadata["stage_name"] = stageName
}
if len(metadata) == 0 {
return nil
}
return metadata
}
var bearerTokenPattern = regexp.MustCompile(`(?i)Bearer\s+[A-Za-z0-9._~+/=-]+`)
func redactScriptoriumError(err error) error {
if err == nil {
return nil
}
return redactedProviderError{err: err}
}
type redactedProviderError struct {
err error
}
func (e redactedProviderError) Error() string {
return bearerTokenPattern.ReplaceAllString(e.err.Error(), "Bearer "+secretReplacement)
}
func (e redactedProviderError) Unwrap() error {
return e.err
}

View File

@@ -0,0 +1,352 @@
package llm
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/scriptorium"
)
func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
client := newTestScriptoriumClient(t, fake)
var out struct {
OK bool `json:"ok"`
}
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
StageName: "test-stage",
PromptID: "adapter.test",
PromptVersion: "v1",
ProfileID: "explicit-profile",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "sha256:source", "file:///source.json"),
},
Vars: map[string]any{"custom": "value"},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", err)
}
if !out.OK {
t.Fatalf("decoded output OK = false, want true")
}
if resp.Provider != scriptoriumProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
t.Fatalf("response metadata = %#v", resp)
}
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
t.Fatalf("usage = %#v, want mapped token counts", resp)
}
if resp.Debug == nil || resp.Debug.Prompt == nil {
t.Fatalf("debug prompt = nil, want prepared prompt material")
}
if resp.Debug.Prompt.PromptID != "adapter.test" || resp.Debug.Prompt.SelectedProfileID != "explicit-profile" {
t.Fatalf("debug prompt metadata = %#v, want prompt/profile", resp.Debug.Prompt)
}
if len(resp.Debug.Prompt.Messages) != 1 || !strings.Contains(resp.Debug.Prompt.Messages[0].Content, `{"source":true}`) {
t.Fatalf("debug prompt messages = %#v, want rendered input content", resp.Debug.Prompt.Messages)
}
if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response)
}
debugJSON, err := json.Marshal(resp.Debug)
if err != nil {
t.Fatalf("marshal debug material: %v", err)
}
if strings.Contains(string(debugJSON), "secret-token") || strings.Contains(string(debugJSON), "sk-") {
t.Fatalf("debug material contains secret material: %s", debugJSON)
}
gotReq := fake.lastRequest()
if gotReq.Prompt.SessionID != "session-123" {
t.Fatalf("session id = %q, want session-123", gotReq.Prompt.SessionID)
}
if gotReq.Target.Model != "explicit-model" {
t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model)
}
if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) {
t.Fatalf("rendered messages = %#v, want transcript input content", gotReq.Prompt.Messages)
}
if gotReq.StructuredOutput == nil {
t.Fatalf("structured output = nil, want JSON schema")
}
manifests := client.LLMProfileManifests()
if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Model != "explicit-model" {
t.Fatalf("profile manifests = %#v", manifests)
}
}
func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
client := newTestScriptoriumClient(t, fake)
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out); err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", err)
}
if got := fake.lastRequest().Target.Model; got != "default-model" {
t.Fatalf("model = %q, want prompt default profile model", got)
}
}
func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"bad":true}`})
var out map[string]any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err == nil || !strings.Contains(err.Error(), "validation failed") {
t.Fatalf("CompleteStructured() error = %v, want validation failure", err)
}
if got := string(resp.Content); got != `{"bad":true}` {
t.Fatalf("response content = %q, want raw failed output", got)
}
if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"bad":true}` {
t.Fatalf("debug response = %#v, want raw failed output", resp.Debug)
}
if resp.Debug.Prompt == nil || len(resp.Debug.Prompt.Messages) == 0 {
t.Fatalf("debug prompt = %#v, want prepared prompt material", resp.Debug.Prompt)
}
}
func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
var out []any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err == nil || !strings.Contains(err.Error(), "decode Scriptorium structured output") {
t.Fatalf("CompleteStructured() error = %v, want decode failure", err)
}
if got := string(resp.Content); got != `{"ok":true}` {
t.Fatalf("response content = %q, want raw decode-failed output", got)
}
if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
t.Fatalf("debug response = %#v, want raw decode-failed output", resp.Debug)
}
}
func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{err: errors.New("provider failed with Bearer secret-token")})
var out map[string]any
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err == nil {
t.Fatalf("CompleteStructured() error = nil, want provider error")
}
if !strings.Contains(err.Error(), `run Scriptorium prompt "adapter.test"`) {
t.Fatalf("error = %q, want operation context", err.Error())
}
if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") {
t.Fatalf("error = %q, want redacted bearer token", err.Error())
}
if resp.Debug != nil {
t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug)
}
}
func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
var out map[string]any
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if !errors.Is(err, context.Canceled) {
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
}
}
func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
fake := &fakeScriptoriumLLM{
content: `{"ok":true}`,
block: make(chan struct{}),
}
client := newTestScriptoriumClient(t, fake)
scheduler, err := NewScheduler(1)
if err != nil {
t.Fatalf("NewScheduler() error = %v, want nil", err)
}
scheduled := NewScheduledClient(client, scheduler)
var wg sync.WaitGroup
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
var out map[string]any
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
SessionID: "session-123",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if callErr != nil {
t.Errorf("CompleteStructured() error = %v, want nil", callErr)
}
}()
}
waitForAtomicAtLeast(t, &fake.calls, 1)
time.Sleep(20 * time.Millisecond)
if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
t.Fatalf("max in-flight calls = %d, want <= 1", got)
}
close(fake.block)
wg.Wait()
}
func TestScriptoriumClientValidatesRequest(t *testing.T) {
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
var out map[string]any
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") {
t.Fatalf("missing prompt id error = %v, want prompt_id validation", err)
}
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test"}, nil); err == nil || !strings.Contains(err.Error(), "non-nil pointer") {
t.Fatalf("nil output error = %v, want output validation", err)
}
}
func newTestScriptoriumClient(t *testing.T, fake *fakeScriptoriumLLM) *ScriptoriumClient {
t.Helper()
registry := NewAssetRegistry()
if err := registry.RegisterPromptFS(fstest.MapFS{
"adapter.test.yaml": {Data: []byte(`id: adapter.test
version: "v1"
default_profile: default-profile
session_id: "{{ .session_id }}"
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content: "Transcript: {{ input \"transcript\" }}"
output:
format: json
validation_mode: json_schema
schema_path: adapter.schema.json
repair_attempts: 0
`)},
}, "."); err != nil {
t.Fatalf("RegisterPromptFS() error = %v", err)
}
if err := registry.RegisterSchemaFS(fstest.MapFS{
"adapter.schema.json": {Data: []byte(`{"type":"object","required":["ok"],"properties":{"ok":{"type":"boolean"}}}`)},
}, "."); err != nil {
t.Fatalf("RegisterSchemaFS() error = %v", err)
}
client, err := NewScriptoriumClient(ScriptoriumClientConfig{
Assets: registry,
EngineOptions: []scriptorium.Option{
scriptorium.WithProfiles(
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "default-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "default-model",
}),
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "explicit-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "explicit-model",
}),
),
scriptorium.WithLLMClient(fake),
},
})
if err != nil {
t.Fatalf("NewScriptoriumClient() error = %v, want nil", err)
}
return client
}
type fakeScriptoriumLLM struct {
content string
err error
block chan struct{}
mu sync.Mutex
last scriptorium.GenerateRequest
calls int32
inFlight int32
maxInFlight int32
}
func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
f.mu.Lock()
f.last = req
f.mu.Unlock()
atomic.AddInt32(&f.calls, 1)
current := atomic.AddInt32(&f.inFlight, 1)
for {
seen := atomic.LoadInt32(&f.maxInFlight)
if current <= seen || atomic.CompareAndSwapInt32(&f.maxInFlight, seen, current) {
break
}
}
defer atomic.AddInt32(&f.inFlight, -1)
if f.block != nil {
select {
case <-f.block:
case <-ctx.Done():
return nil, ctx.Err()
}
}
if f.err != nil {
return nil, f.err
}
content := f.content
if content == "" {
content = `{"ok":true}`
}
if !json.Valid([]byte(content)) {
return nil, errors.New("test fake must return JSON content")
}
return &scriptorium.GenerateResponse{
Content: content,
Usage: scriptorium.TokenUsage{
PromptTokens: 11,
CompletionTokens: 7,
TotalTokens: 18,
},
}, nil
}
func (f *fakeScriptoriumLLM) lastRequest() scriptorium.GenerateRequest {
f.mu.Lock()
defer f.mu.Unlock()
return f.last
}

View File

@@ -0,0 +1,318 @@
package pipeline
import (
"bytes"
"encoding/json"
"fmt"
"reflect"
"sort"
"strings"
"github.com/santhosh-tekuri/jsonschema/v6"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type ArtifactCodecSpec struct {
Kind contracts.ArtifactKind
Schema contracts.ArtifactSchema
SchemaDigest string
MediaType string
}
// ArtifactCodecTypeError reports a value that does not have the exact Go type
// registered for an artifact kind.
type ArtifactCodecTypeError struct {
Operation string
Kind contracts.ArtifactKind
ExpectedType string
ActualType string
}
func (e *ArtifactCodecTypeError) Error() string {
return fmt.Sprintf("%s artifact %q: expected exact Go type %s, got %s", e.Operation, e.Kind, e.ExpectedType, e.ActualType)
}
// ArtifactCodecCompatibilityError reports serialized metadata that does not
// identify the registered representation for an artifact kind.
type ArtifactCodecCompatibilityError struct {
Kind contracts.ArtifactKind
Reason string
}
func (e *ArtifactCodecCompatibilityError) Error() string {
return fmt.Sprintf("decode artifact %q: %s", e.Kind, e.Reason)
}
// ArtifactCodecOperationError preserves an encode or decode failure from the
// registered domain codec.
type ArtifactCodecOperationError struct {
Operation string
Kind contracts.ArtifactKind
Err error
}
func (e *ArtifactCodecOperationError) Error() string {
return fmt.Sprintf("%s artifact %q: %v", e.Operation, e.Kind, e.Err)
}
func (e *ArtifactCodecOperationError) Unwrap() error { return e.Err }
type ArtifactCodecRegistry struct {
entries map[contracts.ArtifactKind]artifactCodecEntry
}
type artifactCodecEntry struct {
spec ArtifactCodecSpec
valueType reflect.Type
encode func(any) ([]byte, error)
encodeCandidate func(any) ([]byte, error)
metadata func(any) map[string]any
decode func([]byte) (any, error)
}
func NewArtifactCodecRegistry() *ArtifactCodecRegistry {
return &ArtifactCodecRegistry{entries: make(map[contracts.ArtifactKind]artifactCodecEntry)}
}
// RegisterArtifactCodec registers one codec for T. The concrete type is kept
// private and checked at every erased encode boundary.
func RegisterArtifactCodec[T any](registry *ArtifactCodecRegistry, codec contracts.ArtifactCodec[T]) error {
if registry == nil {
return fmt.Errorf("artifact codec registry must not be nil")
}
if nilInterface(codec) {
return fmt.Errorf("artifact codec must not be nil")
}
spec, err := artifactCodecSpec(codec.Kind(), codec.Schema(), codec.MediaType())
if err != nil {
return err
}
if _, ok := registry.entries[spec.Kind]; ok {
return fmt.Errorf("artifact codec %q is already registered", spec.Kind)
}
valueType := reflect.TypeFor[T]()
entry := artifactCodecEntry{
spec: cloneArtifactCodecSpec(spec),
valueType: valueType,
encode: func(value any) ([]byte, error) {
actualType := reflect.TypeOf(value)
if actualType != valueType {
return nil, newArtifactCodecTypeError("encode", spec.Kind, valueType, actualType)
}
typed, ok := value.(T)
if !ok {
return nil, newArtifactCodecTypeError("encode", spec.Kind, valueType, actualType)
}
encoded, err := codec.Encode(typed)
if err != nil {
return nil, &ArtifactCodecOperationError{Operation: "encode", Kind: spec.Kind, Err: err}
}
return append([]byte(nil), encoded...), nil
},
decode: func(content []byte) (any, error) {
decoded, err := codec.Decode(append([]byte(nil), content...))
if err != nil {
return nil, &ArtifactCodecOperationError{Operation: "decode", Kind: spec.Kind, Err: err}
}
return decoded, nil
},
}
entry.encodeCandidate = entry.encode
if candidate, ok := any(codec).(interface{ EncodeCandidate(T) ([]byte, error) }); ok {
entry.encodeCandidate = func(value any) ([]byte, error) {
typed, err := exactTypedValue[T]("encode candidate artifact", value)
if err != nil {
return nil, err
}
content, err := candidate.EncodeCandidate(typed)
if err != nil {
return nil, &ArtifactCodecOperationError{Operation: "encode", Kind: spec.Kind, Err: err}
}
return append([]byte(nil), content...), nil
}
}
if provider, ok := any(codec).(interface{ Metadata(T) map[string]any }); ok {
entry.metadata = func(value any) map[string]any {
typed, err := exactTypedValue[T]("artifact metadata", value)
if err != nil {
return nil
}
return cloneMetadata(provider.Metadata(typed))
}
}
if registry.entries == nil {
registry.entries = make(map[contracts.ArtifactKind]artifactCodecEntry)
}
registry.entries[spec.Kind] = entry
return nil
}
func (r *ArtifactCodecRegistry) Spec(kind contracts.ArtifactKind) (ArtifactCodecSpec, bool) {
if r == nil {
return ArtifactCodecSpec{}, false
}
entry, ok := r.entries[normalizeArtifactKind(kind)]
if !ok {
return ArtifactCodecSpec{}, false
}
return cloneArtifactCodecSpec(entry.spec), true
}
func (r *ArtifactCodecRegistry) valueType(kind contracts.ArtifactKind) (reflect.Type, bool) {
if r == nil {
return nil, false
}
entry, ok := r.entries[normalizeArtifactKind(kind)]
if !ok {
return nil, false
}
return entry.valueType, true
}
func (r *ArtifactCodecRegistry) RegisteredKinds() []contracts.ArtifactKind {
if r == nil || len(r.entries) == 0 {
return nil
}
kinds := make([]contracts.ArtifactKind, 0, len(r.entries))
for kind := range r.entries {
kinds = append(kinds, kind)
}
sort.Slice(kinds, func(i, j int) bool { return kinds[i] < kinds[j] })
return kinds
}
// Encode serializes an erased framework value after proving its exact
// registered Go type.
func (r *ArtifactCodecRegistry) Encode(kind contracts.ArtifactKind, value any) (contracts.SerializedArtifact, error) {
entry, normalizedKind, err := r.entry(kind)
if err != nil {
return contracts.SerializedArtifact{}, err
}
content, err := entry.encode(value)
if err != nil {
return contracts.SerializedArtifact{}, err
}
return contracts.SerializedArtifact{
Kind: normalizedKind,
Schema: contracts.CloneArtifactSchema(entry.spec.Schema),
MediaType: entry.spec.MediaType,
Content: content,
}, nil
}
// Decode verifies serialized identity before invoking the registered codec.
func (r *ArtifactCodecRegistry) Decode(artifact contracts.SerializedArtifact) (any, error) {
entry, normalizedKind, err := r.entry(artifact.Kind)
if err != nil {
return nil, err
}
if artifact.Schema.ID != entry.spec.Schema.ID || artifact.Schema.Name != entry.spec.Schema.Name || artifact.Schema.Version != entry.spec.Schema.Version {
return nil, &ArtifactCodecCompatibilityError{Kind: normalizedKind, Reason: "schema identity does not match registered codec"}
}
if contracts.DigestArtifactSchema(artifact.Schema) != entry.spec.SchemaDigest {
return nil, &ArtifactCodecCompatibilityError{Kind: normalizedKind, Reason: "schema digest does not match registered codec"}
}
if strings.TrimSpace(artifact.MediaType) != entry.spec.MediaType {
return nil, &ArtifactCodecCompatibilityError{Kind: normalizedKind, Reason: "media type does not match registered codec"}
}
return entry.decode(artifact.Content)
}
func (r *ArtifactCodecRegistry) entry(kind contracts.ArtifactKind) (artifactCodecEntry, contracts.ArtifactKind, error) {
if r == nil {
return artifactCodecEntry{}, "", fmt.Errorf("artifact codec registry must not be nil")
}
normalizedKind := normalizeArtifactKind(kind)
if normalizedKind == "" {
return artifactCodecEntry{}, "", fmt.Errorf("artifact kind must not be empty")
}
entry, ok := r.entries[normalizedKind]
if !ok {
return artifactCodecEntry{}, normalizedKind, fmt.Errorf("artifact codec %q is not registered", normalizedKind)
}
return entry, normalizedKind, nil
}
func artifactCodecSpec(kind contracts.ArtifactKind, schema contracts.ArtifactSchema, mediaType string) (ArtifactCodecSpec, error) {
kind = normalizeArtifactKind(kind)
schema.ID = strings.TrimSpace(schema.ID)
schema.Name = strings.TrimSpace(schema.Name)
schema.Version = strings.TrimSpace(schema.Version)
mediaType = strings.TrimSpace(mediaType)
switch {
case kind == "":
return ArtifactCodecSpec{}, fmt.Errorf("artifact codec kind must not be empty")
case schema.ID == "":
return ArtifactCodecSpec{}, fmt.Errorf("artifact codec %q schema id must not be empty", kind)
case schema.Name == "":
return ArtifactCodecSpec{}, fmt.Errorf("artifact codec %q schema name must not be empty", kind)
case schema.Version == "":
return ArtifactCodecSpec{}, fmt.Errorf("artifact codec %q schema version must not be empty", kind)
case mediaType == "":
return ArtifactCodecSpec{}, fmt.Errorf("artifact codec %q media type must not be empty", kind)
case len(bytes.TrimSpace(schema.JSONSchema)) == 0:
return ArtifactCodecSpec{}, fmt.Errorf("artifact codec %q JSON Schema must not be empty", kind)
case !json.Valid(schema.JSONSchema):
return ArtifactCodecSpec{}, fmt.Errorf("artifact codec %q JSON Schema must be valid JSON", kind)
}
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schema.JSONSchema))
if err != nil {
return ArtifactCodecSpec{}, fmt.Errorf("artifact codec %q JSON Schema is invalid: %w", kind, err)
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("artifact-schema.json", schemaDocument); err != nil {
return ArtifactCodecSpec{}, fmt.Errorf("artifact codec %q JSON Schema is invalid: %w", kind, err)
}
if _, err := compiler.Compile("artifact-schema.json"); err != nil {
return ArtifactCodecSpec{}, fmt.Errorf("artifact codec %q JSON Schema is invalid: %w", kind, err)
}
schema = contracts.CloneArtifactSchema(schema)
return ArtifactCodecSpec{
Kind: kind,
Schema: schema,
SchemaDigest: contracts.DigestArtifactSchema(schema),
MediaType: mediaType,
}, nil
}
func normalizeArtifactKind(kind contracts.ArtifactKind) contracts.ArtifactKind {
return contracts.ArtifactKind(strings.TrimSpace(string(kind)))
}
func cloneArtifactCodecSpec(spec ArtifactCodecSpec) ArtifactCodecSpec {
spec.Schema = contracts.CloneArtifactSchema(spec.Schema)
return spec
}
func nilInterface(value any) bool {
if value == nil {
return true
}
reflected := reflect.ValueOf(value)
switch reflected.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return reflected.IsNil()
default:
return false
}
}
func newArtifactCodecTypeError(operation string, kind contracts.ArtifactKind, expected reflect.Type, actual reflect.Type) error {
expectedName := "<nil>"
if expected != nil {
expectedName = expected.String()
}
actualName := "<nil>"
if actual != nil {
actualName = actual.String()
}
return &ArtifactCodecTypeError{
Operation: operation,
Kind: kind,
ExpectedType: expectedName,
ActualType: actualName,
}
}

View File

@@ -0,0 +1,307 @@
package pipeline
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type codecNotes struct {
Items []string `json:"items"`
}
type codecScore struct {
Value int `json:"value"`
}
type codecNotesAlias codecNotes
type testArtifactCodec[T any] struct {
kind contracts.ArtifactKind
schema contracts.ArtifactSchema
mediaType string
encodeFunc func(T) ([]byte, error)
decodeFunc func([]byte) (T, error)
}
func (c testArtifactCodec[T]) Kind() contracts.ArtifactKind { return c.kind }
func (c testArtifactCodec[T]) Schema() contracts.ArtifactSchema { return c.schema }
func (c testArtifactCodec[T]) MediaType() string { return c.mediaType }
func (c testArtifactCodec[T]) Encode(value T) ([]byte, error) { return c.encodeFunc(value) }
func (c testArtifactCodec[T]) Decode(content []byte) (T, error) { return c.decodeFunc(content) }
func TestArtifactCodecRegistryStoresHeterogeneousExactTypes(t *testing.T) {
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
t.Fatalf("RegisterArtifactCodec(notes) error = %v, want nil", err)
}
if err := RegisterArtifactCodec(registry, scoreCodec()); err != nil {
t.Fatalf("RegisterArtifactCodec(score) error = %v, want nil", err)
}
if got, want := registry.RegisteredKinds(), []contracts.ArtifactKind{"test/notes", "test/score"}; !reflect.DeepEqual(got, want) {
t.Fatalf("RegisteredKinds() = %#v, want %#v", got, want)
}
notes := codecNotes{Items: []string{"second", "first"}}
first, err := registry.Encode("test/notes", notes)
if err != nil {
t.Fatalf("Encode(notes) error = %v, want nil", err)
}
second, err := registry.Encode(" test/notes ", notes)
if err != nil {
t.Fatalf("Encode(notes again) error = %v, want nil", err)
}
if !bytes.Equal(first.Content, second.Content) {
t.Fatalf("equal values encoded as %q and %q, want deterministic bytes", first.Content, second.Content)
}
decodedNotes, err := registry.Decode(first)
if err != nil {
t.Fatalf("Decode(notes) error = %v, want nil", err)
}
if !reflect.DeepEqual(decodedNotes, notes) {
t.Fatalf("Decode(notes) = %#v, want %#v", decodedNotes, notes)
}
score := codecScore{Value: 17}
serializedScore, err := registry.Encode("test/score", score)
if err != nil {
t.Fatalf("Encode(score) error = %v, want nil", err)
}
decodedScore, err := registry.Decode(serializedScore)
if err != nil {
t.Fatalf("Decode(score) error = %v, want nil", err)
}
if decodedScore != score {
t.Fatalf("Decode(score) = %#v, want %#v", decodedScore, score)
}
_, err = registry.Encode("test/notes", codecNotesAlias(notes))
var typeErr *ArtifactCodecTypeError
if !errors.As(err, &typeErr) {
t.Fatalf("Encode(alias) error = %T %v, want ArtifactCodecTypeError", err, err)
}
if typeErr.ExpectedType == typeErr.ActualType {
t.Fatalf("type error = %#v, want distinct exact types", typeErr)
}
}
func TestArtifactCodecRegistryStoresValidatedSchemaMetadata(t *testing.T) {
registry := NewArtifactCodecRegistry()
codec := notesCodec()
codec.kind = " test/notes "
codec.schema.ID = " notes.v1 "
codec.schema.Name = " notes "
codec.schema.Version = " v1 "
codec.mediaType = " application/json "
if err := RegisterArtifactCodec(registry, codec); err != nil {
t.Fatalf("RegisterArtifactCodec() error = %v, want nil", err)
}
codec.schema.JSONSchema[0] = '['
spec, ok := registry.Spec("test/notes")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
if spec.Kind != "test/notes" || spec.Schema.ID != "notes.v1" || spec.Schema.Name != "notes" || spec.Schema.Version != "v1" || spec.MediaType != "application/json" {
t.Fatalf("Spec() = %#v, want normalized metadata", spec)
}
if spec.SchemaDigest != contracts.DigestArtifactSchema(spec.Schema) {
t.Fatalf("schema digest = %q, want %q", spec.SchemaDigest, contracts.DigestArtifactSchema(spec.Schema))
}
spec.Schema.JSONSchema[0] = '['
again, _ := registry.Spec("test/notes")
if string(again.Schema.JSONSchema) != `{"additionalProperties":false,"properties":{"items":{"items":{"type":"string"},"type":"array"}},"required":["items"],"type":"object"}` {
t.Fatalf("stored JSON Schema changed through Spec result: %q", again.Schema.JSONSchema)
}
}
func TestArtifactCodecRegistryRejectsInvalidRegistration(t *testing.T) {
tests := []struct {
name string
mutate func(*testArtifactCodec[codecNotes])
want string
}{
{name: "kind", mutate: func(codec *testArtifactCodec[codecNotes]) { codec.kind = " " }, want: "kind"},
{name: "schema id", mutate: func(codec *testArtifactCodec[codecNotes]) { codec.schema.ID = "" }, want: "schema id"},
{name: "schema name", mutate: func(codec *testArtifactCodec[codecNotes]) { codec.schema.Name = "" }, want: "schema name"},
{name: "schema version", mutate: func(codec *testArtifactCodec[codecNotes]) { codec.schema.Version = "" }, want: "schema version"},
{name: "media type", mutate: func(codec *testArtifactCodec[codecNotes]) { codec.mediaType = "" }, want: "media type"},
{name: "empty JSON Schema", mutate: func(codec *testArtifactCodec[codecNotes]) { codec.schema.JSONSchema = nil }, want: "JSON Schema"},
{name: "invalid JSON Schema", mutate: func(codec *testArtifactCodec[codecNotes]) { codec.schema.JSONSchema = []byte(`{"type":`) }, want: "valid JSON"},
{name: "non-schema JSON", mutate: func(codec *testArtifactCodec[codecNotes]) { codec.schema.JSONSchema = []byte(`[]`) }, want: "JSON Schema is invalid"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
registry := NewArtifactCodecRegistry()
codec := notesCodec()
test.mutate(&codec)
if err := RegisterArtifactCodec(registry, codec); err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("RegisterArtifactCodec() error = %v, want %q", err, test.want)
}
})
}
}
func TestArtifactCodecRegistryRejectsDuplicateKind(t *testing.T) {
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
t.Fatalf("RegisterArtifactCodec() error = %v, want nil", err)
}
duplicate := notesCodec()
duplicate.kind = " test/notes "
if err := RegisterArtifactCodec(registry, duplicate); err == nil || !strings.Contains(err.Error(), "already registered") {
t.Fatalf("duplicate registration error = %v, want duplicate kind error", err)
}
}
func TestArtifactCodecRegistryRejectsNilRegistryAndCodec(t *testing.T) {
codec := notesCodec()
if err := RegisterArtifactCodec[codecNotes](nil, codec); err == nil || !strings.Contains(err.Error(), "registry") {
t.Fatalf("nil registry error = %v, want registry error", err)
}
var nilCodec *testArtifactCodec[codecNotes]
if err := RegisterArtifactCodec(NewArtifactCodecRegistry(), nilCodec); err == nil || !strings.Contains(err.Error(), "must not be nil") {
t.Fatalf("nil codec error = %v, want codec error", err)
}
}
func TestArtifactCodecRegistryStrictDecodeAndTypedFailures(t *testing.T) {
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
t.Fatalf("RegisterArtifactCodec() error = %v, want nil", err)
}
valid, err := registry.Encode("test/notes", codecNotes{Items: []string{"one"}})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
for _, content := range [][]byte{
[]byte(`{"items":["one"],"unknown":true}`),
[]byte(`{"items":["one"]} {}`),
} {
candidate := contracts.CloneSerializedArtifact(valid)
candidate.Content = content
_, err := registry.Decode(candidate)
var operationErr *ArtifactCodecOperationError
if !errors.As(err, &operationErr) || operationErr.Operation != "decode" {
t.Fatalf("Decode(%q) error = %T %v, want typed decode error", content, err, err)
}
}
wrongSchema := contracts.CloneSerializedArtifact(valid)
wrongSchema.Schema.Version = "v2"
_, err = registry.Decode(wrongSchema)
var compatibilityErr *ArtifactCodecCompatibilityError
if !errors.As(err, &compatibilityErr) {
t.Fatalf("Decode(wrong schema) error = %T %v, want ArtifactCodecCompatibilityError", err, err)
}
}
func TestArtifactCodecRegistryWrapsEncodeFailure(t *testing.T) {
codec := notesCodec()
cause := errors.New("cannot encode notes")
codec.encodeFunc = func(codecNotes) ([]byte, error) { return nil, cause }
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, codec); err != nil {
t.Fatalf("RegisterArtifactCodec() error = %v, want nil", err)
}
_, err := registry.Encode("test/notes", codecNotes{})
var operationErr *ArtifactCodecOperationError
if !errors.As(err, &operationErr) || operationErr.Operation != "encode" || !errors.Is(err, cause) {
t.Fatalf("Encode() error = %T %v, want typed wrapping encode error", err, err)
}
}
func TestArtifactCodecRegistryClonesCodecBytes(t *testing.T) {
shared := []byte(`{"items":["one"]}`)
codec := notesCodec()
codec.encodeFunc = func(codecNotes) ([]byte, error) { return shared, nil }
codec.decodeFunc = func(content []byte) (codecNotes, error) {
content[0] = '['
return codecNotes{Items: []string{"one"}}, nil
}
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, codec); err != nil {
t.Fatalf("RegisterArtifactCodec() error = %v, want nil", err)
}
artifact, err := registry.Encode("test/notes", codecNotes{})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
shared[0] = '['
if string(artifact.Content) != `{"items":["one"]}` {
t.Fatalf("encoded content = %q after codec buffer mutation, want owned bytes", artifact.Content)
}
before := append([]byte(nil), artifact.Content...)
if _, err := registry.Decode(artifact); err != nil {
t.Fatalf("Decode() error = %v, want nil", err)
}
if !bytes.Equal(artifact.Content, before) {
t.Fatalf("serialized content changed during decode: %q", artifact.Content)
}
}
func notesCodec() testArtifactCodec[codecNotes] {
return testArtifactCodec[codecNotes]{
kind: "test/notes",
schema: contracts.ArtifactSchema{
ID: "notes.v1",
Name: "notes",
Version: "v1",
JSONSchema: []byte(`{"additionalProperties":false,"properties":{"items":{"items":{"type":"string"},"type":"array"}},"required":["items"],"type":"object"}`),
},
mediaType: "application/json",
encodeFunc: func(value codecNotes) ([]byte, error) {
return json.Marshal(value)
},
decodeFunc: func(content []byte) (codecNotes, error) {
var value codecNotes
return value, decodeStrictJSON(content, &value)
},
}
}
func scoreCodec() testArtifactCodec[codecScore] {
return testArtifactCodec[codecScore]{
kind: "test/score",
schema: contracts.ArtifactSchema{
ID: "score.v1",
Name: "score",
Version: "v1",
JSONSchema: []byte(`{"additionalProperties":false,"properties":{"value":{"type":"integer"}},"required":["value"],"type":"object"}`),
},
mediaType: "application/json",
encodeFunc: func(value codecScore) ([]byte, error) {
return json.Marshal(value)
},
decodeFunc: func(content []byte) (codecScore, error) {
var value codecScore
return value, decodeStrictJSON(content, &value)
},
}
}
func decodeStrictJSON(content []byte, out any) error {
decoder := json.NewDecoder(bytes.NewReader(content))
decoder.DisallowUnknownFields()
if err := decoder.Decode(out); err != nil {
return err
}
if err := decoder.Decode(&struct{}{}); err != io.EOF {
if err == nil {
return fmt.Errorf("unexpected trailing JSON value")
}
return fmt.Errorf("decode trailing JSON: %w", err)
}
return nil
}

View File

@@ -0,0 +1,221 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type CheckpointFingerprint struct {
Name string `json:"name"`
Value string `json:"value"`
}
type CheckpointRecorder interface {
SourceRunning(moduleKey string) error
SourceSucceeded(moduleKey string, doc *source.SourceDocument) error
SourceFailed(moduleKey string, err error) error
ChunkRunning(moduleKey string, sourceDigest string) error
ChunkSucceeded(moduleKey string, sourceDigest string, chunks []source.Chunk, warnings []contracts.Warning) error
ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error
ChunkFailed(moduleKey string, sourceDigest string, err error) error
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact, warnings []contracts.Warning) error
NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
}
type CheckpointDecision struct {
Reused bool `json:"reused"`
Reason string `json:"reason,omitempty"`
}
type CheckpointEvent struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Action string `json:"action"`
Reason string `json:"reason,omitempty"`
}
type SourceCheckpoint struct {
Document *source.SourceDocument
}
type ChunkCheckpoint struct {
Chunks []source.Chunk
Warnings []contracts.Warning
}
// CheckpointArtifact is the durable, domain-neutral value stored at a lane
// checkpoint boundary.
type CheckpointArtifact struct {
LaneID string
ModuleKey string
SourceID string
ChunkID string
ChunkIndex int
ChunkRef source.SourceRef
Artifact contracts.SerializedArtifact
SchemaDigest string
}
type ExtractCheckpoint struct {
Outputs []CheckpointArtifact
Rejected []contracts.RejectedOutput
Warnings []contracts.Warning
}
type MergeCheckpoint struct {
Output CheckpointArtifact
Warnings []contracts.Warning
}
type NormalizeCheckpoint struct {
Output CheckpointArtifact
Warnings []contracts.Warning
}
type CheckpointLoader interface {
Enabled() bool
Source(moduleKey string) (SourceCheckpoint, CheckpointDecision)
Chunk(moduleKey string, sourceDigest string) (ChunkCheckpoint, CheckpointDecision)
Extract(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision)
Merge(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision)
Normalize(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision)
}
type noopCheckpointRecorder struct{}
type noopCheckpointLoader struct{}
func NoopCheckpointRecorder() CheckpointRecorder { return noopCheckpointRecorder{} }
func NoopCheckpointLoader() CheckpointLoader { return noopCheckpointLoader{} }
func (noopCheckpointRecorder) SourceRunning(string) error { return nil }
func (noopCheckpointRecorder) SourceSucceeded(string, *source.SourceDocument) error { return nil }
func (noopCheckpointRecorder) SourceFailed(string, error) error { return nil }
func (noopCheckpointRecorder) ChunkRunning(string, string) error { return nil }
func (noopCheckpointRecorder) ChunkSucceeded(string, string, []source.Chunk, []contracts.Warning) error {
return nil
}
func (noopCheckpointRecorder) ChunkRejected(string, string, contracts.RejectedOutput) error {
return nil
}
func (noopCheckpointRecorder) ChunkFailed(string, string, error) error { return nil }
func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error {
return nil
}
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []CheckpointArtifact, []contracts.RejectedOutput, []contracts.Warning) error {
return nil
}
func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error {
return nil
}
func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil }
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error {
return nil
}
func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
return nil
}
func (noopCheckpointRecorder) MergeFailed(string, string, []CheckpointFingerprint, error) error {
return nil
}
func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error {
return nil
}
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact, []contracts.Warning) error {
return nil
}
func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
return nil
}
func (noopCheckpointRecorder) NormalizeFailed(string, string, []CheckpointFingerprint, error) error {
return nil
}
func (noopCheckpointLoader) Enabled() bool { return false }
func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) {
return SourceCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
}
func (noopCheckpointLoader) Chunk(string, string) (ChunkCheckpoint, CheckpointDecision) {
return ChunkCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
}
func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return ExtractCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
}
func (noopCheckpointLoader) Merge(string, string, []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
return MergeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
}
func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
}
func digestFingerprints(name string, digest string) []CheckpointFingerprint {
digest = strings.TrimSpace(digest)
if digest == "" {
return nil
}
return []CheckpointFingerprint{{Name: name, Value: digest}}
}
func joinedChunkDigest(chunks []source.Chunk) (string, error) {
if len(chunks) == 0 {
return "", nil
}
values := make([]string, 0, len(chunks))
for _, chunk := range chunks {
digest, err := source.DigestChunk(chunk)
if err != nil {
return "", fmt.Errorf("digest chunk %q: %w", chunk.ID, err)
}
values = append(values, chunk.ID+"="+digest)
}
sort.Strings(values)
sum := sha256.Sum256([]byte(strings.Join(values, "\n")))
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
func normalizeCheckpointFingerprints(values []CheckpointFingerprint) []CheckpointFingerprint {
if len(values) == 0 {
return nil
}
byName := make(map[string]string, len(values))
for _, value := range values {
name := strings.TrimSpace(value.Name)
fingerprint := strings.TrimSpace(value.Value)
if name == "" || fingerprint == "" {
continue
}
byName[name] = fingerprint
}
if len(byName) == 0 {
return nil
}
names := make([]string, 0, len(byName))
for name := range byName {
names = append(names, name)
}
sort.Strings(names)
out := make([]CheckpointFingerprint, 0, len(names))
for _, name := range names {
out = append(out, CheckpointFingerprint{Name: name, Value: byName[name]})
}
return out
}
func checkpointContentDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}

View File

@@ -0,0 +1,109 @@
package pipeline
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []source.Chunk) ([]source.Chunk, error) {
if len(chunks) == 0 {
return nil, fmt.Errorf("chunks must not be empty")
}
sourceUnitIndexes := make(map[int]int, len(doc.Units))
sourceUnits := make(map[int]source.SourceUnit, len(doc.Units))
for index, unit := range doc.Units {
sourceUnitIndexes[unit.ID] = index
sourceUnits[unit.ID] = unit
}
canonicalChunks := make([]source.Chunk, 0, len(chunks))
seenChunkIDs := make(map[string]struct{}, len(chunks))
for chunkIndex, chunk := range chunks {
if strings.TrimSpace(chunk.ID) == "" {
return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
}
if _, ok := seenChunkIDs[chunk.ID]; ok {
return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
}
seenChunkIDs[chunk.ID] = struct{}{}
if chunk.SourceID != doc.ID {
return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
}
if chunk.Index != chunkIndex {
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
}
if len(chunk.Units) == 0 {
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
}
if len(chunk.Content) == 0 {
return nil, fmt.Errorf("chunk %q content must not be empty", chunk.ID)
}
if strings.TrimSpace(chunk.MediaType) == "" {
return nil, fmt.Errorf("chunk %q media_type must not be empty", chunk.ID)
}
if err := source.ValidateRef(doc, chunk.Ref); err != nil {
return nil, fmt.Errorf("chunk %q ref: %w", chunk.ID, err)
}
seenUnitIDs := make(map[int]struct{}, len(chunk.Units))
previousSourceIndex := -1
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
for unitIndex, unit := range chunk.Units {
if unit.ID <= 0 {
return nil, fmt.Errorf("chunk %q unit[%d].id must be positive", chunk.ID, unitIndex)
}
if _, ok := seenUnitIDs[unit.ID]; ok {
return nil, fmt.Errorf("chunk %q repeats source unit %d", chunk.ID, unit.ID)
}
seenUnitIDs[unit.ID] = struct{}{}
sourceIndex, ok := sourceUnitIndexes[unit.ID]
if !ok {
return nil, fmt.Errorf("chunk %q source unit %d was not found in source document %q", chunk.ID, unit.ID, doc.ID)
}
if previousSourceIndex >= 0 && sourceIndex != previousSourceIndex+1 {
return nil, fmt.Errorf("chunk %q source units must form a contiguous range in source document order", chunk.ID)
}
if unit.Ref != sourceUnits[unit.ID].Ref {
return nil, fmt.Errorf("chunk %q source unit %d ref does not match source document", chunk.ID, unit.ID)
}
previousSourceIndex = sourceIndex
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
}
expectedRef := source.SourceRef{
SourceID: doc.ID,
StartUnitID: canonicalUnits[0].Ref.StartUnitID,
EndUnitID: canonicalUnits[len(canonicalUnits)-1].Ref.EndUnitID,
}
if chunk.Ref != expectedRef {
return nil, fmt.Errorf("chunk %q ref %#v does not match unit span %#v", chunk.ID, chunk.Ref, expectedRef)
}
canonicalChunks = append(canonicalChunks, source.Chunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Ref: expectedRef,
Content: append([]byte(nil), chunk.Content...),
MediaType: chunk.MediaType,
Units: canonicalUnits,
Metadata: cloneMetadata(chunk.Metadata),
})
}
return canonicalChunks, nil
}
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
return source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
}
}

View File

@@ -8,15 +8,18 @@ import (
)
type ChunkerConstructor func() (contracts.Chunker, error)
type ChunkerBuilder func(BuildRequest) (contracts.Chunker, error)
type ChunkerRegistry struct {
constructors map[string]ChunkerConstructor
builders map[string]ChunkerBuilder
optionValidators map[string]OptionValidator
specs map[string]ModuleSpec
}
func NewChunkerRegistry() *ChunkerRegistry {
return &ChunkerRegistry{
constructors: make(map[string]ChunkerConstructor),
builders: make(map[string]ChunkerBuilder),
optionValidators: make(map[string]OptionValidator),
specs: make(map[string]ModuleSpec),
}
}
@@ -26,6 +29,15 @@ func (r *ChunkerRegistry) Register(key string, constructor ChunkerConstructor) e
}
func (r *ChunkerRegistry) RegisterWithSpec(spec ModuleSpec, constructor ChunkerConstructor) error {
if constructor == nil {
return fmt.Errorf("chunker constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return r.RegisterBuilderWithSpec(spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Chunker, error) {
return constructor()
})
}
func (r *ChunkerRegistry) RegisterBuilderWithSpec(spec ModuleSpec, validateOptions OptionValidator, builder ChunkerBuilder) error {
if r == nil {
return fmt.Errorf("chunker registry must not be nil")
}
@@ -34,25 +46,36 @@ func (r *ChunkerRegistry) RegisterWithSpec(spec ModuleSpec, constructor ChunkerC
if err := validateModuleSpec("chunker", StageChunk, normalizedSpec); err != nil {
return err
}
if constructor == nil {
return fmt.Errorf("chunker constructor for %q must not be nil", normalizedSpec.Key)
if validateOptions == nil {
return fmt.Errorf("chunker option validator for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.constructors[normalizedSpec.Key]; ok {
if builder == nil {
return fmt.Errorf("chunker builder for %q must not be nil", normalizedSpec.Key)
}
if _, ok := r.builders[normalizedSpec.Key]; ok {
return fmt.Errorf("chunker %q is already registered", normalizedSpec.Key)
}
if r.constructors == nil {
r.constructors = make(map[string]ChunkerConstructor)
if r.builders == nil {
r.builders = make(map[string]ChunkerBuilder)
}
if r.optionValidators == nil {
r.optionValidators = make(map[string]OptionValidator)
}
if r.specs == nil {
r.specs = make(map[string]ModuleSpec)
}
r.constructors[normalizedSpec.Key] = constructor
r.builders[normalizedSpec.Key] = builder
r.optionValidators[normalizedSpec.Key] = validateOptions
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
return nil
}
func (r *ChunkerRegistry) Build(key string) (contracts.Chunker, error) {
return r.BuildWithRequest(key, BuildRequest{})
}
func (r *ChunkerRegistry) BuildWithRequest(key string, request BuildRequest) (contracts.Chunker, error) {
if r == nil {
return nil, fmt.Errorf("chunker registry must not be nil")
}
@@ -62,12 +85,12 @@ func (r *ChunkerRegistry) Build(key string) (contracts.Chunker, error) {
return nil, fmt.Errorf("chunker key must not be empty")
}
constructor, ok := r.constructors[normalizedKey]
builder, ok := r.builders[normalizedKey]
if !ok {
return nil, fmt.Errorf("chunker %q is not registered", normalizedKey)
}
chunker, err := constructor()
chunker, err := builder(cloneBuildRequest(request))
if err != nil {
return nil, fmt.Errorf("build chunker %q: %w", normalizedKey, err)
}
@@ -81,6 +104,18 @@ func (r *ChunkerRegistry) Build(key string) (contracts.Chunker, error) {
return chunker, nil
}
func (r *ChunkerRegistry) ValidateOptions(key string, options map[string]any) error {
if r == nil {
return fmt.Errorf("chunker registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
validator, ok := r.optionValidators[normalizedKey]
if !ok {
return fmt.Errorf("chunker %q is not registered", normalizedKey)
}
return validateRegisteredOptions(validator, options)
}
func (r *ChunkerRegistry) Spec(key string) (ModuleSpec, bool) {
if r == nil {
return ModuleSpec{}, false
@@ -98,5 +133,5 @@ func (r *ChunkerRegistry) RegisteredKeys() []string {
return nil
}
return sortedRegistryKeys(r.constructors)
return sortedRegistryKeys(r.builders)
}

View File

@@ -330,34 +330,14 @@ func (chunker registryChunker) Key() string {
return chunker.key
}
func (chunker registryChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{}, nil
}
type registryMerger struct {
key string
}
func (merger registryMerger) Key() string {
return merger.key
}
func (merger registryMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
return contracts.MergeResult{}, nil
}
type registryNormalizer struct {
key string
}
func (normalizer registryNormalizer) Key() string {
return normalizer.key
}
func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{}, nil
}
type registryOutputEncoder struct {
key string
}
@@ -369,15 +349,3 @@ func (encoder registryOutputEncoder) Key() string {
func (encoder registryOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{}, nil
}
type registryValidator struct {
name string
}
func (validator registryValidator) Name() string {
return validator.name
}
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{}, nil
}

View File

@@ -0,0 +1,69 @@
package pipeline
import (
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
// ModuleDependencies contains run-scoped collaborators shared by constructed
// modules. Implementations retain only the dependencies they use.
type ModuleDependencies struct {
LLM contracts.StructuredLLMClient
}
// BuildRequest contains the stable dependencies and configured options used to
// construct one module or validator for a run.
type BuildRequest struct {
Dependencies ModuleDependencies
Options map[string]any
}
// OptionValidator validates one module binding without constructing it.
type OptionValidator func(map[string]any) error
func rejectUnconfiguredOptions(options map[string]any) error {
return RejectUnknownOptions(options)
}
func validateRegisteredOptions(validator OptionValidator, options map[string]any) error {
if validator == nil {
return fmt.Errorf("option validator must not be nil")
}
return validator(cloneOptions(options))
}
func cloneBuildRequest(request BuildRequest) BuildRequest {
return BuildRequest{
Dependencies: request.Dependencies,
Options: cloneOptions(request.Options),
}
}
// RejectUnknownOptions provides the common strict-map check used by module-
// owned option decoders. Values remain the implementation's responsibility.
func RejectUnknownOptions(options map[string]any, allowed ...string) error {
known := make(map[string]struct{}, len(allowed))
for _, key := range allowed {
key = strings.TrimSpace(key)
if key != "" {
known[key] = struct{}{}
}
}
unknown := make([]string, 0)
for key := range options {
if _, ok := known[key]; !ok {
unknown = append(unknown, key)
}
}
if len(unknown) == 0 {
return nil
}
sort.Strings(unknown)
if len(unknown) == 1 {
return fmt.Errorf("unknown option %q", unknown[0])
}
return fmt.Errorf("unknown options %q", unknown)
}

View File

@@ -0,0 +1,728 @@
package pipeline
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"path"
"regexp"
"strings"
"sync"
"time"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type DebugRecorder interface {
Enabled() bool
WriteJSON(name string, payload any) error
WriteBytes(name string, data []byte) error
}
type noopDebugRecorder struct{}
func NoopDebugRecorder() DebugRecorder { return noopDebugRecorder{} }
func (noopDebugRecorder) Enabled() bool { return false }
func (noopDebugRecorder) WriteJSON(string, any) error { return nil }
func (noopDebugRecorder) WriteBytes(string, []byte) error { return nil }
func debugPathComponent(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return "_"
}
var b strings.Builder
for _, r := range value {
switch {
case r >= 'a' && r <= 'z':
b.WriteRune(r)
case r >= 'A' && r <= 'Z':
b.WriteRune(r)
case r >= '0' && r <= '9':
b.WriteRune(r)
case r == '-' || r == '_' || r == '.':
b.WriteRune(r)
default:
b.WriteString(fmt.Sprintf("~%x", r))
}
}
out := b.String()
if out == "." || out == ".." || strings.Contains(out, "..") {
return "_"
}
return out
}
type debugTimedEnvelope struct {
Stage string `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Attempt int `json:"attempt,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
DurationMS int64 `json:"duration_ms"`
Payload any `json:"payload,omitempty"`
LLMCalls []debugLLMCallReference `json:"llm_calls,omitempty"`
Error string `json:"error,omitempty"`
}
type debugBinaryEnvelope struct {
ContentBase64 string `json:"content_base64,omitempty"`
ContentDigest string `json:"content_digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Warnings []contracts.Warning `json:"warnings,omitempty"`
}
type debugSourceInput struct {
SourceID string `json:"source_id,omitempty"`
Path string `json:"path,omitempty"`
Raw debugBinaryEnvelope `json:"raw,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugSourceDocument struct {
ID string `json:"id"`
Kind string `json:"kind"`
Format string `json:"format,omitempty"`
Digest string `json:"digest,omitempty"`
Units []source.SourceUnit `json:"units,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugSourceChunk struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
Index int `json:"index"`
Ref source.SourceRef `json:"ref"`
Content debugBinaryEnvelope `json:"content"`
Units []source.SourceUnit `json:"units,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugSerializedOutput struct {
LaneID string `json:"lane_id"`
NormalizerKey string `json:"normalizer_key"`
SourceID string `json:"source_id,omitempty"`
Kind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
Schema contracts.ArtifactSchema `json:"schema"`
SchemaDigest string `json:"schema_digest"`
Content debugBinaryEnvelope `json:"content"`
}
type debugLLMInputMaterial struct {
Name string `json:"name"`
MediaType string `json:"media_type,omitempty"`
Content string `json:"content_base64,omitempty"`
Digest string `json:"digest,omitempty"`
OriginURI string `json:"origin_uri,omitempty"`
SizeBytes int64 `json:"size_bytes,omitempty"`
}
type debugStructuredCompletionRequest struct {
StageName string `json:"stage_name"`
PromptID string `json:"prompt_id,omitempty"`
PromptVersion string `json:"prompt_version,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
SessionID string `json:"session_id,omitempty"`
Inputs map[string]debugLLMInputMaterial `json:"inputs,omitempty"`
Vars map[string]any `json:"vars,omitempty"`
}
type debugStructuredCompletionResponse struct {
Content string `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
}
type debugLLMPromptArtifact struct {
CallID string `json:"call_id"`
Prompt *contracts.LLMDebugPrompt `json:"prompt,omitempty"`
}
type debugLLMResponseArtifact struct {
CallID string `json:"call_id"`
Response *contracts.LLMDebugResponse `json:"response,omitempty"`
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
ContentPath string `json:"content_path,omitempty"`
Error string `json:"error,omitempty"`
}
type debugLLMCallReference struct {
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
ResponseContentPath string `json:"response_content_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
Error bool `json:"error,omitempty"`
}
type debugValidationCall struct {
ValidatorName string `json:"validator_name"`
Request any `json:"request"`
Result contracts.ValidationResult `json:"result,omitempty"`
Error string `json:"error,omitempty"`
}
type debugLLMClient struct {
inner contracts.StructuredLLMClient
recorder DebugRecorder
mu sync.Mutex
counter int
}
type debugLLMScope struct {
prefix string
parent *debugLLMScope
mu sync.Mutex
calls []debugLLMCallReference
}
type debugLLMScopeContextKey struct{}
func wrapDebugLLMClient(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient {
if client == nil || recorder == nil || !recorder.Enabled() {
return client
}
if _, ok := client.(*debugLLMClient); ok {
return client
}
return &debugLLMClient{inner: client, recorder: synchronizedDebugRecorder(recorder)}
}
// WithDebugLLMRecording decorates a shared LLM client so calls made by
// construction-injected modules participate in the run's debug recording.
func WithDebugLLMRecording(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient {
return wrapDebugLLMClient(client, recorder)
}
func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.mu.Lock()
client.counter++
callID := fmt.Sprintf("%04d", client.counter)
client.mu.Unlock()
started := time.Now().UTC()
response, err := client.inner.CompleteStructured(ctx, req, out)
completed := time.Now().UTC()
errorText := ""
if err != nil {
errorText = err.Error()
}
scopePrefix := cleanDebugPath(req.StageName)
if scopePrefix == "_" {
scopePrefix = "llm"
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scopePrefix = scope.prefix
}
promptPath := ""
var writeErr error
if response.Debug != nil && response.Debug.Prompt != nil {
promptPath = path.Join(scopePrefix, "prompt-"+callID+".json")
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, promptPath, debugTimedEnvelope{
Stage: req.StageName,
ModuleKey: req.StageName,
StartedAt: started,
CompletedAt: completed,
DurationMS: completed.Sub(started).Milliseconds(),
Payload: debugLLMPromptArtifact{
CallID: callID,
Prompt: response.Debug.Prompt,
},
}))
}
responsePath := path.Join(scopePrefix, "response-"+callID+".json")
responseMaterial := debugResponseMaterial(response)
fallbackMaterial := debugCompletionFallback(response)
responseContentPath, responseForArtifact, fallbackForArtifact, contentErr := writeDebugResponseContent(client.recorder, scopePrefix, callID, responseMaterial, fallbackMaterial)
writeErr = errors.Join(writeErr, contentErr)
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, responsePath, debugTimedEnvelope{
Stage: req.StageName,
ModuleKey: req.StageName,
StartedAt: started,
CompletedAt: completed,
DurationMS: completed.Sub(started).Milliseconds(),
Payload: debugLLMResponseArtifact{
CallID: callID,
Response: responseForArtifact,
Fallback: fallbackForArtifact,
ContentPath: responseContentPath,
Error: errorText,
},
Error: errorText,
}))
callRef := debugLLMCallReference{
CallID: callID,
PromptPath: promptPath,
ResponsePath: responsePath,
ResponseContentPath: responseContentPath,
PromptID: req.PromptID,
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
Error: err != nil,
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scope.record(callRef)
}
if err != nil {
return response, err
}
if writeErr != nil {
return response, fmt.Errorf("write LLM debug artifact: %w", writeErr)
}
return response, err
}
func withDebugLLMScope(ctx context.Context, prefix string) (context.Context, *debugLLMScope) {
if ctx == nil {
ctx = context.Background()
}
prefix = cleanDebugPath(prefix)
scope := &debugLLMScope{
prefix: prefix,
parent: debugLLMScopeFromContext(ctx),
}
return context.WithValue(ctx, debugLLMScopeContextKey{}, scope), scope
}
func withIsolatedDebugLLMScope(ctx context.Context, prefix string) (context.Context, *debugLLMScope) {
if ctx == nil {
ctx = context.Background()
}
prefix = cleanDebugPath(prefix)
scope := &debugLLMScope{prefix: prefix}
return context.WithValue(ctx, debugLLMScopeContextKey{}, scope), scope
}
func debugLLMScopeFromContext(ctx context.Context) *debugLLMScope {
if ctx == nil {
return nil
}
scope, _ := ctx.Value(debugLLMScopeContextKey{}).(*debugLLMScope)
return scope
}
func (scope *debugLLMScope) record(ref debugLLMCallReference) {
if scope == nil {
return
}
scope.mu.Lock()
scope.calls = append(scope.calls, ref)
scope.mu.Unlock()
if scope.parent != nil {
scope.parent.record(ref)
}
}
func (scope *debugLLMScope) references() []debugLLMCallReference {
if scope == nil {
return nil
}
scope.mu.Lock()
defer scope.mu.Unlock()
if len(scope.calls) == 0 {
return nil
}
out := make([]debugLLMCallReference, len(scope.calls))
copy(out, scope.calls)
return out
}
func cleanDebugPath(value string) string {
parts := strings.Split(path.Clean(strings.TrimSpace(value)), "/")
out := make([]string, 0, len(parts))
for _, part := range parts {
out = append(out, debugPathComponent(part))
}
if len(out) == 0 {
return "_"
}
return path.Join(out...)
}
func debugFirstNonEmptyString(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func (client *debugLLMClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
provider, ok := client.inner.(contracts.LLMProfileManifestProvider)
if !ok {
return nil
}
return provider.LLMProfileManifests()
}
func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnvelope) error {
if recorder == nil || !recorder.Enabled() {
return nil
}
if envelope.CompletedAt.IsZero() {
envelope.CompletedAt = time.Now().UTC()
}
if envelope.StartedAt.IsZero() {
envelope.StartedAt = envelope.CompletedAt
}
if envelope.DurationMS == 0 {
envelope.DurationMS = envelope.CompletedAt.Sub(envelope.StartedAt).Milliseconds()
}
return recorder.WriteJSON(name, envelope)
}
func debugEnvelopeWithLLMCalls(envelope debugTimedEnvelope, scope *debugLLMScope) debugTimedEnvelope {
if scope != nil {
envelope.LLMCalls = scope.references()
}
return envelope
}
func writeDebugAttempt(recorder DebugRecorder, attemptPath string, envelope debugTimedEnvelope, scope *debugLLMScope) error {
return writeDebugTimed(recorder, attemptPath+".json", debugEnvelopeWithLLMCalls(envelope, scope))
}
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) debugBinaryEnvelope {
content = redactSecretBytes(content)
return debugBinaryEnvelope{
ContentBase64: base64.StdEncoding.EncodeToString(content),
ContentDigest: debugContentDigest(content),
MediaType: mediaType,
Metadata: redactSensitiveMap(metadata),
Warnings: cloneWarnings(warnings),
}
}
func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocument {
if doc == nil {
return nil
}
return &debugSourceDocument{
ID: doc.ID,
Kind: doc.Kind,
Format: doc.Format,
Digest: doc.Digest,
Units: cloneSourceUnits(doc.Units),
Metadata: redactSensitiveMap(doc.Metadata),
}
}
func debugSourceChunkEnvelope(chunk source.Chunk) debugSourceChunk {
return debugSourceChunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Ref: chunk.Ref,
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
Units: cloneSourceUnits(chunk.Units),
Metadata: redactSensitiveMap(chunk.Metadata),
}
}
func debugSourceChunkEnvelopes(chunks []source.Chunk) []debugSourceChunk {
if len(chunks) == 0 {
return nil
}
out := make([]debugSourceChunk, 0, len(chunks))
for _, chunk := range chunks {
out = append(out, debugSourceChunkEnvelope(chunk))
}
return out
}
func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
digest := contracts.DigestArtifactSchema(schema)
schema.JSONSchema = nil
content := debugContentEnvelope(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)
content.ContentDigest = debugContentDigest(output.Artifact.Content)
return debugSerializedOutput{
LaneID: output.LaneID, NormalizerKey: output.NormalizerKey, SourceID: output.SourceID,
Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: digest,
Content: content,
}
}
func debugSerializedOutputEnvelopes(outputs []contracts.SerializedOutput) []debugSerializedOutput {
if len(outputs) == 0 {
return nil
}
out := make([]debugSerializedOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, debugSerializedOutputEnvelope(output))
}
return out
}
type debugOutputFile struct {
Name string `json:"name"`
ContentType string `json:"content_type,omitempty"`
Content debugBinaryEnvelope `json:"content"`
}
func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
if len(files) == 0 {
return nil
}
out := make([]debugOutputFile, 0, len(files))
for _, file := range files {
out = append(out, debugOutputFile{
Name: file.Name,
ContentType: file.ContentType,
Content: debugContentEnvelope(file.Bytes, file.ContentType, nil, nil),
})
}
return out
}
func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest {
inputs := make(map[string]debugLLMInputMaterial, len(req.Inputs))
for key, material := range req.Inputs {
inputs[key] = debugLLMInputMaterial{
Name: material.Name,
MediaType: material.MediaType,
Content: base64.StdEncoding.EncodeToString(redactSecretBytes(material.Content)),
Digest: material.Digest,
OriginURI: material.OriginURI,
SizeBytes: material.SizeBytes,
}
}
if len(inputs) == 0 {
inputs = nil
}
return debugStructuredCompletionRequest{
StageName: req.StageName,
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID,
SessionID: req.SessionID,
Inputs: inputs,
Vars: redactSensitiveMap(req.Vars),
}
}
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
return debugStructuredCompletionResponse{
Content: string(response.Content),
Provider: response.Provider,
Model: response.Model,
ProfileID: response.ProfileID,
PromptTokens: response.PromptTokens,
CompletionTokens: response.CompletionTokens,
TotalTokens: response.TotalTokens,
}
}
func debugCompletionFallback(response contracts.StructuredCompletionResponse) *debugStructuredCompletionResponse {
if response.Debug != nil && response.Debug.Response != nil {
return nil
}
fallback := debugCompletionResponse(response)
if fallback.Content == "" &&
fallback.Provider == "" &&
fallback.Model == "" &&
fallback.ProfileID == "" &&
fallback.PromptTokens == 0 &&
fallback.CompletionTokens == 0 &&
fallback.TotalTokens == 0 {
return nil
}
return &fallback
}
func writeDebugResponseContent(recorder DebugRecorder, scopePrefix string, callID string, response *contracts.LLMDebugResponse, fallback *debugStructuredCompletionResponse) (string, *contracts.LLMDebugResponse, *debugStructuredCompletionResponse, error) {
responseCopy := cloneDebugResponseWithoutContent(response)
fallbackCopy := cloneDebugFallbackWithoutContent(fallback)
content := ""
if response != nil {
content = response.Content
}
if content == "" && fallback != nil {
content = fallback.Content
}
if content == "" {
return "", responseCopy, fallbackCopy, nil
}
contentPath, data := debugResponseContentFile(scopePrefix, callID, content)
if recorder == nil || !recorder.Enabled() {
return contentPath, responseCopy, fallbackCopy, nil
}
if err := recorder.WriteBytes(contentPath, data); err != nil {
return contentPath, responseCopy, fallbackCopy, err
}
return contentPath, responseCopy, fallbackCopy, nil
}
func cloneDebugResponseWithoutContent(response *contracts.LLMDebugResponse) *contracts.LLMDebugResponse {
if response == nil {
return nil
}
clone := *response
clone.Content = ""
return &clone
}
func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionResponse) *debugStructuredCompletionResponse {
if fallback == nil {
return nil
}
clone := *fallback
clone.Content = ""
return &clone
}
func debugResponseContentFile(scopePrefix string, callID string, content string) (string, []byte) {
raw := []byte(content)
if json.Valid(raw) {
var formatted bytes.Buffer
if err := json.Indent(&formatted, raw, "", " "); err == nil {
formatted.WriteByte('\n')
return path.Join(scopePrefix, "response-content-"+callID+".json"), formatted.Bytes()
}
}
return path.Join(scopePrefix, "response-content-"+callID+".txt"), raw
}
func debugResponseMaterial(response contracts.StructuredCompletionResponse) *contracts.LLMDebugResponse {
if response.Debug == nil {
return nil
}
return response.Debug.Response
}
func debugResponseModel(response contracts.StructuredCompletionResponse) string {
if response.Debug == nil || response.Debug.Response == nil {
return ""
}
return response.Debug.Response.ModelName
}
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
result.Message = string(redactSecretBytes([]byte(result.Message)))
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
for i := range result.Warnings {
result.Warnings[i].Message = string(redactSecretBytes([]byte(result.Warnings[i].Message)))
}
return result
}
func debugWarningEnvelopes(warnings []contracts.Warning) []contracts.Warning {
out := cloneWarnings(warnings)
for i := range out {
out[i].Message = string(redactSecretBytes([]byte(out[i].Message)))
}
return out
}
func debugRejectedOutputEnvelope(rejected contracts.RejectedOutput) contracts.RejectedOutput {
rejected.Message = string(redactSecretBytes([]byte(rejected.Message)))
rejected.DiagnosticArtifactPath = string(redactSecretBytes([]byte(rejected.DiagnosticArtifactPath)))
return rejected
}
func debugRejectedOutputPtr(rejected *contracts.RejectedOutput) any {
if rejected == nil {
return nil
}
out := debugRejectedOutputEnvelope(*rejected)
return out
}
func debugRejectedOutputEnvelopes(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
if len(rejected) == 0 {
return nil
}
out := make([]contracts.RejectedOutput, 0, len(rejected))
for _, item := range rejected {
out = append(out, debugRejectedOutputEnvelope(item))
}
return out
}
func debugContentDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
var secretPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)bearer\s+[a-z0-9._~+/=-]{8,}`),
regexp.MustCompile(`(?i)sk-[a-z0-9_-]{8,}`),
}
func redactSecretBytes(content []byte) []byte {
if len(content) == 0 || !utf8.Valid(content) {
return append([]byte(nil), content...)
}
text := string(content)
for _, pattern := range secretPatterns {
text = pattern.ReplaceAllString(text, "[REDACTED]")
}
return []byte(text)
}
func redactSensitiveMap(values map[string]any) map[string]any {
if len(values) == 0 {
return nil
}
out := make(map[string]any, len(values))
for key, value := range values {
if sensitiveKey(key) {
out[key] = "[REDACTED]"
continue
}
out[key] = redactSensitiveValue(value)
}
return out
}
func redactSensitiveValue(value any) any {
switch typed := value.(type) {
case string:
return string(redactSecretBytes([]byte(typed)))
case map[string]any:
return redactSensitiveMap(typed)
case map[string]string:
out := make(map[string]string, len(typed))
for key, value := range typed {
if sensitiveKey(key) {
out[key] = "[REDACTED]"
} else {
out[key] = string(redactSecretBytes([]byte(value)))
}
}
return out
default:
return value
}
}
func sensitiveKey(key string) bool {
key = strings.ToLower(key)
return strings.Contains(key, "api_key") ||
strings.Contains(key, "apikey") ||
strings.Contains(key, "authorization") ||
strings.Contains(key, "bearer") ||
strings.Contains(key, "password") ||
strings.Contains(key, "secret") ||
strings.Contains(key, "token")
}

View File

@@ -0,0 +1,51 @@
package pipeline
import (
"encoding/json"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) {
doc := validSourceDocument()
envelope := debugSourceDocumentEnvelope(doc)
encoded, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("Marshal(debug source document) error = %v, want nil", err)
}
var decoded debugSourceDocument
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatalf("Unmarshal(debug source document) error = %v, want nil", err)
}
if got, want := decoded.Units[0].Ref, (source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}); got != want {
t.Fatalf("debug source unit ref = %#v, want %#v", got, want)
}
envelope.Units[0].Ref.SourceID = "mutated"
if got := doc.Units[0].Ref.SourceID; got != "source-1" {
t.Fatalf("source document ref = %q after debug mutation, want source-1", got)
}
}
func TestDebugSourceChunkPreservesReference(t *testing.T) {
doc := validSourceDocument()
chunk := source.Chunk{
ID: "chunk-1", SourceID: doc.ID, Index: 0,
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1},
Content: []byte("chunk content"), MediaType: "text/plain", Units: doc.Units[:1],
}
envelope := debugSourceChunkEnvelope(chunk)
encoded, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("Marshal(debug source chunk) error = %v, want nil", err)
}
var decoded debugSourceChunk
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatalf("Unmarshal(debug source chunk) error = %v, want nil", err)
}
if decoded.Ref != chunk.Ref {
t.Fatalf("debug chunk ref = %#v, want %#v", decoded.Ref, chunk.Ref)
}
}

View File

@@ -2,16 +2,17 @@ package pipeline_test
import (
"context"
"encoding/json"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
"gitea.maximumdirect.net/eric/notarius/internal/modules/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/normalize/noop"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/output/json"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
)
func TestPipelineConfigResolvesWithProductionDefaultsRegistered(t *testing.T) {
@@ -34,8 +35,8 @@ func TestPipelineConfigResolvesWithProductionDefaultsRegistered(t *testing.T) {
}
pipeline := resolved.ResolvedPipeline
if pipeline.Chunk.Module != generic.Key {
t.Fatalf("Chunk.Module = %q, want %q", pipeline.Chunk.Module, generic.Key)
if pipeline.Chunk.Module != units.Key {
t.Fatalf("Chunk.Module = %q, want %q", pipeline.Chunk.Module, units.Key)
}
if pipeline.Output.Module != jsonoutput.Key {
t.Fatalf("Output.Module = %q, want %q", pipeline.Output.Module, jsonoutput.Key)
@@ -68,35 +69,47 @@ func defaultModuleCatalog(t *testing.T) pipeline.ModuleCatalog {
}); err != nil {
t.Fatalf("register input: %v", err)
}
if err := generic.Register(chunkers); err != nil {
if err := units.Register(chunkers); err != nil {
t.Fatalf("register generic chunker: %v", err)
}
if err := extractors.RegisterWithSpec(pipeline.ModuleSpec{
if err := pipeline.RegisterExtractor[defaultArtifact](extractors, pipeline.ModuleSpec{
Key: "extract",
Stage: pipeline.StageExtract,
ArtifactKind: defaultArtifactKind,
Requires: []string{"chunks"},
Provides: []string{"records"},
}, func() (contracts.Extractor, error) {
}, func() (contracts.Extractor[defaultArtifact], error) {
return defaultExtractor{}, nil
}); err != nil {
t.Fatalf("register extractor: %v", err)
}
if err := appendorder.Register(mergers); err != nil {
if err := appendorder.RegisterTyped(mergers, defaultArtifactKind, func(values []defaultArtifact) (defaultArtifact, error) {
if len(values) == 0 {
return defaultArtifact{}, nil
}
return values[0], nil
}); err != nil {
t.Fatalf("register appendorder merger: %v", err)
}
if err := noop.Register(normalizers); err != nil {
if err := noop.RegisterTyped[defaultArtifact](normalizers, defaultArtifactKind); err != nil {
t.Fatalf("register noop normalizer: %v", err)
}
if err := jsonoutput.Register(outputs); err != nil {
t.Fatalf("register json output: %v", err)
}
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, defaultArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.ModuleCatalog{
Inputs: inputs,
Chunkers: chunkers,
ArtifactCodecs: codecs,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: outputs,
}
}
@@ -113,12 +126,27 @@ type defaultExtractor struct{}
func (defaultExtractor) Key() string { return "extract" }
func (defaultExtractor) ArtifactType() string { return "record" }
func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (defaultExtractor) SchemaVersion() string { return "v1" }
func (defaultExtractor) Validators() []contracts.Validator { return nil }
func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
func (defaultExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[defaultArtifact], error) {
return contracts.TypedExtractionResult[defaultArtifact]{}, nil
}
const defaultArtifactKind contracts.ArtifactKind = "test/default"
type defaultArtifact struct {
Value string `json:"value"`
}
type defaultArtifactCodec struct{}
func (defaultArtifactCodec) Kind() contracts.ArtifactKind { return defaultArtifactKind }
func (defaultArtifactCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "urn:notarius:test:default", Name: "default", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}
}
func (defaultArtifactCodec) MediaType() string { return "application/json" }
func (defaultArtifactCodec) Encode(value defaultArtifact) ([]byte, error) { return json.Marshal(value) }
func (defaultArtifactCodec) Decode(content []byte) (defaultArtifact, error) {
var value defaultArtifact
err := json.Unmarshal(content, &value)
return value, err
}

View File

@@ -1,102 +1,108 @@
package pipeline
import (
"context"
"fmt"
"reflect"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type ExtractorConstructor func() (contracts.Extractor, error)
type ExtractorRegistry struct {
constructors map[string]ExtractorConstructor
typedEntries map[string]typedExtractorEntry
specs map[string]ModuleSpec
}
type typedExtractorEntry struct {
spec ModuleSpec
valueType reflect.Type
validateOptions OptionValidator
builder func(BuildRequest) (any, error)
extract typedExtractOperation
}
func NewExtractorRegistry() *ExtractorRegistry {
return &ExtractorRegistry{
constructors: make(map[string]ExtractorConstructor),
specs: make(map[string]ModuleSpec),
return &ExtractorRegistry{typedEntries: map[string]typedExtractorEntry{}, specs: map[string]ModuleSpec{}}
}
func RegisterExtractor[T any](registry *ExtractorRegistry, spec ModuleSpec, constructor func() (contracts.Extractor[T], error)) error {
if constructor == nil {
return fmt.Errorf("extractor constructor for %q must not be nil", strings.TrimSpace(spec.Key))
}
return RegisterExtractorBuilder(registry, spec, rejectUnconfiguredOptions, func(BuildRequest) (contracts.Extractor[T], error) { return constructor() })
}
func (r *ExtractorRegistry) Register(key string, constructor ExtractorConstructor) error {
return r.RegisterWithSpec(defaultModuleSpec(key, StageExtract), constructor)
}
func (r *ExtractorRegistry) RegisterWithSpec(spec ModuleSpec, constructor ExtractorConstructor) error {
if r == nil {
func RegisterExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpec, validateOptions OptionValidator, builder func(BuildRequest) (contracts.Extractor[T], error)) error {
if registry == nil {
return fmt.Errorf("extractor registry must not be nil")
}
normalizedSpec := normalizeModuleSpec(spec)
if err := validateModuleSpec("extractor", StageExtract, normalizedSpec); err != nil {
normalized := normalizeModuleSpec(spec)
if err := validateModuleSpec("extractor", StageExtract, normalized); err != nil {
return err
}
if constructor == nil {
return fmt.Errorf("extractor constructor for %q must not be nil", normalizedSpec.Key)
if normalized.ArtifactKind == "" {
return fmt.Errorf("typed extractor %q artifact kind must not be empty", normalized.Key)
}
if _, ok := r.constructors[normalizedSpec.Key]; ok {
return fmt.Errorf("extractor %q is already registered", normalizedSpec.Key)
if validateOptions == nil {
return fmt.Errorf("extractor option validator for %q must not be nil", normalized.Key)
}
if r.constructors == nil {
r.constructors = make(map[string]ExtractorConstructor)
if builder == nil {
return fmt.Errorf("extractor builder for %q must not be nil", normalized.Key)
}
if r.specs == nil {
r.specs = make(map[string]ModuleSpec)
if _, ok := registry.specs[normalized.Key]; ok {
return fmt.Errorf("extractor %q is already registered", normalized.Key)
}
r.constructors[normalizedSpec.Key] = constructor
r.specs[normalizedSpec.Key] = cloneModuleSpec(normalizedSpec)
entry := typedExtractorEntry{spec: cloneModuleSpec(normalized), valueType: reflect.TypeFor[T](), validateOptions: validateOptions, builder: func(request BuildRequest) (any, error) { return builder(cloneBuildRequest(request)) }, extract: func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
extractor, ok := implementation.(contracts.Extractor[T])
if !ok {
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalized.Key, implementation)
}
result, err := extractor.Extract(ctx, request)
if err != nil {
return erasedTypedResult{}, err
}
return erasedTypedResult{Value: result.Value, Warnings: result.Warnings}, nil
}}
if registry.typedEntries == nil {
registry.typedEntries = map[string]typedExtractorEntry{}
}
if registry.specs == nil {
registry.specs = map[string]ModuleSpec{}
}
registry.typedEntries[normalized.Key] = entry
registry.specs[normalized.Key] = cloneModuleSpec(normalized)
return nil
}
func (r *ExtractorRegistry) Build(key string) (contracts.Extractor, error) {
func (r *ExtractorRegistry) validateOptions(key string, options map[string]any) error {
if r == nil {
return nil, fmt.Errorf("extractor registry must not be nil")
return fmt.Errorf("extractor registry must not be nil")
}
normalizedKey := strings.TrimSpace(key)
if normalizedKey == "" {
return nil, fmt.Errorf("extractor key must not be empty")
}
constructor, ok := r.constructors[normalizedKey]
normalized := strings.TrimSpace(key)
entry, ok := r.typedEntries[normalized]
if !ok {
return nil, fmt.Errorf("extractor %q is not registered", normalizedKey)
return fmt.Errorf("extractor %q is not registered", normalized)
}
extractor, err := constructor()
if err != nil {
return nil, fmt.Errorf("build extractor %q: %w", normalizedKey, err)
}
if extractor == nil {
return nil, fmt.Errorf("extractor %q constructor returned nil", normalizedKey)
}
if extractor.Key() != normalizedKey {
return nil, fmt.Errorf("extractor %q returned key %q", normalizedKey, extractor.Key())
}
return extractor, nil
return validateRegisteredOptions(entry.validateOptions, options)
}
func (r *ExtractorRegistry) Spec(key string) (ModuleSpec, bool) {
if r == nil {
return ModuleSpec{}, false
}
spec, ok := r.specs[strings.TrimSpace(key)]
if !ok {
return ModuleSpec{}, false
}
return cloneModuleSpec(spec), true
return cloneModuleSpec(spec), ok
}
func (r *ExtractorRegistry) typedEntry(key string) (typedExtractorEntry, bool) {
if r == nil {
return typedExtractorEntry{}, false
}
entry, ok := r.typedEntries[strings.TrimSpace(key)]
return entry, ok
}
func (r *ExtractorRegistry) RegisteredKeys() []string {
if r == nil {
return nil
}
return sortedRegistryKeys(r.constructors)
return sortedRegistryKeys(r.specs)
}

View File

@@ -1,310 +0,0 @@
package pipeline
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestExtractorRegistryRegisterAndBuild(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
extractor, err := registry.Build("generic-extractor")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
if extractor.Key() != "generic-extractor" {
t.Fatalf("extractor.Key() = %q, want generic-extractor", extractor.Key())
}
}
func TestExtractorRegistryRegisterAndBuildTrimKeys(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
extractor, err := registry.Build("\tgeneric-extractor\n")
if err != nil {
t.Fatalf("Build() error = %v, want nil", err)
}
if extractor.Key() != "generic-extractor" {
t.Fatalf("extractor.Key() = %q, want generic-extractor", extractor.Key())
}
}
func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
registry := NewExtractorRegistry()
spec := ModuleSpec{
Key: " generic-extractor ",
Stage: StageExtract,
Provides: []string{" generic-artifact ", "source-citations", "generic-artifact", ""},
Requires: []string{" source-document ", "source-document", ""},
}
if err := registry.RegisterWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
got, ok := registry.Spec("\tgeneric-extractor\n")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ModuleSpec{
Key: "generic-extractor",
Stage: StageExtract,
Provides: []string{"generic-artifact", "source-citations"},
Requires: []string{"source-document"},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want)
}
got.Provides[0] = "changed"
again, ok := registry.Spec("generic-extractor")
if !ok {
t.Fatal("Spec() after caller mutation ok = false, want true")
}
if !reflect.DeepEqual(again, want) {
t.Fatalf("Spec() after caller mutation = %#v, want %#v", again, want)
}
}
func TestExtractorRegistryRegisterStoresDefaultSpec(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register(" generic-extractor ", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
got, ok := registry.Spec("generic-extractor")
if !ok {
t.Fatal("Spec() ok = false, want true")
}
want := ModuleSpec{Key: "generic-extractor", Stage: StageExtract}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want)
}
}
func TestExtractorRegistryRegisterWithSpecRejectsWrongStage(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterWithSpec(ModuleSpec{Key: "generic-extractor", Stage: StageInput}, fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("RegisterWithSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), "stage") {
t.Fatalf("RegisterWithSpec() error = %q, want stage error", err.Error())
}
}
func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) {
registry := NewExtractorRegistry()
if _, ok := registry.Spec("missing-extractor"); ok {
t.Fatal("Spec() ok = true, want false")
}
}
func TestExtractorRegistryRegisterRejectsEmptyKey(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.Register(" \t", fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("Register() error = nil, want error")
}
if !strings.Contains(err.Error(), "key must not be empty") {
t.Fatalf("Register() error = %q, want empty key error", err.Error())
}
}
func TestExtractorRegistryRegisterRejectsDuplicateKey(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
err := registry.Register(" generic-extractor ", fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("Register() error = nil, want error")
}
if !strings.Contains(err.Error(), "already registered") {
t.Fatalf("Register() error = %q, want duplicate key error", err.Error())
}
}
func TestExtractorRegistryRegisterRejectsNilConstructor(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.Register("generic-extractor", nil)
if err == nil {
t.Fatal("Register() error = nil, want error")
}
if !strings.Contains(err.Error(), "constructor") {
t.Fatalf("Register() error = %q, want constructor error", err.Error())
}
}
func TestExtractorRegistryBuildRejectsUnknownKey(t *testing.T) {
registry := NewExtractorRegistry()
_, err := registry.Build("missing-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "not registered") {
t.Fatalf("Build() error = %q, want unknown key error", err.Error())
}
}
func TestExtractorRegistryBuildWrapsConstructorError(t *testing.T) {
registry := NewExtractorRegistry()
constructorErr := errors.New("constructor failed")
if err := registry.Register("generic-extractor", func() (contracts.Extractor, error) {
return nil, constructorErr
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.Build("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !errors.Is(err, constructorErr) {
t.Fatalf("Build() error = %v, want wrapped constructor error", err)
}
if !strings.Contains(err.Error(), "generic-extractor") {
t.Fatalf("Build() error = %q, want key context", err.Error())
}
}
func TestExtractorRegistryBuildRejectsNilExtractor(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register("generic-extractor", func() (contracts.Extractor, error) {
return nil, nil
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.Build("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "returned nil") {
t.Fatalf("Build() error = %q, want nil extractor error", err.Error())
}
}
func TestExtractorRegistryBuildRejectsExtractorKeyMismatch(t *testing.T) {
registry := NewExtractorRegistry()
if err := registry.Register("generic-extractor", fakeExtractorConstructor("other-extractor")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.Build("generic-extractor")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "returned key") {
t.Fatalf("Build() error = %q, want key mismatch error", err.Error())
}
}
func TestExtractorRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
registry := NewExtractorRegistry()
for _, key := range []string{"zeta", "alpha", "middle"} {
if err := registry.Register(key, fakeExtractorConstructor(key)); err != nil {
t.Fatalf("Register(%q) error = %v, want nil", key, err)
}
}
keys := registry.RegisteredKeys()
want := []string{"alpha", "middle", "zeta"}
if !reflect.DeepEqual(keys, want) {
t.Fatalf("RegisteredKeys() = %#v, want %#v", keys, want)
}
keys[0] = "changed"
if got := registry.RegisteredKeys(); !reflect.DeepEqual(got, want) {
t.Fatalf("RegisteredKeys() after caller mutation = %#v, want %#v", got, want)
}
}
func TestExtractorRegistryNilRegistryBehavior(t *testing.T) {
var registry *ExtractorRegistry
if err := registry.Register("generic-extractor", fakeExtractorConstructor("generic-extractor")); err == nil {
t.Fatal("Register() error = nil, want error")
}
if _, err := registry.Build("generic-extractor"); err == nil {
t.Fatal("Build() error = nil, want error")
}
if _, ok := registry.Spec("generic-extractor"); ok {
t.Fatal("Spec() ok = true, want false")
}
if keys := registry.RegisteredKeys(); keys != nil {
t.Fatalf("RegisteredKeys() = %#v, want nil", keys)
}
}
func TestExtractorRegistryBuildRejectsEmptyKey(t *testing.T) {
registry := NewExtractorRegistry()
_, err := registry.Build(" \n")
if err == nil {
t.Fatal("Build() error = nil, want error")
}
if !strings.Contains(err.Error(), "key must not be empty") {
t.Fatalf("Build() error = %q, want empty key error", err.Error())
}
}
type registryFakeExtractor struct {
key string
}
func fakeExtractorConstructor(key string) ExtractorConstructor {
return func() (contracts.Extractor, error) {
return registryFakeExtractor{key: key}, nil
}
}
func (extractor registryFakeExtractor) Key() string {
return extractor.key
}
func (extractor registryFakeExtractor) ArtifactType() string {
return "generic-artifact"
}
func (extractor registryFakeExtractor) SchemaVersion() string {
return "v1"
}
func (extractor registryFakeExtractor) Validators() []contracts.Validator {
return nil
}
func (extractor registryFakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{}, nil
}

Some files were not shown because too many files have changed in this diff Show More