23 Commits

Author SHA1 Message Date
be6803ffa1 Document extraction reference support 2026-07-05 14:52:52 +00:00
ef4bdd4f9f Use references in D&D spell extraction 2026-07-05 14:49:17 +00:00
2f97895732 Record reference provenance in manifests 2026-07-05 14:44:44 +00:00
9e89b88efc Add prompt reference template functions 2026-07-05 14:37:39 +00:00
a57c6397e3 Materialize extraction reference files 2026-07-05 14:32:35 +00:00
39e071f5ca Add CLI reference binding flags 2026-07-05 14:27:23 +00:00
70d733edaf Resolve extraction reference bindings from config 2026-07-05 14:21:36 +00:00
1c31f56af1 Add reference contracts to extractor metadata 2026-07-05 14:13:48 +00:00
f9999a73df Add a staged implementation plan for background context references 2026-07-05 09:08:31 -05:00
11d8187052 Update documentation to reflect the implemented chunking module 2026-07-05 08:40:24 -05:00
86bff552c1 Update .gitgnore to ignore build artifacts 2026-07-05 08:39:34 -05:00
d3f790095e Trim and validate scene caveats 2026-07-05 13:30:32 +00:00
95218218e2 Canonicalize chunk units before extraction 2026-07-05 13:28:09 +00:00
e700df82d8 Record top-level module metadata in run manifests 2026-07-05 13:23:56 +00:00
e19cc02c4d Add a staged implementation plan to address gaps from the initial implementation of the scene chunking module 2026-07-05 08:15:46 -05:00
8a5419448f Update feature roadmaps to reflect future work 2026-07-05 08:11:54 -05:00
c8217549a8 Verify D&D scene chunker run output 2026-07-04 13:08:45 +00:00
2130414899 Register D&D scene chunker 2026-07-04 13:05:39 +00:00
7f83a20fa6 Implement D&D scene chunker 2026-07-04 13:02:02 +00:00
317ab0472d Add D&D scene chunking assets 2026-07-04 12:57:04 +00:00
e5eb0ba5c8 Make chunk validation a framework contract 2026-07-04 12:52:16 +00:00
b95af4f87d Add a roadmap to implement a D&D-specific chunk module 2026-07-04 07:48:36 -05:00
11073b613c Update AGENTS.md 2026-07-04 07:05:06 -05:00
78 changed files with 6205 additions and 140 deletions

4
.gitignore vendored
View File

@@ -1,3 +1,6 @@
# build artifacts
./notarius
# ---> Go # ---> Go
# If you prefer the allow list template instead of the deny list, see community template: # If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
@@ -49,6 +52,7 @@ go.work.sum
# Icon must end with two \r # Icon must end with two \r
Icon Icon
# Thumbnails # Thumbnails
._* ._*

View File

@@ -1,3 +1,6 @@
Please carefully review the documents in `docs/policy` before making any changes to this repository. Please review `docs/internal/overview.md` for initial orientation in this repository.
- `architecture.md` provides the canonical high-level architecture policy for this repository.
- `documentation.md` provides the canonical documentation policy for this repository. Additionally, please carefully review the relevant documents in `docs/policy` before making any changes to this repository.
- `development.md` defines the contributor workflow for this application.
- `architecture.md` provides the canonical high-level architecture policy for this repository, and should be reviewed before writing or changing any code.
- `documentation.md` provides the canonical documentation policy for this repository, and should be reviewed before writing or changing any documentation.

View File

@@ -20,7 +20,7 @@ a bearer token.
```text ```text
notarius help notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference slot=path] [--without-reference slot]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json] notarius pipelines list --config path/to/config.yml [--json]
``` ```
@@ -46,11 +46,59 @@ Flags:
invocation. invocation.
- `--llm-profile id`: override every effective module binding to use one LLM - `--llm-profile id`: override every effective module binding to use one LLM
profile. profile.
- `--reference slot=path`: bind a reference path to an extractor reference
slot. Repeatable. Use `lane.slot=path` when multiple selected lanes declare
the same slot.
- `--without-reference slot`: remove a configured optional reference binding.
Repeatable. Use `lane.slot` when multiple selected lanes declare the same
slot.
On success, the command prints the completed pipeline ID, approved and rejected On success, the command prints the completed pipeline ID, approved and rejected
artifact counts, and the output directory. If the run completes with warnings, artifact counts, and the output directory. If the run completes with warnings,
the warning count is printed to stderr. the warning count is printed to stderr.
Reference flags are resolved against selected artifact lanes before the run
starts. Flat slot names are accepted only when exactly one selected lane
declares that slot. Bound reference files are read before extraction, validated
as UTF-8 text, and passed only to the lane extractor that declares the slot.
Reference content is not written to diagnostics, logs, errors, or manifests.
Reference binding precedence is:
1. pipeline-level config `references`;
2. lane-level config `references`;
3. `--reference` run flags;
4. `--without-reference` run flags.
`--reference` binds or replaces one slot for one selected lane. Use
`slot=path` when the selected lanes declare the slot unambiguously:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference roster=./campaign-roster.txt
```
Use `lane.slot=path` when multiple selected lanes declare the same slot or when
you want to target a specific lane:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference spells.glossary=./campaign-glossary.txt
```
Use `--without-reference` to remove a configured optional binding for a run:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--without-reference glossary
```
For durable output, diagnostics, retention, and failure inspection, see For durable output, diagnostics, retention, and failure inspection, see
[Operations](operations.md). [Operations](operations.md).
@@ -117,7 +165,7 @@ go run ./cmd/notarius pipelines list \
The production CLI currently registers these module keys: The production CLI currently registers these module keys:
- input: `seriatim` - input: `seriatim`
- chunk: `generic` - chunk: `generic`, `dnd/scenes`
- extract: `dnd/spells` - extract: `dnd/spells`
- merge: `appendorder` - merge: `appendorder`
- normalize: `noop` - normalize: `noop`

View File

@@ -27,6 +27,9 @@ llm_profiles:
pipelines: pipelines:
dnd-session: dnd-session:
input: seriatim input: seriatim
references:
roster: ./dnd-spells-roster.txt
glossary: ./dnd-spells-glossary.txt
chunk: chunk:
module: generic module: generic
options: options:
@@ -121,6 +124,9 @@ Pipeline fields:
- `artifacts`: required for pipeline resolution. It maps artifact lane IDs to - `artifacts`: required for pipeline resolution. It maps artifact lane IDs to
lane definitions. lane definitions.
- `output`: optional module binding. Default module is `json`. - `output`: optional module binding. Default module is `json`.
- `references`: optional map of extractor reference slot names to reference
paths. These bindings are defaults for artifact lanes whose extractor declares
the matching slot.
Artifact lane fields: Artifact lane fields:
@@ -129,11 +135,52 @@ Artifact lane fields:
- `normalize`: optional module binding. Default module is `noop`. - `normalize`: optional module binding. Default module is `noop`.
- `validators`: optional list of module bindings. The production CLI currently - `validators`: optional list of module bindings. The production CLI currently
does not register validator modules. does not register validator modules.
- `references`: optional map of extractor reference slot names to reference
paths. Lane bindings override pipeline-level bindings for the same slot.
`notarius run` and `notarius config validate --pipeline` resolve the pipeline `notarius run` and `notarius config validate --pipeline` resolve the pipeline
against the production module catalog and fail fast for unknown or incompatible against the production module catalog and fail fast for unknown or incompatible
module keys. module keys.
Reference bindings are validated against extractor-declared slots during
pipeline resolution. Required slots must be bound after config defaults,
lane-level bindings, and run-time `--reference` or `--without-reference`
overrides are applied. Config-relative paths are resolved relative to the
config file; CLI reference paths are resolved relative to the current working
directory. Bound files must be UTF-8 text and are passed only to lane
extractors that declare the slot. Reference content is not written to
diagnostics, logs, errors, or manifests.
Pipeline-level `references` are defaults. They apply only to selected lanes
whose extractor declares the slot:
```yaml
pipelines:
dnd-session:
input: seriatim
references:
roster: ./campaign/party-roster.txt
glossary: ./campaign/glossary.txt
artifacts:
spells:
extract: dnd/spells
```
Lane-level `references` override or add bindings for one lane:
```yaml
pipelines:
dnd-session:
input: seriatim
references:
glossary: ./campaign/glossary.txt
artifacts:
spells:
extract: dnd/spells
references:
roster: ./campaign/session-roster.txt
```
## Module Bindings ## Module Bindings
Every module binding may use shorthand: Every module binding may use shorthand:
@@ -167,6 +214,7 @@ one configured profile.
| --- | --- | --- | | --- | --- | --- |
| input | `seriatim` | Reads Seriatim transcript JSON. | | input | `seriatim` | Reads Seriatim transcript JSON. |
| chunk | `generic` | Splits source units into ordered chunks. | | chunk | `generic` | Splits source units into ordered chunks. |
| chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. |
| extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. | | extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. |
| merge | `appendorder` | Keeps candidates in append order. | | merge | `appendorder` | Keeps candidates in append order. |
| normalize | `noop` | Passes merged artifacts through unchanged. | | normalize | `noop` | Passes merged artifacts through unchanged. |
@@ -178,6 +226,17 @@ The `generic` chunker accepts:
- `overlap_units`: non-negative integer, default `0`, and must be less than - `overlap_units`: non-negative integer, default `0`, and must be less than
`max_units`. `max_units`.
The `dnd/scenes` chunker requires transcript source capabilities, calls the
configured structured LLM provider, and does not accept module options.
The `dnd/spells` extractor declares optional text reference slots:
- `roster`
- `glossary`
The extractor uses these references only as supporting disambiguation material;
spell casts still must be present in the source transcript.
## Diagnostics ## Diagnostics
`diagnostics` fields: `diagnostics` fields:
@@ -211,3 +270,5 @@ Pipeline resolution additionally checks:
- required module keys are present; - required module keys are present;
- module keys are registered for the expected slot; - module keys are registered for the expected slot;
- module capability requirements are satisfied. - module capability requirements are satisfied.
- bound reference slots are declared by selected lane extractors;
- required reference slots are bound for selected lanes.

View File

@@ -17,6 +17,14 @@ The extractor requires source chunks and transcript source capability. It
returns generic artifact candidates that are serialized by the JSON output returns generic artifact candidates that are serialized by the JSON output
module. module.
The extractor accepts optional UTF-8 text references:
- `roster`: campaign roster or player-character notes.
- `glossary`: campaign glossary or spell/name notes.
References are supporting disambiguation material only. They are not source
evidence and are not addressable through `source_refs`.
## Artifact Envelope ## Artifact Envelope
Approved artifacts use the generic artifact envelope documented in Approved artifacts use the generic artifact envelope documented in
@@ -120,6 +128,11 @@ Rejection reason codes:
- `invalid_source_ref`: at least one source reference fails generic source - `invalid_source_ref`: at least one source reference fails generic source
reference validation. reference validation.
Warning reason codes:
- `spell_not_near_source`: the extracted spell name was not found in the cited
source text.
Rejected candidates are written to `rejected.json` by the JSON output module. Rejected candidates are written to `rejected.json` by the JSON output module.
## Manifest Metadata ## Manifest Metadata

View File

@@ -57,7 +57,19 @@ approved.
"pipeline_id": "dnd-session", "pipeline_id": "dnd-session",
"pipeline_digest": "sha256:...", "pipeline_digest": "sha256:...",
"input_module": "seriatim", "input_module": "seriatim",
"chunker": "generic", "chunker": "dnd/scenes",
"module_metadata": {
"chunker": {
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:...",
"response_schema_key": "dnd_scenes",
"response_schema_id": "notarius.dnd.scenes",
"response_schema_name": "notarius_dnd_scenes_v1",
"response_schema_version": "v1",
"response_schema_sha256": "sha256:..."
}
},
"source_digests": ["sha256:..."], "source_digests": ["sha256:..."],
"extractors": ["dnd/spells"], "extractors": ["dnd/spells"],
"merger": "appendorder", "merger": "appendorder",
@@ -86,9 +98,39 @@ approved.
Fields with empty values may be omitted by JSON encoding. Fields with empty values may be omitted by JSON encoding.
`source_digests` contains source document digests only. Bound references are
recorded separately under `references`, which contains provenance only:
lane ID, slot name, origin type and URI, digest, media type, byte size, and
binding source. Reference content is not written to durable output.
When references are bound, the manifest section has this shape:
```json
{
"references": [
{
"lane_id": "spells",
"slot_name": "roster",
"origin_type": "file",
"origin_uri": "file:///absolute/path/roster.txt",
"digest": "sha256:...",
"media_type": "text/plain; charset=utf-8",
"size_bytes": 123,
"binding_source": "config"
}
]
}
```
`module_metadata` is omitted when no singleton module provides metadata.
`validation_status` is `approved` when no candidates were rejected and `validation_status` is `approved` when no candidates were rejected and
`rejected` when one or more candidates were rejected. `rejected` when one or more candidates were rejected.
Top-level `module_metadata` is reserved for singleton pipeline modules
(`input`, `chunker`, and `output`). Lane-owned module metadata remains under
`artifact_lanes[].metadata`.
## Artifact Files ## Artifact Files
Each artifact file has this shape: Each artifact file has this shape:

View File

@@ -33,6 +33,7 @@ Implemented artifact names:
- `invocation.json` - `invocation.json`
- `effective-config.json` - `effective-config.json`
- `resolved-pipeline.json` - `resolved-pipeline.json`
- `resolved-references.json`
- `source-document.json` - `source-document.json`
- `run-manifest.json` - `run-manifest.json`
- `run-report.json` - `run-report.json`

View File

@@ -20,6 +20,27 @@ A production module package should provide:
Module specs should describe capabilities accurately. Resolution uses specs to Module specs should describe capabilities accurately. Resolution uses specs to
reject incompatible pipelines before execution. reject incompatible pipelines before execution.
Extractor modules that accept auxiliary reference material must declare slots
through both `ReferenceSlots()` and `ModuleSpec().ReferenceSlots`. The runtime
slot list and registry metadata should match so config validation can inspect
slots without constructing extractor instances. A slot declaration names the
slot, whether it is required, accepted media types, whether multiple items are
allowed, and any byte limit.
Reference content is delivered only to the lane extractor through
`contracts.ExtractionRequest.References`. It is not source evidence and must not
be converted into `SourceRef` values. If a module prompt uses references, load
the prompt bundle with the same declared slots and render with
`RenderUserSystemWithReferences`. Prompt templates may use the `reference`
function for content and the `hasreference` function for conditional sections.
Prompt metadata hashes remain based on template source, not rendered reference
bytes.
Chunk modules receive the structured LLM client through `contracts.ChunkRequest`
when they need model-backed chunking. The pipeline runner validates generic
chunk result invariants before extraction; module-owned policies may be stricter
but must stay within the module package.
## `seriatim` Input ## `seriatim` Input
Package: `internal/modules/input/seriatim` Package: `internal/modules/input/seriatim`
@@ -44,6 +65,10 @@ The `generic` chunker splits source units into ordered chunks. It validates the
source document, clones source units, assigns chunk IDs such as `chunk-000001`, source document, clones source units, assigns chunk IDs such as `chunk-000001`,
and records chunk metadata for start unit, end unit, and unit count. and records chunk metadata for start unit, end unit, and unit count.
The pipeline runner canonicalizes chunk units from the source document by ID
before extractors and mergers run. Chunker-owned context should stay in
`SourceChunk.Metadata`.
Options: Options:
- `max_units`: positive integer, default `50`; - `max_units`: positive integer, default `50`;
@@ -54,6 +79,39 @@ Provides:
- `chunks` - `chunks`
## `dnd/scenes` Chunker
Package: `internal/modules/chunk/dnd/scenes`
The `dnd/scenes` chunker uses the structured LLM client to divide transcript
source units into coherent D&D scenes. It renders embedded prompts, loads the
embedded structured response schema, validates model-authored source-unit
boundaries, and converts each scene into a deterministic source chunk.
Requires:
- `source.transcript`
Provides:
- `chunks`
- `chunks.scenes`
Options: none. Non-empty options are rejected.
The chunker enforces full source-unit coverage from the first source unit to the
last, exact source-unit IDs, sequential contiguous scenes, and no overlap. It
assigns chunk IDs such as `scene-000001` and stores scene metadata including
title, primary mode, participants, summary, boundary note, confidence, boundary
unit IDs, and unit count. Boundary caveats become warnings with reason code
`scene_boundary_caveat`. Whitespace-only caveats are treated as malformed
structured output rather than silently dropped.
Malformed model output fails explicitly rather than falling back to another
chunker. The chunker exposes prompt and response-schema provenance through
top-level `module_metadata.chunker` without raw prompts, raw schemas, source
text, or secrets.
## `dnd/spells` Extractor ## `dnd/spells` Extractor
Package: `internal/modules/extract/dnd/spells` Package: `internal/modules/extract/dnd/spells`
@@ -78,15 +136,22 @@ Artifact type and schema version:
- schema version: `v1` - schema version: `v1`
The extractor adds prompt and response-schema provenance to lane manifest The extractor adds prompt and response-schema provenance to lane manifest
metadata. Durable artifact payload details belong in the metadata under `artifact_lanes[].metadata.extractor`. Durable artifact payload
details belong in the
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md). [D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
The extractor declares optional `roster` and `glossary` reference slots accepting
UTF-8 text. Its prompt frames references as supporting disambiguation material
only; spell-cast artifacts must still be grounded in the source transcript.
## D&D Spell Validators ## D&D Spell Validators
The spell extractor returns two built-in validators: The spell extractor returns two built-in validators:
- `dnd/spells/shape`: rejects malformed payloads and missing required fields. - `dnd/spells/shape`: rejects malformed payloads and missing required fields.
- `dnd/spells/source_refs`: rejects candidates without valid source references. - `dnd/spells/source_refs`: rejects candidates without valid source references.
It also emits a warning when the extracted spell name is not found in the
cited source text.
Reason codes include: Reason codes include:
@@ -94,6 +159,7 @@ Reason codes include:
- `missing_required_field` - `missing_required_field`
- `missing_source_ref` - `missing_source_ref`
- `invalid_source_ref` - `invalid_source_ref`
- `spell_not_near_source`
These validators are supplied by the extractor when no validators are configured These validators are supplied by the extractor when no validators are configured
for the lane. for the lane.

View File

@@ -30,6 +30,33 @@ before execution:
The CLI writes the resolved pipeline and digest to diagnostics. The CLI writes the resolved pipeline and digest to diagnostics.
Pipeline profiles and artifact lanes may include reference binding maps keyed by
extractor reference slot name. During resolution, pipeline-level bindings act as
defaults for selected lanes whose extractor declares the slot, lane-level
bindings override or add lane bindings, runtime `--reference` requests override
config bindings, and runtime unbinds remove optional bindings. Flat runtime slot
names are resolved only when exactly one selected lane declares the slot;
otherwise the CLI requires `lane.slot`. Resolution validates bindings against
extractor specs and records lane-scoped binding metadata. It does not read
reference files or include reference bytes in source digests.
During run preparation, resolved file references are materialized before any
LLM-backed pipeline work. Config bindings resolve relative to the config file,
CLI bindings resolve relative to the current working directory, and materialized
reference content is passed only to the matching lane extractor through
`ExtractionRequest`. Materialization accepts UTF-8 text files, computes
`sha256:` content digests, records file origins, enforces declared byte limits,
and warns for empty bound files. Reference content is omitted from diagnostics
and manifests. The CLI writes provenance-only resolved reference diagnostics,
and the run manifest records lane-scoped reference provenance separately from
source digests.
Prompt bundles can declare reference slots and use `reference` and
`hasreference` template functions. Bundle loading validates string-literal slot
names against the declaration. Rendering receives a lane reference set from the
caller; unbound optional slots render as empty strings, and `hasreference`
returns true only when at least one bound item has content.
## Registries And Module Specs ## Registries And Module Specs
`pipeline.Registries` holds concrete constructors for execution. A `pipeline.Registries` holds concrete constructors for execution. A
@@ -44,6 +71,10 @@ Every production module registers a `ModuleSpec` with:
- `Provides`: capabilities added after that module runs; - `Provides`: capabilities added after that module runs;
- `Requires`: capabilities that must already be available. - `Requires`: capabilities that must already be available.
Extractor specs may also declare reference slots. Slot declarations are
available from registry metadata without constructing extractor instances.
Non-extractor module specs must not declare reference slots.
Capability checks prevent incompatible pipeline composition before a run starts. Capability checks prevent incompatible pipeline composition before a run starts.
## Runner Input And Output ## Runner Input And Output
@@ -73,8 +104,39 @@ The runner:
2. builds the input adapter and parses the raw input into a source document; 2. builds the input adapter and parses the raw input into a source document;
3. validates the source document; 3. validates the source document;
4. builds the chunker and produces source chunks; 4. builds the chunker and produces source chunks;
5. runs each selected artifact lane in sorted resolved order; 5. validates source chunks against framework invariants;
6. builds the output encoder and validates logical output file names. 6. runs each selected artifact lane in sorted resolved order;
7. builds the output encoder and validates logical output file names.
## Chunk Results
Chunkers implement `contracts.Chunker` and receive a `contracts.ChunkRequest`
with the validated source document, the structured LLM client, the configured
LLM profile, module options, and run metadata. Deterministic and LLM-backed
chunkers use the same contract; provider construction stays outside chunk
modules.
After `Chunk` returns, the runner appends chunker warnings before returning any
chunker error. When chunking succeeds, the runner validates generic chunk
invariants before running extractors:
- chunk IDs must be non-empty and unique in the chunk result;
- each chunk `SourceID` must match the source document ID;
- each chunk `Index` must match its zero-based returned order;
- each chunk must contain at least one source unit;
- a chunk must not repeat a source unit;
- every chunk source unit must exist in the source document;
- source units inside each chunk must appear in source-document order.
After validation, the runner rebuilds each chunk from source-document units by
ID, preserving the chunk boundary order and cloning chunk metadata. Extractors
and downstream stages therefore see canonical source units, while
`SourceChunk.Metadata` remains the supported place for chunker-owned context.
The framework does not require complete source-unit coverage and does not reject
overlap between different chunks. Stricter policies, such as full coverage or
non-overlap, belong to individual chunk modules when they are part of that
module's contract.
Within an artifact lane, the runner: Within an artifact lane, the runner:
@@ -119,9 +181,15 @@ On successful execution, the manifest validation status is:
## Manifest Population ## Manifest Population
The manifest records run ID, pipeline ID, pipeline digest, module keys, artifact The manifest records run ID, pipeline ID, pipeline digest, module keys, top-level
lanes, LLM profile metadata, source digest, validation status, and timing. module metadata, artifact lanes, LLM profile metadata, source digest,
reference provenance, validation status, and timing.
Modules can add non-secret manifest metadata by implementing Singleton pipeline modules may add non-secret metadata by implementing
`contracts.ManifestMetadataProvider`. The D&D spell extractor uses this for `contracts.ManifestMetadataProvider`. The runner records that metadata under
`module_metadata` with stable keys for `input`, `chunker`, and `output`.
Lane-owned modules may add non-secret metadata through
`artifact_lanes[].metadata`. The runner records extractor, merger, and
normalizer metadata there. The D&D spell extractor uses lane metadata for
prompt and response-schema provenance. prompt and response-schema provenance.

View File

@@ -34,8 +34,9 @@ The `json` output module writes these files:
- `index.json`: file index with paths to the manifest, artifact files, - `index.json`: file index with paths to the manifest, artifact files,
rejected artifacts, and warnings. rejected artifacts, and warnings.
- `manifest.json`: run manifest with resolved pipeline provenance, module keys, - `manifest.json`: run manifest with resolved pipeline provenance, top-level
validation status, and timing. module metadata, module keys, reference provenance, validation status, and
timing.
- `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact - `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact
type. For the current D&D spell extractor, this includes type. For the current D&D spell extractor, this includes
`artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved. `artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved.
@@ -62,8 +63,11 @@ Implemented diagnostics artifacts:
path, selected lanes, run ID, and pipeline digest when available. path, selected lanes, run ID, and pipeline digest when available.
- `effective-config.json`: resolved config with API keys redacted. - `effective-config.json`: resolved config with API keys redacted.
- `resolved-pipeline.json`: resolved module bindings and pipeline digest. - `resolved-pipeline.json`: resolved module bindings and pipeline digest.
- `resolved-references.json`: lane-scoped resolved reference provenance,
including origin, digest, media type, byte size, and binding source, without
reference content.
- `run-manifest.json`: the same run manifest written to durable output when it - `run-manifest.json`: the same run manifest written to durable output when it
is available. is available, including top-level module metadata when present.
- `warnings.json`: warning list. - `warnings.json`: warning list.
- `run-report.json`: counts, status, output path, diagnostics path, and run ID. - `run-report.json`: counts, status, output path, diagnostics path, and run ID.
- `error.log`: failure message, written after diagnostics directory creation - `error.log`: failure message, written after diagnostics directory creation
@@ -107,6 +111,10 @@ stderr, and writes warnings to durable output and diagnostics when retained.
The run manifest `validation_status` indicates whether final artifacts were The run manifest `validation_status` indicates whether final artifacts were
approved or rejected after validation. approved or rejected after validation.
Reference-related warnings include empty bound reference files and D&D spell
relatedness warnings such as `spell_not_near_source`. Empty references are still
passed to extractors so optional slots can be intentionally blank.
## Cleanup ## Cleanup
It is safe to remove specific old run directories after their output and It is safe to remove specific old run directories after their output and

View File

@@ -0,0 +1,305 @@
# Extraction References Implementation Plan
## Purpose
Implement the extraction-reference feature described in
[references.md](references.md). This plan is decision-complete for an LLM coding
agent: implement each stage in order, keep the repository compiling after each
stage, and do not move planned behavior into non-roadmap docs until the relevant
behavior exists.
Core decisions to preserve:
- references are opaque framework inputs and domain semantics stay in extract
modules;
- references are not evidence and must not be addressable through `SourceRef`;
- reference binding is lane-scoped;
- extractors expose `ReferenceSlots()` directly on the first-class extractor
contract;
- token budgeting is deferred; enforce only UTF-8 text handling, empty-file
warnings, and declared `MaxBytes`;
- run manifests record references in a dedicated section, separate from
`source_digests`;
- CLI unbinding uses `--without-reference`.
## Stage 1: Contracts and Mechanical Adoption
Add the framework contracts needed to describe references without changing
runtime behavior. Slot declarations must be available without constructing
extractor modules, because pipeline/config validation should use registry
metadata rather than runtime module instances.
Implementation steps:
- In `internal/framework/contracts`, add reference model types:
`ReferenceSlot`, `ReferenceOrigin`, `ReferenceItem`,
`ResolvedReferenceSlot`, `ReferenceSet`, and a binding-source enum or string
constants for `config` and `cli`.
- Include `Name`, `Description`, `Required`, `AcceptedMediaTypes`, `Multiple`,
and `MaxBytes` on `ReferenceSlot`.
- Include slot name, media type, content bytes, digest, origin, size bytes, and
binding source on `ReferenceItem`.
- Add `References ReferenceSet` to `contracts.ExtractionRequest`.
- Add `ReferenceSlots() []ReferenceSlot` to `contracts.Extractor`.
- Extend extractor registration metadata so reference slots are also declared
through the extractor's registry spec. Prefer the smallest idiomatic change to
the existing registry model, such as adding `ReferenceSlots` to `ModuleSpec`
with validation that non-extractor modules leave it empty, unless the codebase
shape clearly supports a narrower extractor-specific spec.
- Update every concrete extractor and all extractor fakes/test doubles to
implement `ReferenceSlots()`. Existing extractors without references should
return `nil`.
- Add tests that compare a production extractor's runtime `ReferenceSlots()`
with its registered spec slots so the two declarations cannot drift.
- Add contract tests for empty reference sets, slot copying expectations if
helpers are introduced, and compile-time coverage through existing fakes.
Verification:
- `go test ./internal/framework/contracts`
- `go test ./internal/framework/pipeline`
- `go test ./...`
## Stage 2: Config Shape and Pipeline-Level Resolution
Add unresolved reference bindings to config and resolved lane bindings to the
pipeline model. Do not read reference files in this stage.
Implementation steps:
- Add `references` maps to file config parsing at both pipeline and artifact
lane level.
- Add corresponding fields to `pipeline.PipelineProfile` and
`pipeline.ArtifactLaneProfile`.
- Preserve deterministic map handling and duplicate-after-trim validation.
- Extend config cloning, effective config, redaction, validation, and tests for
the new fields.
- Add resolved reference binding structures to `internal/framework/pipeline`.
They should represent lane ID, slot name, source URI/path, and binding source,
but not file bytes.
- During `pipeline.ResolvePipeline`, collect selected lanes, read each lane
extractor's declared slots from registry metadata, and validate without
building extractor instances:
- every bound slot is declared by the lane extractor;
- required slots are bound after applying pipeline-level and lane-level config;
- required slots remain bound after any CLI unbinds supplied to resolution;
- selected lanes under `--only` are the only lanes considered.
- Apply pipeline-level bindings as defaults only to lanes whose extractor
declares the matching slot.
- Apply lane-level bindings as overrides or additions for that lane.
- Keep reference bindings out of source digests and artifact source references.
- Add tests proving reference-slot validation works through registry specs even
when extractor constructors would fail if called.
Verification:
- `go test ./internal/core/config`
- `go test ./internal/framework/pipeline`
- `go test ./...`
## Stage 3: CLI Reference Overrides and Unbinds
Add run-time CLI syntax for reference binding overrides and optional unbinding.
Implementation steps:
- Add repeatable `--reference` flags to `notarius run`.
Accepted forms:
- `slot=path` for unambiguous slot names across selected lanes;
- `lane.slot=path` for explicit lane-scoped binding.
- Add repeatable `--without-reference` flags to `notarius run`.
Accepted forms:
- `slot`;
- `lane.slot`.
- Reject empty paths for `--reference`; use `--without-reference` for unbinding.
- Reject malformed values with concise CLI errors before expensive work.
- Pass parsed override/unbind requests into config/pipeline resolution.
- Resolve flat CLI names only when exactly one selected lane declares the slot.
If multiple selected lanes declare the same slot, fail and instruct the user
to use `lane.slot`.
- Let CLI bindings override config bindings for the same lane and slot.
- Let CLI unbinds remove config-bound optional slots for the same lane and slot.
- Fail if unbinding leaves a required slot unbound.
- Add CLI tests for flat binding, lane-qualified binding, ambiguous flat
binding, malformed syntax, optional unbind, and required-slot unbind failure.
Verification:
- `go test ./internal/cli`
- `go test ./internal/core/config`
- `go test ./internal/framework/pipeline`
- `go test ./...`
## Stage 4: Run Preparation and Reference Materialization
Read, validate, digest, and materialize resolved file references before any LLM
call.
Implementation steps:
- Add a reference resolver/materializer near pipeline run preparation. Keep file
I/O out of pure config parsing.
- Ensure run preparation receives the loaded config path or config directory so
config-relative reference paths can be resolved after pure config parsing.
- Resolve config-relative paths relative to the config file path and
CLI-relative paths relative to the current working directory.
- For MVP, accept only UTF-8 text files. Reject non-UTF-8 content with an error
naming pipeline, lane, slot, and path.
- Compute `sha256:` content digests over the raw reference bytes.
- Populate `ReferenceItem` values with content bytes, media type, digest,
origin type `file`, normalized origin URI/path, size bytes, and binding source.
- Enforce declared `MaxBytes` when greater than zero. The error should name the
pipeline, lane, slot, actual size, limit, and path.
- Emit a warning for empty bound files, but do not fail.
- Add `ReferenceSet` values to the runner input or resolved pipeline path in a
way that keeps lane-scoped references available when calling each extractor.
- Pass the correct lane-specific `ReferenceSet` into
`contracts.ExtractionRequest`.
- Ensure no reference content is written to ordinary diagnostics, logs, errors,
or manifests.
Verification:
- Focused resolver/materializer tests for path resolution, digest stability,
UTF-8 rejection, empty-file warning, `MaxBytes`, and binding source.
- `go test ./internal/cli`
- `go test ./internal/framework/pipeline`
- `go test ./...`
## Stage 5: Prompt Template Reference Functions
Make references available to module-owned prompt templates.
Implementation steps:
- Extend `internal/framework/prompt` so prompt bundles can be compiled with
declared reference slots.
- Add `reference` and `hasreference` template functions.
- Validate at bundle build time, or the earliest feasible equivalent, that
templates reference only declared slots.
- Render a declared but unbound optional slot as an empty string.
- Ensure `hasreference` returns true only when the slot has at least one bound
item with content.
- Render multiple items deterministically if future `Multiple` support is
enabled; for MVP, reject multiple bindings unless the slot declares
`Multiple`.
- Keep prompt metadata hashes based on template source. Do not include rendered
reference content in prompt identity.
- Add deterministic rendering tests proving byte-identical output across runs
with the same reference bytes and config.
Verification:
- `go test ./internal/framework/prompt`
- `go test ./internal/modules/extract/dnd/spells`
- `go test ./...`
## Stage 6: Manifest and Diagnostics Provenance
Record reference provenance separately from source provenance.
Implementation steps:
- Add a dedicated references section to `artifacts.RunManifest`.
The shape should be lane-scoped and include lane ID, slot name, origin type,
origin URI/path, digest, media type, size bytes, and binding source.
- Do not add reference digests to `source_digests`.
- Include reference digests in any cache/idempotency key if such a key exists.
If no cache/idempotency key exists, add a test or comment documenting that no
additional key needs updating yet.
- Write a diagnostics artifact for resolved references that contains provenance
only, not full content, consistent with redacted effective config behavior.
- Ensure durable JSON output manifests include the new manifest section.
- Add manifest round-trip tests and a CLI/run test where two runs that differ
only in reference bytes produce distinguishable manifests.
Verification:
- `go test ./internal/core/artifacts`
- `go test ./internal/core/diagnostics`
- `go test ./internal/modules/output/json`
- `go test ./internal/cli`
- `go test ./...`
## Stage 7: D&D Spells Consumer
Use the new reference feature in the first production extractor.
Implementation steps:
- Declare optional `roster` and `glossary` slots on `dnd/spells`.
- Set accepted media type to text/UTF-8. Add conservative `MaxBytes` limits only
if a clear module-owned limit is chosen; otherwise leave `MaxBytes` unset.
- Update the D&D spells prompt bundle to include conditional reference sections
using `hasreference` and `reference`.
- Frame references as supporting material only. The prompt must instruct the
model to extract only spell-cast events present in the source transcript and
use references only for disambiguation.
- Update prompt metadata tests as needed while preserving template-hash
semantics.
- Add fixture coverage with no references, with roster/glossary references, and
with a roster that mentions a spell never cast in the transcript. The last
case must assert no spell-cast artifact is produced for the uncast spell.
- If existing deterministic source-reference validation can be extended
cleanly, add warning-level relatedness checks for spell names or close
variants near cited source text. If this becomes large, defer that validator
enhancement to a separate roadmap item and keep the prompt/regression fixture
guard in this stage.
Verification:
- `go test ./internal/modules/extract/dnd/spells`
- `go test ./internal/framework/pipeline`
- `go test ./internal/cli`
- `go test ./...`
## Stage 8: Canonical Documentation and Examples
Move implemented behavior out of roadmap-only status once code exists.
Implementation steps:
- Update `docs/cli.md` with `--reference` and `--without-reference` syntax,
precedence, ambiguity behavior, and examples.
- Update `docs/config.md` with pipeline-level and lane-level `references`
blocks.
- Update `docs/internal/modules.md` or the most appropriate internal docs with
module-author guidance for `ReferenceSlots()`, reference request delivery,
prompt functions, evidence exclusion, and provenance.
- Update `docs/internal/pipeline.md` with reference resolution lifecycle and
lane-scoped delivery.
- Update `docs/integrations/json-output.md` with the manifest reference
provenance shape.
- Update `docs/operations.md` or `docs/troubleshooting.md` for common reference
errors such as unknown slot, ambiguous flat override, missing required slot,
unreadable file, non-UTF-8 content, and `MaxBytes` failures.
- Add maintained example reference files and update `examples/dnd-spells.config.yml`
only after the CLI/config behavior is implemented and covered by tests.
- Keep future-only material in `docs/roadmap/references.md`; do not duplicate
canonical current behavior there after implementation.
Verification:
- `rg -n "references:|--reference|--without-reference|ReferenceSlots|reference \"|hasreference" docs examples`
- `go test ./...`
- `go vet ./...`
- `go build ./cmd/notarius`
## Final Acceptance Criteria
The feature is complete when:
- extractor modules can declare reference slots through the first-class
extractor contract;
- config and CLI can bind and unbind lane-scoped file references;
- selected-pipeline validation catches unknown, ambiguous, or missing required
references before any LLM call;
- run preparation materializes UTF-8 text references with digests, size checks,
and empty-file warnings;
- extractors receive lane-scoped resolved references;
- prompt templates can render `reference` and `hasreference` deterministically;
- D&D spell extraction uses optional roster and glossary references;
- run manifests and diagnostics record reference provenance without recording
full content or treating references as source evidence;
- canonical docs and maintained examples describe only implemented behavior;
- `go test ./...`, `go vet ./...`, and `go build ./cmd/notarius` pass.

238
docs/roadmap/references.md Normal file
View File

@@ -0,0 +1,238 @@
# Feature Roadmap: Extraction References
## Status
This document defines the target state and policy choices for the planned
extraction-reference feature in Notarius. It describes planned behavior, not
implemented behavior. The staged implementation plan lives in
[implementation.md](implementation.md).
## Goal
Extraction quality improves when an LLM-backed extractor receives stable
reference material alongside the source input. For the initial D&D spell
extractor, useful reference material includes a party roster, a player list, and
a campaign glossary.
Notarius should support passing this material to extractors as **named reference
items** without introducing domain-specific concepts into core or framework
packages. The framework should know only that:
- extractors declare named reference slots they accept;
- pipeline config and CLI flags bind content, initially files, to those slots;
- bound content is rendered into module-owned prompt templates;
- bound content is digested and recorded as run provenance.
Only extract modules should know what a "roster" or "glossary" means. Domain
semantics live in module-owned slot declarations and prompt templates.
## Definitions
- **Reference slot**: a named, typed-by-convention input declared by an
extractor, with a human-readable description, required/optional status, and
optional guardrails such as accepted media types and maximum bytes.
- **Reference item**: resolved content bound to a slot for a given run: slot
name, content bytes, media type, content digest, size, origin, and binding
source.
- **Reference binding**: the association of a slot name to a content source,
defined in pipeline config and overridable per run via CLI.
- **Reference set**: the lane-scoped collection of resolved reference items
delivered to an extractor.
## Architectural Principles
- References are opaque to the framework. Core and framework packages must not
interpret reference content or recognize domain slot names.
- References are inputs. Anything that can change extraction output must be
digested into the run manifest and participate in any cache or idempotency key.
- References are not evidence. `SourceRef` values must only ever reference
source units. Reference items must not receive unit IDs and must not be
addressable by source references.
- Slots are declared, not ad hoc. Binding an undeclared slot name, or omitting a
required slot, should fail before any LLM call.
- Reference delivery is lane-scoped. Pipeline-level bindings may apply to
multiple lanes, but each lane receives only the references declared by its
extractor after pipeline, lane, CLI override, and CLI unbind rules are
resolved.
- Optional slots degrade gracefully. Prompt templates should render cleanly
whether or not an optional slot is bound.
- Rendering must be deterministic. Identical source input, config, prompts, and
reference bytes should produce byte-identical rendered prompts. Reference
slots should render in declaration order, with stable binding order within a
slot.
## Target Contracts
Extractors should declare accepted reference slots directly on the extractor
contract. This is a first-class feature, so mechanical updates to existing
extractors and test fakes are acceptable.
The target slot declaration includes:
- `Name`;
- `Description`;
- `Required`;
- `AcceptedMediaTypes`;
- `Multiple`;
- `MaxBytes`.
Extractors with no reference needs return an empty slot list.
Resolved reference items should be content-bearing values, not unresolved file
paths. The MVP producer is "read this file," but the item shape should permit
future producers such as prior-run artifacts, derived summaries, or entity
registries without changing extractor-facing contracts.
The extraction request should carry the lane-scoped resolved reference set.
Framework and core code should treat the set as opaque bytes plus metadata.
## Binding Lifecycle
Reference handling should be split across existing lifecycle boundaries:
1. Config parsing records pipeline-level and lane-level reference bindings
without reading files.
2. Pipeline resolution validates selected lanes, declared extractor slots,
missing required slots, unknown bindings, and ambiguous flat CLI bindings.
3. Run preparation resolves paths, reads files, validates media type and size,
computes digests, and materializes reference items.
4. Extraction receives the lane-specific resolved reference set.
Config-relative paths resolve relative to the config file. CLI-relative paths
resolve relative to the current working directory.
## Configuration and CLI
Reference bindings should live in pipeline config because the initial use cases
are campaign-invariant more often than run-variant. Bindings should be supported
at two levels:
- pipeline level: defaults shared by artifact lanes whose extractors declare
matching slots;
- lane level: additions or overrides for a single artifact lane.
Illustrative config shape:
```yaml
pipelines:
dnd-session:
input: seriatim
references:
roster: ./campaign/party_roster.md
glossary: ./campaign/glossary.md
artifacts:
spells:
extract: dnd/spells
npcs:
extract: dnd/npcs
references:
npc_registry: ./campaign/npcs.md
```
Per-run CLI binding overrides should be repeatable:
```text
notarius run dnd-session --input session-014.json --reference roster=./alt_roster.md
```
Flat CLI slot names are allowed when unambiguous across selected lanes. Lane
qualified names, such as `spells.roster=./alt_roster.md`, disambiguate or target
a specific lane. CLI bindings override config bindings for the same lane and
slot.
Users should also be able to unbind a config-bound optional slot for a run with
an explicit repeatable flag:
```text
notarius run dnd-session --input session-014.json --without-reference roster
```
Unbinding a required slot should fail during pipeline/reference resolution.
## Prompt Template Integration
Prompt templates are module-owned. Template rendering should expose:
- `{{ reference "roster" }}`: renders the content of the bound item;
- `{{ hasreference "glossary" }}`: predicate for conditional sections, so
optional slots can be included only when bound.
Rules:
- Referencing an undeclared slot from a template is a module bug and should fail
at prompt registration/build time or the earliest feasible equivalent.
- Referencing a declared but unbound optional slot should render as empty;
templates should use `hasreference` to avoid dangling section headers.
- Rendering must be deterministic and independent of map iteration order.
- Prompt identity should be computed over the template, not the rendered prompt.
Reference digests are recorded separately in the manifest so a reference edit
is visible as a reference change, not a prompt change.
Reference content is repeated in every per-chunk prompt in the MVP. Per-slot or
per-chunk inclusion policies are deferred until cost data justifies them.
## Provenance
The run manifest must record resolved references separately from source
digests. For every bound lane and slot, it should record:
- lane ID;
- slot name;
- origin type and URI;
- content digest;
- media type;
- size in bytes;
- whether the binding came from config or CLI override.
Reference digests must participate in any idempotency/cache key alongside source
digests, prompt hashes, schema versions, model, and parameters. Two runs that
differ only in reference content must be distinguishable from the manifest
alone.
Diagnostics for a run should include the resolved binding set with digests, not
full reference content, consistent with the existing redacted-effective-config
pattern.
## Validation and Guardrails
### References Are Not Evidence
The primary new failure mode is the model extracting facts from references
rather than from the source input. For example, a roster may list a player
character's known spells, and the model might emit a spell-cast artifact for a
spell that was never cast in the session.
Defenses, in priority order:
1. **Structural.** `SourceRef` remains the only grounding mechanism and can only
reference source units. No contract change should make references
addressable as evidence.
2. **Prompt discipline.** Module templates should frame references explicitly as
reference material, such as "use the roster to resolve speakers to
characters; extract only events that occur in the transcript."
3. **Validator support.** The source-reference validator, or a sibling
deterministic validator, should warn when referenced source text does not
plausibly relate to the extracted fact. Severity should be `warn`, not
`fail`, because transcripts can use paraphrase, nicknames, and abbreviations.
4. **Regression fixtures.** Tests should include a fixture in which a bound
roster mentions a spell that is never cast in the transcript, asserting no
artifact record is produced for it.
### Size and Sanity Guardrails
- MVP accepts UTF-8 text content only. Other media types should be rejected with
a clear error.
- A slot-level `MaxBytes` value should be enforced when declared.
- Empty bound files should produce a warning because they are likely user error.
## Out of Scope
- Token budgeting and model context-window management for references.
- Non-file reference producers, including prior-run artifacts, derived
summaries, and entity registries.
- Per-chunk or per-slot inclusion policies.
- Structured or parsed references such as typed roster schemas. References are
opaque text handed to prompts.
- Reference caching, preprocessing, summarization, embedding, or retrieval.
- Making references addressable as evidence in any form.

View File

@@ -102,6 +102,34 @@ go run ./cmd/notarius run dnd-session \
- For `config validate`, include `--pipeline` when using `--only`. - For `config validate`, include `--pipeline` when using `--only`.
- Confirm the lane ID exists under `pipelines.<id>.artifacts`. - Confirm the lane ID exists under `pipelines.<id>.artifacts`.
## Reference Binding Failure
Symptoms include:
- `reference slot "..." is not declared`
- `reference slot "..." is declared by multiple selected lanes`
- `required reference slot "..." is not bound`
- `--reference must use slot=path or lane.slot=path`
- `--without-reference must use slot or lane.slot`
- `read "...": no such file`
- `must be UTF-8 text`
- `is ... bytes, limit ...`
Fix:
- Confirm the selected extractor declares the slot. The implemented
`dnd/spells` extractor declares optional `roster` and `glossary` slots.
- Use `lane.slot=path` when more than one selected lane declares the same slot.
- Use `--without-reference slot` to remove optional config bindings; do not pass
an empty `--reference slot=`.
- Check whether a path came from config or CLI. Config paths are relative to
the config file. CLI reference paths are relative to the current working
directory.
- Ensure the file is readable UTF-8 text and within any byte limit declared by
the extractor.
- If diagnostics are retained, inspect `resolved-pipeline.json`,
`resolved-references.json`, and `error.log`.
## Seriatim Input Validation Failure ## Seriatim Input Validation Failure
Symptoms include `seriatim input`, `parse JSON`, `segments must not be empty`, Symptoms include `seriatim input`, `parse JSON`, `segments must not be empty`,
@@ -177,6 +205,34 @@ Fix:
Provider error messages are redacted for configured API key values. Provider error messages are redacted for configured API key values.
## Scene Chunking Failure
Symptoms include:
- `dnd scenes chunker`
- `malformed structured output`
- `boundary_caveats`
- `start_unit_id`
- `end_unit_id`
- `gap`
- `overlap`
- `final scene`
- `complete structured output`
Fix:
- Validate the pipeline configuration and confirm the input module provides a
transcript source when using `chunk: dnd/scenes`.
- Confirm the LLM profile has a working OpenAI-compatible `base_url`, `model`,
and credentials.
- Inspect retained diagnostics for the run error and resolved pipeline.
- If the error names malformed structured output, retry with a model that
follows structured response schemas reliably.
- If the error names `boundary_caveats`, check for blank or whitespace-only
caveat text in the scene response.
- Scene boundaries must use exact source-unit IDs, cover the full source
document, be contiguous, and not overlap.
## Output Write Failure ## Output Write Failure
Symptoms include: Symptoms include:

View File

@@ -0,0 +1,2 @@
Cure Wounds: healing spell cast by touch.
Shield: defensive reaction spell.

View File

@@ -0,0 +1,3 @@
Aria: party cleric and recurring healer.
Borin: fighter ally.
Bandit mage: hostile spellcaster.

View File

@@ -7,6 +7,9 @@ llm_profiles:
pipelines: pipelines:
dnd-session: dnd-session:
input: seriatim input: seriatim
references:
roster: ./dnd-spells-roster.txt
glossary: ./dnd-spells-glossary.txt
chunk: chunk:
module: generic module: generic
options: options:

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic" "gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim" "gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
@@ -34,6 +35,9 @@ func productionRegistries() (pipeline.Registries, error) {
if err := generic.Register(registries.Chunkers); err != nil { if err := generic.Register(registries.Chunkers); err != nil {
return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err) return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err)
} }
if err := scenes.Register(registries.Chunkers); err != nil {
return pipeline.Registries{}, fmt.Errorf("register dnd scenes chunker: %w", err)
}
if err := spells.Register(registries.Extractors); err != nil { if err := spells.Register(registries.Extractors); err != nil {
return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err) return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err)
} }

View File

@@ -25,7 +25,7 @@ const defaultOutputRoot = "./notarius-output"
const usage = `Usage: const usage = `Usage:
notarius help notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference slot=path] [--without-reference slot]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json] notarius pipelines list --config path/to/config.yml [--json]
` `
@@ -95,6 +95,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
outputDir := fs.String("output-dir", "", "output directory") outputDir := fs.String("output-dir", "", "output directory")
diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory") diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory")
llmProfile := fs.String("llm-profile", "", "LLM profile override") llmProfile := fs.String("llm-profile", "", "LLM profile override")
referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{}
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path or lane.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, as slot or lane.slot")
if err := fs.Parse(reorderRunArgs(args)); err != nil { if err := fs.Parse(reorderRunArgs(args)); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err) fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2 return 2
@@ -121,6 +125,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stderr, "notarius: %v\n", err) fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2 return 2
} }
referenceRequests, err := parseReferenceFlags(referenceFlags)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
referenceUnbindRequests, err := parseReferenceUnbindFlags(withoutReferenceFlags)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
cfg, loadedConfigPath, err := loadConfig(*configPath, opts) cfg, loadedConfigPath, err := loadConfig(*configPath, opts)
if err != nil { if err != nil {
@@ -155,15 +169,33 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil { if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err) return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
} }
referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests)
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
effective, err := cfg.Resolve(config.ResolveInput{ effective, err := cfg.Resolve(config.ResolveInput{
PipelineID: pipelineID, PipelineID: pipelineID,
Only: only, Only: only,
Catalog: catalog, Catalog: catalog,
LLMProfileOverride: *llmProfile, LLMProfileOverride: *llmProfile,
ReferenceOverrides: referenceOverrides,
ReferenceUnbinds: referenceUnbinds,
}) })
if err != nil { if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err) return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
} }
workingDir, err := os.Getwd()
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("resolve working directory: %w", err))
}
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
ConfigPath: loadedConfigPath,
WorkingDir: workingDir,
})
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
effective.ResolvedPipeline = materialized
invocation.PipelineDigest = effective.ResolvedPipeline.Digest invocation.PipelineDigest = effective.ResolvedPipeline.Digest
if err := runDir.WriteInvocationMetadata(invocation); err != nil { if err := runDir.WriteInvocationMetadata(invocation); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err)) return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
@@ -174,6 +206,9 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err := runDir.WriteResolvedPipeline(effective.ResolvedPipeline); err != nil { if err := runDir.WriteResolvedPipeline(effective.ResolvedPipeline); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved pipeline: %w", err)) return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved pipeline: %w", err))
} }
if err := runDir.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline)); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved references: %w", err))
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline) profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if len(profileIDs) != 1 { if len(profileIDs) != 1 {
@@ -205,6 +240,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
StartedAt: startedAt, StartedAt: startedAt,
LLMProfiles: llmProfiles, LLMProfiles: llmProfiles,
Metadata: runMetadata(*outputDir, *diagnosticsDir), Metadata: runMetadata(*outputDir, *diagnosticsDir),
Warnings: referenceWarnings,
}) })
if err != nil { if err != nil {
if output.Manifest.PipelineID != "" { if output.Manifest.PipelineID != "" {
@@ -412,7 +448,7 @@ func reorderRunArgs(args []string) []string {
func runFlagTakesValue(arg string) bool { func runFlagTakesValue(arg string) bool {
switch arg { switch arg {
case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile": case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile", "--reference", "--without-reference":
return true return true
default: default:
return false return false
@@ -661,6 +697,255 @@ func parseOnly(raw string) ([]string, error) {
return result, nil return result, nil
} }
type stringListFlag []string
func (flag *stringListFlag) String() string {
if flag == nil {
return ""
}
return strings.Join(*flag, ",")
}
func (flag *stringListFlag) Set(value string) error {
*flag = append(*flag, value)
return nil
}
type cliReferenceRequest struct {
LaneID string
SlotName string
Source string
}
type cliReferenceUnbindRequest struct {
LaneID string
SlotName string
}
func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
if len(values) == 0 {
return nil, nil
}
requests := make([]cliReferenceRequest, 0, len(values))
for _, raw := range values {
name, source, ok := strings.Cut(raw, "=")
if !ok {
return nil, fmt.Errorf("--reference must use slot=path or lane.slot=path")
}
if strings.TrimSpace(source) == "" {
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
}
laneID, slotName, err := parseReferenceSelector(name, "--reference")
if err != nil {
return nil, err
}
requests = append(requests, cliReferenceRequest{
LaneID: laneID,
SlotName: slotName,
Source: strings.TrimSpace(source),
})
}
return requests, nil
}
func parseReferenceUnbindFlags(values []string) ([]cliReferenceUnbindRequest, error) {
if len(values) == 0 {
return nil, nil
}
requests := make([]cliReferenceUnbindRequest, 0, len(values))
for _, raw := range values {
if strings.Contains(raw, "=") {
return nil, fmt.Errorf("--without-reference must use slot or lane.slot")
}
laneID, slotName, err := parseReferenceSelector(raw, "--without-reference")
if err != nil {
return nil, err
}
requests = append(requests, cliReferenceUnbindRequest{
LaneID: laneID,
SlotName: slotName,
})
}
return requests, nil
}
func parseReferenceSelector(raw string, flagName string) (string, string, error) {
selector := strings.TrimSpace(raw)
if selector == "" {
return "", "", fmt.Errorf("%s reference slot must not be empty", flagName)
}
if strings.Count(selector, ".") > 1 {
return "", "", fmt.Errorf("%s must use slot or lane.slot", flagName)
}
laneID := ""
slotName := selector
if strings.Contains(selector, ".") {
before, after, _ := strings.Cut(selector, ".")
laneID = strings.TrimSpace(before)
slotName = strings.TrimSpace(after)
if laneID == "" || slotName == "" {
return "", "", fmt.Errorf("%s must use non-empty lane.slot values", flagName)
}
}
return laneID, slotName, nil
}
func resolveCLIReferenceRequests(
cfg config.Config,
pipelineID string,
only []string,
catalog pipeline.ModuleCatalog,
referenceRequests []cliReferenceRequest,
unbindRequests []cliReferenceUnbindRequest,
) ([]pipeline.ReferenceBinding, []pipeline.ReferenceUnbind, error) {
if len(referenceRequests) == 0 && len(unbindRequests) == 0 {
return nil, nil, nil
}
selected, err := selectedReferenceLanes(cfg, pipelineID, only, catalog)
if err != nil {
return nil, nil, err
}
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
for _, request := range referenceRequests {
laneID, err := resolveCLIReferenceLane(selected, request.LaneID, request.SlotName)
if err != nil {
return nil, nil, err
}
overrides = append(overrides, pipeline.ReferenceBinding{
LaneID: laneID,
SlotName: request.SlotName,
Source: request.Source,
BindingSource: contracts.ReferenceBindingSourceCLI,
})
}
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
for _, request := range unbindRequests {
laneID, err := resolveCLIReferenceLane(selected, request.LaneID, request.SlotName)
if err != nil {
return nil, nil, err
}
unbinds = append(unbinds, pipeline.ReferenceUnbind{
LaneID: laneID,
SlotName: request.SlotName,
})
}
return overrides, unbinds, nil
}
type selectedReferenceLane struct {
id string
slots map[string]struct{}
}
func selectedReferenceLanes(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceLane, error) {
profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID)
if !ok {
return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID))
}
lanesByID := make(map[string]pipeline.ArtifactLaneProfile, len(profile.Artifacts))
for rawLaneID, lane := range profile.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
return nil, fmt.Errorf("pipeline %q artifact lane id must not be empty", strings.TrimSpace(pipelineID))
}
if _, ok := lanesByID[laneID]; ok {
return nil, fmt.Errorf("pipeline %q artifact lane %q is duplicated after trimming", strings.TrimSpace(pipelineID), laneID)
}
lanesByID[laneID] = lane
}
selectedIDs := make([]string, 0, len(lanesByID))
if len(only) == 0 {
for laneID := range lanesByID {
selectedIDs = append(selectedIDs, laneID)
}
} else {
seen := make(map[string]struct{}, len(only))
for _, rawLaneID := range only {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
return nil, fmt.Errorf("pipeline %q selected artifact lane id must not be empty", strings.TrimSpace(pipelineID))
}
if _, ok := lanesByID[laneID]; !ok {
return nil, fmt.Errorf("pipeline %q selected artifact lane %q is not declared", strings.TrimSpace(pipelineID), laneID)
}
if _, ok := seen[laneID]; !ok {
selectedIDs = append(selectedIDs, laneID)
seen[laneID] = struct{}{}
}
}
}
sort.Strings(selectedIDs)
selected := make([]selectedReferenceLane, 0, len(selectedIDs))
for _, laneID := range selectedIDs {
lane := lanesByID[laneID]
extractModule := strings.TrimSpace(lane.Extract.Module)
if extractModule == "" {
return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID)
}
if catalog.Extractors == nil {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", strings.TrimSpace(pipelineID), laneID, extractModule, extractModule)
}
spec, ok := catalog.Extractors.Spec(extractModule)
if !ok {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", strings.TrimSpace(pipelineID), laneID, extractModule, extractModule)
}
slotSet := make(map[string]struct{}, len(spec.ReferenceSlots))
for _, slot := range spec.ReferenceSlots {
slotSet[slot.Name] = struct{}{}
}
selected = append(selected, selectedReferenceLane{id: laneID, slots: slotSet})
}
return selected, nil
}
func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
pipelineID = strings.TrimSpace(pipelineID)
for rawID, profile := range profiles {
if strings.TrimSpace(rawID) == pipelineID {
return profile, true
}
}
return pipeline.PipelineProfile{}, false
}
func resolveCLIReferenceLane(selected []selectedReferenceLane, requestedLaneID string, slotName string) (string, error) {
slotName = strings.TrimSpace(slotName)
requestedLaneID = strings.TrimSpace(requestedLaneID)
if requestedLaneID != "" {
for _, lane := range selected {
if lane.id == requestedLaneID {
if _, ok := lane.slots[slotName]; !ok {
return "", fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, requestedLaneID)
}
return requestedLaneID, nil
}
}
return "", fmt.Errorf("reference lane %q is not selected", requestedLaneID)
}
matches := make([]string, 0, 1)
for _, lane := range selected {
if _, ok := lane.slots[slotName]; ok {
matches = append(matches, lane.id)
}
}
switch len(matches) {
case 0:
return "", fmt.Errorf("reference slot %q is not declared by any selected lane", slotName)
case 1:
return matches[0], nil
default:
return "", fmt.Errorf("reference slot %q is declared by multiple selected lanes (%s); use lane.slot", slotName, strings.Join(matches, ", "))
}
}
func sortedPipelineIDs(cfg config.Config) []string { func sortedPipelineIDs(cfg config.Config) []string {
ids := make([]string, 0, len(cfg.Pipelines)) ids := make([]string, 0, len(cfg.Pipelines))
for id := range cfg.Pipelines { for id := range cfg.Pipelines {

View File

@@ -8,14 +8,17 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"reflect" "reflect"
"sort"
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic" "gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim" "gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
@@ -136,6 +139,11 @@ func TestProductionCatalogIncludesDefaultModules(t *testing.T) {
got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(generic.Key) }, got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(generic.Key) },
want: generic.ModuleSpec(), want: generic.ModuleSpec(),
}, },
{
name: "dnd scenes chunker",
got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(scenes.Key) },
want: scenes.ModuleSpec(),
},
{ {
name: "dnd spells extractor", name: "dnd spells extractor",
got: func() (pipeline.ModuleSpec, bool) { return catalog.Extractors.Spec(spells.Key) }, got: func() (pipeline.ModuleSpec, bool) { return catalog.Extractors.Spec(spells.Key) },
@@ -186,6 +194,21 @@ func TestRunConfigValidateUsesProductionCatalogByDefault(t *testing.T) {
} }
} }
func TestRunConfigValidateAcceptsDNDScenesChunker(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells"))
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "is valid for pipeline") {
t.Fatalf("stdout = %q, want validation success", stdout.String())
}
}
func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) { func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "missing/extract")) configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "missing/extract"))
var stdout bytes.Buffer var stdout bytes.Buffer
@@ -716,6 +739,219 @@ func TestRunPipelineLLMProfileOverrideSelectsFactoryProfile(t *testing.T) {
} }
} }
func TestRunPipelineReferenceFlagBindsUnambiguousSlot(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := filepath.Join(t.TempDir(), "missing.json")
referencePath := writeFile(t, "roster.yml", "Aria\n")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--reference", "roster=" + referencePath,
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
}),
})
if code != 1 || !strings.Contains(stderr.String(), "read input") {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
refs := resolved.ArtifactLanes[0].References
want := []pipeline.ReferenceBinding{
{LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI},
}
if !reflect.DeepEqual(refs, want) {
t.Fatalf("resolved references = %#v, want %#v", refs, want)
}
}
func TestRunPipelineReferenceFlagBindsLaneQualifiedSlot(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events", "notes"))
inputPath := filepath.Join(t.TempDir(), "missing.json")
referencePath := writeFile(t, "notes.yml", "Notes\n")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--only", "events,notes",
"--diagnostics-dir", diagnosticsDir,
"--reference", "notes.roster=" + referencePath,
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
}),
})
if code != 1 || !strings.Contains(stderr.String(), "read input") {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
events := resolvedArtifactLane(t, resolved, "events")
if len(events.References) != 0 {
t.Fatalf("events references = %#v, want none", events.References)
}
notes := resolvedArtifactLane(t, resolved, "notes")
if len(notes.References) != 1 || notes.References[0].Source != referencePath {
t.Fatalf("notes references = %#v, want lane-qualified binding", notes.References)
}
}
func TestRunPipelineReferenceFlagRejectsAmbiguousFlatSlot(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events", "notes"))
inputPath := writeSeriatimInput(t)
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--reference", "roster=./roster.yml",
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
}),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "multiple selected lanes") || !strings.Contains(stderr.String(), "lane.slot") {
t.Fatalf("stderr = %q, want ambiguous reference error", stderr.String())
}
}
func TestRunPipelineReferenceFlagsRejectMalformedValues(t *testing.T) {
tests := []struct {
name string
args []string
want string
}{
{name: "missing equals", args: []string{"--reference", "roster"}, want: "slot=path"},
{name: "empty path", args: []string{"--reference", "roster="}, want: "path must not be empty"},
{name: "empty slot", args: []string{"--reference", "=./roster.yml"}, want: "slot must not be empty"},
{name: "too many selector parts", args: []string{"--reference", "a.b.c=./roster.yml"}, want: "lane.slot"},
{name: "unbind with equals", args: []string{"--without-reference", "roster=./roster.yml"}, want: "slot or lane.slot"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAML("example", "events"))
inputPath := writeSeriatimInput(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
args := []string{"run", "example", "--config", configPath, "--input", inputPath}
args = append(args, test.args...)
code := RunWithOptions(args, &stdout, &stderr, Options{Catalog: fakeCatalog(t)})
if code != 2 {
t.Fatalf("RunWithOptions() code = %d, want 2", code)
}
if !strings.Contains(stderr.String(), test.want) {
t.Fatalf("stderr = %q, want substring %q", stderr.String(), test.want)
}
})
}
}
func TestRunPipelineWithoutReferenceRemovesOptionalConfigBinding(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAMLWithReferences("example", "events", map[string]string{"roster": "./config-roster.yml"}))
inputPath := filepath.Join(t.TempDir(), "missing.json")
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--without-reference", "roster",
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
}),
})
if code != 1 || !strings.Contains(stderr.String(), "read input") {
t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String())
}
resolved := readResolvedPipeline(t, diagnosticsDir)
if refs := resolved.ArtifactLanes[0].References; len(refs) != 0 {
t.Fatalf("references = %#v, want unbound optional slot", refs)
}
}
func TestRunPipelineWithoutReferenceFailsWhenRequiredSlotWouldBeMissing(t *testing.T) {
configPath := writeTestConfig(t, testConfigYAMLWithReferences("example", "events", map[string]string{"roster": "./config-roster.yml"}))
inputPath := writeSeriatimInput(t)
diagnosticsDir := t.TempDir()
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "example",
"--config", configPath,
"--input", inputPath,
"--diagnostics-dir", diagnosticsDir,
"--without-reference", "roster",
}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
}),
})
if code != 1 {
t.Fatalf("RunWithOptions() code = %d, want 1", code)
}
if !strings.Contains(stderr.String(), "required reference slot") || !strings.Contains(stderr.String(), "roster") {
t.Fatalf("stderr = %q, want required reference error", stderr.String())
}
}
func TestRunPipelineWritesDurableOutputFiles(t *testing.T) { func TestRunPipelineWritesDurableOutputFiles(t *testing.T) {
diagnosticsDir := t.TempDir() diagnosticsDir := t.TempDir()
outputDir := t.TempDir() outputDir := t.TempDir()
@@ -749,6 +985,74 @@ func TestRunPipelineWritesDurableOutputFiles(t *testing.T) {
assertNoTemporaryFiles(t, runOutputDir) assertNoTemporaryFiles(t, runOutputDir)
} }
func TestRunPipelineReferenceBytesProduceDistinctManifests(t *testing.T) {
run := func(t *testing.T, referenceText string) artifacts.RunManifest {
t.Helper()
diagnosticsDir := t.TempDir()
outputDir := t.TempDir()
configDir := t.TempDir()
referencePath := filepath.Join(configDir, "roster.txt")
if err := os.WriteFile(referencePath, []byte(referenceText), 0o644); err != nil {
t.Fatalf("write reference: %v", err)
}
configPath := filepath.Join(configDir, "config.yml")
if err := os.WriteFile(configPath, []byte(testConfigYAMLWithReferencesAndDiagnostics("example", "events", diagnosticsDir, map[string]string{"roster": "roster.txt"})), 0o644); err != nil {
t.Fatalf("write config: %v", err)
}
inputPath := writeFile(t, "source.txt", "source text")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "example", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
}),
Registries: fakeExecutionRegistries(t),
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
var manifest artifacts.RunManifest
readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest)
if len(manifest.References) != 1 {
t.Fatalf("manifest references = %#v, want one entry", manifest.References)
}
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) {
t.Fatalf("source digests = %#v, want source-only digest", manifest.SourceDigests)
}
var resolvedReferences []artifacts.ReferenceProvenance
readJSONFile(t, filepath.Join(onlyChildDir(t, diagnosticsDir), diagnostics.ArtifactResolvedReferences), &resolvedReferences)
if !reflect.DeepEqual(resolvedReferences, manifest.References) {
t.Fatalf("resolved references = %#v, want manifest references %#v", resolvedReferences, manifest.References)
}
resolvedReferenceJSON := string(readFile(t, filepath.Join(onlyChildDir(t, diagnosticsDir), diagnostics.ArtifactResolvedReferences)))
if strings.Contains(resolvedReferenceJSON, referenceText) || strings.Contains(resolvedReferenceJSON, "content") {
t.Fatalf("resolved references diagnostics contains content: %s", resolvedReferenceJSON)
}
return manifest
}
first := run(t, "first roster")
second := run(t, "second roster")
if first.References[0].Digest == second.References[0].Digest {
t.Fatalf("reference digests match for different bytes: %q", first.References[0].Digest)
}
if first.PipelineDigest != second.PipelineDigest {
t.Fatalf("pipeline digests differ = %q vs %q, want reference bytes outside pipeline identity", first.PipelineDigest, second.PipelineDigest)
}
}
func TestRunPipelineRejectsUnsafeOutputFileName(t *testing.T) { func TestRunPipelineRejectsUnsafeOutputFileName(t *testing.T) {
diagnosticsDir := t.TempDir() diagnosticsDir := t.TempDir()
outputDir := t.TempDir() outputDir := t.TempDir()
@@ -1041,6 +1345,81 @@ func TestExampleFixtureRunWritesExpectedJSON(t *testing.T) {
} }
} }
func TestExampleFixtureRunWithDNDScenesRecordsChunkerAndWarnings(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells"))
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
outputDir := t.TempDir()
diagnosticsDir := t.TempDir()
client := newSceneRunLLMClient("Scene boundary was ambiguous.")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(client, nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if client.calls != 2 {
t.Fatalf("LLM calls = %d, want chunking and extraction calls", client.calls)
}
if !strings.Contains(stderr.String(), "1 warning") {
t.Fatalf("stderr = %q, want warning count", stderr.String())
}
runOutputDir := onlyChildDir(t, outputDir)
manifestBytes := readFile(t, filepath.Join(runOutputDir, "manifest.json"))
var manifest artifacts.RunManifest
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
if manifest.Chunker != scenes.Key {
t.Fatalf("manifest chunker = %q, want %q", manifest.Chunker, scenes.Key)
}
chunkerMetadata := manifest.ModuleMetadata["chunker"]
if chunkerMetadata == nil {
t.Fatalf("module metadata chunker = %#v, want object", manifest.ModuleMetadata["chunker"])
}
wantMetadataKeys := []string{
"prompt_id",
"prompt_version",
"prompt_sha256",
"response_schema_key",
"response_schema_id",
"response_schema_name",
"response_schema_version",
"response_schema_sha256",
}
if len(chunkerMetadata) != len(wantMetadataKeys) {
t.Fatalf("chunker metadata keys = %#v, want %d keys", chunkerMetadata, len(wantMetadataKeys))
}
for _, key := range wantMetadataKeys {
value, ok := chunkerMetadata[key]
if !ok {
t.Fatalf("chunker metadata missing key %q: %#v", key, chunkerMetadata)
}
if _, ok := value.(string); !ok {
t.Fatalf("chunker metadata[%q] = %#v, want string", key, value)
}
}
for _, forbidden := range []string{"prompt", "schema", "source", "text", "payload", "api_key", "secret", "token"} {
if _, ok := chunkerMetadata[forbidden]; ok {
t.Fatalf("chunker metadata leaked forbidden key %q: %#v", forbidden, chunkerMetadata)
}
}
for _, forbidden := range []string{"Source document ID:", "Aria casts Cure Wounds.", "spell_casts", "Scene boundary was ambiguous."} {
if strings.Contains(string(manifestBytes), forbidden) {
t.Fatalf("manifest leaked %q: %s", forbidden, manifestBytes)
}
}
warnings := string(readFile(t, filepath.Join(runOutputDir, "warnings.json")))
if !strings.Contains(warnings, "scene_boundary_caveat") || !strings.Contains(warnings, "Scene boundary was ambiguous.") {
t.Fatalf("warnings output = %s, want scene boundary caveat", warnings)
}
}
func TestExampleFixtureRunOnlySpells(t *testing.T) { func TestExampleFixtureRunOnlySpells(t *testing.T) {
configPath := fixturePath(t, "examples/dnd-spells.config.yml") configPath := fixturePath(t, "examples/dnd-spells.config.yml")
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json") inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
@@ -1209,6 +1588,51 @@ func testConfigYAMLForPipelines(pipelines map[string][]string) string {
return b.String() return b.String()
} }
func testConfigYAMLWithReferences(pipelineID string, laneID string, references map[string]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("pipelines:\n")
b.WriteString(" " + pipelineID + ":\n")
b.WriteString(" input: fake/input\n")
b.WriteString(" artifacts:\n")
b.WriteString(" " + laneID + ":\n")
b.WriteString(" extract: fake/extract\n")
b.WriteString(" references:\n")
keys := make([]string, 0, len(references))
for key := range references {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
b.WriteString(" " + key + ": " + references[key] + "\n")
}
return b.String()
}
func testConfigYAMLWithReferencesAndDiagnostics(pipelineID string, laneID string, diagnosticsDir string, references map[string]string) string {
var b strings.Builder
b.WriteString("version: 1\n")
b.WriteString("diagnostics:\n")
b.WriteString(" work_dir: " + diagnosticsDir + "\n")
b.WriteString(" retention: always\n")
b.WriteString("pipelines:\n")
b.WriteString(" " + pipelineID + ":\n")
b.WriteString(" input: fake/input\n")
b.WriteString(" artifacts:\n")
b.WriteString(" " + laneID + ":\n")
b.WriteString(" extract: fake/extract\n")
b.WriteString(" references:\n")
keys := make([]string, 0, len(references))
for key := range references {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
b.WriteString(" " + key + ": " + references[key] + "\n")
}
return b.String()
}
func mvpConfigYAML(pipelineID string, extractor string) string { func mvpConfigYAML(pipelineID string, extractor string) string {
return `version: 1 return `version: 1
pipelines: pipelines:
@@ -1220,6 +1644,18 @@ pipelines:
` `
} }
func mvpConfigYAMLWithChunk(pipelineID string, chunker string, extractor string) string {
return `version: 1
pipelines:
` + pipelineID + `:
input: seriatim
chunk: ` + chunker + `
artifacts:
spells:
extract: ` + extractor + `
`
}
func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string { func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
var b strings.Builder var b strings.Builder
b.WriteString("version: 1\n") b.WriteString("version: 1\n")
@@ -1287,6 +1723,7 @@ type fakeRunLLMClient struct {
calls int calls int
err error err error
payload map[string]any payload map[string]any
sceneCaveat string
} }
func newFakeRunLLMClient(invalidSourceRef bool) *fakeRunLLMClient { func newFakeRunLLMClient(invalidSourceRef bool) *fakeRunLLMClient {
@@ -1301,11 +1738,43 @@ func newMalformedRunLLMClient() *fakeRunLLMClient {
return &fakeRunLLMClient{payload: map[string]any{}} return &fakeRunLLMClient{payload: map[string]any{}}
} }
func newSceneRunLLMClient(caveat string) *fakeRunLLMClient {
return &fakeRunLLMClient{sceneCaveat: caveat}
}
func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.calls++ client.calls++
if client.err != nil { if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err return contracts.StructuredCompletionResponse{}, client.err
} }
if req.StageName == scenes.Key && client.payload == nil {
payload := map[string]any{
"scenes": []map[string]any{
{
"start_unit_id": "seg-001",
"end_unit_id": "seg-002",
"short_title": "Opening spell",
"primary_mode": "Narrative",
"main_participants": []string{"Aria"},
"summary": "Aria casts a spell.",
"boundary_note": "The provided source units form one scene.",
"boundary_confidence": "High",
},
},
"boundary_caveats": []string{},
}
if client.sceneCaveat != "" {
payload["boundary_caveats"] = []string{client.sceneCaveat}
}
encoded, err := json.Marshal(payload)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
if err := json.Unmarshal(encoded, out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: encoded}, nil
}
startUnitID := "seg-001" startUnitID := "seg-001"
if client.invalidSourceRef { if client.invalidSourceRef {
startUnitID = "missing-segment" startUnitID = "missing-segment"
@@ -1412,6 +1881,153 @@ func registriesWithOutput(t *testing.T, encoder contracts.OutputEncoder) pipelin
return registries return registries
} }
func fakeExecutionRegistries(t *testing.T) pipeline.Registries {
t.Helper()
inputs := pipeline.NewInputAdapterRegistry()
chunkers := pipeline.NewChunkerRegistry()
extractors := pipeline.NewExtractorRegistry()
mergers := pipeline.NewMergerRegistry()
normalizers := pipeline.NewNormalizerRegistry()
outputs := pipeline.NewOutputEncoderRegistry()
if err := inputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "fake/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func() (contracts.InputAdapter, error) {
return fakeRunInputAdapter{}, nil
}); err != nil {
t.Fatalf("register fake input: %v", err)
}
if err := chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "generic", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}}, func() (contracts.Chunker, error) {
return fakeRunChunker{}, nil
}); err != nil {
t.Fatalf("register fake chunker: %v", err)
}
if err := extractors.RegisterWithSpec(pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
}, func() (contracts.Extractor, error) {
return fakeRunExtractor{}, nil
}); err != nil {
t.Fatalf("register fake extractor: %v", err)
}
if err := mergers.RegisterWithSpec(pipeline.ModuleSpec{Key: "appendorder", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}}, func() (contracts.Merger, error) {
return fakeRunMerger{}, nil
}); err != nil {
t.Fatalf("register fake merger: %v", err)
}
if err := normalizers.RegisterWithSpec(pipeline.ModuleSpec{Key: "noop", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}}, func() (contracts.Normalizer, error) {
return fakeRunNormalizer{}, nil
}); err != nil {
t.Fatalf("register fake normalizer: %v", err)
}
if err := jsonoutput.Register(outputs); err != nil {
t.Fatalf("register json output: %v", err)
}
return pipeline.Registries{
Inputs: inputs,
Chunkers: chunkers,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Outputs: outputs,
}
}
type fakeRunInputAdapter struct{}
func (fakeRunInputAdapter) Key() string {
return "fake/input"
}
func (fakeRunInputAdapter) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
return &source.SourceDocument{
ID: "source",
Kind: "text",
Format: "test",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: "unit-1", Kind: "text", Text: string(req.Raw)},
},
}, nil
}
type fakeRunChunker struct{}
func (fakeRunChunker) Key() string {
return "generic"
}
func (fakeRunChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{
{ID: "chunk-1", SourceID: req.Source.ID, Index: 0, Units: req.Source.Units},
},
}, nil
}
type fakeRunExtractor struct{}
func (fakeRunExtractor) Key() string {
return "fake/extract"
}
func (fakeRunExtractor) ArtifactType() string {
return "fake.artifact"
}
func (fakeRunExtractor) SchemaVersion() string {
return "v1"
}
func (fakeRunExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return []contracts.ReferenceSlot{{Name: "roster"}}
}
func (fakeRunExtractor) Validators() []contracts.Validator {
return nil
}
func (fakeRunExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
return contracts.ExtractionResult{
Candidates: []artifacts.ArtifactCandidate{
{
Payload: []byte(`{"value":true}`),
SourceRefs: []source.SourceRef{
{SourceID: "source", StartUnitID: "unit-1", EndUnitID: "unit-1"},
},
},
},
}, nil
}
type fakeRunMerger struct{}
func (fakeRunMerger) Key() string {
return "appendorder"
}
func (fakeRunMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
var candidates []artifacts.ArtifactCandidate
for _, chunkArtifacts := range req.ChunkArtifacts {
candidates = append(candidates, chunkArtifacts.Candidates...)
}
return contracts.MergeResult{Candidates: candidates}, nil
}
type fakeRunNormalizer struct{}
func (fakeRunNormalizer) Key() string {
return "noop"
}
func (fakeRunNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: append([]artifacts.ArtifactCandidate(nil), req.Candidates...)}, nil
}
func onlyChildDir(t *testing.T, root string) string { func onlyChildDir(t *testing.T, root string) string {
t.Helper() t.Helper()
children := childDirs(t, root) children := childDirs(t, root)
@@ -1455,6 +2071,25 @@ func readJSONFile(t *testing.T, path string, out any) {
} }
} }
func readResolvedPipeline(t *testing.T, diagnosticsDir string) pipeline.ResolvedPipeline {
t.Helper()
runDir := onlyChildDir(t, diagnosticsDir)
var resolved pipeline.ResolvedPipeline
readJSONFile(t, filepath.Join(runDir, diagnostics.ArtifactResolvedPipeline), &resolved)
return resolved
}
func resolvedArtifactLane(t *testing.T, resolved pipeline.ResolvedPipeline, laneID string) pipeline.ResolvedArtifactLane {
t.Helper()
for _, lane := range resolved.ArtifactLanes {
if lane.ID == laneID {
return lane
}
}
t.Fatalf("lane %q not found in resolved pipeline", laneID)
return pipeline.ResolvedArtifactLane{}
}
func assertNoTemporaryFiles(t *testing.T, root string) { func assertNoTemporaryFiles(t *testing.T, root string) {
t.Helper() t.Helper()
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
@@ -1470,7 +2105,7 @@ func assertNoTemporaryFiles(t *testing.T, root string) {
} }
} }
func fakeCatalog(t *testing.T) pipeline.ModuleCatalog { func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.ModuleCatalog {
t.Helper() t.Helper()
inputs := pipeline.NewInputAdapterRegistry() inputs := pipeline.NewInputAdapterRegistry()
chunkers := pipeline.NewChunkerRegistry() chunkers := pipeline.NewChunkerRegistry()
@@ -1480,12 +2115,24 @@ func fakeCatalog(t *testing.T) pipeline.ModuleCatalog {
validators := pipeline.NewValidatorRegistry() validators := pipeline.NewValidatorRegistry()
outputs := pipeline.NewOutputEncoderRegistry() outputs := pipeline.NewOutputEncoderRegistry()
mustRegisterInput(t, inputs, pipeline.ModuleSpec{Key: "fake/input", Stage: pipeline.StageInput, Provides: []string{"source"}}) specs := map[string]pipeline.ModuleSpec{
mustRegisterChunker(t, chunkers, pipeline.ModuleSpec{Key: "generic", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}}) "fake/input": {Key: "fake/input", Stage: pipeline.StageInput, Provides: []string{"source"}},
mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{Key: "fake/extract", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}}) "generic": {Key: "generic", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}},
mustRegisterMerger(t, mergers, pipeline.ModuleSpec{Key: "appendorder", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}}) "fake/extract": {Key: "fake/extract", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}},
mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{Key: "noop", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}}) "appendorder": {Key: "appendorder", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}},
mustRegisterOutput(t, outputs, pipeline.ModuleSpec{Key: "json", Stage: pipeline.StageOutput, Requires: []string{"normalized"}}) "noop": {Key: "noop", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}},
"json": {Key: "json", Stage: pipeline.StageOutput, Requires: []string{"normalized"}},
}
for _, override := range overrides {
specs[override.Key] = override
}
mustRegisterInput(t, inputs, specs["fake/input"])
mustRegisterChunker(t, chunkers, specs["generic"])
mustRegisterExtractor(t, extractors, specs["fake/extract"])
mustRegisterMerger(t, mergers, specs["appendorder"])
mustRegisterNormalizer(t, normalizers, specs["noop"])
mustRegisterOutput(t, outputs, specs["json"])
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
Inputs: inputs, Inputs: inputs,

View File

@@ -48,23 +48,36 @@ type LLMProfileManifest struct {
Model string `json:"model,omitempty"` Model string `json:"model,omitempty"`
} }
type ReferenceProvenance struct {
LaneID string `json:"lane_id"`
SlotName string `json:"slot_name"`
OriginType string `json:"origin_type"`
OriginURI string `json:"origin_uri,omitempty"`
Digest string `json:"digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
SizeBytes int64 `json:"size_bytes,omitempty"`
BindingSource string `json:"binding_source,omitempty"`
}
type RunManifest struct { type RunManifest struct {
RunID string `json:"run_id,omitempty"` RunID string `json:"run_id,omitempty"`
PipelineID string `json:"pipeline_id,omitempty"` PipelineID string `json:"pipeline_id,omitempty"`
PipelineDigest string `json:"pipeline_digest,omitempty"` PipelineDigest string `json:"pipeline_digest,omitempty"`
InputModule string `json:"input_module,omitempty"` InputModule string `json:"input_module,omitempty"`
Chunker string `json:"chunker,omitempty"` Chunker string `json:"chunker,omitempty"`
SourceDigests []string `json:"source_digests,omitempty"` SourceDigests []string `json:"source_digests,omitempty"`
Extractors []string `json:"extractors,omitempty"` Extractors []string `json:"extractors,omitempty"`
Merger string `json:"merger,omitempty"` Merger string `json:"merger,omitempty"`
Normalizer string `json:"normalizer,omitempty"` Normalizer string `json:"normalizer,omitempty"`
OutputEncoder string `json:"output_encoder,omitempty"` OutputEncoder string `json:"output_encoder,omitempty"`
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"` ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"` ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
SchemaVersion string `json:"schema_version,omitempty"` References []ReferenceProvenance `json:"references,omitempty"`
ValidationStatus string `json:"validation_status,omitempty"` LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"` SchemaVersion string `json:"schema_version,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"` ValidationStatus string `json:"validation_status,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
} }
func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact { func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact {

View File

@@ -183,6 +183,79 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata") assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
} }
func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
manifest := RunManifest{
References: []ReferenceProvenance{
{
LaneID: "events",
SlotName: "roster",
OriginType: "file",
OriginURI: "file:///tmp/roster.txt",
Digest: "sha256:reference",
MediaType: "text/plain; charset=utf-8",
SizeBytes: 12,
BindingSource: "config",
},
},
}
gotJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var got RunManifest
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if len(got.References) != 1 {
t.Fatalf("len(References) = %d, want 1", len(got.References))
}
reference := got.References[0]
if reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference)
}
if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" {
t.Fatalf("reference provenance = %#v, want digest/media/size/source details", reference)
}
}
func TestRunManifestIncludesTopLevelModuleMetadata(t *testing.T) {
manifest := RunManifest{
ModuleMetadata: map[string]map[string]any{
"chunker": {
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:abc123",
"response_schema_key": "dnd_scenes",
"response_schema_name": "dnd_scenes",
},
},
}
gotJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var got map[string]any
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
moduleMetadata, ok := got["module_metadata"].(map[string]any)
if !ok {
t.Fatalf("module_metadata = %#v, want object", got["module_metadata"])
}
assertHasKeys(t, moduleMetadata, "chunker")
chunkerMetadata, ok := moduleMetadata["chunker"].(map[string]any)
if !ok {
t.Fatalf("module_metadata.chunker = %#v, want object", moduleMetadata["chunker"])
}
assertHasKeys(t, chunkerMetadata, "prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_name")
}
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) { func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
t.Helper() t.Helper()

View File

@@ -73,6 +73,7 @@ func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile
out.Input = cloneModuleBinding(in.Input) out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk) out.Chunk = cloneModuleBinding(in.Chunk)
out.Output = cloneModuleBinding(in.Output) out.Output = cloneModuleBinding(in.Output)
out.References = cloneStringMap(in.References)
if len(in.Artifacts) > 0 { if len(in.Artifacts) > 0 {
out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts)) out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts))
for key, lane := range in.Artifacts { for key, lane := range in.Artifacts {
@@ -87,6 +88,7 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
out.Extract = cloneModuleBinding(in.Extract) out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge) out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize) out.Normalize = cloneModuleBinding(in.Normalize)
out.References = cloneStringMap(in.References)
if len(in.Validators) > 0 { if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators)) out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators { for i, binding := range in.Validators {
@@ -96,6 +98,17 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
return out return out
} }
func cloneStringMap(in map[string]string) map[string]string {
if len(in) == 0 {
return nil
}
out := make(map[string]string, len(in))
for key, value := range in {
out[key] = value
}
return out
}
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding { func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out := in out := in
if len(in.Options) > 0 { if len(in.Options) > 0 {

View File

@@ -14,13 +14,17 @@ type ResolveInput struct {
Only []string Only []string
Catalog pipeline.ModuleCatalog Catalog pipeline.ModuleCatalog
LLMProfileOverride string LLMProfileOverride string
ReferenceOverrides []pipeline.ReferenceBinding
ReferenceUnbinds []pipeline.ReferenceUnbind
} }
type EffectiveConfig struct { type EffectiveConfig struct {
Config Config Config Config
PipelineID string PipelineID string
Only []string Only []string
ResolvedPipeline pipeline.ResolvedPipeline ReferenceOverrides []pipeline.ReferenceBinding
ReferenceUnbinds []pipeline.ReferenceUnbind
ResolvedPipeline pipeline.ResolvedPipeline
} }
func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) { func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
@@ -46,16 +50,22 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
applyLLMProfileOverride(&profile, override) applyLLMProfileOverride(&profile, override)
} }
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{Only: input.Only}, input.Catalog) resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{
Only: input.Only,
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
}, input.Catalog)
if err != nil { if err != nil {
return EffectiveConfig{}, fmt.Errorf("resolve pipeline %q: %w", pipelineID, err) return EffectiveConfig{}, fmt.Errorf("resolve pipeline %q: %w", pipelineID, err)
} }
return EffectiveConfig{ return EffectiveConfig{
Config: cloneConfig(c), Config: cloneConfig(c),
PipelineID: pipelineID, PipelineID: pipelineID,
Only: append([]string(nil), input.Only...), Only: append([]string(nil), input.Only...),
ResolvedPipeline: resolved, ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
ResolvedPipeline: resolved,
}, nil }, nil
} }

View File

@@ -98,6 +98,43 @@ func TestResolveSurfacesMissingCapabilityThroughCatalog(t *testing.T) {
} }
} }
func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
profile.Chunk = pipeline.Binding("dnd/scenes")
lane := profile.Artifacts["events"]
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"events": lane}
cfg.Pipelines["example"] = profile
catalog := fakeCatalog(t,
pipeline.ModuleSpec{
Key: "fake/input",
Stage: pipeline.StageInput,
Provides: []string{"source.transcript"},
},
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"artifact"},
},
)
mustRegisterChunker(t, catalog.Chunkers, pipeline.ModuleSpec{
Key: "dnd/scenes",
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks", "chunks.scenes"},
})
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
if got := effective.ResolvedPipeline.Chunk.Module; got != "dnd/scenes" {
t.Fatalf("Chunk.Module = %q, want dnd/scenes", got)
}
}
func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) { func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
cfg := validConfig() cfg := validConfig()
first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)}) first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})

View File

@@ -35,10 +35,11 @@ type FileLLMProfile struct {
} }
type FilePipelineProfile struct { type FilePipelineProfile struct {
Input fileModuleBinding `yaml:"input"` Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"` Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"` Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"` Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]string `yaml:"references,omitempty"`
} }
type FileArtifactLaneProfile struct { type FileArtifactLaneProfile struct {
@@ -46,6 +47,7 @@ type FileArtifactLaneProfile struct {
Merge *fileModuleBinding `yaml:"merge,omitempty"` Merge *fileModuleBinding `yaml:"merge,omitempty"`
Normalize *fileModuleBinding `yaml:"normalize,omitempty"` Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
Validators []fileModuleBinding `yaml:"validators,omitempty"` Validators []fileModuleBinding `yaml:"validators,omitempty"`
References map[string]string `yaml:"references,omitempty"`
} }
type FileConcurrencyConfig struct { type FileConcurrencyConfig struct {
@@ -212,6 +214,18 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil { if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
return err return err
} }
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
return err
}
for rawLaneID, fileLane := range filePipeline.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
continue
}
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
return err
}
}
} }
for _, profileID := range profileIDs { for _, profileID := range profileIDs {
@@ -253,9 +267,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
return err return err
} }
profile := pipeline.PipelineProfile{ profile := pipeline.PipelineProfile{
ID: pipelineID, ID: pipelineID,
Input: filePipeline.Input.toPipelineBinding(), Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)), Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: normalizedStringMap(filePipeline.References),
} }
if filePipeline.Chunk != nil { if filePipeline.Chunk != nil {
profile.Chunk = filePipeline.Chunk.toPipelineBinding() profile.Chunk = filePipeline.Chunk.toPipelineBinding()
@@ -266,7 +281,8 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
for _, laneID := range laneIDs { for _, laneID := range laneIDs {
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]] fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
lane := pipeline.ArtifactLaneProfile{ lane := pipeline.ArtifactLaneProfile{
Extract: fileLane.Extract.toPipelineBinding(), Extract: fileLane.Extract.toPipelineBinding(),
References: normalizedStringMap(fileLane.References),
} }
if fileLane.Merge != nil { if fileLane.Merge != nil {
lane.Merge = fileLane.Merge.toPipelineBinding() lane.Merge = fileLane.Merge.toPipelineBinding()
@@ -318,6 +334,25 @@ func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, ma
return keys, rawByNormalized, nil return keys, rawByNormalized, nil
} }
func normalizedStringMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
}
return out
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) { func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName) name := strings.TrimSpace(envName)
if name == "" { if name == "" {

View File

@@ -144,6 +144,31 @@ pipelines:
} }
} }
func TestParseFileConfigReferenceMaps(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
pipelines:
example:
input: fake/input
references:
" roster ": " ./shared-roster.yml "
artifacts:
events:
extract: fake/extract
references:
" lore ": " ./lore.md "
`)
profile := cfg.Pipelines["example"]
if !reflect.DeepEqual(profile.References, map[string]string{"roster": "./shared-roster.yml"}) {
t.Fatalf("pipeline references = %#v, want trimmed map", profile.References)
}
gotLaneRefs := profile.Artifacts["events"].References
if !reflect.DeepEqual(gotLaneRefs, map[string]string{"lore": "./lore.md"}) {
t.Fatalf("lane references = %#v, want trimmed map", gotLaneRefs)
}
}
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) { func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
cfg := parseAndApplyConfig(t, ` cfg := parseAndApplyConfig(t, `
version: 1 version: 1
@@ -297,6 +322,58 @@ pipelines:
} }
} }
func TestApplyFileConfigRejectsDuplicateTrimmedReferenceSlots(t *testing.T) {
tests := []struct {
name string
raw string
want string
}{
{
name: "pipeline",
raw: `
version: 1
pipelines:
example:
input: fake/input
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" reference slot`,
},
{
name: "lane",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract: fake/extract
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" reference slot`,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(tc.raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML: %v", err)
}
cfg := Default()
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "duplicated") {
t.Fatalf("expected duplicate reference slot error, got %v", err)
}
})
}
}
func TestApplyFileConfigAllowsRetryOnlyLLMProfile(t *testing.T) { func TestApplyFileConfigAllowsRetryOnlyLLMProfile(t *testing.T) {
cfg := parseAndApplyConfig(t, ` cfg := parseAndApplyConfig(t, `
version: 1 version: 1

View File

@@ -21,10 +21,12 @@ func (c Config) RedactedDiagnosticsPayload() any {
func (e EffectiveConfig) RedactedDiagnosticsPayload() any { func (e EffectiveConfig) RedactedDiagnosticsPayload() any {
return EffectiveConfig{ return EffectiveConfig{
Config: e.Config.Redacted(), Config: e.Config.Redacted(),
PipelineID: e.PipelineID, PipelineID: e.PipelineID,
Only: append([]string(nil), e.Only...), Only: append([]string(nil), e.Only...),
ResolvedPipeline: cloneResolvedPipeline(e.ResolvedPipeline), ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), e.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), e.ReferenceUnbinds...),
ResolvedPipeline: cloneResolvedPipeline(e.ResolvedPipeline),
} }
} }
@@ -47,6 +49,8 @@ func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.Resolv
out.Extract = cloneModuleBinding(in.Extract) out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge) out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize) out.Normalize = cloneModuleBinding(in.Normalize)
out.References = append([]pipeline.ReferenceBinding(nil), in.References...)
out.ReferenceSet = pipeline.CloneReferenceSet(in.ReferenceSet)
if len(in.Validators) > 0 { if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators)) out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators { for i, binding := range in.Validators {

View File

@@ -3,6 +3,7 @@ package config
import ( import (
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
@@ -65,12 +66,21 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile
lane := cfg.Pipelines["example"].Artifacts["events"] lane := cfg.Pipelines["example"].Artifacts["events"]
lane.Extract.Options = map[string]any{"temperature": 0.2} lane.Extract.Options = map[string]any{"temperature": 0.2}
lane.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"].Artifacts["events"] = lane cfg.Pipelines["example"].Artifacts["events"] = lane
effective, err := cfg.Resolve(ResolveInput{ effective, err := cfg.Resolve(ResolveInput{
PipelineID: "example", PipelineID: "example",
Only: []string{"events"}, Only: []string{"events"},
Catalog: fakeCatalog(t), Catalog: fakeCatalog(t, pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
}),
}) })
if err != nil { if err != nil {
t.Fatalf("Resolve: %v", err) t.Fatalf("Resolve: %v", err)
@@ -98,4 +108,8 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 { if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 {
t.Fatalf("expected resolved pipeline options to be copied") t.Fatalf("expected resolved pipeline options to be copied")
} }
payload.ResolvedPipeline.ArtifactLanes[0].References[0].Source = "./changed.yml"
if effective.ResolvedPipeline.ArtifactLanes[0].References[0].Source != "./roster.yml" {
t.Fatalf("expected resolved pipeline references to be copied")
}
} }

View File

@@ -98,11 +98,17 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil { if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil {
return err return err
} }
if err := validateReferenceMap(id, "", profile.References); err != nil {
return err
}
for rawLaneID, lane := range profile.Artifacts { for rawLaneID, lane := range profile.Artifacts {
laneID := strings.TrimSpace(rawLaneID) laneID := strings.TrimSpace(rawLaneID)
if laneID == "" { if laneID == "" {
return fmt.Errorf("pipeline %q artifact lane id must not be empty", id) return fmt.Errorf("pipeline %q artifact lane id must not be empty", id)
} }
if err := validateReferenceMap(id, laneID, lane.References); err != nil {
return err
}
if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil { if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil {
return err return err
} }
@@ -122,6 +128,33 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
return nil return nil
} }
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
seen := make(map[string]struct{}, len(references))
for rawSlotName, rawSource := range references {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q reference slot name must not be empty", pipelineID, laneID)
}
return fmt.Errorf("pipeline %q reference slot name must not be empty", pipelineID)
}
if _, ok := seen[slotName]; ok {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q reference slot %q is duplicated after trimming", pipelineID, laneID, slotName)
}
return fmt.Errorf("pipeline %q reference slot %q is duplicated after trimming", pipelineID, slotName)
}
seen[slotName] = struct{}{}
if strings.TrimSpace(rawSource) == "" {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q reference slot %q source must not be empty", pipelineID, laneID, slotName)
}
return fmt.Errorf("pipeline %q reference slot %q source must not be empty", pipelineID, slotName)
}
}
return nil
}
func validateBindingLLMProfile( func validateBindingLLMProfile(
pipelineID string, pipelineID string,
laneID string, laneID string,

View File

@@ -116,6 +116,73 @@ func TestValidateRejectsInvalidDiagnosticsRetention(t *testing.T) {
} }
} }
func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
tests := []struct {
name string
mutate func(Config) Config
want []string
}{
{
name: "empty pipeline slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.References = map[string]string{" ": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "reference slot", "empty"},
},
{
name: "empty pipeline source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.References = map[string]string{"roster": " "}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "roster", "source", "empty"},
},
{
name: "empty lane slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "reference slot", "empty"},
},
{
name: "empty lane source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "roster", "source", "empty"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := tc.mutate(validConfig()).Validate()
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
})
}
}
func TestValidateRejectsEmptyIDs(t *testing.T) { func TestValidateRejectsEmptyIDs(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -4,6 +4,7 @@ const (
ArtifactInvocationMetadata = "invocation.json" ArtifactInvocationMetadata = "invocation.json"
ArtifactEffectiveConfig = "effective-config.json" ArtifactEffectiveConfig = "effective-config.json"
ArtifactResolvedPipeline = "resolved-pipeline.json" ArtifactResolvedPipeline = "resolved-pipeline.json"
ArtifactResolvedReferences = "resolved-references.json"
ArtifactSourceDocument = "source-document.json" ArtifactSourceDocument = "source-document.json"
ArtifactRunManifest = "run-manifest.json" ArtifactRunManifest = "run-manifest.json"
ArtifactRunReport = "run-report.json" ArtifactRunReport = "run-report.json"

View File

@@ -7,6 +7,7 @@ func TestArtifactNamesUseExtractionOrientedNames(t *testing.T) {
ArtifactInvocationMetadata, ArtifactInvocationMetadata,
ArtifactEffectiveConfig, ArtifactEffectiveConfig,
ArtifactResolvedPipeline, ArtifactResolvedPipeline,
ArtifactResolvedReferences,
ArtifactSourceDocument, ArtifactSourceDocument,
ArtifactRunManifest, ArtifactRunManifest,
ArtifactRunReport, ArtifactRunReport,

View File

@@ -149,6 +149,10 @@ func (r *RunDirectory) WriteResolvedPipeline(payload any) error {
return r.WriteJSONArtifact(ArtifactResolvedPipeline, payload) return r.WriteJSONArtifact(ArtifactResolvedPipeline, payload)
} }
func (r *RunDirectory) WriteResolvedReferences(payload any) error {
return r.WriteJSONArtifact(ArtifactResolvedReferences, payload)
}
func (r *RunDirectory) WriteSourceDocument(payload any) error { func (r *RunDirectory) WriteSourceDocument(payload any) error {
return r.WriteJSONArtifact(ArtifactSourceDocument, payload) return r.WriteJSONArtifact(ArtifactSourceDocument, payload)
} }

View File

@@ -191,6 +191,9 @@ func TestWriteTypedArtifacts(t *testing.T) {
if err := runDir.WriteResolvedPipeline(map[string]any{"pipeline": "test"}); err != nil { if err := runDir.WriteResolvedPipeline(map[string]any{"pipeline": "test"}); err != nil {
t.Fatalf("WriteResolvedPipeline: %v", err) t.Fatalf("WriteResolvedPipeline: %v", err)
} }
if err := runDir.WriteResolvedReferences([]artifacts.ReferenceProvenance{{LaneID: "events", SlotName: "roster"}}); err != nil {
t.Fatalf("WriteResolvedReferences: %v", err)
}
if err := runDir.WriteSourceDocument(map[string]any{"source_id": "source-1"}); err != nil { if err := runDir.WriteSourceDocument(map[string]any{"source_id": "source-1"}); err != nil {
t.Fatalf("WriteSourceDocument: %v", err) t.Fatalf("WriteSourceDocument: %v", err)
} }
@@ -207,6 +210,7 @@ func TestWriteTypedArtifacts(t *testing.T) {
for _, name := range []string{ for _, name := range []string{
ArtifactEffectiveConfig, ArtifactEffectiveConfig,
ArtifactResolvedPipeline, ArtifactResolvedPipeline,
ArtifactResolvedReferences,
ArtifactSourceDocument, ArtifactSourceDocument,
ArtifactRunManifest, ArtifactRunManifest,
ArtifactRunReport, ArtifactRunReport,

View File

@@ -17,6 +17,7 @@ var _ contracts.Extractor = compositionExtractor{}
var _ contracts.Merger = compositionMerger{} var _ contracts.Merger = compositionMerger{}
var _ contracts.Normalizer = compositionNormalizer{} var _ contracts.Normalizer = compositionNormalizer{}
var _ contracts.Validator = compositionValidator{} var _ contracts.Validator = compositionValidator{}
var _ contracts.StructuredLLMClient = compositionLLMClient{}
var _ contracts.OutputEncoder = compositionOutputEncoder{} var _ contracts.OutputEncoder = compositionOutputEncoder{}
func TestContractsComposeAcrossPackages(t *testing.T) { func TestContractsComposeAcrossPackages(t *testing.T) {
@@ -38,8 +39,9 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
} }
chunking, err := chunker.Chunk(ctx, contracts.ChunkRequest{ chunking, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc, Source: doc,
Metadata: map[string]any{"max_units": 2}, LLMClient: compositionLLMClient{},
Metadata: map[string]any{"max_units": 2},
}) })
if err != nil { if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err) t.Fatalf("Chunk() error = %v, want nil", err)
@@ -164,6 +166,9 @@ func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.Chunk
if req.Source == nil { if req.Source == nil {
return contracts.ChunkResult{}, errors.New("source document is required") return contracts.ChunkResult{}, errors.New("source document is required")
} }
if req.LLMClient == nil {
return contracts.ChunkResult{}, errors.New("structured llm client is required")
}
return contracts.ChunkResult{ return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{ Chunks: []contracts.SourceChunk{
@@ -178,6 +183,12 @@ func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.Chunk
}, nil }, nil
} }
type compositionLLMClient struct{}
func (client compositionLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
return contracts.StructuredCompletionResponse{}, nil
}
type compositionExtractor struct{} type compositionExtractor struct{}
func (extractor compositionExtractor) Key() string { func (extractor compositionExtractor) Key() string {
@@ -192,6 +203,10 @@ func (extractor compositionExtractor) SchemaVersion() string {
return "v1" return "v1"
} }
func (extractor compositionExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor compositionExtractor) Validators() []contracts.Validator { func (extractor compositionExtractor) Validators() []contracts.Validator {
return []contracts.Validator{compositionValidator{}} return []contracts.Validator{compositionValidator{}}
} }

View File

@@ -58,6 +58,7 @@ type SourceChunk struct {
type ChunkRequest struct { type ChunkRequest struct {
Source *source.SourceDocument `json:"-"` Source *source.SourceDocument `json:"-"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
@@ -73,10 +74,49 @@ type Chunker interface {
Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
} }
const (
ReferenceBindingSourceConfig = "config"
ReferenceBindingSourceCLI = "cli"
)
type ReferenceSlot struct {
Name string `json:"name"`
Description string `json:"description,omitempty"`
Required bool `json:"required,omitempty"`
AcceptedMediaTypes []string `json:"accepted_media_types,omitempty"`
Multiple bool `json:"multiple,omitempty"`
MaxBytes int64 `json:"max_bytes,omitempty"`
}
type ReferenceOrigin struct {
Type string `json:"type"`
URI string `json:"uri,omitempty"`
}
type ReferenceItem struct {
SlotName string `json:"slot_name"`
MediaType string `json:"media_type,omitempty"`
Content []byte `json:"-"`
Digest string `json:"digest,omitempty"`
Origin ReferenceOrigin `json:"origin"`
SizeBytes int64 `json:"size_bytes,omitempty"`
BindingSource string `json:"binding_source,omitempty"`
}
type ResolvedReferenceSlot struct {
Slot ReferenceSlot `json:"slot"`
Items []ReferenceItem `json:"items,omitempty"`
}
type ReferenceSet struct {
Slots map[string]ResolvedReferenceSlot `json:"slots,omitempty"`
}
type ExtractionRequest struct { type ExtractionRequest struct {
Source *source.SourceDocument `json:"-"` Source *source.SourceDocument `json:"-"`
Chunk *SourceChunk `json:"chunk,omitempty"` Chunk *SourceChunk `json:"chunk,omitempty"`
AmbientContext map[string]any `json:"ambient_context,omitempty"` AmbientContext map[string]any `json:"ambient_context,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"` LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
@@ -92,6 +132,7 @@ type Extractor interface {
Key() string Key() string
ArtifactType() string ArtifactType() string
SchemaVersion() string SchemaVersion() string
ReferenceSlots() []ReferenceSlot
Validators() []Validator Validators() []Validator
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
} }

View File

@@ -117,6 +117,27 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
} }
} }
func TestFakeChunkerReceivesLLMClient(t *testing.T) {
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "Source text."},
},
}
client := fakeLLMClient{}
chunker := &recordingChunker{key: "llm-chunker"}
if _, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc, LLMClient: client}); err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
if chunker.request.LLMClient == nil {
t.Fatal("ChunkRequest.LLMClient = nil, want structured LLM client")
}
}
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) { func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
extractor := fakeExtractor{ extractor := fakeExtractor{
key: "generic-extractor", key: "generic-extractor",
@@ -165,6 +186,71 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
} }
} }
func TestReferenceSetDataTypes(t *testing.T) {
references := ReferenceSet{
Slots: map[string]ResolvedReferenceSlot{
"roster": {
Slot: ReferenceSlot{
Name: "roster",
Description: "Known characters",
Required: true,
AcceptedMediaTypes: []string{"text/plain"},
Multiple: true,
MaxBytes: 4096,
},
Items: []ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain",
Content: []byte("Aria\nBryn\n"),
Digest: "sha256:reference",
Origin: ReferenceOrigin{
Type: "file",
URI: "file:///tmp/roster.txt",
},
SizeBytes: 10,
BindingSource: ReferenceBindingSourceConfig,
},
},
},
},
}
item := references.Slots["roster"].Items[0]
if item.SlotName != "roster" || item.MediaType != "text/plain" || string(item.Content) != "Aria\nBryn\n" {
t.Fatalf("reference item = %#v, want constructed item fields", item)
}
if item.BindingSource != ReferenceBindingSourceConfig {
t.Fatalf("BindingSource = %q, want %q", item.BindingSource, ReferenceBindingSourceConfig)
}
}
func TestReferenceItemJSONOmitsContent(t *testing.T) {
item := ReferenceItem{
SlotName: "roster",
MediaType: "text/plain",
Content: []byte("reference content"),
Digest: "sha256:reference",
Origin: ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
}
encoded, err := json.Marshal(item)
if err != nil {
t.Fatalf("json.Marshal() error = %v, want nil", err)
}
var got map[string]any
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
}
if _, ok := got["content"]; ok {
t.Fatalf("encoded reference item leaked content: %s", encoded)
}
if _, ok := got["Content"]; ok {
t.Fatalf("encoded reference item leaked Content: %s", encoded)
}
}
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) { func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
candidate := artifacts.ArtifactCandidate{ candidate := artifacts.ArtifactCandidate{
Index: 0, Index: 0,
@@ -305,6 +391,20 @@ func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkRe
}, nil }, nil
} }
type recordingChunker struct {
key string
request ChunkRequest
}
func (chunker *recordingChunker) Key() string {
return chunker.key
}
func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
chunker.request = req
return fakeChunker{key: chunker.key}.Chunk(ctx, req)
}
type fakeExtractor struct { type fakeExtractor struct {
key string key string
artifactType string artifactType string
@@ -324,6 +424,10 @@ func (extractor fakeExtractor) SchemaVersion() string {
return extractor.schemaVersion return extractor.schemaVersion
} }
func (extractor fakeExtractor) ReferenceSlots() []ReferenceSlot {
return nil
}
func (extractor fakeExtractor) Validators() []Validator { func (extractor fakeExtractor) Validators() []Validator {
return extractor.validators return extractor.validators
} }

View File

@@ -0,0 +1,82 @@
package pipeline
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
sourceUnitIndexes := make(map[string]int, len(doc.Units))
sourceUnits := make(map[string]source.SourceUnit, len(doc.Units))
for index, unit := range doc.Units {
sourceUnitIndexes[unit.ID] = index
sourceUnits[unit.ID] = unit
}
canonicalChunks := make([]contracts.SourceChunk, 0, len(chunks))
seenChunkIDs := make(map[string]struct{}, len(chunks))
for chunkIndex, chunk := range chunks {
if strings.TrimSpace(chunk.ID) == "" {
return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
}
if _, ok := seenChunkIDs[chunk.ID]; ok {
return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
}
seenChunkIDs[chunk.ID] = struct{}{}
if chunk.SourceID != doc.ID {
return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
}
if chunk.Index != chunkIndex {
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
}
if len(chunk.Units) == 0 {
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
}
seenUnitIDs := make(map[string]struct{}, len(chunk.Units))
previousSourceIndex := -1
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
for unitIndex, unit := range chunk.Units {
if strings.TrimSpace(unit.ID) == "" {
return nil, fmt.Errorf("chunk %q unit[%d].id must not be empty", chunk.ID, unitIndex)
}
if _, ok := seenUnitIDs[unit.ID]; ok {
return nil, fmt.Errorf("chunk %q repeats source unit %q", chunk.ID, unit.ID)
}
seenUnitIDs[unit.ID] = struct{}{}
sourceIndex, ok := sourceUnitIndexes[unit.ID]
if !ok {
return nil, fmt.Errorf("chunk %q source unit %q was not found in source document %q", chunk.ID, unit.ID, doc.ID)
}
if sourceIndex <= previousSourceIndex {
return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
}
previousSourceIndex = sourceIndex
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
}
canonicalChunks = append(canonicalChunks, contracts.SourceChunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Units: canonicalUnits,
Metadata: cloneMetadata(chunk.Metadata),
})
}
return canonicalChunks, nil
}
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
return source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Metadata: cloneMetadata(unit.Metadata),
}
}

View File

@@ -117,6 +117,8 @@ func (defaultExtractor) ArtifactType() string { return "record" }
func (defaultExtractor) SchemaVersion() string { return "v1" } func (defaultExtractor) SchemaVersion() string { return "v1" }
func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (defaultExtractor) Validators() []contracts.Validator { return nil } func (defaultExtractor) Validators() []contracts.Validator { return nil }
func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) { func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {

View File

@@ -49,6 +49,20 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
Stage: StageExtract, Stage: StageExtract,
Provides: []string{" generic-artifact ", "source-citations", "generic-artifact", ""}, Provides: []string{" generic-artifact ", "source-citations", "generic-artifact", ""},
Requires: []string{" source-document ", "source-document", ""}, Requires: []string{" source-document ", "source-document", ""},
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: " glossary ",
Description: " Supporting terms ",
AcceptedMediaTypes: []string{" text/plain ", "text/markdown", "text/plain", ""},
MaxBytes: 1024,
},
{
Name: " roster ",
Description: " Characters ",
Required: true,
Multiple: true,
},
},
} }
if err := registry.RegisterWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil { if err := registry.RegisterWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil {
@@ -64,12 +78,28 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
Stage: StageExtract, Stage: StageExtract,
Provides: []string{"generic-artifact", "source-citations"}, Provides: []string{"generic-artifact", "source-citations"},
Requires: []string{"source-document"}, Requires: []string{"source-document"},
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Supporting terms",
AcceptedMediaTypes: []string{"text/markdown", "text/plain"},
MaxBytes: 1024,
},
{
Name: "roster",
Description: "Characters",
Required: true,
Multiple: true,
},
},
} }
if !reflect.DeepEqual(got, want) { if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want) t.Fatalf("Spec() = %#v, want %#v", got, want)
} }
got.Provides[0] = "changed" got.Provides[0] = "changed"
got.ReferenceSlots[0].Name = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
again, ok := registry.Spec("generic-extractor") again, ok := registry.Spec("generic-extractor")
if !ok { if !ok {
t.Fatal("Spec() after caller mutation ok = false, want true") t.Fatal("Spec() after caller mutation ok = false, want true")
@@ -109,6 +139,50 @@ func TestExtractorRegistryRegisterWithSpecRejectsWrongStage(t *testing.T) {
} }
} }
func TestExtractorRegistryRejectsInvalidReferenceSlots(t *testing.T) {
tests := []struct {
name string
slots []contracts.ReferenceSlot
want string
}{
{
name: "empty name",
slots: []contracts.ReferenceSlot{{Name: " "}},
want: "name",
},
{
name: "duplicate name after trim",
slots: []contracts.ReferenceSlot{
{Name: "roster"},
{Name: " roster "},
},
want: "duplicated",
},
{
name: "negative max bytes",
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
want: "max_bytes",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterWithSpec(ModuleSpec{
Key: "generic-extractor",
Stage: StageExtract,
ReferenceSlots: test.slots,
}, fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("RegisterWithSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), test.want) {
t.Fatalf("RegisterWithSpec() error = %q, want %q", err.Error(), test.want)
}
})
}
}
func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) { func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) {
registry := NewExtractorRegistry() registry := NewExtractorRegistry()
@@ -301,6 +375,10 @@ func (extractor registryFakeExtractor) SchemaVersion() string {
return "v1" return "v1"
} }
func (extractor registryFakeExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor registryFakeExtractor) Validators() []contracts.Validator { func (extractor registryFakeExtractor) Validators() []contracts.Validator {
return nil return nil
} }

View File

@@ -4,6 +4,8 @@ import (
"fmt" "fmt"
"sort" "sort"
"strings" "strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
type ModuleStage string type ModuleStage string
@@ -19,10 +21,11 @@ const (
) )
type ModuleSpec struct { type ModuleSpec struct {
Key string Key string
Stage ModuleStage Stage ModuleStage
Provides []string Provides []string
Requires []string Requires []string
ReferenceSlots []contracts.ReferenceSlot
} }
func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec { func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec {
@@ -34,10 +37,11 @@ func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec {
func normalizeModuleSpec(spec ModuleSpec) ModuleSpec { func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
return ModuleSpec{ return ModuleSpec{
Key: strings.TrimSpace(spec.Key), Key: strings.TrimSpace(spec.Key),
Stage: spec.Stage, Stage: spec.Stage,
Provides: normalizeCapabilities(spec.Provides), Provides: normalizeCapabilities(spec.Provides),
Requires: normalizeCapabilities(spec.Requires), Requires: normalizeCapabilities(spec.Requires),
ReferenceSlots: normalizeReferenceSlots(spec.ReferenceSlots),
} }
} }
@@ -68,10 +72,11 @@ func normalizeCapabilities(values []string) []string {
func cloneModuleSpec(spec ModuleSpec) ModuleSpec { func cloneModuleSpec(spec ModuleSpec) ModuleSpec {
return ModuleSpec{ return ModuleSpec{
Key: spec.Key, Key: spec.Key,
Stage: spec.Stage, Stage: spec.Stage,
Provides: append([]string(nil), spec.Provides...), Provides: append([]string(nil), spec.Provides...),
Requires: append([]string(nil), spec.Requires...), Requires: append([]string(nil), spec.Requires...),
ReferenceSlots: cloneReferenceSlots(spec.ReferenceSlots),
} }
} }
@@ -82,6 +87,12 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
if spec.Stage != expectedStage { if spec.Stage != expectedStage {
return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage) return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage)
} }
if spec.Stage != StageExtract && len(spec.ReferenceSlots) > 0 {
return fmt.Errorf("%s %q must not declare reference slots", kind, spec.Key)
}
if err := validateReferenceSlots(spec.ReferenceSlots); err != nil {
return fmt.Errorf("%s %q reference slots: %w", kind, spec.Key, err)
}
return nil return nil
} }
@@ -97,3 +108,74 @@ func sortedRegistryKeys[C any](constructors map[string]C) []string {
sort.Strings(keys) sort.Strings(keys)
return keys return keys
} }
func normalizeReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
normalized := make([]contracts.ReferenceSlot, 0, len(slots))
for _, slot := range slots {
slot.Name = strings.TrimSpace(slot.Name)
slot.Description = strings.TrimSpace(slot.Description)
slot.AcceptedMediaTypes = normalizeStringSet(slot.AcceptedMediaTypes)
normalized = append(normalized, slot)
}
sort.SliceStable(normalized, func(i, j int) bool {
return normalized[i].Name < normalized[j].Name
})
return normalized
}
func normalizeStringSet(values []string) []string {
if len(values) == 0 {
return nil
}
seen := make(map[string]struct{}, len(values))
for _, value := range values {
normalized := strings.TrimSpace(value)
if normalized == "" {
continue
}
seen[normalized] = struct{}{}
}
if len(seen) == 0 {
return nil
}
out := make([]string, 0, len(seen))
for value := range seen {
out = append(out, value)
}
sort.Strings(out)
return out
}
func validateReferenceSlots(slots []contracts.ReferenceSlot) error {
seen := make(map[string]struct{}, len(slots))
for i, slot := range slots {
if slot.Name == "" {
return fmt.Errorf("slot[%d].name must not be empty", i)
}
if _, ok := seen[slot.Name]; ok {
return fmt.Errorf("slot name %q is duplicated", slot.Name)
}
seen[slot.Name] = struct{}{}
if slot.MaxBytes < 0 {
return fmt.Errorf("slot %q max_bytes must not be negative", slot.Name)
}
}
return nil
}
func cloneReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
out := make([]contracts.ReferenceSlot, 0, len(slots))
for _, slot := range slots {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
out = append(out, slot)
}
return out
}

View File

@@ -0,0 +1,25 @@
package pipeline
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestValidateModuleSpecRejectsReferenceSlotsForNonExtractors(t *testing.T) {
err := validateModuleSpec("chunker", StageChunk, ModuleSpec{
Key: "generic",
Stage: StageChunk,
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
})
if err == nil {
t.Fatal("validateModuleSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), "reference slots") {
t.Fatalf("validateModuleSpec() error = %q, want reference slots context", err.Error())
}
}

View File

@@ -7,6 +7,8 @@ import (
"fmt" "fmt"
"sort" "sort"
"strings" "strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
const ( const (
@@ -24,30 +26,48 @@ type ModuleBinding struct {
} }
type ArtifactLaneProfile struct { type ArtifactLaneProfile struct {
Extract ModuleBinding `json:"extract"` Extract ModuleBinding `json:"extract"`
Merge ModuleBinding `json:"merge,omitempty"` Merge ModuleBinding `json:"merge,omitempty"`
Normalize ModuleBinding `json:"normalize,omitempty"` Normalize ModuleBinding `json:"normalize,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"` Validators []ModuleBinding `json:"validators,omitempty"`
References map[string]string `json:"references,omitempty"`
} }
type PipelineProfile struct { type PipelineProfile struct {
ID string `json:"id"` ID string `json:"id"`
Input ModuleBinding `json:"input"` Input ModuleBinding `json:"input"`
Chunk ModuleBinding `json:"chunk,omitempty"` Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"` Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
Output ModuleBinding `json:"output,omitempty"` Output ModuleBinding `json:"output,omitempty"`
References map[string]string `json:"references,omitempty"`
} }
type ResolveOptions struct { type ResolveOptions struct {
Only []string Only []string
ReferenceOverrides []ReferenceBinding
ReferenceUnbinds []ReferenceUnbind
}
type ReferenceBinding struct {
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
Source string `json:"source"`
BindingSource string `json:"binding_source,omitempty"`
}
type ReferenceUnbind struct {
LaneID string `json:"lane_id"`
SlotName string `json:"slot_name"`
} }
type ResolvedArtifactLane struct { type ResolvedArtifactLane struct {
ID string ID string
Extract ModuleBinding Extract ModuleBinding
Merge ModuleBinding Merge ModuleBinding
Normalize ModuleBinding Normalize ModuleBinding
Validators []ModuleBinding Validators []ModuleBinding
References []ReferenceBinding `json:"references,omitempty"`
ReferenceSet contracts.ReferenceSet `json:"-"`
} }
type ResolvedPipeline struct { type ResolvedPipeline struct {
@@ -126,7 +146,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
for _, laneID := range selectedLaneIDs { for _, laneID := range selectedLaneIDs {
laneProfile := lanesByID[laneID] laneProfile := lanesByID[laneID]
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, capabilities, catalog) lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
if err != nil { if err != nil {
return ResolvedPipeline{}, err return ResolvedPipeline{}, err
} }
@@ -150,7 +170,15 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
return resolved, nil return resolved, nil
} }
func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile, inherited capabilitySet, catalog ModuleCatalog) (ResolvedArtifactLane, capabilitySet, error) { func resolveArtifactLane(
pipelineID string,
laneID string,
profile ArtifactLaneProfile,
pipelineReferences map[string]string,
options ResolveOptions,
inherited capabilitySet,
catalog ModuleCatalog,
) (ResolvedArtifactLane, capabilitySet, error) {
lane := ResolvedArtifactLane{ lane := ResolvedArtifactLane{
ID: laneID, ID: laneID,
Extract: resolveBinding(profile.Extract, ""), Extract: resolveBinding(profile.Extract, ""),
@@ -171,6 +199,11 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
if missing, ok := capabilities.missing(extractSpec.Requires); ok { if missing, ok := capabilities.missing(extractSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing) return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
} }
references, err := resolveReferenceBindings(pipelineID, laneID, lane.Extract.Module, extractSpec.ReferenceSlots, pipelineReferences, profile.References, options)
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.References = references
capabilities.add(extractSpec.Provides...) capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module) mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
@@ -205,6 +238,162 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
return lane, capabilities, nil return lane, capabilities, nil
} }
func resolveReferenceBindings(
pipelineID string,
laneID string,
extractorModule string,
slots []contracts.ReferenceSlot,
pipelineReferences map[string]string,
laneReferences map[string]string,
options ResolveOptions,
) ([]ReferenceBinding, error) {
slotByName := make(map[string]contracts.ReferenceSlot, len(slots))
for _, slot := range slots {
slotByName[slot.Name] = slot
}
bindings := make(map[string]ReferenceBinding)
addBinding := func(slotName, source, bindingSource string) error {
slotName = strings.TrimSpace(slotName)
source = strings.TrimSpace(source)
if slotName == "" {
return fmt.Errorf("pipeline %q lane %q reference slot name must not be empty", pipelineID, laneID)
}
if source == "" {
return fmt.Errorf("pipeline %q lane %q reference slot %q source must not be empty", pipelineID, laneID, slotName)
}
if _, ok := slotByName[slotName]; !ok {
return fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, laneID, slotName, extractorModule)
}
bindings[slotName] = ReferenceBinding{
LaneID: laneID,
SlotName: slotName,
Source: source,
BindingSource: bindingSource,
}
return nil
}
normalizedPipelineReferences, err := normalizedReferenceMap(pipelineReferences, fmt.Sprintf("pipeline %q reference slot", pipelineID))
if err != nil {
return nil, err
}
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
if _, ok := slotByName[slotName]; !ok {
continue
}
if err := addBinding(slotName, normalizedPipelineReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
return nil, err
}
}
normalizedLaneReferences, err := normalizedReferenceMap(laneReferences, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID))
if err != nil {
return nil, err
}
for _, slotName := range sortedStringMapKeys(normalizedLaneReferences) {
if err := addBinding(slotName, normalizedLaneReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
return nil, err
}
}
for _, override := range options.ReferenceOverrides {
optionLaneID := strings.TrimSpace(override.LaneID)
if optionLaneID == "" {
return nil, fmt.Errorf("pipeline %q reference override lane id must not be empty", pipelineID)
}
if optionLaneID != laneID {
continue
}
source := override.BindingSource
if strings.TrimSpace(source) == "" {
source = contracts.ReferenceBindingSourceCLI
}
if err := addBinding(override.SlotName, override.Source, strings.TrimSpace(source)); err != nil {
return nil, err
}
}
for _, unbind := range options.ReferenceUnbinds {
optionLaneID := strings.TrimSpace(unbind.LaneID)
if optionLaneID == "" {
return nil, fmt.Errorf("pipeline %q reference unbind lane id must not be empty", pipelineID)
}
if optionLaneID != laneID {
continue
}
slotName := strings.TrimSpace(unbind.SlotName)
if slotName == "" {
return nil, fmt.Errorf("pipeline %q lane %q reference unbind slot name must not be empty", pipelineID, laneID)
}
if _, ok := slotByName[slotName]; !ok {
return nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared", pipelineID, laneID, slotName)
}
delete(bindings, slotName)
}
for _, slot := range slots {
if slot.Required {
if _, ok := bindings[slot.Name]; !ok {
return nil, fmt.Errorf("pipeline %q lane %q required reference slot %q is not bound", pipelineID, laneID, slot.Name)
}
}
}
keys := sortedReferenceBindingKeys(bindings)
resolved := make([]ReferenceBinding, 0, len(keys))
for _, slotName := range keys {
resolved = append(resolved, bindings[slotName])
}
return resolved, nil
}
func normalizedReferenceMap(values map[string]string, keyName string) (map[string]string, error) {
if len(values) == 0 {
return nil, nil
}
out := make(map[string]string, len(values))
for rawSlotName, rawSource := range values {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
return nil, fmt.Errorf("%s must not be empty", keyName)
}
if _, ok := out[slotName]; ok {
return nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, slotName)
}
source := strings.TrimSpace(rawSource)
if source == "" {
return nil, fmt.Errorf("%s %q source must not be empty", keyName, slotName)
}
out[slotName] = source
}
return out, nil
}
func sortedStringMapKeys(values map[string]string) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func sortedReferenceBindingKeys(values map[string]ReferenceBinding) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding { func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
module := strings.TrimSpace(binding.Module) module := strings.TrimSpace(binding.Module)
if module == "" { if module == "" {

View File

@@ -3,6 +3,7 @@ package pipeline
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
@@ -125,6 +126,137 @@ func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
} }
} }
func TestResolvePipelineAppliesReferenceBindings(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{
" roster ": " ./shared-roster.yml ",
"unclaimed": "./ignored.yml",
}
lane := profile.Artifacts["events"]
lane.References = map[string]string{
"roster": "./lane-roster.yml",
" lore ": " ./lore.md ",
}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
{Name: "lore"},
},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events", "summaries"}}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
events := resolvedLane(t, resolved.ArtifactLanes, "events")
want := []ReferenceBinding{
{LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig},
{LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig},
}
if !reflect.DeepEqual(events.References, want) {
t.Fatalf("events references = %#v, want %#v", events.References, want)
}
summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries")
if len(summaries.References) != 0 {
t.Fatalf("summaries references = %#v, want none", summaries.References)
}
}
func TestResolvePipelineRejectsUndeclaredReferenceSlot(t *testing.T) {
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.References = map[string]string{"missing": "./missing.yml"}
profile.Artifacts["events"] = lane
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "events", "missing", "not declared")
}
func TestResolvePipelineRequiresBoundReferenceSlotsForSelectedLanes(t *testing.T) {
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
})
if _, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"notes"}}, catalog); err != nil {
t.Fatalf("ResolvePipeline(unselected required slot) error = %v, want nil", err)
}
_, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"events"}}, catalog)
if err == nil {
t.Fatal("ResolvePipeline(selected required slot) error = nil, want error")
}
assertErrorContains(t, err, "events", "required", "roster", "not bound")
}
func TestResolvePipelineReferenceUnbindCanLeaveRequiredSlotMissing(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
})
_, err := ResolvePipeline(profile, ResolveOptions{
ReferenceUnbinds: []ReferenceUnbind{{LaneID: "events", SlotName: "roster"}},
}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "events", "required", "roster", "not bound")
}
func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := emptyProfileCatalog()
for _, spec := range defaultProfileSpecs() {
if spec.Key != "event-extractor" {
registerProfileSpecs(t, catalog, spec)
}
}
if err := catalog.Extractors.RegisterWithSpec(ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
}, func() (contracts.Extractor, error) {
return nil, errors.New("constructor should not run")
}); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := resolved.ArtifactLanes[0].References[0].Source; got != "./roster.yml" {
t.Fatalf("reference source = %q, want ./roster.yml", got)
}
}
func TestResolvePipelineRejectsUnknownOnlyLane(t *testing.T) { func TestResolvePipelineRejectsUnknownOnlyLane(t *testing.T) {
_, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"missing"}}, newProfileCatalog(t)) _, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"missing"}}, newProfileCatalog(t))
if err == nil { if err == nil {
@@ -463,6 +595,17 @@ func laneIDs(lanes []ResolvedArtifactLane) []string {
return ids return ids
} }
func resolvedLane(t *testing.T, lanes []ResolvedArtifactLane, laneID string) ResolvedArtifactLane {
t.Helper()
for _, lane := range lanes {
if lane.ID == laneID {
return lane
}
}
t.Fatalf("lane %q not found in %#v", laneID, laneIDs(lanes))
return ResolvedArtifactLane{}
}
func assertErrorContains(t *testing.T, err error, values ...string) { func assertErrorContains(t *testing.T, err error, values ...string) {
t.Helper() t.Helper()

View File

@@ -0,0 +1,218 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
const (
referenceOriginFile = "file"
referenceMediaType = "text/plain; charset=utf-8"
)
type ReferenceMaterializationOptions struct {
ConfigPath string
WorkingDir string
}
func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) {
out := resolved
if len(resolved.ArtifactLanes) == 0 {
return out, nil, nil
}
warnings := []contracts.Warning(nil)
out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
for i, lane := range resolved.ArtifactLanes {
materializedLane := lane
referenceSet, laneWarnings, err := materializeLaneReferences(resolved.ID, lane, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
materializedLane.ReferenceSet = referenceSet
out.ArtifactLanes[i] = materializedLane
warnings = append(warnings, laneWarnings...)
}
return out, warnings, nil
}
func materializeLaneReferences(
pipelineID string,
lane ResolvedArtifactLane,
catalog ModuleCatalog,
options ReferenceMaterializationOptions,
) (contracts.ReferenceSet, []contracts.Warning, error) {
if len(lane.References) == 0 {
return contracts.ReferenceSet{}, nil, nil
}
if catalog.Extractors == nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
}
spec, ok := catalog.Extractors.Spec(lane.Extract.Module)
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q extract module %q: module %q is not registered", pipelineID, lane.ID, lane.Extract.Module, lane.Extract.Module)
}
slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots))
for _, slot := range spec.ReferenceSlots {
slotByName[slot.Name] = slot
}
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(lane.References))}
var warnings []contracts.Warning
for _, binding := range lane.References {
slotName := strings.TrimSpace(binding.SlotName)
slot, ok := slotByName[slotName]
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q is not declared by extractor %q", pipelineID, lane.ID, slotName, lane.Extract.Module)
}
path, err := referencePath(binding, options)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q: %w", pipelineID, lane.ID, slotName, binding.Source, err)
}
content, err := os.ReadFile(path)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q read %q: %w", pipelineID, lane.ID, slotName, path, err)
}
if !utf8.Valid(content) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q must be UTF-8 text", pipelineID, lane.ID, slotName, path)
}
if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes {
return contracts.ReferenceSet{}, nil, fmt.Errorf("pipeline %q lane %q reference slot %q path %q is %d bytes, limit %d", pipelineID, lane.ID, slotName, path, len(content), slot.MaxBytes)
}
if len(content) == 0 {
warnings = append(warnings, contracts.Warning{
Scope: fmt.Sprintf("pipeline.%s.lane.%s.reference.%s", pipelineID, lane.ID, slotName),
ReasonCode: "empty_reference",
Message: fmt.Sprintf("reference slot %q for lane %q is bound to an empty file", slotName, lane.ID),
})
}
item := contracts.ReferenceItem{
SlotName: slotName,
MediaType: referenceMediaType,
Content: append([]byte(nil), content...),
Digest: referenceDigest(content),
Origin: contracts.ReferenceOrigin{Type: referenceOriginFile, URI: fileURI(path)},
SizeBytes: int64(len(content)),
BindingSource: strings.TrimSpace(binding.BindingSource),
}
set.Slots[slotName] = contracts.ResolvedReferenceSlot{
Slot: cloneReferenceSlot(slot),
Items: []contracts.ReferenceItem{item},
}
}
return set, warnings, nil
}
func referencePath(binding ReferenceBinding, options ReferenceMaterializationOptions) (string, error) {
source := strings.TrimSpace(binding.Source)
if source == "" {
return "", fmt.Errorf("must not be empty")
}
if filepath.IsAbs(source) {
return filepath.Clean(source), nil
}
base := strings.TrimSpace(options.WorkingDir)
if strings.TrimSpace(binding.BindingSource) == contracts.ReferenceBindingSourceConfig {
base = filepath.Dir(strings.TrimSpace(options.ConfigPath))
}
if base == "" {
var err error
base, err = os.Getwd()
if err != nil {
return "", fmt.Errorf("resolve working directory: %w", err)
}
}
return filepath.Clean(filepath.Join(base, source)), nil
}
func referenceDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
func fileURI(path string) string {
absolute, err := filepath.Abs(path)
if err != nil {
absolute = path
}
absolute = filepath.ToSlash(filepath.Clean(absolute))
if strings.HasPrefix(absolute, "/") {
return "file://" + (&url.URL{Path: absolute}).EscapedPath()
}
return "file:///" + (&url.URL{Path: absolute}).EscapedPath()
}
func cloneReferenceSlot(slot contracts.ReferenceSlot) contracts.ReferenceSlot {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
return slot
}
func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet {
if len(in.Slots) == 0 {
return contracts.ReferenceSet{}
}
out := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(in.Slots))}
keys := make([]string, 0, len(in.Slots))
for key := range in.Slots {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
slot := in.Slots[key]
slot.Slot = cloneReferenceSlot(slot.Slot)
if len(slot.Items) > 0 {
items := make([]contracts.ReferenceItem, len(slot.Items))
for i, item := range slot.Items {
item.Content = append([]byte(nil), item.Content...)
items[i] = item
}
slot.Items = items
}
out.Slots[key] = slot
}
return out
}
func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance {
provenance := []artifacts.ReferenceProvenance{}
for _, lane := range resolved.ArtifactLanes {
if len(lane.ReferenceSet.Slots) == 0 {
continue
}
slotNames := make([]string, 0, len(lane.ReferenceSet.Slots))
for slotName := range lane.ReferenceSet.Slots {
slotNames = append(slotNames, slotName)
}
sort.Strings(slotNames)
for _, slotName := range slotNames {
slot := lane.ReferenceSet.Slots[slotName]
for _, item := range slot.Items {
provenance = append(provenance, artifacts.ReferenceProvenance{
LaneID: lane.ID,
SlotName: item.SlotName,
OriginType: item.Origin.Type,
OriginURI: item.Origin.URI,
Digest: item.Digest,
MediaType: item.MediaType,
SizeBytes: item.SizeBytes,
BindingSource: item.BindingSource,
})
}
}
}
return provenance
}

View File

@@ -0,0 +1,189 @@
package pipeline
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
configDir := t.TempDir()
workingDir := t.TempDir()
configReference := filepath.Join(configDir, "config-reference.txt")
cliReference := filepath.Join(workingDir, "cli-reference.txt")
writeReferenceFile(t, configReference, []byte("config text"))
writeReferenceFile(t, cliReference, []byte("cli text"))
pipeline := baselineProfile()
pipeline.References = map[string]string{"roster": "config-reference.txt"}
lane := pipeline.Artifacts["events"]
lane.References = map[string]string{"glossary": "cli-reference.txt"}
pipeline.Artifacts["events"] = lane
catalog := referenceCatalog(t, []contracts.ReferenceSlot{
{Name: "roster"},
{Name: "glossary"},
})
resolved, err := ResolvePipeline(pipeline, ResolveOptions{
ReferenceOverrides: []ReferenceBinding{
{LaneID: "events", SlotName: "glossary", Source: "cli-reference.txt", BindingSource: contracts.ReferenceBindingSourceCLI},
},
}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
first, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
WorkingDir: workingDir,
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
if len(warnings) != 0 {
t.Fatalf("warnings = %#v, want none", warnings)
}
second, _, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
WorkingDir: workingDir,
})
if err != nil {
t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
}
referenceSet := first.ArtifactLanes[0].ReferenceSet
roster := referenceSet.Slots["roster"].Items[0]
if string(roster.Content) != "config text" {
t.Fatalf("roster content = %q, want config text", roster.Content)
}
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Digest {
t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
}
if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
t.Fatalf("roster binding source = %q, want config", roster.BindingSource)
}
if roster.MediaType != referenceMediaType || roster.Origin.Type != referenceOriginFile || roster.SizeBytes != int64(len("config text")) {
t.Fatalf("roster metadata = %#v, want text file metadata", roster)
}
if !strings.Contains(roster.Origin.URI, "config-reference.txt") {
t.Fatalf("roster origin URI = %q, want config reference path", roster.Origin.URI)
}
glossary := referenceSet.Slots["glossary"].Items[0]
if string(glossary.Content) != "cli text" {
t.Fatalf("glossary content = %q, want cli text", glossary.Content)
}
if glossary.BindingSource != contracts.ReferenceBindingSourceCLI {
t.Fatalf("glossary binding source = %q, want cli", glossary.BindingSource)
}
if !strings.Contains(glossary.Origin.URI, "cli-reference.txt") {
t.Fatalf("glossary origin URI = %q, want cli reference path", glossary.Origin.URI)
}
provenance := ReferenceProvenance(first)
if len(provenance) != 2 {
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
}
if provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
}
if provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
}
encoded, err := json.Marshal(first)
if err != nil {
t.Fatalf("json.Marshal(materialized) error = %v, want nil", err)
}
if strings.Contains(string(encoded), "config text") || strings.Contains(string(encoded), "cli text") {
t.Fatalf("materialized pipeline JSON contains reference content: %s", encoded)
}
}
func TestMaterializeReferencesRejectsNonUTF8Content(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "bad.txt")
writeReferenceFile(t, path, []byte{0xff, 0xfe})
resolved := resolvedPipelineWithReference(t, "roster", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), path) {
t.Fatalf("error = %v, want UTF-8 path error", err)
}
}
func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "empty.txt")
writeReferenceFile(t, path, nil)
resolved := resolvedPipelineWithReference(t, "roster", "empty.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
materialized, warnings, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
t.Fatalf("warnings = %#v, want empty reference warning", warnings)
}
item := materialized.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0]
if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
t.Fatalf("empty item = %#v, want zero size and empty digest", item)
}
}
func TestMaterializeReferencesEnforcesMaxBytes(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "large.txt")
writeReferenceFile(t, path, []byte("too large"))
slot := contracts.ReferenceSlot{Name: "roster", MaxBytes: 3}
resolved := resolvedPipelineWithReference(t, "roster", "large.txt", contracts.ReferenceBindingSourceConfig, slot)
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "9 bytes") || !strings.Contains(err.Error(), "limit 3") || !strings.Contains(err.Error(), "roster") {
t.Fatalf("error = %v, want max bytes error", err)
}
}
func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
t.Helper()
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.References = map[string]string{slotName: source}
profile.Artifacts["events"] = lane
catalog := referenceCatalog(t, []contracts.ReferenceSlot{slot})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if bindingSource != contracts.ReferenceBindingSourceConfig {
resolved.ArtifactLanes[0].References[0].BindingSource = bindingSource
}
return resolved
}
func referenceCatalog(t *testing.T, slots []contracts.ReferenceSlot) ModuleCatalog {
t.Helper()
return newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: slots,
})
}
func writeReferenceFile(t *testing.T, path string, content []byte) {
t.Helper()
if err := os.WriteFile(path, content, 0o644); err != nil {
t.Fatalf("write reference %q: %v", path, err)
}
}

View File

@@ -149,6 +149,10 @@ func (extractor integrationExtractor) SchemaVersion() string {
return "v1" return "v1"
} }
func (extractor integrationExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor integrationExtractor) Validators() []contracts.Validator { func (extractor integrationExtractor) Validators() []contracts.Validator {
return extractor.validators return extractor.validators
} }

View File

@@ -41,6 +41,7 @@ type RunInput struct {
StartedAt time.Time StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any Metadata map[string]any
Warnings []contracts.Warning
} }
type RunOutput struct { type RunOutput struct {
@@ -63,12 +64,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
return output, err return output, err
} }
output.Warnings = append(output.Warnings, cloneWarnings(input.Warnings)...)
output.Manifest = manifestFromPipeline(input) output.Manifest = manifestFromPipeline(input)
adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module) adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module)
if err != nil { if err != nil {
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err) return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
} }
attachModuleManifestMetadata(&output, "input", adapter)
doc, err := adapter.Parse(ctx, contracts.ParseRequest{ doc, err := adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID, SourceID: input.SourceID,
Path: input.Path, Path: input.Path,
@@ -89,8 +92,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil { if err != nil {
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err) return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
} }
attachModuleManifestMetadata(&output, "chunker", chunker)
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{ chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc, Source: doc,
LLMClient: input.LLMClient,
LLMProfile: input.Pipeline.Chunk.LLMProfile, LLMProfile: input.Pipeline.Chunk.LLMProfile,
Options: cloneOptions(input.Pipeline.Chunk.Options), Options: cloneOptions(input.Pipeline.Chunk.Options),
Metadata: input.Metadata, Metadata: input.Metadata,
@@ -102,10 +107,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if len(chunkResult.Chunks) == 0 { if len(chunkResult.Chunks) == 0 {
return failOutput(output), fmt.Errorf("chunker %q returned no chunks", chunker.Key()) return failOutput(output), fmt.Errorf("chunker %q returned no chunks", chunker.Key())
} }
canonicalChunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
if err != nil {
return failOutput(output), fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
}
nextCandidateIndex := 0 nextCandidateIndex := 0
for _, lane := range input.Pipeline.ArtifactLanes { for _, lane := range input.Pipeline.ArtifactLanes {
if err := r.runLane(ctx, input, doc, chunkResult.Chunks, lane, &output, &nextCandidateIndex); err != nil { if err := r.runLane(ctx, input, doc, canonicalChunks, lane, &output, &nextCandidateIndex); err != nil {
return failOutput(output), err return failOutput(output), err
} }
} }
@@ -121,6 +130,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil { if err != nil {
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err) return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
} }
attachModuleManifestMetadata(&output, "output", encoder)
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{ encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: output.Manifest, Manifest: output.Manifest,
Approved: output.Approved, Approved: output.Approved,
@@ -176,6 +186,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{ result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
Source: doc, Source: doc,
Chunk: &chunk, Chunk: &chunk,
References: CloneReferenceSet(lane.ReferenceSet),
LLMClient: input.LLMClient, LLMClient: input.LLMClient,
LLMProfile: lane.Extract.LLMProfile, LLMProfile: lane.Extract.LLMProfile,
Options: cloneOptions(lane.Extract.Options), Options: cloneOptions(lane.Extract.Options),
@@ -345,8 +356,12 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)), ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
RunID: runID, RunID: runID,
StartedAt: timePtr(startedAt), StartedAt: timePtr(startedAt),
References: ReferenceProvenance(pipeline),
LLMProfiles: cloneLLMProfiles(input.LLMProfiles), LLMProfiles: cloneLLMProfiles(input.LLMProfiles),
} }
// The runner does not currently maintain a cache or idempotency key. Reference
// digests are recorded in manifest provenance and intentionally kept separate
// from source_digests.
for _, lane := range pipeline.ArtifactLanes { for _, lane := range pipeline.ArtifactLanes {
laneManifest := artifacts.ArtifactLaneManifest{ laneManifest := artifacts.ArtifactLaneManifest{
@@ -382,14 +397,10 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
metadata := make(map[string]any) metadata := make(map[string]any)
for _, module := range modules { for _, module := range modules {
provider, ok := module.(contracts.ManifestMetadataProvider) moduleMetadata, ok := moduleManifestMetadata(module)
if !ok { if !ok {
continue continue
} }
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
if len(moduleMetadata) == 0 {
continue
}
key := manifestMetadataKey(module) key := manifestMetadataKey(module)
if key == "" { if key == "" {
continue continue
@@ -403,6 +414,20 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
} }
} }
func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
if output == nil {
return
}
moduleMetadata, ok := moduleManifestMetadata(module)
if !ok {
return
}
if output.Manifest.ModuleMetadata == nil {
output.Manifest.ModuleMetadata = make(map[string]map[string]any)
}
output.Manifest.ModuleMetadata[moduleKey] = moduleMetadata
}
func manifestMetadataKey(module any) string { func manifestMetadataKey(module any) string {
switch module.(type) { switch module.(type) {
case contracts.Extractor: case contracts.Extractor:
@@ -416,6 +441,19 @@ func manifestMetadataKey(module any) string {
} }
} }
func moduleManifestMetadata(module any) (map[string]any, bool) {
provider, ok := module.(contracts.ManifestMetadataProvider)
if !ok {
return nil, false
}
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
if len(moduleMetadata) == 0 {
return nil, false
}
return moduleMetadata, true
}
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) { func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
out := make([]contracts.OutputFile, 0, len(result.Files)) out := make([]contracts.OutputFile, 0, len(result.Files))
for _, file := range result.Files { for _, file := range result.Files {
@@ -469,6 +507,13 @@ func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMPr
return append([]artifacts.LLMProfileManifest(nil), profiles...) return append([]artifacts.LLMProfileManifest(nil), profiles...)
} }
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
if len(warnings) == 0 {
return nil
}
return append([]contracts.Warning(nil), warnings...)
}
func timePtr(t time.Time) *time.Time { func timePtr(t time.Time) *time.Time {
return &t return &t
} }

View File

@@ -256,6 +256,191 @@ func TestRunRejectsChunkerBuildChunkAndEmptyChunkErrors(t *testing.T) {
} }
} }
func TestRunRejectsInvalidChunks(t *testing.T) {
tests := []struct {
name string
chunks []contracts.SourceChunk
want string
}{
{
name: "empty chunk id",
chunks: []contracts.SourceChunk{{ID: "", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}}},
want: "id must not be empty",
},
{
name: "duplicate chunk id",
chunks: []contracts.SourceChunk{
{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}},
{ID: "chunk-0", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u2")}},
},
want: "duplicated",
},
{
name: "wrong source id",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "other-source", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}}},
want: "source_id",
},
{
name: "wrong index",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u1")}}},
want: "index",
},
{
name: "empty units",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0}},
want: "units must not be empty",
},
{
name: "repeated unit inside chunk",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1"), unitWithID("u1")}}},
want: "repeats source unit",
},
{
name: "unknown unit",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u9")}}},
want: "was not found",
},
{
name: "units out of source order",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u2"), unitWithID("u1")}}},
want: "source document order",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = test.chunks
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, test.want)
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
if len(modules.extractors["extract-alpha"].requests) != 0 {
t.Fatalf("extractor calls = %d, want none after invalid chunks", len(modules.extractors["extract-alpha"].requests))
}
})
}
}
func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = []contracts.SourceChunk{
{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1"), unitWithID("u2")}},
{ID: "chunk-1", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u2")}},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want one candidate per accepted chunk", len(output.Approved))
}
}
func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
modules := defaultRunnerModules()
modules.input.doc = sourceDocumentWithUnitMetadata()
modules.chunker.chunks = []contracts.SourceChunk{
{
ID: "chunk-0",
SourceID: "source-1",
Index: 0,
Units: []source.SourceUnit{
{
ID: "u1",
Kind: "mutated-kind",
Text: "mutated text",
Metadata: map[string]any{
"speaker": "chunker-speaker",
"note": "chunker note",
},
},
},
},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
}
extractor := modules.extractors["extract-alpha"]
if len(extractor.requests) != 1 {
t.Fatalf("len(extractor requests) = %d, want 1", len(extractor.requests))
}
chunk := extractor.requests[0].Chunk
if chunk == nil {
t.Fatal("extractor chunk = nil, want canonical chunk")
}
if chunk.Units[0].ID != "u1" || chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" {
t.Fatalf("chunk unit = %#v, want source document unit values", chunk.Units[0])
}
if got := chunk.Units[0].Metadata["speaker"]; got != "source-speaker" {
t.Fatalf("chunk unit metadata = %#v, want source document metadata", chunk.Units[0].Metadata)
}
if got := chunk.Units[0].Metadata["topic"]; got != "source-topic" {
t.Fatalf("chunk unit metadata = %#v, want cloned source document metadata", chunk.Units[0].Metadata)
}
modules.input.doc.Units[0].Kind = "changed-kind"
modules.input.doc.Units[0].Text = "changed text"
modules.input.doc.Units[0].Metadata["speaker"] = "changed-speaker"
if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Metadata["speaker"] != "source-speaker" {
t.Fatalf("chunk unit changed after source mutation: %#v", chunk.Units[0])
}
}
func TestRunPreservesChunkMetadataDuringCanonicalization(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = []contracts.SourceChunk{
{
ID: "chunk-0",
SourceID: "source-1",
Index: 0,
Units: []source.SourceUnit{
unitWithID("u1"),
},
Metadata: map[string]any{
"scene_title": "Original scene",
"boundary_note": "Chunker note",
},
},
}
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
extractor := modules.extractors["extract-alpha"]
if len(extractor.requests) != 1 || extractor.requests[0].Chunk == nil {
t.Fatalf("extractor requests = %#v, want one canonical chunk", extractor.requests)
}
if got := extractor.requests[0].Chunk.Metadata["scene_title"]; got != "Original scene" {
t.Fatalf("chunk metadata = %#v, want chunker metadata", extractor.requests[0].Chunk.Metadata)
}
if got := extractor.requests[0].Chunk.Metadata["boundary_note"]; got != "Chunker note" {
t.Fatalf("chunk metadata = %#v, want chunker metadata", extractor.requests[0].Chunk.Metadata)
}
modules.chunker.chunks[0].Metadata["scene_title"] = "changed"
modules.chunker.chunks[0].Metadata["boundary_note"] = "changed"
if got := extractor.requests[0].Chunk.Metadata["scene_title"]; got != "Original scene" {
t.Fatalf("chunk metadata aliased to chunker map: %#v", extractor.requests[0].Chunk.Metadata)
}
if got := extractor.requests[0].Chunk.Metadata["boundary_note"]; got != "Chunker note" {
t.Fatalf("chunk metadata aliased to chunker map: %#v", extractor.requests[0].Chunk.Metadata)
}
}
func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) { func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
llmClient := fakeLLMClient{} llmClient := fakeLLMClient{}
@@ -273,6 +458,9 @@ func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
if !reflect.DeepEqual(extractor.seenChunkIDs, []string{"chunk-0", "chunk-1"}) { if !reflect.DeepEqual(extractor.seenChunkIDs, []string{"chunk-0", "chunk-1"}) {
t.Fatalf("seen chunks = %#v, want both chunks", extractor.seenChunkIDs) t.Fatalf("seen chunks = %#v, want both chunks", extractor.seenChunkIDs)
} }
if len(modules.chunker.requests) != 1 || modules.chunker.requests[0].LLMClient == nil {
t.Fatalf("chunker LLM client = %#v, want client on chunk request", modules.chunker.requests)
}
if len(extractor.seenLLMClients) != 2 || extractor.seenLLMClients[0] == nil || extractor.seenLLMClients[1] == nil { if len(extractor.seenLLMClients) != 2 || extractor.seenLLMClients[0] == nil || extractor.seenLLMClients[1] == nil {
t.Fatalf("seen LLM clients = %#v, want client for each chunk", extractor.seenLLMClients) t.Fatalf("seen LLM clients = %#v, want client for each chunk", extractor.seenLLMClients)
} }
@@ -371,6 +559,101 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
} }
} }
func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain; charset=utf-8",
Content: []byte("reference text"),
Digest: "sha256:test",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/reference.txt"},
SizeBytes: int64(len("reference text")),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
req := modules.extractors["extract-alpha"].requests[0]
item := req.References.Slots["roster"].Items[0]
if string(item.Content) != "reference text" {
t.Fatalf("reference content = %q, want reference text", item.Content)
}
item.Content[0] = 'R'
if got := string(pipeline.ArtifactLanes[0].ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
t.Fatalf("runner mutated reference set content = %q", got)
}
}
func TestRunIncludesInputWarnings(t *testing.T) {
modules := defaultRunnerModules()
warning := contracts.Warning{Scope: "reference", ReasonCode: "empty_reference", Message: "empty reference"}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
Warnings: []contracts.Warning{warning},
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Warnings) != 1 || output.Warnings[0] != warning {
t.Fatalf("warnings = %#v, want input warning", output.Warnings)
}
}
func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) {
modules := defaultRunnerModules()
modules.input.manifestMetadata = map[string]any{
"input_profile": "input-metadata",
}
modules.chunker.manifestMetadata = map[string]any{
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:chunker-prompt",
"response_schema_key": "dnd_scenes",
"response_schema_id": "schema-dnd-scenes",
"response_schema_name": "dnd_scenes",
}
modules.output.manifestMetadata = map[string]any{
"output_profile": "output-metadata",
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if output.Manifest.ModuleMetadata == nil {
t.Fatal("ModuleMetadata = nil, want module metadata map")
}
if got := output.Manifest.ModuleMetadata["input"]; !reflect.DeepEqual(got, modules.input.manifestMetadata) {
t.Fatalf("input module metadata = %#v, want %#v", got, modules.input.manifestMetadata)
}
if got := output.Manifest.ModuleMetadata["chunker"]; !reflect.DeepEqual(got, modules.chunker.manifestMetadata) {
t.Fatalf("chunker module metadata = %#v, want %#v", got, modules.chunker.manifestMetadata)
}
if got := output.Manifest.ModuleMetadata["output"]; !reflect.DeepEqual(got, modules.output.manifestMetadata) {
t.Fatalf("output module metadata = %#v, want %#v", got, modules.output.manifestMetadata)
}
modules.chunker.manifestMetadata["prompt_id"] = "changed"
if output.Manifest.ModuleMetadata["chunker"]["prompt_id"] != "dnd.scenes" {
t.Fatalf("chunker module metadata aliased to provider map: %#v", output.Manifest.ModuleMetadata["chunker"])
}
}
func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) { func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
@@ -677,7 +960,27 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
} }
func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) { func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolvedPipelineWithValidators("configured")}) resolved := resolvedPipelineWithValidators("configured")
resolved.ArtifactLanes[0].ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain; charset=utf-8",
Content: []byte("reference content"),
Digest: "sha256:reference",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
SizeBytes: int64(len("reference content")),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolved})
if err != nil { if err != nil {
t.Fatalf("Run() error = %v, want nil", err) t.Fatalf("Run() error = %v, want nil", err)
} }
@@ -692,6 +995,16 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) { if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) {
t.Fatalf("SourceDigests = %#v, want source digest", manifest.SourceDigests) t.Fatalf("SourceDigests = %#v, want source digest", manifest.SourceDigests)
} }
if len(manifest.References) != 1 {
t.Fatalf("References = %#v, want one reference provenance entry", manifest.References)
}
reference := manifest.References[0]
if reference.LaneID != "alpha" || reference.SlotName != "roster" || reference.Digest != "sha256:reference" {
t.Fatalf("reference provenance = %#v, want lane slot digest", reference)
}
if reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != int64(len("reference content")) || reference.BindingSource != contracts.ReferenceBindingSourceConfig {
t.Fatalf("reference provenance = %#v, want origin/media/size/source", reference)
}
if manifest.ValidationStatus != "approved" { if manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved", manifest.ValidationStatus) t.Fatalf("ValidationStatus = %q, want approved", manifest.ValidationStatus)
} }
@@ -775,6 +1088,11 @@ func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" { if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata) t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
} }
if output.Manifest.ModuleMetadata != nil {
if _, ok := output.Manifest.ModuleMetadata["extractor"]; ok {
t.Fatalf("top-level module metadata includes lane metadata key: %#v", output.Manifest.ModuleMetadata)
}
}
} }
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) { func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
@@ -963,10 +1281,11 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries {
} }
type runnerInputAdapter struct { type runnerInputAdapter struct {
key string key string
doc *source.SourceDocument doc *source.SourceDocument
err error err error
requests []contracts.ParseRequest manifestMetadata map[string]any
requests []contracts.ParseRequest
} }
func (adapter *runnerInputAdapter) Key() string { func (adapter *runnerInputAdapter) Key() string {
@@ -978,12 +1297,17 @@ func (adapter *runnerInputAdapter) Parse(ctx context.Context, req contracts.Pars
return adapter.doc, adapter.err return adapter.doc, adapter.err
} }
func (adapter *runnerInputAdapter) ManifestMetadata() map[string]any {
return adapter.manifestMetadata
}
type runnerChunker struct { type runnerChunker struct {
key string key string
chunks []contracts.SourceChunk chunks []contracts.SourceChunk
warnings []contracts.Warning warnings []contracts.Warning
err error err error
requests []contracts.ChunkRequest manifestMetadata map[string]any
requests []contracts.ChunkRequest
} }
func (chunker *runnerChunker) Key() string { func (chunker *runnerChunker) Key() string {
@@ -998,6 +1322,10 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ
}, chunker.err }, chunker.err
} }
func (chunker *runnerChunker) ManifestMetadata() map[string]any {
return chunker.manifestMetadata
}
type runnerExtractor struct { type runnerExtractor struct {
key string key string
artifactType string artifactType string
@@ -1025,6 +1353,10 @@ func (extractor *runnerExtractor) SchemaVersion() string {
return extractor.schemaVersion return extractor.schemaVersion
} }
func (extractor *runnerExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor *runnerExtractor) ManifestMetadata() map[string]any { func (extractor *runnerExtractor) ManifestMetadata() map[string]any {
return extractor.manifestMetadata return extractor.manifestMetadata
} }
@@ -1138,11 +1470,12 @@ func (validator *runnerValidator) Validate(ctx context.Context, req contracts.Va
} }
type runnerOutputEncoder struct { type runnerOutputEncoder struct {
key string key string
files []contracts.OutputFile files []contracts.OutputFile
warnings []contracts.Warning warnings []contracts.Warning
err error err error
requests []contracts.OutputRequest manifestMetadata map[string]any
requests []contracts.OutputRequest
} }
func (encoder *runnerOutputEncoder) Key() string { func (encoder *runnerOutputEncoder) Key() string {
@@ -1157,6 +1490,10 @@ func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.Ou
}, encoder.err }, encoder.err
} }
func (encoder *runnerOutputEncoder) ManifestMetadata() map[string]any {
return encoder.manifestMetadata
}
type fakeLLMClient struct{} type fakeLLMClient struct{}
func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
@@ -1179,6 +1516,36 @@ func validSourceDocument() *source.SourceDocument {
Digest: "sha256:source", Digest: "sha256:source",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "unit", Text: "Source unit."}, {ID: "u1", Kind: "unit", Text: "Source unit."},
{ID: "u2", Kind: "unit", Text: "Second source unit."},
{ID: "u3", Kind: "unit", Text: "Third source unit."},
},
}
}
func sourceDocumentWithUnitMetadata() *source.SourceDocument {
return &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{
{
ID: "u1",
Kind: "source-kind",
Text: "source text",
Metadata: map[string]any{
"speaker": "source-speaker",
"topic": "source-topic",
},
},
{
ID: "u2",
Kind: "source-kind",
Text: "second source text",
Metadata: map[string]any{
"speaker": "source-speaker-2",
},
},
}, },
} }
} }
@@ -1194,6 +1561,19 @@ func sourceChunkWithID(id string, index int) contracts.SourceChunk {
} }
} }
func unitWithID(id string) source.SourceUnit {
switch id {
case "u1":
return source.SourceUnit{ID: "u1", Kind: "unit", Text: "Source unit."}
case "u2":
return source.SourceUnit{ID: "u2", Kind: "unit", Text: "Second source unit."}
case "u3":
return source.SourceUnit{ID: "u3", Kind: "unit", Text: "Third source unit."}
default:
return source.SourceUnit{ID: id, Kind: "unit", Text: "Unknown source unit."}
}
}
func warningReasons(warnings []contracts.Warning) []string { func warningReasons(warnings []contracts.Warning) []string {
reasons := make([]string, 0, len(warnings)) reasons := make([]string, 0, len(warnings))
for _, warning := range warnings { for _, warning := range warnings {

View File

@@ -252,6 +252,10 @@ func (extractor walkingSkeletonExtractor) SchemaVersion() string {
return "v1" return "v1"
} }
func (extractor walkingSkeletonExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor walkingSkeletonExtractor) Validators() []contracts.Validator { func (extractor walkingSkeletonExtractor) Validators() []contracts.Validator {
return nil return nil
} }

View File

@@ -7,9 +7,13 @@ import (
"fmt" "fmt"
"io/fs" "io/fs"
"path" "path"
"reflect"
"sort" "sort"
"strings" "strings"
"text/template" "text/template"
"text/template/parse"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
//go:embed assets/** //go:embed assets/**
@@ -43,18 +47,20 @@ func (m Metadata) DiagnosticsMap() map[string]any {
// Definition identifies a caller-owned system/user prompt bundle. // Definition identifies a caller-owned system/user prompt bundle.
type Definition struct { type Definition struct {
PromptID string PromptID string
Version string Version string
EmbeddedPath string EmbeddedPath string
SystemPath string SystemPath string
UserPath string UserPath string
ReferenceSlots []contracts.ReferenceSlot
} }
// Bundle is a compiled system/user prompt pair. // Bundle is a compiled system/user prompt pair.
type Bundle struct { type Bundle struct {
systemTmpl *template.Template systemTmpl *template.Template
userTmpl *template.Template userTmpl *template.Template
metadata Metadata metadata Metadata
referenceSlots map[string]contracts.ReferenceSlot
} }
// Metadata returns metadata for the compiled prompt bundle. // Metadata returns metadata for the compiled prompt bundle.
@@ -168,7 +174,9 @@ func LoadBundle(fsys fs.FS, def Definition) (*Bundle, error) {
} }
funcs := template.FuncMap{ funcs := template.FuncMap{
"hardening": func() string { return sharedHardening }, "hardening": func() string { return sharedHardening },
"reference": func(string) (string, error) { return "", nil },
"hasreference": func(string) (bool, error) { return false, nil },
} }
systemTmpl, err := template.New(path.Base(systemPath)).Option("missingkey=error").Funcs(funcs).Parse(systemSource) systemTmpl, err := template.New(path.Base(systemPath)).Option("missingkey=error").Funcs(funcs).Parse(systemSource)
if err != nil { if err != nil {
@@ -178,6 +186,13 @@ func LoadBundle(fsys fs.FS, def Definition) (*Bundle, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("parse embedded user prompt %q: %w", userPath, err) return nil, fmt.Errorf("parse embedded user prompt %q: %w", userPath, err)
} }
referenceSlots := referenceSlotMap(def.ReferenceSlots)
if err := validateTemplateReferenceSlots(systemTmpl, referenceSlots); err != nil {
return nil, fmt.Errorf("validate embedded system prompt %q: %w", systemPath, err)
}
if err := validateTemplateReferenceSlots(userTmpl, referenceSlots); err != nil {
return nil, fmt.Errorf("validate embedded user prompt %q: %w", userPath, err)
}
hashInput := systemSource + "\n\n" + userSource hashInput := systemSource + "\n\n" + userSource
hash := sha256.Sum256([]byte(hashInput)) hash := sha256.Sum256([]byte(hashInput))
@@ -190,12 +205,127 @@ func LoadBundle(fsys fs.FS, def Definition) (*Bundle, error) {
} }
return &Bundle{ return &Bundle{
systemTmpl: systemTmpl, systemTmpl: systemTmpl,
userTmpl: userTmpl, userTmpl: userTmpl,
metadata: metadata, metadata: metadata,
referenceSlots: referenceSlots,
}, nil }, nil
} }
func referenceSlotMap(slots []contracts.ReferenceSlot) map[string]contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
out := make(map[string]contracts.ReferenceSlot, len(slots))
for _, slot := range slots {
name := strings.TrimSpace(slot.Name)
if name == "" {
continue
}
slot.Name = name
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
out[name] = slot
}
return out
}
func validateTemplateReferenceSlots(tmpl *template.Template, declared map[string]contracts.ReferenceSlot) error {
if tmpl == nil || tmpl.Tree == nil || tmpl.Tree.Root == nil {
return nil
}
return validateReferenceNodes(tmpl.Tree.Root, declared)
}
func validateReferenceNodes(node parse.Node, declared map[string]contracts.ReferenceSlot) error {
if node == nil || reflect.ValueOf(node).IsNil() {
return nil
}
switch typed := node.(type) {
case *parse.ListNode:
for _, child := range typed.Nodes {
if err := validateReferenceNodes(child, declared); err != nil {
return err
}
}
case *parse.ActionNode:
return validateReferencePipeline(typed.Pipe, declared)
case *parse.IfNode:
if err := validateReferencePipeline(typed.Pipe, declared); err != nil {
return err
}
if err := validateReferenceNodes(typed.List, declared); err != nil {
return err
}
return validateReferenceNodes(typed.ElseList, declared)
case *parse.RangeNode:
if err := validateReferencePipeline(typed.Pipe, declared); err != nil {
return err
}
if err := validateReferenceNodes(typed.List, declared); err != nil {
return err
}
return validateReferenceNodes(typed.ElseList, declared)
case *parse.WithNode:
if err := validateReferencePipeline(typed.Pipe, declared); err != nil {
return err
}
if err := validateReferenceNodes(typed.List, declared); err != nil {
return err
}
return validateReferenceNodes(typed.ElseList, declared)
case *parse.TemplateNode:
return nil
}
return nil
}
func validateReferencePipeline(pipe *parse.PipeNode, declared map[string]contracts.ReferenceSlot) error {
if pipe == nil {
return nil
}
for _, cmd := range pipe.Cmds {
if err := validateReferenceCommand(cmd, declared); err != nil {
return err
}
}
return nil
}
func validateReferenceCommand(cmd *parse.CommandNode, declared map[string]contracts.ReferenceSlot) error {
if cmd == nil || len(cmd.Args) == 0 {
return nil
}
for _, arg := range cmd.Args[1:] {
if nested, ok := arg.(*parse.PipeNode); ok {
if err := validateReferencePipeline(nested, declared); err != nil {
return err
}
}
}
identifier, ok := cmd.Args[0].(*parse.IdentifierNode)
if !ok {
return nil
}
if identifier.Ident != "reference" && identifier.Ident != "hasreference" {
return nil
}
if len(cmd.Args) != 2 {
return fmt.Errorf("%s requires one string slot name", identifier.Ident)
}
slotArg, ok := cmd.Args[1].(*parse.StringNode)
if !ok {
return fmt.Errorf("%s requires a string literal slot name", identifier.Ident)
}
slotName := strings.TrimSpace(slotArg.Text)
if slotName == "" {
return fmt.Errorf("%s slot name must not be empty", identifier.Ident)
}
if _, ok := declared[slotName]; !ok {
return fmt.Errorf("%s slot %q is not declared", identifier.Ident, slotName)
}
return nil
}
func readPromptAsset(fsys fs.FS, assetPath string) (string, error) { func readPromptAsset(fsys fs.FS, assetPath string) (string, error) {
if strings.TrimSpace(assetPath) == "" { if strings.TrimSpace(assetPath) == "" {
return "", fmt.Errorf("prompt asset path must not be empty") return "", fmt.Errorf("prompt asset path must not be empty")

View File

@@ -4,6 +4,9 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"strings" "strings"
"text/template"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
// RenderUserSystem renders the system and user prompt pair for promptID. // RenderUserSystem renders the system and user prompt pair for promptID.
@@ -16,20 +19,105 @@ func RenderUserSystem(promptID string, data any) (system string, user string, me
return compiled.RenderUserSystem(data) return compiled.RenderUserSystem(data)
} }
// RenderUserSystemWithReferences renders the system and user prompt pair for promptID with reference template functions.
func RenderUserSystemWithReferences(promptID string, data any, references contracts.ReferenceSet) (system string, user string, metadata Metadata, err error) {
trimmedID := strings.TrimSpace(promptID)
compiled, ok := promptRegistry[trimmedID]
if !ok {
return "", "", Metadata{}, fmt.Errorf("unknown prompt id %q", promptID)
}
return compiled.RenderUserSystemWithReferences(data, references)
}
// RenderUserSystem renders the bundle's system and user prompts. // RenderUserSystem renders the bundle's system and user prompts.
func (b *Bundle) RenderUserSystem(data any) (system string, user string, metadata Metadata, err error) { func (b *Bundle) RenderUserSystem(data any) (system string, user string, metadata Metadata, err error) {
return b.RenderUserSystemWithReferences(data, contracts.ReferenceSet{})
}
// RenderUserSystemWithReferences renders the bundle's system and user prompts with reference template functions.
func (b *Bundle) RenderUserSystemWithReferences(data any, references contracts.ReferenceSet) (system string, user string, metadata Metadata, err error) {
if b == nil { if b == nil {
return "", "", Metadata{}, fmt.Errorf("prompt bundle must not be nil") return "", "", Metadata{}, fmt.Errorf("prompt bundle must not be nil")
} }
systemTmpl, userTmpl, err := b.renderTemplates(references)
if err != nil {
return "", "", Metadata{}, err
}
var systemBuf bytes.Buffer var systemBuf bytes.Buffer
if err := b.systemTmpl.Execute(&systemBuf, data); err != nil { if err := systemTmpl.Execute(&systemBuf, data); err != nil {
return "", "", Metadata{}, fmt.Errorf("render system prompt %q: %w", b.metadata.PromptID, err) return "", "", Metadata{}, fmt.Errorf("render system prompt %q: %w", b.metadata.PromptID, err)
} }
var userBuf bytes.Buffer var userBuf bytes.Buffer
if err := b.userTmpl.Execute(&userBuf, data); err != nil { if err := userTmpl.Execute(&userBuf, data); err != nil {
return "", "", Metadata{}, fmt.Errorf("render user prompt %q: %w", b.metadata.PromptID, err) return "", "", Metadata{}, fmt.Errorf("render user prompt %q: %w", b.metadata.PromptID, err)
} }
return strings.TrimSpace(systemBuf.String()), strings.TrimSpace(userBuf.String()), b.metadata, nil return strings.TrimSpace(systemBuf.String()), strings.TrimSpace(userBuf.String()), b.metadata, nil
} }
func (b *Bundle) renderTemplates(references contracts.ReferenceSet) (*template.Template, *template.Template, error) {
funcs := b.referenceFuncs(references)
systemTmpl, err := b.systemTmpl.Clone()
if err != nil {
return nil, nil, fmt.Errorf("clone system prompt %q: %w", b.metadata.PromptID, err)
}
userTmpl, err := b.userTmpl.Clone()
if err != nil {
return nil, nil, fmt.Errorf("clone user prompt %q: %w", b.metadata.PromptID, err)
}
systemTmpl.Funcs(funcs)
userTmpl.Funcs(funcs)
return systemTmpl, userTmpl, nil
}
func (b *Bundle) referenceFuncs(references contracts.ReferenceSet) template.FuncMap {
return template.FuncMap{
"hardening": func() string { return sharedHardening },
"hasreference": func(slotName string) (bool, error) {
items, _, err := b.referenceItems(slotName, references)
if err != nil {
return false, err
}
for _, item := range items {
if len(item.Content) > 0 {
return true, nil
}
}
return false, nil
},
"reference": func(slotName string) (string, error) {
items, slot, err := b.referenceItems(slotName, references)
if err != nil {
return "", err
}
if len(items) == 0 {
return "", nil
}
if len(items) > 1 && !slot.Multiple {
return "", fmt.Errorf("reference slot %q has %d bound items but does not allow multiple", slot.Name, len(items))
}
parts := make([]string, 0, len(items))
for _, item := range items {
parts = append(parts, string(item.Content))
}
return strings.Join(parts, "\n"), nil
},
}
}
func (b *Bundle) referenceItems(slotName string, references contracts.ReferenceSet) ([]contracts.ReferenceItem, contracts.ReferenceSlot, error) {
slotName = strings.TrimSpace(slotName)
slot, ok := b.referenceSlots[slotName]
if !ok {
return nil, contracts.ReferenceSlot{}, fmt.Errorf("reference slot %q is not declared", slotName)
}
if len(references.Slots) == 0 {
return nil, slot, nil
}
resolved, ok := references.Slots[slotName]
if !ok {
return nil, slot, nil
}
return append([]contracts.ReferenceItem(nil), resolved.Items...), slot, nil
}

View File

@@ -1,8 +1,13 @@
package prompt package prompt
import ( import (
"crypto/sha256"
"encoding/hex"
"strings" "strings"
"testing" "testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
func TestRenderUserSystemReturnsTextAndMetadata(t *testing.T) { func TestRenderUserSystemReturnsTextAndMetadata(t *testing.T) {
@@ -64,3 +69,175 @@ func TestRenderUserSystemIncludesHardeningText(t *testing.T) {
t.Fatalf("expected rendered system prompt to include hardening text: %q", system) t.Fatalf("expected rendered system prompt to include hardening text: %q", system)
} }
} }
func TestRenderUserSystemWithReferencesRendersDeclaredSlots(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster"}, {Name: "glossary"}},
`System has roster={{ hasreference "roster" }} has glossary={{ hasreference "glossary" }}`,
`Roster={{ reference "roster" }} Glossary={{ reference "glossary" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria")},
},
},
}}
system, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if !strings.Contains(system, "has roster=true") || !strings.Contains(system, "has glossary=false") {
t.Fatalf("system = %q, want reference presence flags", system)
}
if !strings.Contains(user, "Roster=Aria") || !strings.Contains(user, "Glossary=") {
t.Fatalf("user = %q, want rendered and empty optional references", user)
}
}
func TestRenderUserSystemReferenceHasReferenceRequiresContent(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster"}},
`System`,
`{{ hasreference "roster" }} {{ reference "roster" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: nil},
},
},
}}
_, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if user != "false" {
t.Fatalf("user = %q, want false with empty reference content", user)
}
}
func TestLoadBundleRejectsUndeclaredReferenceSlots(t *testing.T) {
_, err := LoadBundle(referenceBundleFS(`System`, `{{ reference "roster" }}`), referenceBundleDefinition(nil))
if err == nil || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("LoadBundle() error = %v, want undeclared reference slot error", err)
}
}
func TestLoadBundleRejectsDynamicReferenceSlotNames(t *testing.T) {
_, err := LoadBundle(referenceBundleFS(`System`, `{{ reference .SlotName }}`), referenceBundleDefinition([]contracts.ReferenceSlot{{Name: "roster"}}))
if err == nil || !strings.Contains(err.Error(), "string literal") {
t.Fatalf("LoadBundle() error = %v, want string literal error", err)
}
}
func TestLoadBundleRejectsNestedUndeclaredReferenceSlots(t *testing.T) {
_, err := LoadBundle(referenceBundleFS(`System`, `{{ printf "%s" (reference "roster") }}`), referenceBundleDefinition(nil))
if err == nil || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("LoadBundle() error = %v, want nested undeclared reference slot error", err)
}
}
func TestRenderUserSystemRejectsMultipleReferenceItemsUnlessDeclared(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster"}},
`System`,
`{{ reference "roster" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria")},
{SlotName: "roster", Content: []byte("Bryn")},
},
},
}}
_, _, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err == nil || !strings.Contains(err.Error(), "does not allow multiple") {
t.Fatalf("RenderUserSystemWithReferences() error = %v, want multiple item error", err)
}
}
func TestRenderUserSystemRendersMultipleReferenceItemsDeterministicallyWhenDeclared(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster", Multiple: true}},
`System`,
`{{ reference "roster" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster", Multiple: true},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria")},
{SlotName: "roster", Content: []byte("Bryn")},
},
},
}}
_, first, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences(first): %v", err)
}
_, second, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences(second): %v", err)
}
if first != "Aria\nBryn" || first != second {
t.Fatalf("rendered references = %q/%q, want deterministic item order", first, second)
}
}
func TestPromptMetadataHashIgnoresRenderedReferenceContent(t *testing.T) {
systemSource := `System`
userSource := `{{ reference "roster" }}`
bundle := loadReferenceBundle(t, []contracts.ReferenceSlot{{Name: "roster"}}, systemSource, userSource)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{{SlotName: "roster", Content: []byte("Aria")}},
},
}}
_, _, metadata, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
hash := sha256.Sum256([]byte(systemSource + "\n\n" + userSource))
want := "sha256:" + hex.EncodeToString(hash[:])
if metadata.SHA256 != want {
t.Fatalf("metadata.SHA256 = %q, want template source hash %q", metadata.SHA256, want)
}
}
func loadReferenceBundle(t *testing.T, slots []contracts.ReferenceSlot, systemSource string, userSource string) *Bundle {
t.Helper()
bundle, err := LoadBundle(referenceBundleFS(systemSource, userSource), referenceBundleDefinition(slots))
if err != nil {
t.Fatalf("LoadBundle() error = %v, want nil", err)
}
return bundle
}
func referenceBundleDefinition(slots []contracts.ReferenceSlot) Definition {
return Definition{
PromptID: "test.references",
Version: VersionV1,
EmbeddedPath: "assets/test/references",
SystemPath: "assets/test/references/system.md",
UserPath: "assets/test/references/user.md",
ReferenceSlots: slots,
}
}
func referenceBundleFS(systemSource string, userSource string) fstest.MapFS {
return fstest.MapFS{
"assets/test/references/system.md": {Data: []byte(systemSource)},
"assets/test/references/user.md": {Data: []byte(userSource)},
}
}

View File

@@ -0,0 +1,6 @@
package scenes
import "embed"
//go:embed assets/prompts/*.md assets/schemas/*.json
var embeddedAssets embed.FS

View File

@@ -0,0 +1,9 @@
You identify coherent scenes in Dungeons & Dragons session source units.
{{ hardening }}
Use only the provided source units. Source text may contain transcription
errors, repeated lines, incomplete sentences, and misheard proper nouns. Speaker
metadata, when present, may be treated as accurate.
Return only valid JSON matching the provided response schema.

View File

@@ -0,0 +1,71 @@
Source document ID: {{ .SourceID }}
Ordered source units:
{{ range .Units }}
- Unit ID: {{ .ID }}
Text: {{ .Text }}
{{ if .Metadata }}
Metadata:
{{ range .Metadata }}
- {{ .Key }}: {{ .Value }}
{{ end }}
{{ end }}
{{ end }}
Divide these source units into D&D scenes for the dnd/scenes chunk module.
A scene is a coherent unit of play. Start a new scene when there is a meaningful
change in location, objective, threat, activity, encounter, or mode of play.
Good reasons to start a new scene include:
- the party moves to a new location;
- a combat encounter begins or ends;
- combat changes into a substantially different phase;
- the party shifts between combat, exploration, social interaction, discussion,
planning, travel, rest, or downtime;
- a new NPC, faction, threat, or objective becomes central;
- the party completes one immediate goal and begins another;
- a major table-level rules discussion interrupts and materially changes play.
Do not start a new scene merely because:
- the speaker changes;
- a new combat round begins;
- a player asks a brief rules question;
- there is a joke, aside, or short table comment;
- a character takes a routine turn;
- the same encounter continues without a meaningful change in situation.
dnd/scenes boundary policy:
- cover the full provided source document from the first source unit to the last
source unit;
- return sequential scenes with no gaps;
- do not overlap scenes;
- preserve source-unit order;
- use exact source-unit IDs from the ordered source units;
- each scene must have start_unit_id and end_unit_id;
- do not include final chunk IDs or chunk indexes.
For each scene:
- short_title should be brief and factual;
- primary_mode must be Recap, Discussion, Combat, or Narrative;
- main_participants should include only principal characters, NPCs, factions, or
groups involved;
- summary should be factual and compact, usually one to three sentences;
- boundary_note should explain why the scene begins at start_unit_id and ends at
end_unit_id;
- boundary_confidence must be High, Medium, or Low.
Primary mode guidance:
- Use Recap for opening recap, initiative setup, session framing, or immediate
continuation from prior events.
- Use Discussion when the party is primarily discussing options or choosing a
course of action.
- Use Combat when active combat or combat-resolution mechanics dominate.
- Use Narrative for all other non-combat gameplay, including exploration, social
interactions, shopping, preparation, travel, rest, and downtime.
In boundary_caveats, list overall caveats about scene divisions. Include scenes
that could reasonably be split differently, combat phases that were kept
together, gradual transitions, or places where map context would have helped.
Return exactly one JSON object and no explanatory text.

View File

@@ -0,0 +1,84 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "notarius.dnd.scenes",
"type": "object",
"additionalProperties": false,
"required": [
"scenes",
"boundary_caveats"
],
"properties": {
"scenes": {
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"start_unit_id",
"end_unit_id",
"short_title",
"primary_mode",
"main_participants",
"summary",
"boundary_note",
"boundary_confidence"
],
"properties": {
"start_unit_id": {
"type": "string",
"minLength": 1
},
"end_unit_id": {
"type": "string",
"minLength": 1
},
"short_title": {
"type": "string",
"minLength": 1
},
"primary_mode": {
"type": "string",
"enum": [
"Recap",
"Discussion",
"Combat",
"Narrative"
]
},
"main_participants": {
"type": "array",
"minItems": 1,
"items": {
"type": "string",
"minLength": 1
}
},
"summary": {
"type": "string",
"minLength": 1
},
"boundary_note": {
"type": "string",
"minLength": 1
},
"boundary_confidence": {
"type": "string",
"enum": [
"High",
"Medium",
"Low"
]
}
}
}
},
"boundary_caveats": {
"type": "array",
"items": {
"type": "string",
"minLength": 1
}
}
}
}

View File

@@ -0,0 +1,311 @@
package scenes
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "dnd/scenes"
var requiredCapabilities = []string{
"source.transcript",
}
var providedCapabilities = []string{
"chunks",
"chunks.scenes",
}
var _ contracts.Chunker = (*Chunker)(nil)
var _ contracts.ManifestMetadataProvider = (*Chunker)(nil)
type Chunker struct{}
func New() *Chunker {
return &Chunker{}
}
func (c *Chunker) Key() string {
return Key
}
func (c *Chunker) ManifestMetadata() map[string]any {
promptMetadata := scenesPromptBundle.Metadata()
metadata := map[string]any{
"prompt_id": PromptID,
"prompt_version": promptMetadata.PromptVersion,
"prompt_sha256": promptMetadata.SHA256,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
}
if schema, err := loadResponseSchema(); err == nil {
metadata["response_schema_version"] = schema.Version
metadata["response_schema_sha256"] = schema.SHA256
}
return metadata
}
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if c == nil {
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")
}
if ctx == nil {
return contracts.ChunkResult{}, chunkerErrorf("context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("context error before chunking: %w", err)
}
if req.Source == nil {
return contracts.ChunkResult{}, chunkerErrorf("source must not be nil")
}
if len(req.Source.Units) == 0 {
return contracts.ChunkResult{}, chunkerErrorf("source units must not be empty")
}
if err := source.ValidateDocument(req.Source); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("validate source document: %w", err)
}
if req.LLMClient == nil {
return contracts.ChunkResult{}, chunkerErrorf("LLM client must not be nil")
}
if len(req.Options) > 0 {
return contracts.ChunkResult{}, chunkerErrorf("options are not supported")
}
system, user, _, err := renderPrompt(req)
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("render prompt: %w", err)
}
schema, err := loadResponseSchema()
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("load response schema %q: %w", ResponseSchemaKey, err)
}
var response chunkResponse
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key,
Messages: []contracts.LLMMessage{
{Role: "system", Content: system},
{Role: "user", Content: user},
},
ResponseSchemaName: schema.Name,
ResponseSchema: schema.JSONSchema,
}, &response); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("complete structured output: %w", err)
}
warnings, err := warningsFromCaveats(response.BoundaryCaveats)
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
}
chunks, err := chunksFromResponse(req.Source, response)
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
}
return contracts.ChunkResult{
Chunks: chunks,
Warnings: warnings,
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageChunk,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
}
}
func Register(registry *pipeline.ChunkerRegistry) error {
return registry.RegisterWithSpec(ModuleSpec(), func() (contracts.Chunker, error) {
return New(), nil
})
}
func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]contracts.SourceChunk, error) {
if response.Scenes == nil {
return nil, fmt.Errorf("scenes must be present")
}
if len(response.Scenes) == 0 {
return nil, fmt.Errorf("scenes must not be empty")
}
unitIndexes := make(map[string]int, len(doc.Units))
for i, unit := range doc.Units {
unitIndexes[unit.ID] = i
}
chunks := make([]contracts.SourceChunk, 0, len(response.Scenes))
previousEnd := -1
for i, scene := range response.Scenes {
normalized, err := normalizeScene(i, scene)
if err != nil {
return nil, err
}
startIndex, ok := unitIndexes[normalized.StartUnitID]
if !ok {
return nil, fmt.Errorf("scene[%d] start_unit_id %q was not found", i, normalized.StartUnitID)
}
endIndex, ok := unitIndexes[normalized.EndUnitID]
if !ok {
return nil, fmt.Errorf("scene[%d] end_unit_id %q was not found", i, normalized.EndUnitID)
}
if startIndex > endIndex {
return nil, fmt.Errorf("scene[%d] start_unit_id %q appears after end_unit_id %q", i, normalized.StartUnitID, normalized.EndUnitID)
}
if i == 0 && startIndex != 0 {
return nil, fmt.Errorf("first scene must start at first source unit %q", doc.Units[0].ID)
}
if i > 0 {
if startIndex <= previousEnd {
return nil, fmt.Errorf("scene[%d] overlaps previous scene", i)
}
if startIndex > previousEnd+1 {
return nil, fmt.Errorf("scene[%d] leaves a gap after previous scene", i)
}
}
previousEnd = endIndex
units := cloneUnits(doc.Units[startIndex : endIndex+1])
chunks = append(chunks, contracts.SourceChunk{
ID: fmt.Sprintf("scene-%06d", i+1),
SourceID: doc.ID,
Index: i,
Units: units,
Metadata: map[string]any{
"scene_title": normalized.ShortTitle,
"primary_mode": normalized.PrimaryMode,
"main_participants": append([]string(nil), normalized.MainParticipants...),
"summary": normalized.Summary,
"boundary_note": normalized.BoundaryNote,
"boundary_confidence": normalized.BoundaryConfidence,
"start_unit_id": normalized.StartUnitID,
"end_unit_id": normalized.EndUnitID,
"unit_count": len(units),
},
})
}
if previousEnd != len(doc.Units)-1 {
return nil, fmt.Errorf("final scene must end at final source unit %q", doc.Units[len(doc.Units)-1].ID)
}
return chunks, nil
}
func normalizeScene(index int, scene sceneResponse) (sceneResponse, error) {
out := sceneResponse{
StartUnitID: strings.TrimSpace(scene.StartUnitID),
EndUnitID: strings.TrimSpace(scene.EndUnitID),
ShortTitle: strings.TrimSpace(scene.ShortTitle),
PrimaryMode: strings.TrimSpace(scene.PrimaryMode),
Summary: strings.TrimSpace(scene.Summary),
BoundaryNote: strings.TrimSpace(scene.BoundaryNote),
BoundaryConfidence: strings.TrimSpace(scene.BoundaryConfidence),
}
required := map[string]string{
"start_unit_id": out.StartUnitID,
"end_unit_id": out.EndUnitID,
"short_title": out.ShortTitle,
"primary_mode": out.PrimaryMode,
"summary": out.Summary,
"boundary_note": out.BoundaryNote,
"boundary_confidence": out.BoundaryConfidence,
}
for field, value := range required {
if value == "" {
return sceneResponse{}, fmt.Errorf("scene[%d] %s must not be empty", index, field)
}
}
if !validPrimaryMode(out.PrimaryMode) {
return sceneResponse{}, fmt.Errorf("scene[%d] primary_mode %q is not supported", index, out.PrimaryMode)
}
if !validBoundaryConfidence(out.BoundaryConfidence) {
return sceneResponse{}, fmt.Errorf("scene[%d] boundary_confidence %q is not supported", index, out.BoundaryConfidence)
}
if len(scene.MainParticipants) == 0 {
return sceneResponse{}, fmt.Errorf("scene[%d] main_participants must not be empty", index)
}
out.MainParticipants = make([]string, 0, len(scene.MainParticipants))
for participantIndex, participant := range scene.MainParticipants {
trimmed := strings.TrimSpace(participant)
if trimmed == "" {
return sceneResponse{}, fmt.Errorf("scene[%d] main_participants[%d] must not be empty", index, participantIndex)
}
out.MainParticipants = append(out.MainParticipants, trimmed)
}
return out, nil
}
func validPrimaryMode(value string) bool {
switch value {
case "Recap", "Discussion", "Combat", "Narrative":
return true
default:
return false
}
}
func validBoundaryConfidence(value string) bool {
switch value {
case "High", "Medium", "Low":
return true
default:
return false
}
}
func warningsFromCaveats(caveats []string) ([]contracts.Warning, error) {
if len(caveats) == 0 {
return nil, nil
}
warnings := make([]contracts.Warning, 0, len(caveats))
for i, caveat := range caveats {
trimmed := strings.TrimSpace(caveat)
if trimmed == "" {
return nil, fmt.Errorf("boundary_caveats[%d] must not be empty after trimming", i)
}
warnings = append(warnings, contracts.Warning{
Scope: Key,
ReasonCode: "scene_boundary_caveat",
Message: trimmed,
})
}
return warnings, nil
}
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
out := make([]source.SourceUnit, 0, len(units))
for _, unit := range units {
out = append(out, source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Metadata: cloneMetadata(unit.Metadata),
})
}
return out
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
func chunkerErrorf(format string, args ...any) error {
return fmt.Errorf("dnd scenes chunker: "+format, args...)
}

View File

@@ -0,0 +1,484 @@
package scenes
import (
"bytes"
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestNewModuleSpecAndRegister(t *testing.T) {
chunker := New()
if chunker == nil {
t.Fatal("New() = nil, want chunker")
}
if chunker.Key() != Key {
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
}
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks", "chunks.scenes"},
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
got := ModuleSpec()
got.Requires[0] = "changed"
got.Provides[0] = "changed"
if again := ModuleSpec(); !reflect.DeepEqual(again, want) {
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
}
registry := pipeline.NewChunkerRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
registered, ok := registry.Spec(Key)
if !ok {
t.Fatalf("Spec(%q) ok = false, want true", Key)
}
if !reflect.DeepEqual(registered, want) {
t.Fatalf("registered spec = %#v, want %#v", registered, want)
}
built, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if built.Key() != Key {
t.Fatalf("built Key() = %q, want %q", built.Key(), Key)
}
}
func TestRegisterNilRegistryReturnsError(t *testing.T) {
err := Register(nil)
if err == nil {
t.Fatal("Register(nil) error = nil, want error")
}
if !strings.Contains(err.Error(), "chunker registry") {
t.Fatalf("Register(nil) error = %q, want registry context", err.Error())
}
}
func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
client := &fakeScenesLLMClient{
response: chunkResponse{
Scenes: []sceneResponse{
{
StartUnitID: "seg-001",
EndUnitID: "seg-002",
ShortTitle: " Goblin parley ",
PrimaryMode: "Discussion",
MainParticipants: []string{" Aria ", "Goblin scout"},
Summary: " The party negotiates with a scout. ",
BoundaryNote: " The scene covers the discussion before fighting starts. ",
BoundaryConfidence: "High",
},
{
StartUnitID: "seg-003",
EndUnitID: "seg-004",
ShortTitle: "Ambush at the gate",
PrimaryMode: "Combat",
MainParticipants: []string{"Aria", "Goblin ambushers"},
Summary: "The goblins attack at the gate.",
BoundaryNote: "Combat begins and resolves the immediate threat.",
BoundaryConfidence: "Medium",
},
},
BoundaryCaveats: []string{" The transition into combat is gradual. "},
},
}
result, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
if len(client.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
req := client.requests[0]
if req.StageName != Key {
t.Fatalf("StageName = %q, want %q", req.StageName, Key)
}
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if req.ResponseSchemaName != schema.Name {
t.Fatalf("ResponseSchemaName = %q, want %q", req.ResponseSchemaName, schema.Name)
}
if !bytes.Equal(req.ResponseSchema, schema.JSONSchema) {
t.Fatal("ResponseSchema does not match D&D scenes schema")
}
if len(req.Messages) != 2 || req.Messages[0].Role != "system" || req.Messages[1].Role != "user" {
t.Fatalf("Messages = %#v, want system then user", req.Messages)
}
for _, want := range []string{"session-alpha", "seg-001", "seg-004", "start_unit_id", "boundary_confidence"} {
if !strings.Contains(req.Messages[1].Content, want) {
t.Fatalf("user message = %q, want substring %q", req.Messages[1].Content, want)
}
}
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"scene-000001", "scene-000002"}) {
t.Fatalf("chunk IDs = %#v, want deterministic scene IDs", got)
}
gotUnits := [][]string{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units)}
wantUnits := [][]string{{"seg-001", "seg-002"}, {"seg-003", "seg-004"}}
if !reflect.DeepEqual(gotUnits, wantUnits) {
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
}
first := result.Chunks[0]
if first.SourceID != "session-alpha" || first.Index != 0 {
t.Fatalf("first chunk = %#v, want source and index fields", first)
}
if first.Metadata["scene_title"] != "Goblin parley" ||
first.Metadata["primary_mode"] != "Discussion" ||
first.Metadata["summary"] != "The party negotiates with a scout." ||
first.Metadata["boundary_note"] != "The scene covers the discussion before fighting starts." ||
first.Metadata["boundary_confidence"] != "High" ||
first.Metadata["start_unit_id"] != "seg-001" ||
first.Metadata["end_unit_id"] != "seg-002" ||
first.Metadata["unit_count"] != 2 {
t.Fatalf("first metadata = %#v, want scene metadata", first.Metadata)
}
if got, ok := first.Metadata["main_participants"].([]string); !ok || !reflect.DeepEqual(got, []string{"Aria", "Goblin scout"}) {
t.Fatalf("main_participants = %#v, want trimmed participant slice", first.Metadata["main_participants"])
}
if got := result.Warnings; len(got) != 1 ||
got[0].Scope != Key ||
got[0].ReasonCode != "scene_boundary_caveat" ||
got[0].Message != "The transition into combat is gradual." {
t.Fatalf("Warnings = %#v, want boundary caveat warning", got)
}
}
func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
client := &fakeScenesLLMClient{
response: chunkResponse{
Scenes: validSceneResponse().Scenes,
BoundaryCaveats: []string{
" ",
},
},
}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want malformed structured output error")
}
if !strings.Contains(err.Error(), "dnd scenes chunker") || !strings.Contains(err.Error(), "malformed structured output") || !strings.Contains(err.Error(), "boundary_caveats[0]") {
t.Fatalf("Chunk() error = %q, want malformed boundary caveat context", err.Error())
}
}
func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
doc := sceneSourceDocument()
client := &fakeScenesLLMClient{response: validSceneResponse()}
result, err := New().Chunk(context.Background(), contracts.ChunkRequest{
Source: doc,
LLMClient: client,
})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
doc.Units[0].ID = "mutated"
doc.Units[0].Metadata["speaker"] = "mutated"
client.response.Scenes[0].MainParticipants[0] = "mutated"
if result.Chunks[0].Units[0].ID != "seg-001" {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
}
if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" {
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
}
participants, ok := result.Chunks[0].Metadata["main_participants"].([]string)
if !ok || participants[0] != "Aria" {
t.Fatalf("participants = %#v, want defensive copy", result.Chunks[0].Metadata["main_participants"])
}
}
func TestChunkerManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) {
metadata := New().ManifestMetadata()
tests := map[string]string{
"prompt_id": PromptID,
"prompt_version": ResponseSchemaVersion,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": ResponseSchemaVersion,
}
for key, want := range tests {
if metadata[key] != want {
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
}
}
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
value, ok := metadata[key].(string)
if !ok || !strings.HasPrefix(value, "sha256:") {
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
}
}
for _, forbidden := range []string{"prompt", "schema", "source", "text"} {
if _, ok := metadata[forbidden]; ok {
t.Fatalf("metadata includes raw %q field: %#v", forbidden, metadata)
}
}
}
func TestChunkRejectsInvalidRequests(t *testing.T) {
validClient := &fakeScenesLLMClient{response: validSceneResponse()}
validReq := chunkRequestWithClient(validClient)
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()
invalidDoc := sceneSourceDocument()
invalidDoc.Units[0].ID = ""
emptyDoc := sceneSourceDocument()
emptyDoc.Units = nil
tests := []struct {
name string
chunker *Chunker
ctx context.Context
req contracts.ChunkRequest
want string
}{
{name: "nil chunker", chunker: nil, ctx: context.Background(), req: validReq, want: "chunker"},
{name: "nil context", chunker: New(), ctx: nil, req: validReq, want: "context"},
{name: "canceled context", chunker: New(), ctx: canceledCtx, req: validReq, want: "context"},
{name: "nil source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{LLMClient: validClient}, want: "source"},
{name: "empty source units", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: emptyDoc, LLMClient: validClient}, want: "units"},
{name: "invalid source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: invalidDoc, LLMClient: validClient}, want: "validate source document"},
{name: "nil LLM client", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: sceneSourceDocument()}, want: "LLM client"},
{name: "unsupported options", chunker: New(), ctx: context.Background(), req: requestWithOptions(validReq), want: "options"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := tt.chunker.Chunk(tt.ctx, tt.req)
if err == nil {
t.Fatal("Chunk() error = nil, want error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
}
})
}
}
func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
tests := []struct {
name string
response chunkResponse
want string
}{
{name: "missing scenes", response: chunkResponse{}, want: "scenes"},
{name: "empty scenes", response: chunkResponse{Scenes: []sceneResponse{}}, want: "scenes"},
{
name: "unknown boundary id",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-999"),
}),
want: "was not found",
},
{
name: "out of order boundaries",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-003", "seg-002"),
}),
want: "appears after",
},
{
name: "gap",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-001"),
scene("seg-003", "seg-004"),
}),
want: "gap",
},
{
name: "overlap",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-002"),
scene("seg-002", "seg-004"),
}),
want: "overlap",
},
{
name: "incomplete coverage",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-003"),
}),
want: "final scene",
},
{
name: "empty metadata field",
response: replaceScenes(validSceneResponse(), []sceneResponse{
{
StartUnitID: "seg-001",
EndUnitID: "seg-004",
ShortTitle: " ",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"},
Summary: "Summary.",
BoundaryNote: "Note.",
BoundaryConfidence: "High",
},
}),
want: "short_title",
},
{
name: "empty participant",
response: replaceScenes(validSceneResponse(), []sceneResponse{
{
StartUnitID: "seg-001",
EndUnitID: "seg-004",
ShortTitle: "Title",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria", " "},
Summary: "Summary.",
BoundaryNote: "Note.",
BoundaryConfidence: "High",
},
}),
want: "main_participants",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := &fakeScenesLLMClient{response: tt.response}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
}
})
}
}
func TestChunkWrapsLLMClientError(t *testing.T) {
client := &fakeScenesLLMClient{err: errors.New("provider unavailable")}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want LLM error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), "provider unavailable") {
t.Fatalf("Chunk() error = %q, want wrapped LLM context", err.Error())
}
}
func chunkRequestWithClient(client contracts.StructuredLLMClient) contracts.ChunkRequest {
return contracts.ChunkRequest{
Source: sceneSourceDocument(),
LLMClient: client,
}
}
func requestWithOptions(req contracts.ChunkRequest) contracts.ChunkRequest {
req.Options = map[string]any{"max_units": 2}
return req
}
func sceneSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "session-alpha",
Kind: "transcript",
Format: "application/vnd.seriatim.minimal+json",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: "seg-001", Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}},
{ID: "seg-002", Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."},
{ID: "seg-003", Kind: "transcript_segment", Text: "The guards rush out with blades drawn."},
{ID: "seg-004", Kind: "transcript_segment", Text: "The party defeats the ambushers."},
},
}
}
func validSceneResponse() chunkResponse {
return chunkResponse{
Scenes: []sceneResponse{
scene("seg-001", "seg-004"),
},
BoundaryCaveats: []string{},
}
}
func replaceScenes(response chunkResponse, scenes []sceneResponse) chunkResponse {
response.Scenes = scenes
return response
}
func scene(startUnitID string, endUnitID string) sceneResponse {
return sceneResponse{
StartUnitID: startUnitID,
EndUnitID: endUnitID,
ShortTitle: "Scene title",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"},
Summary: "A compact summary.",
BoundaryNote: "The source units form one coherent scene.",
BoundaryConfidence: "High",
}
}
func chunkIDs(chunks []contracts.SourceChunk) []string {
ids := make([]string, 0, len(chunks))
for _, chunk := range chunks {
ids = append(ids, chunk.ID)
}
return ids
}
func unitIDs(units []source.SourceUnit) []string {
ids := make([]string, 0, len(units))
for _, unit := range units {
ids = append(ids, unit.ID)
}
return ids
}
type fakeScenesLLMClient struct {
response chunkResponse
err error
requests []contracts.StructuredCompletionRequest
}
func (client *fakeScenesLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, contracts.StructuredCompletionRequest{
StageName: req.StageName,
Messages: append([]contracts.LLMMessage(nil), req.Messages...),
Model: req.Model,
ResponseSchemaName: req.ResponseSchemaName,
ResponseSchema: append(json.RawMessage(nil), req.ResponseSchema...),
})
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
}
target, ok := out.(*chunkResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
}
*target = client.response
content, err := json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}

View File

@@ -0,0 +1,17 @@
package scenes
type chunkResponse struct {
Scenes []sceneResponse `json:"scenes"`
BoundaryCaveats []string `json:"boundary_caveats"`
}
type sceneResponse struct {
StartUnitID string `json:"start_unit_id"`
EndUnitID string `json:"end_unit_id"`
ShortTitle string `json:"short_title"`
PrimaryMode string `json:"primary_mode"`
MainParticipants []string `json:"main_participants"`
Summary string `json:"summary"`
BoundaryNote string `json:"boundary_note"`
BoundaryConfidence string `json:"boundary_confidence"`
}

View File

@@ -0,0 +1,97 @@
package scenes
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/prompt"
)
type promptData struct {
SourceID string
Units []promptUnit
}
type promptUnit struct {
ID string
Text string
Metadata []promptMetadata
}
type promptMetadata struct {
Key string
Value string
}
var scenesPromptBundle = mustLoadPromptBundle()
func mustLoadPromptBundle() *prompt.Bundle {
bundle, err := prompt.LoadBundle(embeddedAssets, prompt.Definition{
PromptID: PromptID,
Version: ResponseSchemaVersion,
EmbeddedPath: "assets/prompts",
SystemPath: "assets/prompts/system.md",
UserPath: "assets/prompts/user.md",
})
if err != nil {
panic(err)
}
return bundle
}
func buildPromptData(req contracts.ChunkRequest) (promptData, error) {
if req.Source == nil {
return promptData{}, fmt.Errorf("dnd scenes prompt: source must not be nil")
}
data := promptData{
SourceID: req.Source.ID,
Units: make([]promptUnit, 0, len(req.Source.Units)),
}
for _, unit := range req.Source.Units {
data.Units = append(data.Units, promptUnit{
ID: unit.ID,
Text: unit.Text,
Metadata: selectedMetadata(unit),
})
}
return data, nil
}
func renderPrompt(req contracts.ChunkRequest) (system string, user string, metadata prompt.Metadata, err error) {
data, err := buildPromptData(req)
if err != nil {
return "", "", prompt.Metadata{}, err
}
system, user, metadata, err = scenesPromptBundle.RenderUserSystem(data)
if err != nil {
return "", "", prompt.Metadata{}, fmt.Errorf("dnd scenes prompt: %w", err)
}
return system, user, metadata, nil
}
func selectedMetadata(unit source.SourceUnit) []promptMetadata {
if len(unit.Metadata) == 0 {
return nil
}
keys := []string{"speaker", "start", "end"}
metadata := make([]promptMetadata, 0, len(keys))
for _, key := range keys {
value, ok := unit.Metadata[key]
if !ok {
continue
}
rendered := strings.TrimSpace(fmt.Sprint(value))
if rendered == "" {
continue
}
metadata = append(metadata, promptMetadata{
Key: key,
Value: rendered,
})
}
return metadata
}

View File

@@ -0,0 +1,155 @@
package scenes
import (
"encoding/json"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/prompt"
)
func TestBuildPromptDataFromSourceDocument(t *testing.T) {
req := promptChunkRequest()
data, err := buildPromptData(req)
if err != nil {
t.Fatalf("buildPromptData() error = %v, want nil", err)
}
if data.SourceID != "session-alpha" {
t.Fatalf("SourceID = %q, want session-alpha", data.SourceID)
}
if len(data.Units) != 2 {
t.Fatalf("len(Units) = %d, want 2", len(data.Units))
}
first := data.Units[0]
if first.ID != "seg-001" || first.Text != "Aria and Bram discuss whether to enter the ruins." {
t.Fatalf("first unit = %#v, want source unit data", first)
}
wantMetadata := []promptMetadata{
{Key: "speaker", Value: "Alice"},
{Key: "start", Value: "1.25"},
{Key: "end", Value: "3.5"},
}
if !reflect.DeepEqual(first.Metadata, wantMetadata) {
t.Fatalf("first.Metadata = %#v, want %#v", first.Metadata, wantMetadata)
}
if len(data.Units[1].Metadata) != 0 {
t.Fatalf("second.Metadata = %#v, want no selected metadata", data.Units[1].Metadata)
}
}
func TestBuildPromptDataDoesNotMutateRequest(t *testing.T) {
req := promptChunkRequest()
beforeSource := mustJSON(t, req.Source)
beforeRequest := mustJSON(t, req)
if _, err := buildPromptData(req); err != nil {
t.Fatalf("buildPromptData() error = %v, want nil", err)
}
afterSource := mustJSON(t, req.Source)
afterRequest := mustJSON(t, req)
if beforeSource != afterSource || beforeRequest != afterRequest {
t.Fatalf(
"request mutated:\nsource before: %s\nsource after: %s\nrequest before: %s\nrequest after: %s",
beforeSource,
afterSource,
beforeRequest,
afterRequest,
)
}
}
func TestRenderPromptIncludesSourceUnitsAndMetadata(t *testing.T) {
system, user, metadata, err := renderPrompt(promptChunkRequest())
if err != nil {
t.Fatalf("renderPrompt() error = %v, want nil", err)
}
if !strings.Contains(system, prompt.HardeningText()) {
t.Fatalf("system prompt = %q, want hardening text", system)
}
for _, want := range []string{
"session-alpha",
"seg-001",
"seg-002",
"Aria and Bram discuss whether to enter the ruins.",
"The goblins rush out and initiative begins.",
"speaker: Alice",
"start: 1.25",
"end: 3.5",
"start_unit_id",
"end_unit_id",
"primary_mode",
"boundary_confidence",
"Recap, Discussion, Combat, or Narrative",
"High, Medium, or Low",
"no gaps",
"do not overlap",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
if metadata.PromptID != PromptID {
t.Fatalf("metadata.PromptID = %q, want %q", metadata.PromptID, PromptID)
}
if metadata.PromptVersion != ResponseSchemaVersion {
t.Fatalf("metadata.PromptVersion = %q, want %q", metadata.PromptVersion, ResponseSchemaVersion)
}
if metadata.EmbeddedPath != "assets/prompts" {
t.Fatalf("metadata.EmbeddedPath = %q, want assets/prompts", metadata.EmbeddedPath)
}
}
func TestBuildPromptDataRejectsMissingSourceContext(t *testing.T) {
if _, err := buildPromptData(contracts.ChunkRequest{}); err == nil || !strings.Contains(err.Error(), "source") {
t.Fatalf("buildPromptData() error = %v, want source error", err)
}
}
func promptChunkRequest() contracts.ChunkRequest {
return contracts.ChunkRequest{
Source: promptSourceDocument(),
}
}
func promptSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "session-alpha",
Kind: "transcript",
Format: "application/vnd.seriatim.minimal+json",
Digest: "sha256:test",
Units: []source.SourceUnit{
{
ID: "seg-001",
Kind: "transcript_segment",
Text: "Aria and Bram discuss whether to enter the ruins.",
Metadata: map[string]any{
"speaker": "Alice",
"start": json.Number("1.25"),
"end": json.Number("3.5"),
"ignored": "not rendered",
},
},
{
ID: "seg-002",
Kind: "transcript_segment",
Text: "The goblins rush out and initiative begins.",
Metadata: map[string]any{"ignored": "not rendered"},
},
},
}
}
func mustJSON(t *testing.T, value any) string {
t.Helper()
encoded, err := json.Marshal(value)
if err != nil {
t.Fatalf("Marshal() error = %v, want nil", err)
}
return string(encoded)
}

View File

@@ -0,0 +1,21 @@
package scenes
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.scenes"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scenes")
ResponseSchemaID = "notarius.dnd.scenes"
ResponseSchemaVersion = "v1"
ResponseSchemaName = "notarius_dnd_scenes_v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: ResponseSchemaVersion,
Name: ResponseSchemaName,
AssetPath: "assets/schemas/dnd_scenes.v1.json",
})
}

View File

@@ -0,0 +1,146 @@
package scenes
import (
"encoding/json"
"strings"
"testing"
)
func TestLoadResponseSchemaForScenes(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if schema.Key != ResponseSchemaKey {
t.Fatalf("schema.Key = %q, want %q", schema.Key, ResponseSchemaKey)
}
if schema.ID != ResponseSchemaID {
t.Fatalf("schema.ID = %q, want %q", schema.ID, ResponseSchemaID)
}
if schema.Version != ResponseSchemaVersion {
t.Fatalf("schema.Version = %q, want %q", schema.Version, ResponseSchemaVersion)
}
if schema.Name != ResponseSchemaName {
t.Fatalf("schema.Name = %q, want %q", schema.Name, ResponseSchemaName)
}
if !strings.HasPrefix(schema.SHA256, "sha256:") {
t.Fatalf("schema.SHA256 = %q, want sha256 prefix", schema.SHA256)
}
if !json.Valid(schema.JSONSchema) {
t.Fatalf("schema.JSONSchema is invalid JSON: %s", schema.JSONSchema)
}
}
func TestResponseSchemaShapeUsesSourceUnitBoundaries(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
var decoded map[string]any
if err := json.Unmarshal(schema.JSONSchema, &decoded); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if decoded["$id"] != ResponseSchemaID {
t.Fatalf("$id = %#v, want %q", decoded["$id"], ResponseSchemaID)
}
if decoded["additionalProperties"] != false {
t.Fatalf("additionalProperties = %#v, want false", decoded["additionalProperties"])
}
properties := decoded["properties"].(map[string]any)
if _, ok := properties["artifact_type"]; ok {
t.Fatal("schema includes artifact_type, want only scene response fields")
}
if _, ok := properties["session_scope"]; ok {
t.Fatal("schema includes session_scope, want no session wrapper")
}
sceneProperties := properties["scenes"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any)
for _, field := range []string{"scene_id", "start_segment_id", "end_segment_id"} {
if _, ok := sceneProperties[field]; ok {
t.Fatalf("scene schema includes old field %q", field)
}
}
for _, field := range []string{"start_unit_id", "end_unit_id"} {
property := sceneProperties[field].(map[string]any)
if property["type"] != "string" {
t.Fatalf("%s type = %#v, want string", field, property["type"])
}
}
modeEnum := sceneProperties["primary_mode"].(map[string]any)["enum"].([]any)
if !sameStrings(modeEnum, []string{"Recap", "Discussion", "Combat", "Narrative"}) {
t.Fatalf("primary_mode enum = %#v, want Recap/Discussion/Combat/Narrative", modeEnum)
}
confidenceEnum := sceneProperties["boundary_confidence"].(map[string]any)["enum"].([]any)
if !sameStrings(confidenceEnum, []string{"High", "Medium", "Low"}) {
t.Fatalf("boundary_confidence enum = %#v, want High/Medium/Low", confidenceEnum)
}
boundaryCaveatItems := decoded["properties"].(map[string]any)["boundary_caveats"].(map[string]any)["items"].(map[string]any)
if boundaryCaveatItems["type"] != "string" {
t.Fatalf("boundary_caveats.items.type = %#v, want string", boundaryCaveatItems["type"])
}
if boundaryCaveatItems["minLength"] != float64(1) {
t.Fatalf("boundary_caveats.items.minLength = %#v, want 1", boundaryCaveatItems["minLength"])
}
}
func TestResponseStructRejectsIntegerBoundaries(t *testing.T) {
raw := []byte(`{
"scenes": [
{
"start_unit_id": 1,
"end_unit_id": 3,
"short_title": "Ambush",
"primary_mode": "Combat",
"main_participants": ["Aria"],
"summary": "The party fights.",
"boundary_note": "Combat starts and resolves.",
"boundary_confidence": "High"
}
],
"boundary_caveats": []
}`)
var response chunkResponse
err := json.Unmarshal(raw, &response)
if err == nil {
t.Fatalf("Unmarshal() error = nil, want integer boundary type error: %#v", response)
}
if !strings.Contains(err.Error(), "string") {
t.Fatalf("Unmarshal() error = %v, want string type error", err)
}
}
func TestResponseSchemaJSONIsMutationSafe(t *testing.T) {
first, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
first.JSONSchema[0] = '['
second, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if !json.Valid(second.JSONSchema) {
t.Fatalf("schema JSON was mutated: %s", second.JSONSchema)
}
if len(second.JSONSchema) > 0 && second.JSONSchema[0] == '[' {
t.Fatalf("schema JSON did not use defensive copy")
}
}
func sameStrings(got []any, want []string) bool {
if len(got) != len(want) {
return false
}
for i := range want {
if got[i] != want[i] {
return false
}
}
return true
}

View File

@@ -6,4 +6,9 @@ Extract only spell casts that are supported by the provided source text. Do not
infer spells from general D&D knowledge or from table chatter that does not infer spells from general D&D knowledge or from table chatter that does not
identify a spell being cast. identify a spell being cast.
Reference material, when present, is supporting context only. Use it only to
disambiguate names, aliases, speakers, campaign terms, or spell names already
present in the source text. Do not extract a spell cast solely because it appears
in reference material.
Source references must use the source-unit IDs exactly as provided. Source references must use the source-unit IDs exactly as provided.

View File

@@ -16,6 +16,19 @@ Source units:
{{ end }} {{ end }}
{{ end }} {{ end }}
{{ if hasreference "roster" }}
Roster reference material:
{{ reference "roster" }}
{{ end }}
{{ if hasreference "glossary" }}
Glossary reference material:
{{ reference "glossary" }}
{{ end }}
Return only D&D spell-cast artifacts. For each spell cast, identify the in-world Return only D&D spell-cast artifacts. For each spell cast, identify the in-world
caster, spell name, effect, narrative description, and source references using caster, spell name, effect, narrative description, and source references using
source_id, start_unit_id, and end_unit_id. source_id, start_unit_id, and end_unit_id.
Use roster and glossary reference material only to clarify source text. Do not
return spells, casters, or effects that are mentioned only in reference material.

View File

@@ -25,6 +25,19 @@ var providedCapabilities = []string{
"dnd.spell_casts", "dnd.spell_casts",
} }
var referenceSlots = []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Optional campaign glossary reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
}
var _ contracts.Extractor = (*Extractor)(nil) var _ contracts.Extractor = (*Extractor)(nil)
type Extractor struct{} type Extractor struct{}
@@ -45,6 +58,10 @@ func (e *Extractor) SchemaVersion() string {
return SchemaVersion return SchemaVersion
} }
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot {
return cloneReferenceSlots(referenceSlots)
}
func (e *Extractor) ManifestMetadata() map[string]any { func (e *Extractor) ManifestMetadata() map[string]any {
promptMetadata := spellsPromptBundle.Metadata() promptMetadata := spellsPromptBundle.Metadata()
metadata := map[string]any{ metadata := map[string]any{
@@ -136,10 +153,11 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.ExtractionRequest
func ModuleSpec() pipeline.ModuleSpec { func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{ return pipeline.ModuleSpec{
Key: Key, Key: Key,
Stage: pipeline.StageExtract, Stage: pipeline.StageExtract,
Requires: append([]string(nil), requiredCapabilities...), Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...), Provides: append([]string(nil), providedCapabilities...),
ReferenceSlots: cloneReferenceSlots(referenceSlots),
} }
} }
@@ -161,3 +179,15 @@ func spellCastPayload(spellCast spellCastResponse) (json.RawMessage, error) {
func extractorErrorf(format string, args ...any) error { func extractorErrorf(format string, args ...any) error {
return fmt.Errorf("dnd spells extractor: "+format, args...) return fmt.Errorf("dnd spells extractor: "+format, args...)
} }
func cloneReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
out := make([]contracts.ReferenceSlot, len(slots))
for i, slot := range slots {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
out[i] = slot
}
return out
}

View File

@@ -116,6 +116,56 @@ func TestExtractorManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T
} }
} }
func TestExtractIncludesReferencesInPrompt(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
req := extractionRequestWithClient(client)
req.References = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria Brightmantle: party cleric")},
},
},
"glossary": {
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{SlotName: "glossary", Content: []byte("Brightmantle: local temple name")},
},
},
},
}
if _, err := New().Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(client.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
system := client.requests[0].Messages[0].Content
for _, want := range []string{
"Reference material, when present, is supporting context only.",
"in reference material.",
} {
if !strings.Contains(system, want) {
t.Fatalf("system prompt = %q, want substring %q", system, want)
}
}
user := client.requests[0].Messages[1].Content
for _, want := range []string{
"Roster reference material:",
"Aria Brightmantle: party cleric",
"Glossary reference material:",
"Brightmantle: local temple name",
"mentioned only in reference material.",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
}
func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) { func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}} client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}

View File

@@ -32,11 +32,12 @@ var spellsPromptBundle = mustLoadPromptBundle()
func mustLoadPromptBundle() *prompt.Bundle { func mustLoadPromptBundle() *prompt.Bundle {
bundle, err := prompt.LoadBundle(embeddedAssets, prompt.Definition{ bundle, err := prompt.LoadBundle(embeddedAssets, prompt.Definition{
PromptID: PromptID, PromptID: PromptID,
Version: SchemaVersion, Version: SchemaVersion,
EmbeddedPath: "assets/prompts", EmbeddedPath: "assets/prompts",
SystemPath: "assets/prompts/system.md", SystemPath: "assets/prompts/system.md",
UserPath: "assets/prompts/user.md", UserPath: "assets/prompts/user.md",
ReferenceSlots: cloneReferenceSlots(referenceSlots),
}) })
if err != nil { if err != nil {
panic(err) panic(err)
@@ -74,7 +75,7 @@ func renderPrompt(req contracts.ExtractionRequest) (system string, user string,
if err != nil { if err != nil {
return "", "", prompt.Metadata{}, err return "", "", prompt.Metadata{}, err
} }
system, user, metadata, err = spellsPromptBundle.RenderUserSystem(data) system, user, metadata, err = spellsPromptBundle.RenderUserSystemWithReferences(data, req.References)
if err != nil { if err != nil {
return "", "", prompt.Metadata{}, fmt.Errorf("dnd spells prompt: %w", err) return "", "", prompt.Metadata{}, fmt.Errorf("dnd spells prompt: %w", err)
} }

View File

@@ -101,6 +101,50 @@ func TestRenderPromptIncludesSourceContext(t *testing.T) {
if metadata.EmbeddedPath != "assets/prompts" { if metadata.EmbeddedPath != "assets/prompts" {
t.Fatalf("metadata.EmbeddedPath = %q, want assets/prompts", metadata.EmbeddedPath) t.Fatalf("metadata.EmbeddedPath = %q, want assets/prompts", metadata.EmbeddedPath)
} }
if strings.Contains(user, "Roster reference material") || strings.Contains(user, "Glossary reference material") {
t.Fatalf("user prompt = %q, want no optional reference sections without bindings", user)
}
}
func TestRenderPromptIncludesBoundReferences(t *testing.T) {
req := promptExtractionRequest()
req.References = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria: cleric, also known as Sister Aria")},
},
},
"glossary": {
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{SlotName: "glossary", Content: []byte("Cure Wounds: healing spell")},
},
},
},
}
_, user, metadata, err := renderPrompt(req)
if err != nil {
t.Fatalf("renderPrompt() error = %v, want nil", err)
}
for _, want := range []string{
"Roster reference material:",
"Aria: cleric, also known as Sister Aria",
"Glossary reference material:",
"Cure Wounds: healing spell",
"Use roster and glossary reference material only to clarify source text.",
"mentioned only in reference material.",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
if metadata.SHA256 != spellsPromptBundle.Metadata().SHA256 {
t.Fatalf("metadata.SHA256 = %q, want template hash %q", metadata.SHA256, spellsPromptBundle.Metadata().SHA256)
}
} }
func TestBuildPromptDataRejectsMissingSourceContext(t *testing.T) { func TestBuildPromptDataRejectsMissingSourceContext(t *testing.T) {

View File

@@ -5,6 +5,7 @@ import (
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
@@ -36,6 +37,18 @@ func TestModuleSpec(t *testing.T) {
Provides: []string{ Provides: []string{
"dnd.spell_casts", "dnd.spell_casts",
}, },
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Optional campaign glossary reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
},
} }
if !reflect.DeepEqual(got, want) { if !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
@@ -43,6 +56,7 @@ func TestModuleSpec(t *testing.T) {
got.Requires[0] = "changed" got.Requires[0] = "changed"
got.Provides[0] = "changed" got.Provides[0] = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
again := ModuleSpec() again := ModuleSpec()
if !reflect.DeepEqual(again, want) { if !reflect.DeepEqual(again, want) {
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want) t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
@@ -82,6 +96,15 @@ func TestRegisterStoresModuleSpec(t *testing.T) {
} }
} }
func TestRuntimeReferenceSlotsMatchModuleSpec(t *testing.T) {
extractor := New()
spec := ModuleSpec()
if !reflect.DeepEqual(extractor.ReferenceSlots(), spec.ReferenceSlots) {
t.Fatalf("ReferenceSlots() = %#v, want spec slots %#v", extractor.ReferenceSlots(), spec.ReferenceSlots)
}
}
func TestRegisterNilRegistryReturnsError(t *testing.T) { func TestRegisterNilRegistryReturnsError(t *testing.T) {
err := Register(nil) err := Register(nil)
if err == nil { if err == nil {

View File

@@ -110,6 +110,96 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
} }
} }
func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
raw := readDNDSpellsFixture(t)
expectedDoc := parseDNDSpellsFixture(t, raw)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet(
"Aria: party cleric\nBorin: fighter",
"Fire Bolt: evocation cantrip",
)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
{
Caster: "Borin",
Spell: "Fire Bolt",
Effect: "Scorches the wight.",
NarrativeDescription: "Borin hurls fire at the wight.",
SourceRefs: []source.SourceRef{
{SourceID: expectedDoc.ID, StartUnitID: "seg-003", EndUnitID: "seg-003"},
},
},
},
},
}
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
Pipeline: resolved.ResolvedPipeline,
RawInput: raw,
LLMClient: llmClient,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
}
if len(output.Manifest.References) != 2 {
t.Fatalf("manifest references = %#v, want roster and glossary provenance", output.Manifest.References)
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
}
user := llmClient.requests[0].Messages[1].Content
for _, want := range []string{
"Roster reference material:",
"Aria: party cleric",
"Glossary reference material:",
"Fire Bolt: evocation cantrip",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
}
func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ReferenceSet = dndSpellsReferenceSet(
"Mira: wizard who can cast Lightning Bolt",
"",
)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{SpellCasts: []spellCastResponse{}},
}
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
Pipeline: resolved.ResolvedPipeline,
RawInput: raw,
LLMClient: llmClient,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 0 {
t.Fatalf("approved artifacts = %#v, want no roster-only spell casts", output.Approved)
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
}
user := llmClient.requests[0].Messages[1].Content
if !strings.Contains(user, "Lightning Bolt") {
t.Fatalf("user prompt = %q, want roster-only spell in reference section", user)
}
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved empty extraction", output.Manifest.ValidationStatus)
}
}
func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) { func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
raw := readDNDSpellsFixture(t) raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t) resolved := resolveDNDSpellsPipeline(t)
@@ -155,6 +245,43 @@ func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
} }
} }
func dndSpellsReferenceSet(roster string, glossary string) contracts.ReferenceSet {
slots := make(map[string]contracts.ResolvedReferenceSlot)
if strings.TrimSpace(roster) != "" {
slots["roster"] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain; charset=utf-8",
Content: []byte(roster),
Digest: "sha256:roster",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
SizeBytes: int64(len(roster)),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
}
}
if strings.TrimSpace(glossary) != "" {
slots["glossary"] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{
SlotName: "glossary",
MediaType: "text/plain; charset=utf-8",
Content: []byte(glossary),
Digest: "sha256:glossary",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/glossary.txt"},
SizeBytes: int64(len(glossary)),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
}
}
return contracts.ReferenceSet{Slots: slots}
}
func TestRunnerFailsWhenDNDSpellsExtractorReturnsMalformedOutput(t *testing.T) { func TestRunnerFailsWhenDNDSpellsExtractorReturnsMalformedOutput(t *testing.T) {
raw := readDNDSpellsFixture(t) raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t) resolved := resolveDNDSpellsPipeline(t)

View File

@@ -20,6 +20,7 @@ const (
reasonMissingRequiredField = "missing_required_field" reasonMissingRequiredField = "missing_required_field"
reasonMissingSourceRef = "missing_source_ref" reasonMissingSourceRef = "missing_source_ref"
reasonInvalidSourceRef = "invalid_source_ref" reasonInvalidSourceRef = "invalid_source_ref"
reasonSpellNotNearSource = "spell_not_near_source"
) )
var _ contracts.Validator = ShapeValidator{} var _ contracts.Validator = ShapeValidator{}
@@ -54,12 +55,15 @@ func (validator SourceRefValidator) Validate(ctx context.Context, req contracts.
} }
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates)) decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
var warnings []contracts.Warning
for _, candidate := range req.Candidates { for _, candidate := range req.Candidates {
decisions = append(decisions, validateSourceRefs(req.Source, candidate)) decisions = append(decisions, validateSourceRefs(req.Source, candidate))
warnings = append(warnings, sourceRelatednessWarnings(req.Source, candidate)...)
} }
return contracts.ValidationResult{ return contracts.ValidationResult{
ValidatorName: validator.Name(), ValidatorName: validator.Name(),
Decisions: decisions, Decisions: decisions,
Warnings: warnings,
}, nil }, nil
} }
@@ -88,6 +92,66 @@ func validateSourceRefs(doc *source.SourceDocument, candidate artifacts.Artifact
return validate.Approved(candidate.Index) return validate.Approved(candidate.Index)
} }
func sourceRelatednessWarnings(doc *source.SourceDocument, candidate artifacts.ArtifactCandidate) []contracts.Warning {
if doc == nil || len(candidate.SourceRefs) == 0 {
return nil
}
var payload SpellCast
if err := json.Unmarshal(candidate.Payload, &payload); err != nil {
return nil
}
spell := strings.TrimSpace(payload.Spell)
if spell == "" {
return nil
}
needle := strings.ToLower(spell)
for _, ref := range candidate.SourceRefs {
text, ok := sourceRefText(doc, ref)
if !ok {
continue
}
if strings.Contains(strings.ToLower(text), needle) {
return nil
}
}
return []contracts.Warning{
{
Scope: fmt.Sprintf("candidate.%d", candidate.Index),
ReasonCode: reasonSpellNotNearSource,
Message: fmt.Sprintf("spell %q was not found in the cited source text", spell),
},
}
}
func sourceRefText(doc *source.SourceDocument, ref source.SourceRef) (string, bool) {
if err := source.ValidateRef(doc, ref); err != nil {
return "", false
}
start := -1
end := -1
for i, unit := range doc.Units {
if unit.ID == ref.StartUnitID {
start = i
}
if unit.ID == ref.EndUnitID {
end = i
}
}
if start < 0 || end < start {
return "", false
}
var b strings.Builder
for i := start; i <= end; i++ {
if b.Len() > 0 {
b.WriteString("\n")
}
b.WriteString(doc.Units[i].Text)
}
return b.String(), true
}
func requiredSpellCastFields(payload SpellCast) []struct { func requiredSpellCastFields(payload SpellCast) []struct {
name string name string
value string value string

View File

@@ -51,6 +51,9 @@ func TestValidatorsApproveValidCandidate(t *testing.T) {
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err) t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
} }
assertSingleDecision(t, sourceRefResult, sourceRefValidatorName, 7, true, validate.ReasonApproved) assertSingleDecision(t, sourceRefResult, sourceRefValidatorName, 7, true, validate.ReasonApproved)
if len(sourceRefResult.Warnings) != 0 {
t.Fatalf("warnings = %#v, want none", sourceRefResult.Warnings)
}
} }
func TestShapeValidatorRejectsMalformedPayload(t *testing.T) { func TestShapeValidatorRejectsMalformedPayload(t *testing.T) {
@@ -167,6 +170,29 @@ func TestSourceRefValidatorRejectsInvalidRefs(t *testing.T) {
} }
} }
func TestSourceRefValidatorWarnsWhenSpellNameIsNotInCitedSource(t *testing.T) {
candidate := validSpellCandidate(31)
payload := validSpellPayload()
payload.Spell = "Shield"
candidate.Payload = mustSpellPayload(t, payload)
result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Source: promptSourceDocument(),
Candidates: []artifacts.ArtifactCandidate{candidate},
})
if err != nil {
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, result, sourceRefValidatorName, 31, true, validate.ReasonApproved)
if len(result.Warnings) != 1 {
t.Fatalf("warnings = %#v, want one relatedness warning", result.Warnings)
}
warning := result.Warnings[0]
if warning.ReasonCode != reasonSpellNotNearSource || !strings.Contains(warning.Message, "Shield") {
t.Fatalf("warning = %#v, want spell relatedness warning", warning)
}
}
func TestSourceRefValidatorRequiresSourceDocument(t *testing.T) { func TestSourceRefValidatorRequiresSourceDocument(t *testing.T) {
_, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{ _, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Candidates: []artifacts.ArtifactCandidate{validSpellCandidate(19)}, Candidates: []artifacts.ArtifactCandidate{validSpellCandidate(19)},

View File

@@ -221,6 +221,8 @@ func (fakeExtractor) ArtifactType() string { return "fake" }
func (fakeExtractor) SchemaVersion() string { return "v1" } func (fakeExtractor) SchemaVersion() string { return "v1" }
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeExtractor) Validators() []contracts.Validator { return nil } func (fakeExtractor) Validators() []contracts.Validator { return nil }
func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) { func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {

View File

@@ -186,6 +186,10 @@ func (e *runnerSeriatimExtractor) SchemaVersion() string {
return "v1" return "v1"
} }
func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (e *runnerSeriatimExtractor) Validators() []contracts.Validator { func (e *runnerSeriatimExtractor) Validators() []contracts.Validator {
return nil return nil
} }

View File

@@ -142,6 +142,42 @@ func TestEncodePrettyPrintsJSON(t *testing.T) {
} }
} }
func TestEncodeIncludesManifestReferences(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{
RunID: "run-1",
References: []artifacts.ReferenceProvenance{
{
LaneID: "events",
SlotName: "roster",
OriginType: "file",
OriginURI: "file:///tmp/roster.txt",
Digest: "sha256:reference",
MediaType: "text/plain; charset=utf-8",
SizeBytes: 12,
BindingSource: "config",
},
},
},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
manifest := decodeObject(t, fileBytes(t, result.Files, "manifest.json"))
references := manifest["references"].([]any)
if len(references) != 1 {
t.Fatalf("references = %#v, want one entry", references)
}
reference := references[0].(map[string]any)
if reference["lane_id"] != "events" || reference["slot_name"] != "roster" || reference["digest"] != "sha256:reference" {
t.Fatalf("reference manifest = %#v, want lane slot digest", reference)
}
if _, ok := reference["content"]; ok {
t.Fatalf("reference manifest = %#v, want no content field", reference)
}
}
func TestEncodeRejectsArtifactTypeWithoutSafeFileName(t *testing.T) { func TestEncodeRejectsArtifactTypeWithoutSafeFileName(t *testing.T) {
_, err := New().Encode(context.Background(), contracts.OutputRequest{ _, err := New().Encode(context.Background(), contracts.OutputRequest{
Approved: []artifacts.Artifact{artifact("///", "unsafe")}, Approved: []artifacts.Artifact{artifact("///", "unsafe")},