35 Commits

Author SHA1 Message Date
3aa88ab9d3 Document completed LLM and reference expansion 2026-07-05 16:42:15 +00:00
c1ba94192d Cover reference prompts for chunk and normalize usage 2026-07-05 16:39:41 +00:00
22032dfd6d Deliver references to eligible runtime targets 2026-07-05 16:38:17 +00:00
4cafde2502 Materialize references for all eligible targets 2026-07-05 16:35:24 +00:00
8c623b7ad8 Support target-aware reference selectors 2026-07-05 16:31:23 +00:00
43dc954440 Resolve references across eligible pipeline targets 2026-07-05 16:24:49 +00:00
51053d390d Add stage-local reference config bindings 2026-07-05 16:18:00 +00:00
9278797aa9 Introduce target-aware resolved reference storage 2026-07-05 16:12:38 +00:00
39e49d7f77 Expand module contracts for references and normalizer LLM access 2026-07-05 16:06:47 +00:00
84c4c06712 Add a staged implementation plan to extend first-class LLM access and reference support across all chunk, extract, and normalize stages 2026-07-05 11:01:32 -05:00
eab640aa21 Update .gitignore to ignore build and testing output artifacts 2026-07-05 10:54:16 -05:00
a516944086 Finish the references implementation for the extraction module and update roadmap documentation 2026-07-05 10:53:16 -05:00
be6803ffa1 Document extraction reference support 2026-07-05 14:52:52 +00:00
ef4bdd4f9f Use references in D&D spell extraction 2026-07-05 14:49:17 +00:00
2f97895732 Record reference provenance in manifests 2026-07-05 14:44:44 +00:00
9e89b88efc Add prompt reference template functions 2026-07-05 14:37:39 +00:00
a57c6397e3 Materialize extraction reference files 2026-07-05 14:32:35 +00:00
39e071f5ca Add CLI reference binding flags 2026-07-05 14:27:23 +00:00
70d733edaf Resolve extraction reference bindings from config 2026-07-05 14:21:36 +00:00
1c31f56af1 Add reference contracts to extractor metadata 2026-07-05 14:13:48 +00:00
f9999a73df Add a staged implementation plan for background context references 2026-07-05 09:08:31 -05:00
11d8187052 Update documentation to reflect the implemented chunking module 2026-07-05 08:40:24 -05:00
86bff552c1 Update .gitgnore to ignore build artifacts 2026-07-05 08:39:34 -05:00
d3f790095e Trim and validate scene caveats 2026-07-05 13:30:32 +00:00
95218218e2 Canonicalize chunk units before extraction 2026-07-05 13:28:09 +00:00
e700df82d8 Record top-level module metadata in run manifests 2026-07-05 13:23:56 +00:00
e19cc02c4d Add a staged implementation plan to address gaps from the initial implementation of the scene chunking module 2026-07-05 08:15:46 -05:00
8a5419448f Update feature roadmaps to reflect future work 2026-07-05 08:11:54 -05:00
c8217549a8 Verify D&D scene chunker run output 2026-07-04 13:08:45 +00:00
2130414899 Register D&D scene chunker 2026-07-04 13:05:39 +00:00
7f83a20fa6 Implement D&D scene chunker 2026-07-04 13:02:02 +00:00
317ab0472d Add D&D scene chunking assets 2026-07-04 12:57:04 +00:00
e5eb0ba5c8 Make chunk validation a framework contract 2026-07-04 12:52:16 +00:00
b95af4f87d Add a roadmap to implement a D&D-specific chunk module 2026-07-04 07:48:36 -05:00
11073b613c Update AGENTS.md 2026-07-04 07:05:06 -05:00
85 changed files with 8142 additions and 190 deletions

5
.gitignore vendored
View File

@@ -1,3 +1,7 @@
# build and testing artifacts
notarius
notarius-output
# ---> Go
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
@@ -49,6 +53,7 @@ go.work.sum
# Icon must end with two \r
Icon
# Thumbnails
._*

View File

@@ -1,3 +1,6 @@
Please carefully review the documents in `docs/policy` before making any changes to this repository.
- `architecture.md` provides the canonical high-level architecture policy for this repository.
- `documentation.md` provides the canonical documentation policy for this repository.
Please review `docs/internal/overview.md` for initial orientation in this repository.
Additionally, please carefully review the relevant documents in `docs/policy` before making any changes to this repository.
- `development.md` defines the contributor workflow for this application.
- `architecture.md` provides the canonical high-level architecture policy for this repository, and should be reviewed before writing or changing any code.
- `documentation.md` provides the canonical documentation policy for this repository, and should be reviewed before writing or changing any documentation.

View File

@@ -20,7 +20,7 @@ a bearer token.
```text
notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b]
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference selector=path] [--without-reference selector]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json]
```
@@ -46,11 +46,82 @@ Flags:
invocation.
- `--llm-profile id`: override every effective module binding to use one LLM
profile.
- `--reference selector=path`: bind a reference path to a chunk, extractor, or
normalizer reference slot. Repeatable.
- `--without-reference selector`: remove a configured optional reference binding.
Repeatable. It accepts the same selector forms as `--reference`, without
`=path`.
On success, the command prints the completed pipeline ID, approved and rejected
artifact counts, and the output directory. If the run completes with warnings,
the warning count is printed to stderr.
Reference flags are resolved against selected chunk, extractor, and normalizer
targets before the run starts. Flat slot names are accepted only when exactly
one selected target declares that slot. Bound reference files are read before
pipeline work starts, validated as UTF-8 text, and recorded as provenance for
the target that declares the slot. Runtime reference content is passed to the
chunker, extractor, or normalizer target that declares the slot. Notarius infers
reference media types from file extensions for provenance and for optional slot
checks. Reference content is not written to diagnostics, logs, errors, or
manifests.
Reference binding precedence is:
1. pipeline-level config `references`;
2. target-local config references, including legacy lane-level extractor
`references`;
3. `--reference` run flags;
4. `--without-reference` run flags.
`--reference` binds or replaces one slot for one selected target. Selectors are:
- `slot=path`: valid when exactly one selected target declares `slot`;
- `chunk.slot=path`: target the chunker;
- `lane.slot=path`: valid when exactly one selected extractor or normalizer in
that lane declares `slot`;
- `lane.extract.slot=path`: target a lane extractor;
- `lane.normalize.slot=path`: target a lane normalizer.
Use `slot=path` when the selected targets declare the slot unambiguously:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference roster=./campaign-roster.txt
```
Use an explicit selector when multiple selected targets declare the same slot or
when you want to target a specific target:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference spells.extract.glossary=./campaign-glossary.txt
```
The same grammar can target chunk and normalize slots when the configured
modules declare them:
```sh
go run ./cmd/notarius run dnd-session \
--config path/to/config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference chunk.scene_guide=./campaign-scenes.txt \
--reference spells.normalize.normalization_notes=./normalization-notes.txt
```
Use `--without-reference` to remove a configured optional binding for a run:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--without-reference glossary
```
For durable output, diagnostics, retention, and failure inspection, see
[Operations](operations.md).
@@ -117,7 +188,7 @@ go run ./cmd/notarius pipelines list \
The production CLI currently registers these module keys:
- input: `seriatim`
- chunk: `generic`
- chunk: `generic`, `dnd/scenes`
- extract: `dnd/spells`
- merge: `appendorder`
- normalize: `noop`

View File

@@ -27,6 +27,9 @@ llm_profiles:
pipelines:
dnd-session:
input: seriatim
references:
roster: ./dnd-spells-roster.txt
glossary: ./dnd-spells-glossary.txt
chunk:
module: generic
options:
@@ -121,6 +124,9 @@ Pipeline fields:
- `artifacts`: required for pipeline resolution. It maps artifact lane IDs to
lane definitions.
- `output`: optional module binding. Default module is `json`.
- `references`: optional map of reference slot names to reference paths. These
bindings are defaults for eligible pipeline targets that declare the matching
slot.
Artifact lane fields:
@@ -129,11 +135,78 @@ Artifact lane fields:
- `normalize`: optional module binding. Default module is `noop`.
- `validators`: optional list of module bindings. The production CLI currently
does not register validator modules.
- `references`: optional compatibility alias for extractor reference bindings.
Lane bindings override pipeline-level bindings for the same slot.
`notarius run` and `notarius config validate --pipeline` resolve the pipeline
against the production module catalog and fail fast for unknown or incompatible
module keys.
Reference bindings are validated against reference slots declared by eligible
chunk, extract, and normalize targets during pipeline resolution. Required slots
must be bound after config defaults, target-local references, lane-level
compatibility bindings, and run-time `--reference` or `--without-reference`
overrides are applied. Config-relative paths are resolved relative to the config
file; CLI reference paths are resolved relative to the current working
directory. Materialized bound files must be UTF-8 text. Materialized reference
provenance is recorded for chunk, extractor, and normalizer targets, and runtime
reference content is passed to the target that declares the slot. Reference
media types are inferred from file extensions, recorded as canonical base media
types, and checked only when a module declares
`AcceptedMediaTypes`; unknown extensions are recorded as
`application/octet-stream`. Reference content is not written to diagnostics,
logs, errors, or manifests.
Pipeline-level `references` are defaults. They are valid when at least one
eligible target in the full configured pipeline declares the slot, including
chunk, extractor, and normalizer targets. During a run, they apply only to the
selected targets that declare the slot:
```yaml
pipelines:
dnd-session:
input: seriatim
references:
roster: ./campaign/party-roster.txt
glossary: ./campaign/glossary.txt
artifacts:
spells:
extract: dnd/spells
```
Extractor binding `references` are the canonical lane-local location. The
legacy lane-level `references` field remains supported as an alias; when both
bind the same slot, `extract.references` wins:
```yaml
pipelines:
dnd-session:
input: seriatim
references:
glossary: ./campaign/glossary.txt
artifacts:
spells:
references:
roster: ./campaign/legacy-roster.txt
extract:
module: dnd/spells
references:
roster: ./campaign/session-roster.txt
```
`chunk.references` and `normalize.references` are accepted in object-form
bindings. They override pipeline-level defaults for slots declared by the chunk
or normalizer module. Extractor-local references apply only to the extractor,
and normalizer-local references apply only to the normalizer.
Stage-local reference fields use the same map shape at:
- `pipelines.<id>.chunk.references`
- `pipelines.<id>.artifacts.<lane>.extract.references`
- `pipelines.<id>.artifacts.<lane>.normalize.references`
Each binding is valid only when that target module declares the slot.
## Module Bindings
Every module binding may use shorthand:
@@ -157,6 +230,9 @@ Binding fields:
- `module`: module key.
- `llm_profile`: optional LLM profile ID. Empty means `default`.
- `options`: optional module-specific settings.
- `references`: optional reference bindings. Supported only for `chunk`,
`extract`, and `normalize` bindings. `input`, `merge`, validator, and
`output` bindings reject this field during validation.
The `--llm-profile` run flag overrides every effective module binding to use
one configured profile.
@@ -167,6 +243,7 @@ one configured profile.
| --- | --- | --- |
| input | `seriatim` | Reads Seriatim transcript JSON. |
| chunk | `generic` | Splits source units into ordered chunks. |
| chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. |
| extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. |
| merge | `appendorder` | Keeps candidates in append order. |
| normalize | `noop` | Passes merged artifacts through unchanged. |
@@ -178,6 +255,17 @@ The `generic` chunker accepts:
- `overlap_units`: non-negative integer, default `0`, and must be less than
`max_units`.
The `dnd/scenes` chunker requires transcript source capabilities, calls the
configured structured LLM provider, and does not accept module options.
The `dnd/spells` extractor declares optional text reference slots:
- `roster`
- `glossary`
The extractor uses these references only as supporting disambiguation material;
spell casts still must be present in the source transcript.
## Diagnostics
`diagnostics` fields:
@@ -211,3 +299,5 @@ Pipeline resolution additionally checks:
- required module keys are present;
- module keys are registered for the expected slot;
- module capability requirements are satisfied.
- bound reference slots are declared by selected lane extractors;
- required reference slots are bound for selected lanes.

View File

@@ -17,6 +17,14 @@ The extractor requires source chunks and transcript source capability. It
returns generic artifact candidates that are serialized by the JSON output
module.
The extractor accepts optional UTF-8 text references:
- `roster`: campaign roster or player-character notes.
- `glossary`: campaign glossary or spell/name notes.
References are supporting disambiguation material only. They are not source
evidence and are not addressable through `source_refs`.
## Artifact Envelope
Approved artifacts use the generic artifact envelope documented in
@@ -120,6 +128,11 @@ Rejection reason codes:
- `invalid_source_ref`: at least one source reference fails generic source
reference validation.
Warning reason codes:
- `spell_not_near_source`: the extracted spell name was not found in the cited
source text.
Rejected candidates are written to `rejected.json` by the JSON output module.
## Manifest Metadata

View File

@@ -57,7 +57,19 @@ approved.
"pipeline_id": "dnd-session",
"pipeline_digest": "sha256:...",
"input_module": "seriatim",
"chunker": "generic",
"chunker": "dnd/scenes",
"module_metadata": {
"chunker": {
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:...",
"response_schema_key": "dnd_scenes",
"response_schema_id": "notarius.dnd.scenes",
"response_schema_name": "notarius_dnd_scenes_v1",
"response_schema_version": "v1",
"response_schema_sha256": "sha256:..."
}
},
"source_digests": ["sha256:..."],
"extractors": ["dnd/spells"],
"merger": "appendorder",
@@ -86,9 +98,48 @@ approved.
Fields with empty values may be omitted by JSON encoding.
`source_digests` contains source document digests only. Bound references are
recorded separately under `references`, which contains provenance only:
target stage, lane ID when present, slot name, origin type and URI, digest,
media type, byte size, and binding source. Reference content is not written to
durable output.
Reference `stage` is `chunk`, `extract`, or `normalize`. `lane_id` is omitted
for chunk references and present for extract and normalize references.
When references are bound, the manifest section has this shape:
```json
{
"references": [
{
"stage": "extract",
"lane_id": "spells",
"slot_name": "roster",
"origin_type": "file",
"origin_uri": "file:///absolute/path/roster.txt",
"digest": "sha256:...",
"media_type": "text/plain",
"size_bytes": 123,
"binding_source": "config"
}
]
}
```
Reference media types are inferred from file extensions and recorded as
canonical base media types. Unknown extensions are recorded as
`application/octet-stream`.
`module_metadata` is omitted when no singleton module provides metadata.
`validation_status` is `approved` when no candidates were rejected and
`rejected` when one or more candidates were rejected.
Top-level `module_metadata` is reserved for singleton pipeline modules
(`input`, `chunker`, and `output`). Lane-owned module metadata remains under
`artifact_lanes[].metadata`.
## Artifact Files
Each artifact file has this shape:

View File

@@ -33,6 +33,7 @@ Implemented artifact names:
- `invocation.json`
- `effective-config.json`
- `resolved-pipeline.json`
- `resolved-references.json`
- `source-document.json`
- `run-manifest.json`
- `run-report.json`

View File

@@ -17,8 +17,8 @@ The request contains messages, optional model override, response schema name,
and response schema JSON. The caller supplies a pointer target for decoded
structured output.
Extractors own prompts and schemas. Provider adapters should not contain
domain-specific prompt logic.
Modules that call the LLM own their prompts and schemas. Provider adapters
should not contain domain-specific prompt logic.
## Production Client Construction

View File

@@ -20,6 +20,36 @@ A production module package should provide:
Module specs should describe capabilities accurately. Resolution uses specs to
reject incompatible pipelines before execution.
Chunk, extract, and normalize modules that accept auxiliary reference material
must declare slots through both `ReferenceSlots()` and
`ModuleSpec().ReferenceSlots`. The runtime slot list and registry metadata
should match so config validation can inspect slots without constructing module
instances. A slot declaration names the slot, whether it is required, accepted
media types, whether multiple items are allowed, and any byte limit. Empty
`AcceptedMediaTypes` means any inferred media type is accepted, though the file
must still be UTF-8 text. When a slot declares accepted media types, Notarius
compares the canonical base media type inferred from the file extension,
case-insensitively and without parameters.
The resolver materializes reference content for chunk, extractor, and
normalizer targets. Runtime delivery uses `contracts.ChunkRequest.References`,
`contracts.ExtractionRequest.References`, and
`contracts.NormalizeRequest.References`. Reference material is not source
evidence and must not be converted into `SourceRef` values. If a module prompt
uses references, load the prompt bundle with the same declared slots and render
with `RenderUserSystemWithReferences`. Prompt templates may use the `reference`
function for content and the `hasreference` function for conditional sections.
Prompt metadata hashes remain based on template source, not rendered reference
bytes.
Chunk modules receive the structured LLM client through `contracts.ChunkRequest`
when they need model-backed chunking. The pipeline runner validates generic
chunk result invariants before extraction; module-owned policies may be stricter
but must stay within the module package.
Normalize modules receive the structured LLM client through
`contracts.NormalizeRequest` when they need model-backed reconciliation.
## `seriatim` Input
Package: `internal/modules/input/seriatim`
@@ -44,6 +74,10 @@ The `generic` chunker splits source units into ordered chunks. It validates the
source document, clones source units, assigns chunk IDs such as `chunk-000001`,
and records chunk metadata for start unit, end unit, and unit count.
The pipeline runner canonicalizes chunk units from the source document by ID
before extractors and mergers run. Chunker-owned context should stay in
`SourceChunk.Metadata`.
Options:
- `max_units`: positive integer, default `50`;
@@ -54,6 +88,39 @@ Provides:
- `chunks`
## `dnd/scenes` Chunker
Package: `internal/modules/chunk/dnd/scenes`
The `dnd/scenes` chunker uses the structured LLM client to divide transcript
source units into coherent D&D scenes. It renders embedded prompts, loads the
embedded structured response schema, validates model-authored source-unit
boundaries, and converts each scene into a deterministic source chunk.
Requires:
- `source.transcript`
Provides:
- `chunks`
- `chunks.scenes`
Options: none. Non-empty options are rejected.
The chunker enforces full source-unit coverage from the first source unit to the
last, exact source-unit IDs, sequential contiguous scenes, and no overlap. It
assigns chunk IDs such as `scene-000001` and stores scene metadata including
title, primary mode, participants, summary, boundary note, confidence, boundary
unit IDs, and unit count. Boundary caveats become warnings with reason code
`scene_boundary_caveat`. Whitespace-only caveats are treated as malformed
structured output rather than silently dropped.
Malformed model output fails explicitly rather than falling back to another
chunker. The chunker exposes prompt and response-schema provenance through
top-level `module_metadata.chunker` without raw prompts, raw schemas, source
text, or secrets.
## `dnd/spells` Extractor
Package: `internal/modules/extract/dnd/spells`
@@ -78,15 +145,23 @@ Artifact type and schema version:
- schema version: `v1`
The extractor adds prompt and response-schema provenance to lane manifest
metadata. Durable artifact payload details belong in the
metadata under `artifact_lanes[].metadata.extractor`. Durable artifact payload
details belong in the
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
The extractor declares optional `roster` and `glossary` reference slots
accepting UTF-8 text without narrowing accepted media types. Its prompt frames
references as supporting disambiguation material only; spell-cast artifacts must
still be grounded in the source transcript.
## D&D Spell Validators
The spell extractor returns two built-in validators:
- `dnd/spells/shape`: rejects malformed payloads and missing required fields.
- `dnd/spells/source_refs`: rejects candidates without valid source references.
It also emits a warning when the extracted spell name is not found in the
cited source text.
Reason codes include:
@@ -94,6 +169,7 @@ Reason codes include:
- `missing_required_field`
- `missing_source_ref`
- `invalid_source_ref`
- `spell_not_near_source`
These validators are supplied by the extractor when no validators are configured
for the lane.

View File

@@ -30,6 +30,38 @@ before execution:
The CLI writes the resolved pipeline and digest to diagnostics.
Pipeline profiles and artifact lanes may include reference binding maps keyed by
reference slot name. During resolution, pipeline-level bindings act as defaults
for selected chunk, extractor, and normalizer targets that declare the slot;
target-local bindings override or add bindings for that target. Runtime
`--reference` requests override target config bindings, and runtime unbinds
remove optional target bindings. Flat runtime slot names are resolved only when
exactly one selected target declares the slot; otherwise the CLI requires a more
specific selector such as `chunk.slot`, `lane.extract.slot`, or
`lane.normalize.slot`. Resolution validates bindings against the declaring
target specs and stores the bindings in target-aware resolved reference holders.
It does not read reference files or include reference bytes in source digests.
During run preparation, resolved file references for chunk, extractor, and
normalizer targets are materialized before any LLM-backed pipeline work. Config
bindings resolve relative to the config file, and CLI bindings resolve relative
to the current working directory. Materialization accepts UTF-8 text files,
computes `sha256:` content digests, records file origins, infers canonical base
media types from file extensions, enforces declared byte limits, and warns for
empty bound files. Media-type acceptance is checked only when a slot declares
`AcceptedMediaTypes`; unknown extensions are recorded as
`application/octet-stream`. Reference content is omitted from diagnostics and
manifests. The CLI writes provenance-only resolved reference diagnostics, and
the run manifest records target-stage reference provenance separately from
source digests. Runtime reference content is passed to the matching chunker,
extractor, or normalizer request.
Prompt bundles can declare reference slots and use `reference` and
`hasreference` template functions. Bundle loading validates string-literal slot
names against the declaration. Rendering receives a target reference set from the
caller; unbound optional slots render as empty strings, and `hasreference`
returns true only when at least one bound item has content.
## Registries And Module Specs
`pipeline.Registries` holds concrete constructors for execution. A
@@ -44,6 +76,11 @@ Every production module registers a `ModuleSpec` with:
- `Provides`: capabilities added after that module runs;
- `Requires`: capabilities that must already be available.
Chunk, extract, and normalize specs may also declare reference slots. Slot
declarations are available from registry metadata without constructing module
instances. Input, merge, validate, and output specs must not declare reference
slots.
Capability checks prevent incompatible pipeline composition before a run starts.
## Runner Input And Output
@@ -73,8 +110,39 @@ The runner:
2. builds the input adapter and parses the raw input into a source document;
3. validates the source document;
4. builds the chunker and produces source chunks;
5. runs each selected artifact lane in sorted resolved order;
6. builds the output encoder and validates logical output file names.
5. validates source chunks against framework invariants;
6. runs each selected artifact lane in sorted resolved order;
7. builds the output encoder and validates logical output file names.
## Chunk Results
Chunkers implement `contracts.Chunker` and receive a `contracts.ChunkRequest`
with the validated source document, reference set, structured LLM client, the
configured LLM profile, module options, and run metadata. Deterministic and
LLM-backed chunkers use the same contract; provider construction stays outside
chunk modules.
After `Chunk` returns, the runner appends chunker warnings before returning any
chunker error. When chunking succeeds, the runner validates generic chunk
invariants before running extractors:
- chunk IDs must be non-empty and unique in the chunk result;
- each chunk `SourceID` must match the source document ID;
- each chunk `Index` must match its zero-based returned order;
- each chunk must contain at least one source unit;
- a chunk must not repeat a source unit;
- every chunk source unit must exist in the source document;
- source units inside each chunk must appear in source-document order.
After validation, the runner rebuilds each chunk from source-document units by
ID, preserving the chunk boundary order and cloning chunk metadata. Extractors
and downstream stages therefore see canonical source units, while
`SourceChunk.Metadata` remains the supported place for chunker-owned context.
The framework does not require complete source-unit coverage and does not reject
overlap between different chunks. Stricter policies, such as full coverage or
non-overlap, belong to individual chunk modules when they are part of that
module's contract.
Within an artifact lane, the runner:
@@ -119,9 +187,15 @@ On successful execution, the manifest validation status is:
## Manifest Population
The manifest records run ID, pipeline ID, pipeline digest, module keys, artifact
lanes, LLM profile metadata, source digest, validation status, and timing.
The manifest records run ID, pipeline ID, pipeline digest, module keys, top-level
module metadata, artifact lanes, LLM profile metadata, source digest,
reference provenance, validation status, and timing.
Modules can add non-secret manifest metadata by implementing
`contracts.ManifestMetadataProvider`. The D&D spell extractor uses this for
Singleton pipeline modules may add non-secret metadata by implementing
`contracts.ManifestMetadataProvider`. The runner records that metadata under
`module_metadata` with stable keys for `input`, `chunker`, and `output`.
Lane-owned modules may add non-secret metadata through
`artifact_lanes[].metadata`. The runner records extractor, merger, and
normalizer metadata there. The D&D spell extractor uses lane metadata for
prompt and response-schema provenance.

View File

@@ -34,8 +34,9 @@ The `json` output module writes these files:
- `index.json`: file index with paths to the manifest, artifact files,
rejected artifacts, and warnings.
- `manifest.json`: run manifest with resolved pipeline provenance, module keys,
validation status, and timing.
- `manifest.json`: run manifest with resolved pipeline provenance, top-level
module metadata, module keys, reference provenance, validation status, and
timing.
- `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact
type. For the current D&D spell extractor, this includes
`artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved.
@@ -62,8 +63,11 @@ Implemented diagnostics artifacts:
path, selected lanes, run ID, and pipeline digest when available.
- `effective-config.json`: resolved config with API keys redacted.
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
- `resolved-references.json`: resolved reference provenance, including target
stage, lane ID when present, origin, digest, media type, byte size, and
binding source, without reference content.
- `run-manifest.json`: the same run manifest written to durable output when it
is available.
is available, including top-level module metadata when present.
- `warnings.json`: warning list.
- `run-report.json`: counts, status, output path, diagnostics path, and run ID.
- `error.log`: failure message, written after diagnostics directory creation
@@ -107,6 +111,10 @@ stderr, and writes warnings to durable output and diagnostics when retained.
The run manifest `validation_status` indicates whether final artifacts were
approved or rejected after validation.
Reference-related warnings include empty bound reference files and D&D spell
relatedness warnings such as `spell_not_near_source`. Empty references are still
passed to extractors so optional slots can be intentionally blank.
## Cleanup
It is safe to remove specific old run directories after their output and

View File

@@ -8,6 +8,16 @@ future work only.
- Additional input adapters, such as Markdown or note-export formats.
- Additional D&D extractors beyond spell casts.
- Add non-file reference producers, such as prior-run artifacts, derived
summaries, or entity registries, without changing module-facing reference
item contracts.
- Add token budgeting and model context-window management for reference content.
- Add per-slot or per-chunk inclusion policies so modules can avoid repeating
large reference content in every prompt when that becomes important.
- Add structured or parsed references, such as typed roster schemas, when a
module has a clear need for more than opaque UTF-8 text.
- Add reference caching, preprocessing, summarization, embedding, or retrieval
if references become large enough to require preprocessing.
- Cross-lane entity normalization.
- Cross-chunk semantic deduplication.
- Configurable validator chains with production validator modules.
@@ -15,6 +25,13 @@ future work only.
- Parallel execution where it preserves deterministic manifests and diagnostics.
- Additional output encoders.
## Candidate Architecture Work
- Evaluate replacing the local LLM adapter with an import from
`gitea.maximumdirect.net/eric/scriptorium`, provided it preserves Notarius
boundaries around provider plumbing, prompt ownership, diagnostics, and secret
handling.
## Candidate Operational Work
- Packaged release artifacts for alpha distribution.

View File

@@ -0,0 +1,21 @@
# LLM And Reference Expansion Completed
This roadmap item is implemented.
Notarius now supports first-class LLM access for chunk, extract, and normalize
modules, plus target-aware reference declaration, binding, CLI override,
materialization, provenance, and runtime delivery for chunker, extractor, and
normalizer targets.
Current behavior is documented in:
- [CLI Reference](../cli.md)
- [Configuration](../config.md)
- [Pipeline Internals](../internal/pipeline.md)
- [Module Internals](../internal/modules.md)
- [JSON Output](../integrations/json-output.md)
- [Troubleshooting](../troubleshooting.md)
Deferred reference work remains tracked in [Future Work](future.md), including
non-file reference producers, token budgeting, structured references, and
reference preprocessing or retrieval.

View File

@@ -102,6 +102,45 @@ go run ./cmd/notarius run dnd-session \
- For `config validate`, include `--pipeline` when using `--only`.
- Confirm the lane ID exists under `pipelines.<id>.artifacts`.
## Reference Binding Failure
Symptoms include:
- `reference slot "..." is not declared`
- `reference slot "..." is declared by multiple selected targets`
- `required reference slot "..." is not bound`
- `--reference must use slot=path`
- `--without-reference must use a reference selector without =path`
- `read "...": no such file`
- `must be UTF-8 text`
- `media type "..." is not accepted`
- `is ... bytes, limit ...`
Fix:
- Confirm the selected chunker, extractor, or normalizer declares the slot. The
implemented `dnd/spells` extractor declares optional `roster` and `glossary`
slots.
- Use a specific selector when more than one selected target declares the same
slot. Examples include `chunk.context=./context.txt`,
`spells.extract.context=./extract-context.txt`, and
`spells.normalize.context=./normalize-context.txt`.
- `lane.slot=path` is valid only when exactly one selected extractor or
normalizer in that lane declares the slot. If both do, use
`lane.extract.slot=path` or `lane.normalize.slot=path`.
- Use `--without-reference selector` to remove optional config bindings; do not
pass an empty `--reference selector=`.
- Check whether a path came from config or CLI. Config paths are relative to
the config file. CLI reference paths are relative to the current working
directory.
- Ensure the file is readable UTF-8 text and within any byte limit declared by
the declaring module.
- If the declaring module narrows accepted media types, use a file extension that
infers an accepted type such as `text/markdown` or `application/json`.
Unknown extensions infer `application/octet-stream`.
- If diagnostics are retained, inspect `resolved-pipeline.json`,
`resolved-references.json`, and `error.log`.
## Seriatim Input Validation Failure
Symptoms include `seriatim input`, `parse JSON`, `segments must not be empty`,
@@ -177,6 +216,34 @@ Fix:
Provider error messages are redacted for configured API key values.
## Scene Chunking Failure
Symptoms include:
- `dnd scenes chunker`
- `malformed structured output`
- `boundary_caveats`
- `start_unit_id`
- `end_unit_id`
- `gap`
- `overlap`
- `final scene`
- `complete structured output`
Fix:
- Validate the pipeline configuration and confirm the input module provides a
transcript source when using `chunk: dnd/scenes`.
- Confirm the LLM profile has a working OpenAI-compatible `base_url`, `model`,
and credentials.
- Inspect retained diagnostics for the run error and resolved pipeline.
- If the error names malformed structured output, retry with a model that
follows structured response schemas reliably.
- If the error names `boundary_caveats`, check for blank or whitespace-only
caveat text in the scene response.
- Scene boundaries must use exact source-unit IDs, cover the full source
document, be contiguous, and not overlap.
## Output Write Failure
Symptoms include:

View File

@@ -0,0 +1,2 @@
Cure Wounds: healing spell cast by touch.
Shield: defensive reaction spell.

View File

@@ -0,0 +1,3 @@
Aria: party cleric and recurring healer.
Borin: fighter ally.
Bandit mage: hostile spellcaster.

View File

@@ -7,6 +7,9 @@ llm_profiles:
pipelines:
dnd-session:
input: seriatim
references:
roster: ./dnd-spells-roster.txt
glossary: ./dnd-spells-glossary.txt
chunk:
module: generic
options:

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
@@ -34,6 +35,9 @@ func productionRegistries() (pipeline.Registries, error) {
if err := generic.Register(registries.Chunkers); err != nil {
return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err)
}
if err := scenes.Register(registries.Chunkers); err != nil {
return pipeline.Registries{}, fmt.Errorf("register dnd scenes chunker: %w", err)
}
if err := spells.Register(registries.Extractors); err != nil {
return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err)
}

View File

@@ -25,7 +25,7 @@ const defaultOutputRoot = "./notarius-output"
const usage = `Usage:
notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b]
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference selector=path] [--without-reference selector]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json]
`
@@ -95,6 +95,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
outputDir := fs.String("output-dir", "", "output directory")
diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory")
llmProfile := fs.String("llm-profile", "", "LLM profile override")
referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{}
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, or lane.normalize.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
if err := fs.Parse(reorderRunArgs(args)); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
@@ -121,6 +125,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
referenceRequests, err := parseReferenceFlags(referenceFlags)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
referenceUnbindRequests, err := parseReferenceUnbindFlags(withoutReferenceFlags)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
cfg, loadedConfigPath, err := loadConfig(*configPath, opts)
if err != nil {
@@ -155,15 +169,33 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests)
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
effective, err := cfg.Resolve(config.ResolveInput{
PipelineID: pipelineID,
Only: only,
Catalog: catalog,
LLMProfileOverride: *llmProfile,
ReferenceOverrides: referenceOverrides,
ReferenceUnbinds: referenceUnbinds,
})
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
workingDir, err := os.Getwd()
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("resolve working directory: %w", err))
}
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
ConfigPath: loadedConfigPath,
WorkingDir: workingDir,
})
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
effective.ResolvedPipeline = materialized
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
if err := runDir.WriteInvocationMetadata(invocation); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
@@ -174,6 +206,9 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err := runDir.WriteResolvedPipeline(effective.ResolvedPipeline); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved pipeline: %w", err))
}
if err := runDir.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline)); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved references: %w", err))
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if len(profileIDs) != 1 {
@@ -205,6 +240,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
StartedAt: startedAt,
LLMProfiles: llmProfiles,
Metadata: runMetadata(*outputDir, *diagnosticsDir),
Warnings: referenceWarnings,
})
if err != nil {
if output.Manifest.PipelineID != "" {
@@ -412,7 +448,7 @@ func reorderRunArgs(args []string) []string {
func runFlagTakesValue(arg string) bool {
switch arg {
case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile":
case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile", "--reference", "--without-reference":
return true
default:
return false
@@ -661,6 +697,417 @@ func parseOnly(raw string) ([]string, error) {
return result, nil
}
type stringListFlag []string
func (flag *stringListFlag) String() string {
if flag == nil {
return ""
}
return strings.Join(*flag, ",")
}
func (flag *stringListFlag) Set(value string) error {
*flag = append(*flag, value)
return nil
}
type cliReferenceRequest struct {
Selector cliReferenceSelector
Source string
}
type cliReferenceUnbindRequest struct {
Selector cliReferenceSelector
}
type cliReferenceSelector struct {
LaneID string
Stage pipeline.ModuleStage
SlotName string
}
func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
if len(values) == 0 {
return nil, nil
}
requests := make([]cliReferenceRequest, 0, len(values))
for _, raw := range values {
name, source, ok := strings.Cut(raw, "=")
if !ok {
return nil, fmt.Errorf("--reference must use slot=path or lane.slot=path")
}
if strings.TrimSpace(source) == "" {
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
}
selector, err := parseReferenceSelector(name, "--reference")
if err != nil {
return nil, err
}
requests = append(requests, cliReferenceRequest{
Selector: selector,
Source: strings.TrimSpace(source),
})
}
return requests, nil
}
func parseReferenceUnbindFlags(values []string) ([]cliReferenceUnbindRequest, error) {
if len(values) == 0 {
return nil, nil
}
requests := make([]cliReferenceUnbindRequest, 0, len(values))
for _, raw := range values {
if strings.Contains(raw, "=") {
return nil, fmt.Errorf("--without-reference must use a reference selector without =path")
}
selector, err := parseReferenceSelector(raw, "--without-reference")
if err != nil {
return nil, err
}
requests = append(requests, cliReferenceUnbindRequest{
Selector: selector,
})
}
return requests, nil
}
func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector, error) {
selector := strings.TrimSpace(raw)
if selector == "" {
return cliReferenceSelector{}, fmt.Errorf("%s reference slot must not be empty", flagName)
}
parts := strings.Split(selector, ".")
for _, part := range parts {
if strings.TrimSpace(part) == "" {
return cliReferenceSelector{}, fmt.Errorf("%s must use non-empty reference selector values", flagName)
}
}
switch len(parts) {
case 1:
return cliReferenceSelector{SlotName: strings.TrimSpace(parts[0])}, nil
case 2:
first := strings.TrimSpace(parts[0])
slotName := strings.TrimSpace(parts[1])
if first == string(pipeline.StageChunk) {
return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil
}
return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
case 3:
laneID := strings.TrimSpace(parts[0])
stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
slotName := strings.TrimSpace(parts[2])
if stage != pipeline.StageExtract && stage != pipeline.StageNormalize {
return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot or lane.normalize.slot", flagName)
}
return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil
default:
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, lane.slot, lane.extract.slot, or lane.normalize.slot", flagName)
}
}
func resolveCLIReferenceRequests(
cfg config.Config,
pipelineID string,
only []string,
catalog pipeline.ModuleCatalog,
referenceRequests []cliReferenceRequest,
unbindRequests []cliReferenceUnbindRequest,
) ([]pipeline.ReferenceBinding, []pipeline.ReferenceUnbind, error) {
if len(referenceRequests) == 0 && len(unbindRequests) == 0 {
return nil, nil, nil
}
targets, err := selectedReferenceTargets(cfg, pipelineID, only, catalog)
if err != nil {
return nil, nil, err
}
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
for _, request := range referenceRequests {
target, err := resolveCLIReferenceTarget(targets, request.Selector)
if err != nil {
return nil, nil, err
}
overrides = append(overrides, pipeline.ReferenceBinding{
Stage: target.stage,
LaneID: target.laneID,
SlotName: request.Selector.SlotName,
Source: request.Source,
BindingSource: contracts.ReferenceBindingSourceCLI,
})
}
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
for _, request := range unbindRequests {
target, err := resolveCLIReferenceTarget(targets, request.Selector)
if err != nil {
return nil, nil, err
}
unbinds = append(unbinds, pipeline.ReferenceUnbind{
Stage: target.stage,
LaneID: target.laneID,
SlotName: request.Selector.SlotName,
})
}
return overrides, unbinds, nil
}
type selectedReferenceTarget struct {
laneID string
stage pipeline.ModuleStage
module string
slots map[string]struct{}
}
func selectedReferenceTargets(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceTarget, error) {
profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID)
if !ok {
return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID))
}
lanesByID := make(map[string]pipeline.ArtifactLaneProfile, len(profile.Artifacts))
for rawLaneID, lane := range profile.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
return nil, fmt.Errorf("pipeline %q artifact lane id must not be empty", strings.TrimSpace(pipelineID))
}
if _, ok := lanesByID[laneID]; ok {
return nil, fmt.Errorf("pipeline %q artifact lane %q is duplicated after trimming", strings.TrimSpace(pipelineID), laneID)
}
lanesByID[laneID] = lane
}
selectedIDs := make([]string, 0, len(lanesByID))
if len(only) == 0 {
for laneID := range lanesByID {
selectedIDs = append(selectedIDs, laneID)
}
} else {
seen := make(map[string]struct{}, len(only))
for _, rawLaneID := range only {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
return nil, fmt.Errorf("pipeline %q selected artifact lane id must not be empty", strings.TrimSpace(pipelineID))
}
if _, ok := lanesByID[laneID]; !ok {
return nil, fmt.Errorf("pipeline %q selected artifact lane %q is not declared", strings.TrimSpace(pipelineID), laneID)
}
if _, ok := seen[laneID]; !ok {
selectedIDs = append(selectedIDs, laneID)
seen[laneID] = struct{}{}
}
}
}
sort.Strings(selectedIDs)
targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*2)
chunk := pipeline.Binding(profile.Chunk.Module)
chunk.Module = strings.TrimSpace(profile.Chunk.Module)
if chunk.Module == "" {
chunk.Module = pipeline.DefaultChunkModule
}
chunkSpec, err := cliReferenceChunkerSpec(catalog, chunk.Module)
if err != nil {
return nil, fmt.Errorf("pipeline %q chunk module %q: %w", strings.TrimSpace(pipelineID), chunk.Module, err)
}
targets = append(targets, selectedReferenceTarget{
stage: pipeline.StageChunk,
module: chunk.Module,
slots: referenceSlotSet(chunkSpec.ReferenceSlots),
})
for _, laneID := range selectedIDs {
lane := lanesByID[laneID]
extractModule := strings.TrimSpace(lane.Extract.Module)
if extractModule == "" {
return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID)
}
extractSpec, err := cliReferenceExtractorSpec(catalog, extractModule)
if err != nil {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: %w", strings.TrimSpace(pipelineID), laneID, extractModule, err)
}
targets = append(targets, selectedReferenceTarget{
laneID: laneID,
stage: pipeline.StageExtract,
module: extractModule,
slots: referenceSlotSet(extractSpec.ReferenceSlots),
})
normalizeModule := strings.TrimSpace(lane.Normalize.Module)
if normalizeModule == "" {
normalizeModule = pipeline.DefaultNormalizeModule
}
normalizeSpec, err := cliReferenceNormalizerSpec(catalog, normalizeModule)
if err != nil {
return nil, fmt.Errorf("pipeline %q lane %q normalize module %q: %w", strings.TrimSpace(pipelineID), laneID, normalizeModule, err)
}
targets = append(targets, selectedReferenceTarget{
laneID: laneID,
stage: pipeline.StageNormalize,
module: normalizeModule,
slots: referenceSlotSet(normalizeSpec.ReferenceSlots),
})
}
return targets, nil
}
func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
pipelineID = strings.TrimSpace(pipelineID)
for rawID, profile := range profiles {
if strings.TrimSpace(rawID) == pipelineID {
return profile, true
}
}
return pipeline.PipelineProfile{}, false
}
func cliReferenceChunkerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Chunkers == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Chunkers.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func cliReferenceExtractorSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Extractors == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Extractors.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Normalizers == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Normalizers.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} {
slotSet := make(map[string]struct{}, len(slots))
for _, slot := range slots {
slotSet[slot.Name] = struct{}{}
}
return slotSet
}
func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliReferenceSelector) (selectedReferenceTarget, error) {
slotName := strings.TrimSpace(selector.SlotName)
if slotName == "" {
return selectedReferenceTarget{}, fmt.Errorf("reference slot must not be empty")
}
if selector.Stage == pipeline.StageChunk {
for _, target := range targets {
if target.stage != pipeline.StageChunk {
continue
}
if _, ok := target.slots[slotName]; !ok {
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
}
return target, nil
}
return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
}
if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageNormalize {
for _, target := range targets {
if target.laneID == selector.LaneID && target.stage == selector.Stage {
if _, ok := target.slots[slotName]; !ok {
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
}
return target, nil
}
}
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
}
if strings.TrimSpace(selector.LaneID) != "" {
return resolveCLIReferenceLaneTarget(targets, strings.TrimSpace(selector.LaneID), slotName)
}
return resolveCLIReferenceFlatTarget(targets, slotName)
}
func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
laneSelected := false
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if target.laneID != laneID {
continue
}
laneSelected = true
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
if !laneSelected {
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", laneID)
}
switch len(matches) {
case 0:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, laneID)
case 1:
return matches[0], nil
default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use %s.extract.%s or %s.normalize.%s", slotName, laneID, targetList(matches), laneID, slotName, laneID, slotName)
}
}
func resolveCLIReferenceFlatTarget(targets []selectedReferenceTarget, slotName string) (selectedReferenceTarget, error) {
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
switch len(matches) {
case 0:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected reference target", slotName)
case 1:
return matches[0], nil
default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets (%s); use a more specific selector such as %s", slotName, targetList(matches), selectorSuggestions(matches, slotName))
}
}
func targetList(targets []selectedReferenceTarget) string {
labels := make([]string, 0, len(targets))
for _, target := range targets {
labels = append(labels, targetLabel(target))
}
sort.Strings(labels)
return strings.Join(labels, ", ")
}
func targetLabel(target selectedReferenceTarget) string {
if target.stage == pipeline.StageChunk {
return "chunk"
}
return target.laneID + "." + string(target.stage)
}
func selectorSuggestions(targets []selectedReferenceTarget, slotName string) string {
suggestions := make([]string, 0, len(targets))
for _, target := range targets {
if target.stage == pipeline.StageChunk {
suggestions = append(suggestions, "chunk."+slotName)
continue
}
suggestions = append(suggestions, target.laneID+"."+string(target.stage)+"."+slotName)
}
sort.Strings(suggestions)
return strings.Join(suggestions, " or ")
}
func sortedPipelineIDs(cfg config.Config) []string {
ids := make([]string, 0, len(cfg.Pipelines))
for id := range cfg.Pipelines {

File diff suppressed because it is too large Load Diff

View File

@@ -48,23 +48,37 @@ type LLMProfileManifest struct {
Model string `json:"model,omitempty"`
}
type ReferenceProvenance struct {
Stage string `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
OriginType string `json:"origin_type"`
OriginURI string `json:"origin_uri,omitempty"`
Digest string `json:"digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
SizeBytes int64 `json:"size_bytes,omitempty"`
BindingSource string `json:"binding_source,omitempty"`
}
type RunManifest struct {
RunID string `json:"run_id,omitempty"`
PipelineID string `json:"pipeline_id,omitempty"`
PipelineDigest string `json:"pipeline_digest,omitempty"`
InputModule string `json:"input_module,omitempty"`
Chunker string `json:"chunker,omitempty"`
SourceDigests []string `json:"source_digests,omitempty"`
Extractors []string `json:"extractors,omitempty"`
Merger string `json:"merger,omitempty"`
Normalizer string `json:"normalizer,omitempty"`
OutputEncoder string `json:"output_encoder,omitempty"`
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
SchemaVersion string `json:"schema_version,omitempty"`
ValidationStatus string `json:"validation_status,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
RunID string `json:"run_id,omitempty"`
PipelineID string `json:"pipeline_id,omitempty"`
PipelineDigest string `json:"pipeline_digest,omitempty"`
InputModule string `json:"input_module,omitempty"`
Chunker string `json:"chunker,omitempty"`
SourceDigests []string `json:"source_digests,omitempty"`
Extractors []string `json:"extractors,omitempty"`
Merger string `json:"merger,omitempty"`
Normalizer string `json:"normalizer,omitempty"`
OutputEncoder string `json:"output_encoder,omitempty"`
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
References []ReferenceProvenance `json:"references,omitempty"`
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
SchemaVersion string `json:"schema_version,omitempty"`
ValidationStatus string `json:"validation_status,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
}
func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact {

View File

@@ -183,6 +183,80 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
}
func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
manifest := RunManifest{
References: []ReferenceProvenance{
{
Stage: "extract",
LaneID: "events",
SlotName: "roster",
OriginType: "file",
OriginURI: "file:///tmp/roster.txt",
Digest: "sha256:reference",
MediaType: "text/plain; charset=utf-8",
SizeBytes: 12,
BindingSource: "config",
},
},
}
gotJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var got RunManifest
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if len(got.References) != 1 {
t.Fatalf("len(References) = %d, want 1", len(got.References))
}
reference := got.References[0]
if reference.Stage != "extract" || reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference)
}
if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" {
t.Fatalf("reference provenance = %#v, want digest/media/size/source details", reference)
}
}
func TestRunManifestIncludesTopLevelModuleMetadata(t *testing.T) {
manifest := RunManifest{
ModuleMetadata: map[string]map[string]any{
"chunker": {
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:abc123",
"response_schema_key": "dnd_scenes",
"response_schema_name": "dnd_scenes",
},
},
}
gotJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var got map[string]any
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
moduleMetadata, ok := got["module_metadata"].(map[string]any)
if !ok {
t.Fatalf("module_metadata = %#v, want object", got["module_metadata"])
}
assertHasKeys(t, moduleMetadata, "chunker")
chunkerMetadata, ok := moduleMetadata["chunker"].(map[string]any)
if !ok {
t.Fatalf("module_metadata.chunker = %#v, want object", moduleMetadata["chunker"])
}
assertHasKeys(t, chunkerMetadata, "prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_name")
}
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
t.Helper()

View File

@@ -73,6 +73,7 @@ func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile
out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk)
out.Output = cloneModuleBinding(in.Output)
out.References = cloneStringMap(in.References)
if len(in.Artifacts) > 0 {
out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts))
for key, lane := range in.Artifacts {
@@ -87,6 +88,7 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize)
out.References = cloneStringMap(in.References)
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators {
@@ -96,11 +98,23 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
return out
}
func cloneStringMap(in map[string]string) map[string]string {
if len(in) == 0 {
return nil
}
out := make(map[string]string, len(in))
for key, value := range in {
out[key] = value
}
return out
}
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out := in
if len(in.Options) > 0 {
out.Options = cloneOptions(in.Options)
}
out.References = cloneStringMap(in.References)
return out
}

View File

@@ -14,13 +14,17 @@ type ResolveInput struct {
Only []string
Catalog pipeline.ModuleCatalog
LLMProfileOverride string
ReferenceOverrides []pipeline.ReferenceBinding
ReferenceUnbinds []pipeline.ReferenceUnbind
}
type EffectiveConfig struct {
Config Config
PipelineID string
Only []string
ResolvedPipeline pipeline.ResolvedPipeline
Config Config
PipelineID string
Only []string
ReferenceOverrides []pipeline.ReferenceBinding
ReferenceUnbinds []pipeline.ReferenceUnbind
ResolvedPipeline pipeline.ResolvedPipeline
}
func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
@@ -46,16 +50,22 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
applyLLMProfileOverride(&profile, override)
}
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{Only: input.Only}, input.Catalog)
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{
Only: input.Only,
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
}, input.Catalog)
if err != nil {
return EffectiveConfig{}, fmt.Errorf("resolve pipeline %q: %w", pipelineID, err)
}
return EffectiveConfig{
Config: cloneConfig(c),
PipelineID: pipelineID,
Only: append([]string(nil), input.Only...),
ResolvedPipeline: resolved,
Config: cloneConfig(c),
PipelineID: pipelineID,
Only: append([]string(nil), input.Only...),
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
ResolvedPipeline: resolved,
}, nil
}

View File

@@ -98,6 +98,43 @@ func TestResolveSurfacesMissingCapabilityThroughCatalog(t *testing.T) {
}
}
func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
profile.Chunk = pipeline.Binding("dnd/scenes")
lane := profile.Artifacts["events"]
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"events": lane}
cfg.Pipelines["example"] = profile
catalog := fakeCatalog(t,
pipeline.ModuleSpec{
Key: "fake/input",
Stage: pipeline.StageInput,
Provides: []string{"source.transcript"},
},
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"artifact"},
},
)
mustRegisterChunker(t, catalog.Chunkers, pipeline.ModuleSpec{
Key: "dnd/scenes",
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks", "chunks.scenes"},
})
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
if got := effective.ResolvedPipeline.Chunk.Module; got != "dnd/scenes" {
t.Fatalf("Chunk.Module = %q, want dnd/scenes", got)
}
}
func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
cfg := validConfig()
first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})

View File

@@ -35,10 +35,11 @@ type FileLLMProfile struct {
}
type FilePipelineProfile struct {
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]string `yaml:"references,omitempty"`
}
type FileArtifactLaneProfile struct {
@@ -46,6 +47,7 @@ type FileArtifactLaneProfile struct {
Merge *fileModuleBinding `yaml:"merge,omitempty"`
Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
Validators []fileModuleBinding `yaml:"validators,omitempty"`
References map[string]string `yaml:"references,omitempty"`
}
type FileConcurrencyConfig struct {
@@ -97,6 +99,7 @@ type fileModuleBinding struct {
Module string
LLMProfile string
Options map[string]any
References map[string]string
}
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
@@ -131,6 +134,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err
}
b.Options = normalizeOptions(options)
case "references":
var references map[string]string
if err := valueNode.Decode(&references); err != nil {
return err
}
b.References = references
default:
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
}
@@ -146,6 +155,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile),
Options: cloneOptions(b.Options),
References: normalizedStringMap(b.References),
}
}
@@ -212,6 +222,49 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
return err
}
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Chunk != nil {
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
return err
}
}
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Output != nil {
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
return err
}
}
for rawLaneID, fileLane := range filePipeline.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
continue
}
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
return err
}
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
return err
}
if fileLane.Merge != nil {
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
return err
}
}
if fileLane.Normalize != nil {
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
return err
}
}
for i, validator := range fileLane.Validators {
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
return err
}
}
}
}
for _, profileID := range profileIDs {
@@ -253,9 +306,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
return err
}
profile := pipeline.PipelineProfile{
ID: pipelineID,
Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
ID: pipelineID,
Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: normalizedStringMap(filePipeline.References),
}
if filePipeline.Chunk != nil {
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
@@ -265,8 +319,11 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
}
for _, laneID := range laneIDs {
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
extract := fileLane.Extract.toPipelineBinding()
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
lane := pipeline.ArtifactLaneProfile{
Extract: fileLane.Extract.toPipelineBinding(),
Extract: extract,
References: normalizedStringMap(fileLane.References),
}
if fileLane.Merge != nil {
lane.Merge = fileLane.Merge.toPipelineBinding()
@@ -318,6 +375,39 @@ func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, ma
return keys, rawByNormalized, nil
}
func normalizedStringMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
}
return out
}
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
for key, value := range base {
out[key] = value
}
for key, value := range override {
out[key] = value
}
return out
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName)
if name == "" {

View File

@@ -144,6 +144,78 @@ pipelines:
}
}
func TestParseFileConfigReferenceMaps(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
pipelines:
example:
input: fake/input
references:
" roster ": " ./shared-roster.yml "
artifacts:
events:
extract: fake/extract
references:
" lore ": " ./lore.md "
`)
profile := cfg.Pipelines["example"]
if !reflect.DeepEqual(profile.References, map[string]string{"roster": "./shared-roster.yml"}) {
t.Fatalf("pipeline references = %#v, want trimmed map", profile.References)
}
gotLaneRefs := profile.Artifacts["events"].References
if !reflect.DeepEqual(gotLaneRefs, map[string]string{"lore": "./lore.md"}) {
t.Fatalf("lane references = %#v, want trimmed map", gotLaneRefs)
}
}
func TestParseFileConfigStageLocalReferenceMaps(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
pipelines:
example:
input: fake/input
chunk:
module: generic
references:
" scene_guide ": " ./scenes.md "
artifacts:
events:
extract:
module: fake/extract
references:
" glossary ": " ./glossary.md "
" roster ": " ./extract-roster.yml "
references:
roster: ./legacy-roster.yml
lore: ./lore.md
normalize:
module: noop
references:
" normalization_notes ": " ./normalization.md "
`)
profile := cfg.Pipelines["example"]
if !reflect.DeepEqual(profile.Chunk.References, map[string]string{"scene_guide": "./scenes.md"}) {
t.Fatalf("chunk references = %#v, want trimmed map", profile.Chunk.References)
}
lane := profile.Artifacts["events"]
if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) {
t.Fatalf("legacy lane references = %#v, want trimmed map", lane.References)
}
wantExtract := map[string]string{
"glossary": "./glossary.md",
"lore": "./lore.md",
"roster": "./extract-roster.yml",
}
if !reflect.DeepEqual(lane.Extract.References, wantExtract) {
t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract)
}
if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) {
t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References)
}
}
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
@@ -297,6 +369,108 @@ pipelines:
}
}
func TestApplyFileConfigRejectsDuplicateTrimmedReferenceSlots(t *testing.T) {
tests := []struct {
name string
raw string
want string
}{
{
name: "pipeline",
raw: `
version: 1
pipelines:
example:
input: fake/input
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" reference slot`,
},
{
name: "lane",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract: fake/extract
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" reference slot`,
},
{
name: "chunk",
raw: `
version: 1
pipelines:
example:
input: fake/input
chunk:
module: generic
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" chunk reference slot`,
},
{
name: "extract",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract:
module: fake/extract
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" extract reference slot`,
},
{
name: "normalize",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract: fake/extract
normalize:
module: noop
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" normalize reference slot`,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(tc.raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML: %v", err)
}
cfg := Default()
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "duplicated") {
t.Fatalf("expected duplicate reference slot error, got %v", err)
}
})
}
}
func TestApplyFileConfigAllowsRetryOnlyLLMProfile(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1

View File

@@ -21,10 +21,12 @@ func (c Config) RedactedDiagnosticsPayload() any {
func (e EffectiveConfig) RedactedDiagnosticsPayload() any {
return EffectiveConfig{
Config: e.Config.Redacted(),
PipelineID: e.PipelineID,
Only: append([]string(nil), e.Only...),
ResolvedPipeline: cloneResolvedPipeline(e.ResolvedPipeline),
Config: e.Config.Redacted(),
PipelineID: e.PipelineID,
Only: append([]string(nil), e.Only...),
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), e.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), e.ReferenceUnbinds...),
ResolvedPipeline: cloneResolvedPipeline(e.ResolvedPipeline),
}
}
@@ -32,6 +34,7 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
out := in
out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk)
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
out.Output = cloneModuleBinding(in.Output)
if len(in.ArtifactLanes) > 0 {
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
@@ -47,6 +50,8 @@ func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.Resolv
out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize)
out.ExtractReferences = pipeline.CloneReferenceTarget(in.ExtractReferences)
out.NormalizeReferences = pipeline.CloneReferenceTarget(in.NormalizeReferences)
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators {

View File

@@ -3,6 +3,7 @@ package config
import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -65,12 +66,47 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile
lane := cfg.Pipelines["example"].Artifacts["events"]
lane.Extract.Options = map[string]any{"temperature": 0.2}
lane.References = map[string]string{"roster": "./roster.yml"}
lane.Extract.References = map[string]string{"glossary": "./glossary.md"}
lane.Normalize.References = map[string]string{"notes": "./normalize.md"}
cfg.Pipelines["example"].Artifacts["events"] = lane
pipelineProfile := cfg.Pipelines["example"]
pipelineProfile.Chunk.References = map[string]string{"scene_guide": "./scene.md"}
cfg.Pipelines["example"] = pipelineProfile
effective, err := cfg.Resolve(ResolveInput{
PipelineID: "example",
Only: []string{"events"},
Catalog: fakeCatalog(t),
Catalog: fakeCatalog(t,
pipeline.ModuleSpec{
Key: "generic",
Stage: pipeline.StageChunk,
Requires: []string{"source"},
Provides: []string{"chunks"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "scene_guide"},
},
},
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "glossary"},
{Name: "roster"},
},
},
pipeline.ModuleSpec{
Key: "noop",
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes"},
},
},
),
})
if err != nil {
t.Fatalf("Resolve: %v", err)
@@ -98,4 +134,52 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 {
t.Fatalf("expected resolved pipeline options to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml"
if referenceBindingSource(effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings, "roster") != "./roster.yml" {
t.Fatalf("expected resolved pipeline references to be copied")
}
payload.ResolvedPipeline.Chunk.References["scene_guide"] = "./changed-scene.md"
if effective.ResolvedPipeline.Chunk.References["scene_guide"] != "./scene.md" {
t.Fatalf("expected chunk references to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] = "./changed-glossary.md"
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] != "./glossary.md" {
t.Fatalf("expected extract references to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] = "./changed-normalize.md"
if effective.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] != "./normalize.md" {
t.Fatalf("expected normalize references to be copied")
}
effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
Content: []byte("reference content"),
},
},
},
},
}
payload, ok = effective.RedactedDiagnosticsPayload().(EffectiveConfig)
if !ok {
t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload())
}
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content[0] = 'X'
got := effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content
if string(got) != "reference content" {
t.Fatalf("expected materialized reference content to be copied, got %q", got)
}
}
func referenceBindingSource(bindings []pipeline.ReferenceBinding, slotName string) string {
for _, binding := range bindings {
if binding.SlotName == slotName {
return binding.Source
}
}
return ""
}

View File

@@ -89,13 +89,16 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
}
if err := validateBindingLLMProfile(id, "", "input", profile.Input, llmProfiles); err != nil {
if err := validateBinding(id, "", "input", profile.Input, llmProfiles, false); err != nil {
return err
}
if err := validateBindingLLMProfile(id, "", "chunk", profile.Chunk, llmProfiles); err != nil {
if err := validateBinding(id, "", "chunk", profile.Chunk, llmProfiles, true); err != nil {
return err
}
if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil {
if err := validateBinding(id, "", "output", profile.Output, llmProfiles, false); err != nil {
return err
}
if err := validateReferenceMap(id, "", profile.References); err != nil {
return err
}
for rawLaneID, lane := range profile.Artifacts {
@@ -103,17 +106,20 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if laneID == "" {
return fmt.Errorf("pipeline %q artifact lane id must not be empty", id)
}
if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil {
if err := validateReferenceMap(id, laneID, lane.References); err != nil {
return err
}
if err := validateBindingLLMProfile(id, laneID, "merge", lane.Merge, llmProfiles); err != nil {
if err := validateBinding(id, laneID, "extract", lane.Extract, llmProfiles, true); err != nil {
return err
}
if err := validateBindingLLMProfile(id, laneID, "normalize", lane.Normalize, llmProfiles); err != nil {
if err := validateBinding(id, laneID, "merge", lane.Merge, llmProfiles, false); err != nil {
return err
}
if err := validateBinding(id, laneID, "normalize", lane.Normalize, llmProfiles, true); err != nil {
return err
}
for i, validator := range lane.Validators {
if err := validateBindingLLMProfile(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles); err != nil {
if err := validateBinding(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles, false); err != nil {
return err
}
}
@@ -122,6 +128,64 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
return nil
}
func validateBinding(
pipelineID string,
laneID string,
slot string,
binding pipeline.ModuleBinding,
profiles map[string]LLMProfile,
referencesAllowed bool,
) error {
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding, profiles); err != nil {
return err
}
if len(binding.References) == 0 {
return nil
}
if !referencesAllowed {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q %s references are not supported", pipelineID, laneID, slot)
}
return fmt.Errorf("pipeline %q %s references are not supported", pipelineID, slot)
}
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
}
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
return validateReferenceMapForContext(pipelineID, laneID, "", references)
}
func validateReferenceMapForContext(pipelineID string, laneID string, slot string, references map[string]string) error {
seen := make(map[string]struct{}, len(references))
for rawSlotName, rawSource := range references {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
return fmt.Errorf("%s reference slot name must not be empty", referenceContext(pipelineID, laneID, slot))
}
if _, ok := seen[slotName]; ok {
return fmt.Errorf("%s reference slot %q is duplicated after trimming", referenceContext(pipelineID, laneID, slot), slotName)
}
seen[slotName] = struct{}{}
if strings.TrimSpace(rawSource) == "" {
return fmt.Errorf("%s reference slot %q source must not be empty", referenceContext(pipelineID, laneID, slot), slotName)
}
}
return nil
}
func referenceContext(pipelineID string, laneID string, slot string) string {
if laneID != "" && slot != "" {
return fmt.Sprintf("pipeline %q lane %q %s", pipelineID, laneID, slot)
}
if laneID != "" {
return fmt.Sprintf("pipeline %q lane %q", pipelineID, laneID)
}
if slot != "" {
return fmt.Sprintf("pipeline %q %s", pipelineID, slot)
}
return fmt.Sprintf("pipeline %q", pipelineID)
}
func validateBindingLLMProfile(
pipelineID string,
laneID string,

View File

@@ -116,6 +116,211 @@ func TestValidateRejectsInvalidDiagnosticsRetention(t *testing.T) {
}
}
func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
tests := []struct {
name string
mutate func(Config) Config
want []string
}{
{
name: "empty chunk slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Chunk.References = map[string]string{" ": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "chunk", "reference slot", "empty"},
},
{
name: "empty chunk source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Chunk.References = map[string]string{"roster": " "}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "chunk", "roster", "source", "empty"},
},
{
name: "empty extract slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "extract", "reference slot", "empty"},
},
{
name: "empty extract source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "extract", "roster", "source", "empty"},
},
{
name: "empty normalize slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "normalize", "reference slot", "empty"},
},
{
name: "empty normalize source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "normalize", "roster", "source", "empty"},
},
{
name: "empty pipeline slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.References = map[string]string{" ": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "reference slot", "empty"},
},
{
name: "empty pipeline source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.References = map[string]string{"roster": " "}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "roster", "source", "empty"},
},
{
name: "empty lane slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "reference slot", "empty"},
},
{
name: "empty lane source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "roster", "source", "empty"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := tc.mutate(validConfig()).Validate()
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
})
}
}
func TestValidateRejectsReferencesOnUnsupportedBindings(t *testing.T) {
tests := []struct {
name string
mutate func(Config) Config
want []string
}{
{
name: "input",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Input.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "input", "references", "not supported"},
},
{
name: "merge",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Merge.References = map[string]string{"roster": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "merge", "references", "not supported"},
},
{
name: "validator",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Validators = []pipeline.ModuleBinding{{
Module: "fake/validator",
References: map[string]string{"roster": "./roster.yml"},
}}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "validator[0]", "references", "not supported"},
},
{
name: "output",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Output.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "output", "references", "not supported"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := tc.mutate(validConfig()).Validate()
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
})
}
}
func TestValidateRejectsEmptyIDs(t *testing.T) {
tests := []struct {
name string

View File

@@ -4,6 +4,7 @@ const (
ArtifactInvocationMetadata = "invocation.json"
ArtifactEffectiveConfig = "effective-config.json"
ArtifactResolvedPipeline = "resolved-pipeline.json"
ArtifactResolvedReferences = "resolved-references.json"
ArtifactSourceDocument = "source-document.json"
ArtifactRunManifest = "run-manifest.json"
ArtifactRunReport = "run-report.json"

View File

@@ -7,6 +7,7 @@ func TestArtifactNamesUseExtractionOrientedNames(t *testing.T) {
ArtifactInvocationMetadata,
ArtifactEffectiveConfig,
ArtifactResolvedPipeline,
ArtifactResolvedReferences,
ArtifactSourceDocument,
ArtifactRunManifest,
ArtifactRunReport,

View File

@@ -149,6 +149,10 @@ func (r *RunDirectory) WriteResolvedPipeline(payload any) error {
return r.WriteJSONArtifact(ArtifactResolvedPipeline, payload)
}
func (r *RunDirectory) WriteResolvedReferences(payload any) error {
return r.WriteJSONArtifact(ArtifactResolvedReferences, payload)
}
func (r *RunDirectory) WriteSourceDocument(payload any) error {
return r.WriteJSONArtifact(ArtifactSourceDocument, payload)
}

View File

@@ -191,6 +191,9 @@ func TestWriteTypedArtifacts(t *testing.T) {
if err := runDir.WriteResolvedPipeline(map[string]any{"pipeline": "test"}); err != nil {
t.Fatalf("WriteResolvedPipeline: %v", err)
}
if err := runDir.WriteResolvedReferences([]artifacts.ReferenceProvenance{{LaneID: "events", SlotName: "roster"}}); err != nil {
t.Fatalf("WriteResolvedReferences: %v", err)
}
if err := runDir.WriteSourceDocument(map[string]any{"source_id": "source-1"}); err != nil {
t.Fatalf("WriteSourceDocument: %v", err)
}
@@ -207,6 +210,7 @@ func TestWriteTypedArtifacts(t *testing.T) {
for _, name := range []string{
ArtifactEffectiveConfig,
ArtifactResolvedPipeline,
ArtifactResolvedReferences,
ArtifactSourceDocument,
ArtifactRunManifest,
ArtifactRunReport,

View File

@@ -17,6 +17,7 @@ var _ contracts.Extractor = compositionExtractor{}
var _ contracts.Merger = compositionMerger{}
var _ contracts.Normalizer = compositionNormalizer{}
var _ contracts.Validator = compositionValidator{}
var _ contracts.StructuredLLMClient = compositionLLMClient{}
var _ contracts.OutputEncoder = compositionOutputEncoder{}
func TestContractsComposeAcrossPackages(t *testing.T) {
@@ -38,8 +39,9 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
}
chunking, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc,
Metadata: map[string]any{"max_units": 2},
Source: doc,
LLMClient: compositionLLMClient{},
Metadata: map[string]any{"max_units": 2},
})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
@@ -160,10 +162,17 @@ func (chunker compositionChunker) Key() string {
return "generic-chunker"
}
func (chunker compositionChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if req.Source == nil {
return contracts.ChunkResult{}, errors.New("source document is required")
}
if req.LLMClient == nil {
return contracts.ChunkResult{}, errors.New("structured llm client is required")
}
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{
@@ -178,6 +187,12 @@ func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.Chunk
}, nil
}
type compositionLLMClient struct{}
func (client compositionLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
return contracts.StructuredCompletionResponse{}, nil
}
type compositionExtractor struct{}
func (extractor compositionExtractor) Key() string {
@@ -192,6 +207,10 @@ func (extractor compositionExtractor) SchemaVersion() string {
return "v1"
}
func (extractor compositionExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor compositionExtractor) Validators() []contracts.Validator {
return []contracts.Validator{compositionValidator{}}
}
@@ -249,6 +268,10 @@ func (normalizer compositionNormalizer) Key() string {
return "generic-normalizer"
}
func (normalizer compositionNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
}

View File

@@ -58,6 +58,8 @@ type SourceChunk struct {
type ChunkRequest struct {
Source *source.SourceDocument `json:"-"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
@@ -70,13 +72,53 @@ type ChunkResult struct {
type Chunker interface {
Key() string
ReferenceSlots() []ReferenceSlot
Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
}
const (
ReferenceBindingSourceConfig = "config"
ReferenceBindingSourceCLI = "cli"
)
type ReferenceSlot struct {
Name string `json:"name"`
Description string `json:"description,omitempty"`
Required bool `json:"required,omitempty"`
AcceptedMediaTypes []string `json:"accepted_media_types,omitempty"`
Multiple bool `json:"multiple,omitempty"`
MaxBytes int64 `json:"max_bytes,omitempty"`
}
type ReferenceOrigin struct {
Type string `json:"type"`
URI string `json:"uri,omitempty"`
}
type ReferenceItem struct {
SlotName string `json:"slot_name"`
MediaType string `json:"media_type,omitempty"`
Content []byte `json:"-"`
Digest string `json:"digest,omitempty"`
Origin ReferenceOrigin `json:"origin"`
SizeBytes int64 `json:"size_bytes,omitempty"`
BindingSource string `json:"binding_source,omitempty"`
}
type ResolvedReferenceSlot struct {
Slot ReferenceSlot `json:"slot"`
Items []ReferenceItem `json:"items,omitempty"`
}
type ReferenceSet struct {
Slots map[string]ResolvedReferenceSlot `json:"slots,omitempty"`
}
type ExtractionRequest struct {
Source *source.SourceDocument `json:"-"`
Chunk *SourceChunk `json:"chunk,omitempty"`
AmbientContext map[string]any `json:"ambient_context,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
@@ -92,6 +134,7 @@ type Extractor interface {
Key() string
ArtifactType() string
SchemaVersion() string
ReferenceSlots() []ReferenceSlot
Validators() []Validator
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
}
@@ -124,6 +167,8 @@ type NormalizeRequest struct {
Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
@@ -136,6 +181,7 @@ type NormalizeResult struct {
type Normalizer interface {
Key() string
ReferenceSlots() []ReferenceSlot
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
}

View File

@@ -117,6 +117,27 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
}
}
func TestFakeChunkerReceivesLLMClient(t *testing.T) {
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "Source text."},
},
}
client := fakeLLMClient{}
chunker := &recordingChunker{key: "llm-chunker"}
if _, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc, LLMClient: client}); err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
if chunker.request.LLMClient == nil {
t.Fatal("ChunkRequest.LLMClient = nil, want structured LLM client")
}
}
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
extractor := fakeExtractor{
key: "generic-extractor",
@@ -165,6 +186,71 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
}
}
func TestReferenceSetDataTypes(t *testing.T) {
references := ReferenceSet{
Slots: map[string]ResolvedReferenceSlot{
"roster": {
Slot: ReferenceSlot{
Name: "roster",
Description: "Known characters",
Required: true,
AcceptedMediaTypes: []string{"text/plain"},
Multiple: true,
MaxBytes: 4096,
},
Items: []ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain",
Content: []byte("Aria\nBryn\n"),
Digest: "sha256:reference",
Origin: ReferenceOrigin{
Type: "file",
URI: "file:///tmp/roster.txt",
},
SizeBytes: 10,
BindingSource: ReferenceBindingSourceConfig,
},
},
},
},
}
item := references.Slots["roster"].Items[0]
if item.SlotName != "roster" || item.MediaType != "text/plain" || string(item.Content) != "Aria\nBryn\n" {
t.Fatalf("reference item = %#v, want constructed item fields", item)
}
if item.BindingSource != ReferenceBindingSourceConfig {
t.Fatalf("BindingSource = %q, want %q", item.BindingSource, ReferenceBindingSourceConfig)
}
}
func TestReferenceItemJSONOmitsContent(t *testing.T) {
item := ReferenceItem{
SlotName: "roster",
MediaType: "text/plain",
Content: []byte("reference content"),
Digest: "sha256:reference",
Origin: ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
}
encoded, err := json.Marshal(item)
if err != nil {
t.Fatalf("json.Marshal() error = %v, want nil", err)
}
var got map[string]any
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
}
if _, ok := got["content"]; ok {
t.Fatalf("encoded reference item leaked content: %s", encoded)
}
if _, ok := got["Content"]; ok {
t.Fatalf("encoded reference item leaked Content: %s", encoded)
}
}
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
candidate := artifacts.ArtifactCandidate{
Index: 0,
@@ -292,6 +378,10 @@ func (chunker fakeChunker) Key() string {
return chunker.key
}
func (chunker fakeChunker) ReferenceSlots() []ReferenceSlot {
return nil
}
func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
return ChunkResult{
Chunks: []SourceChunk{
@@ -305,6 +395,24 @@ func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkRe
}, nil
}
type recordingChunker struct {
key string
request ChunkRequest
}
func (chunker *recordingChunker) Key() string {
return chunker.key
}
func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
return nil
}
func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
chunker.request = req
return fakeChunker{key: chunker.key}.Chunk(ctx, req)
}
type fakeExtractor struct {
key string
artifactType string
@@ -324,6 +432,10 @@ func (extractor fakeExtractor) SchemaVersion() string {
return extractor.schemaVersion
}
func (extractor fakeExtractor) ReferenceSlots() []ReferenceSlot {
return nil
}
func (extractor fakeExtractor) Validators() []Validator {
return extractor.validators
}
@@ -383,6 +495,10 @@ func (normalizer fakeNormalizer) Key() string {
return normalizer.key
}
func (normalizer fakeNormalizer) ReferenceSlots() []ReferenceSlot {
return nil
}
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) {
return NormalizeResult{Candidates: req.Candidates}, nil
}

View File

@@ -0,0 +1,82 @@
package pipeline
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
sourceUnitIndexes := make(map[string]int, len(doc.Units))
sourceUnits := make(map[string]source.SourceUnit, len(doc.Units))
for index, unit := range doc.Units {
sourceUnitIndexes[unit.ID] = index
sourceUnits[unit.ID] = unit
}
canonicalChunks := make([]contracts.SourceChunk, 0, len(chunks))
seenChunkIDs := make(map[string]struct{}, len(chunks))
for chunkIndex, chunk := range chunks {
if strings.TrimSpace(chunk.ID) == "" {
return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
}
if _, ok := seenChunkIDs[chunk.ID]; ok {
return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
}
seenChunkIDs[chunk.ID] = struct{}{}
if chunk.SourceID != doc.ID {
return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
}
if chunk.Index != chunkIndex {
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
}
if len(chunk.Units) == 0 {
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
}
seenUnitIDs := make(map[string]struct{}, len(chunk.Units))
previousSourceIndex := -1
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
for unitIndex, unit := range chunk.Units {
if strings.TrimSpace(unit.ID) == "" {
return nil, fmt.Errorf("chunk %q unit[%d].id must not be empty", chunk.ID, unitIndex)
}
if _, ok := seenUnitIDs[unit.ID]; ok {
return nil, fmt.Errorf("chunk %q repeats source unit %q", chunk.ID, unit.ID)
}
seenUnitIDs[unit.ID] = struct{}{}
sourceIndex, ok := sourceUnitIndexes[unit.ID]
if !ok {
return nil, fmt.Errorf("chunk %q source unit %q was not found in source document %q", chunk.ID, unit.ID, doc.ID)
}
if sourceIndex <= previousSourceIndex {
return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
}
previousSourceIndex = sourceIndex
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
}
canonicalChunks = append(canonicalChunks, contracts.SourceChunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Units: canonicalUnits,
Metadata: cloneMetadata(chunk.Metadata),
})
}
return canonicalChunks, nil
}
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
return source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Metadata: cloneMetadata(unit.Metadata),
}
}

View File

@@ -330,6 +330,10 @@ func (chunker registryChunker) Key() string {
return chunker.key
}
func (chunker registryChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{}, nil
}
@@ -354,6 +358,10 @@ func (normalizer registryNormalizer) Key() string {
return normalizer.key
}
func (normalizer registryNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{}, nil
}

View File

@@ -117,6 +117,8 @@ func (defaultExtractor) ArtifactType() string { return "record" }
func (defaultExtractor) SchemaVersion() string { return "v1" }
func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (defaultExtractor) Validators() []contracts.Validator { return nil }
func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {

View File

@@ -49,6 +49,20 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
Stage: StageExtract,
Provides: []string{" generic-artifact ", "source-citations", "generic-artifact", ""},
Requires: []string{" source-document ", "source-document", ""},
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: " glossary ",
Description: " Supporting terms ",
AcceptedMediaTypes: []string{" text/plain ", "text/markdown", "text/plain", ""},
MaxBytes: 1024,
},
{
Name: " roster ",
Description: " Characters ",
Required: true,
Multiple: true,
},
},
}
if err := registry.RegisterWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil {
@@ -64,12 +78,28 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
Stage: StageExtract,
Provides: []string{"generic-artifact", "source-citations"},
Requires: []string{"source-document"},
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Supporting terms",
AcceptedMediaTypes: []string{"text/markdown", "text/plain"},
MaxBytes: 1024,
},
{
Name: "roster",
Description: "Characters",
Required: true,
Multiple: true,
},
},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want)
}
got.Provides[0] = "changed"
got.ReferenceSlots[0].Name = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
again, ok := registry.Spec("generic-extractor")
if !ok {
t.Fatal("Spec() after caller mutation ok = false, want true")
@@ -109,6 +139,50 @@ func TestExtractorRegistryRegisterWithSpecRejectsWrongStage(t *testing.T) {
}
}
func TestExtractorRegistryRejectsInvalidReferenceSlots(t *testing.T) {
tests := []struct {
name string
slots []contracts.ReferenceSlot
want string
}{
{
name: "empty name",
slots: []contracts.ReferenceSlot{{Name: " "}},
want: "name",
},
{
name: "duplicate name after trim",
slots: []contracts.ReferenceSlot{
{Name: "roster"},
{Name: " roster "},
},
want: "duplicated",
},
{
name: "negative max bytes",
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
want: "max_bytes",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterWithSpec(ModuleSpec{
Key: "generic-extractor",
Stage: StageExtract,
ReferenceSlots: test.slots,
}, fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("RegisterWithSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), test.want) {
t.Fatalf("RegisterWithSpec() error = %q, want %q", err.Error(), test.want)
}
})
}
}
func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) {
registry := NewExtractorRegistry()
@@ -301,6 +375,10 @@ func (extractor registryFakeExtractor) SchemaVersion() string {
return "v1"
}
func (extractor registryFakeExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor registryFakeExtractor) Validators() []contracts.Validator {
return nil
}

View File

@@ -4,6 +4,8 @@ import (
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type ModuleStage string
@@ -19,10 +21,11 @@ const (
)
type ModuleSpec struct {
Key string
Stage ModuleStage
Provides []string
Requires []string
Key string
Stage ModuleStage
Provides []string
Requires []string
ReferenceSlots []contracts.ReferenceSlot
}
func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec {
@@ -34,10 +37,11 @@ func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec {
func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
return ModuleSpec{
Key: strings.TrimSpace(spec.Key),
Stage: spec.Stage,
Provides: normalizeCapabilities(spec.Provides),
Requires: normalizeCapabilities(spec.Requires),
Key: strings.TrimSpace(spec.Key),
Stage: spec.Stage,
Provides: normalizeCapabilities(spec.Provides),
Requires: normalizeCapabilities(spec.Requires),
ReferenceSlots: normalizeReferenceSlots(spec.ReferenceSlots),
}
}
@@ -68,10 +72,11 @@ func normalizeCapabilities(values []string) []string {
func cloneModuleSpec(spec ModuleSpec) ModuleSpec {
return ModuleSpec{
Key: spec.Key,
Stage: spec.Stage,
Provides: append([]string(nil), spec.Provides...),
Requires: append([]string(nil), spec.Requires...),
Key: spec.Key,
Stage: spec.Stage,
Provides: append([]string(nil), spec.Provides...),
Requires: append([]string(nil), spec.Requires...),
ReferenceSlots: cloneReferenceSlots(spec.ReferenceSlots),
}
}
@@ -82,9 +87,19 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
if spec.Stage != expectedStage {
return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage)
}
if !referenceSlotStage(spec.Stage) && len(spec.ReferenceSlots) > 0 {
return fmt.Errorf("%s %q must not declare reference slots", kind, spec.Key)
}
if err := validateReferenceSlots(spec.ReferenceSlots); err != nil {
return fmt.Errorf("%s %q reference slots: %w", kind, spec.Key, err)
}
return nil
}
func referenceSlotStage(stage ModuleStage) bool {
return stage == StageChunk || stage == StageExtract || stage == StageNormalize
}
func sortedRegistryKeys[C any](constructors map[string]C) []string {
if len(constructors) == 0 {
return nil
@@ -97,3 +112,74 @@ func sortedRegistryKeys[C any](constructors map[string]C) []string {
sort.Strings(keys)
return keys
}
func normalizeReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
normalized := make([]contracts.ReferenceSlot, 0, len(slots))
for _, slot := range slots {
slot.Name = strings.TrimSpace(slot.Name)
slot.Description = strings.TrimSpace(slot.Description)
slot.AcceptedMediaTypes = normalizeStringSet(slot.AcceptedMediaTypes)
normalized = append(normalized, slot)
}
sort.SliceStable(normalized, func(i, j int) bool {
return normalized[i].Name < normalized[j].Name
})
return normalized
}
func normalizeStringSet(values []string) []string {
if len(values) == 0 {
return nil
}
seen := make(map[string]struct{}, len(values))
for _, value := range values {
normalized := strings.TrimSpace(value)
if normalized == "" {
continue
}
seen[normalized] = struct{}{}
}
if len(seen) == 0 {
return nil
}
out := make([]string, 0, len(seen))
for value := range seen {
out = append(out, value)
}
sort.Strings(out)
return out
}
func validateReferenceSlots(slots []contracts.ReferenceSlot) error {
seen := make(map[string]struct{}, len(slots))
for i, slot := range slots {
if slot.Name == "" {
return fmt.Errorf("slot[%d].name must not be empty", i)
}
if _, ok := seen[slot.Name]; ok {
return fmt.Errorf("slot name %q is duplicated", slot.Name)
}
seen[slot.Name] = struct{}{}
if slot.MaxBytes < 0 {
return fmt.Errorf("slot %q max_bytes must not be negative", slot.Name)
}
}
return nil
}
func cloneReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
out := make([]contracts.ReferenceSlot, 0, len(slots))
for _, slot := range slots {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
out = append(out, slot)
}
return out
}

View File

@@ -0,0 +1,123 @@
package pipeline
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestValidateModuleSpecAllowsReferenceSlotsForEligibleStages(t *testing.T) {
tests := []struct {
name string
kind string
stage ModuleStage
}{
{name: "chunker", kind: "chunker", stage: StageChunk},
{name: "extractor", kind: "extractor", stage: StageExtract},
{name: "normalizer", kind: "normalizer", stage: StageNormalize},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
spec := normalizeModuleSpec(ModuleSpec{
Key: "module",
Stage: test.stage,
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Description: "Character roster", MaxBytes: 1024},
},
})
err := validateModuleSpec(test.kind, test.stage, spec)
if err != nil {
t.Fatalf("validateModuleSpec() error = %v, want nil", err)
}
})
}
}
func TestValidateModuleSpecRejectsReferenceSlotsForIneligibleStages(t *testing.T) {
tests := []struct {
name string
kind string
stage ModuleStage
}{
{name: "input", kind: "input adapter", stage: StageInput},
{name: "merge", kind: "merger", stage: StageMerge},
{name: "validate", kind: "validator", stage: StageValidate},
{name: "output", kind: "output encoder", stage: StageOutput},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
spec := normalizeModuleSpec(ModuleSpec{
Key: "module",
Stage: test.stage,
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
})
err := validateModuleSpec(test.kind, test.stage, spec)
if err == nil {
t.Fatal("validateModuleSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), "reference slots") {
t.Fatalf("validateModuleSpec() error = %q, want reference slots context", err.Error())
}
})
}
}
func TestValidateModuleSpecRejectsInvalidReferenceSlotsForEligibleStages(t *testing.T) {
invalidSlots := []struct {
name string
slots []contracts.ReferenceSlot
want string
}{
{
name: "empty name",
slots: []contracts.ReferenceSlot{{Name: " "}},
want: "name",
},
{
name: "duplicate name after trim",
slots: []contracts.ReferenceSlot{
{Name: "roster"},
{Name: " roster "},
},
want: "duplicated",
},
{
name: "negative max bytes",
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
want: "max_bytes",
},
}
eligibleStages := []struct {
name string
kind string
stage ModuleStage
}{
{name: "chunk", kind: "chunker", stage: StageChunk},
{name: "extract", kind: "extractor", stage: StageExtract},
{name: "normalize", kind: "normalizer", stage: StageNormalize},
}
for _, stage := range eligibleStages {
for _, invalid := range invalidSlots {
t.Run(stage.name+"/"+invalid.name, func(t *testing.T) {
spec := normalizeModuleSpec(ModuleSpec{
Key: "module",
Stage: stage.stage,
ReferenceSlots: invalid.slots,
})
err := validateModuleSpec(stage.kind, stage.stage, spec)
if err == nil {
t.Fatal("validateModuleSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), invalid.want) {
t.Fatalf("validateModuleSpec() error = %q, want %q", err.Error(), invalid.want)
}
})
}
}
}

View File

@@ -7,6 +7,8 @@ import (
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
const (
@@ -18,45 +20,75 @@ const (
)
type ModuleBinding struct {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"`
}
type ArtifactLaneProfile struct {
Extract ModuleBinding `json:"extract"`
Merge ModuleBinding `json:"merge,omitempty"`
Normalize ModuleBinding `json:"normalize,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"`
Extract ModuleBinding `json:"extract"`
Merge ModuleBinding `json:"merge,omitempty"`
Normalize ModuleBinding `json:"normalize,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"`
References map[string]string `json:"references,omitempty"`
}
type PipelineProfile struct {
ID string `json:"id"`
Input ModuleBinding `json:"input"`
Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
Output ModuleBinding `json:"output,omitempty"`
ID string `json:"id"`
Input ModuleBinding `json:"input"`
Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
Output ModuleBinding `json:"output,omitempty"`
References map[string]string `json:"references,omitempty"`
}
type ResolveOptions struct {
Only []string
Only []string
ReferenceOverrides []ReferenceBinding
ReferenceUnbinds []ReferenceUnbind
}
type ReferenceBinding struct {
Stage ModuleStage `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
Source string `json:"source"`
BindingSource string `json:"binding_source,omitempty"`
}
type ReferenceUnbind struct {
Stage ModuleStage `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
}
type ResolvedReferenceTarget struct {
Stage ModuleStage `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
Module string `json:"module"`
Bindings []ReferenceBinding `json:"bindings,omitempty"`
ReferenceSet contracts.ReferenceSet `json:"-"`
}
type ResolvedArtifactLane struct {
ID string
Extract ModuleBinding
Merge ModuleBinding
Normalize ModuleBinding
Validators []ModuleBinding
ID string
Extract ModuleBinding
Merge ModuleBinding
Normalize ModuleBinding
Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
}
type ResolvedPipeline struct {
ID string
Digest string
Input ModuleBinding
Chunk ModuleBinding
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
ID string
Digest string
Input ModuleBinding
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
}
type ModuleCatalog struct {
@@ -115,18 +147,34 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
if len(selectedLaneIDs) == 0 {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q must select at least one artifact lane", pipelineID)
}
if err := validatePipelineReferenceDefaults(pipelineID, profile.References, chunkSpec, lanesByID, catalog); err != nil {
return ResolvedPipeline{}, err
}
chunkReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
Stage: StageChunk,
Module: chunk.Module,
Slots: chunkSpec.ReferenceSlots,
PipelineReferences: profile.References,
LocalReferences: chunk.References,
Options: options,
})
if err != nil {
return ResolvedPipeline{}, err
}
resolved := ResolvedPipeline{
ID: pipelineID,
Input: input,
Chunk: chunk,
Output: resolveBinding(profile.Output, DefaultOutputModule),
ID: pipelineID,
Input: input,
Chunk: chunk,
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: resolveBinding(profile.Output, DefaultOutputModule),
}
outputCapabilities := capabilities.clone()
for _, laneID := range selectedLaneIDs {
laneProfile := lanesByID[laneID]
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, capabilities, catalog)
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
if err != nil {
return ResolvedPipeline{}, err
}
@@ -150,7 +198,15 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
return resolved, nil
}
func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile, inherited capabilitySet, catalog ModuleCatalog) (ResolvedArtifactLane, capabilitySet, error) {
func resolveArtifactLane(
pipelineID string,
laneID string,
profile ArtifactLaneProfile,
pipelineReferences map[string]string,
options ResolveOptions,
inherited capabilitySet,
catalog ModuleCatalog,
) (ResolvedArtifactLane, capabilitySet, error) {
lane := ResolvedArtifactLane{
ID: laneID,
Extract: resolveBinding(profile.Extract, ""),
@@ -171,6 +227,21 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
}
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
LaneID: laneID,
Stage: StageExtract,
Module: lane.Extract.Module,
Slots: extractSpec.ReferenceSlots,
PipelineReferences: pipelineReferences,
LocalReferences: extractReferences,
Options: options,
})
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
@@ -189,6 +260,20 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
}
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
LaneID: laneID,
Stage: StageNormalize,
Module: lane.Normalize.Module,
Slots: normalizeSpec.ReferenceSlots,
PipelineReferences: pipelineReferences,
LocalReferences: lane.Normalize.References,
Options: options,
})
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
capabilities.add(normalizeSpec.Provides...)
for _, validator := range lane.Validators {
@@ -205,6 +290,307 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
return lane, capabilities, nil
}
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
return ResolvedReferenceTarget{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
Module: strings.TrimSpace(module),
Bindings: append([]ReferenceBinding(nil), bindings...),
}
}
func mergeReferenceMaps(base map[string]string, override map[string]string) map[string]string {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
for key, value := range base {
out[key] = value
}
for key, value := range override {
out[key] = value
}
return out
}
func validatePipelineReferenceDefaults(
pipelineID string,
pipelineReferences map[string]string,
chunkSpec ModuleSpec,
lanesByID map[string]ArtifactLaneProfile,
catalog ModuleCatalog,
) error {
normalizedPipelineReferences, err := normalizedReferenceMap(pipelineReferences, fmt.Sprintf("pipeline %q reference slot", pipelineID))
if err != nil {
return err
}
if len(normalizedPipelineReferences) == 0 {
return nil
}
declaredByAnyTarget := make(map[string]struct{}, len(normalizedPipelineReferences))
for _, slot := range chunkSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
}
for _, laneID := range sortedArtifactLaneProfileKeys(lanesByID) {
laneProfile := lanesByID[laneID]
extract := resolveBinding(laneProfile.Extract, "")
if extract.Module == "" {
return fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
}
extractSpec, err := extractorSpec(catalog, extract.Module)
if err != nil {
return moduleLookupError(pipelineID, laneID, StageExtract, extract.Module, err)
}
for _, slot := range extractSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
}
normalize := resolveBinding(laneProfile.Normalize, DefaultNormalizeModule)
normalizeSpec, err := normalizerSpec(catalog, normalize.Module)
if err != nil {
return moduleLookupError(pipelineID, laneID, StageNormalize, normalize.Module, err)
}
for _, slot := range normalizeSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
}
}
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
if _, ok := declaredByAnyTarget[slotName]; !ok {
return fmt.Errorf("pipeline %q reference slot %q is not declared by any eligible reference target", pipelineID, slotName)
}
}
return nil
}
type referenceResolutionTarget struct {
PipelineID string
LaneID string
Stage ModuleStage
Module string
Slots []contracts.ReferenceSlot
PipelineReferences map[string]string
LocalReferences map[string]string
Options ResolveOptions
}
func resolveReferenceTargetBindings(target referenceResolutionTarget) ([]ReferenceBinding, error) {
slotByName := make(map[string]contracts.ReferenceSlot, len(target.Slots))
for _, slot := range target.Slots {
slotByName[slot.Name] = slot
}
bindings := make(map[string]ReferenceBinding)
addBinding := func(slotName, source, bindingSource string) error {
slotName = strings.TrimSpace(slotName)
source = strings.TrimSpace(source)
if slotName == "" {
return fmt.Errorf("%s reference slot name must not be empty", referenceTargetErrorContext(target))
}
if source == "" {
return fmt.Errorf("%s reference slot %q source must not be empty", referenceTargetErrorContext(target), slotName)
}
if _, ok := slotByName[slotName]; !ok {
return fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetErrorContext(target), slotName, target.Stage, target.Module)
}
bindings[slotName] = ReferenceBinding{
LaneID: target.LaneID,
SlotName: slotName,
Source: source,
BindingSource: bindingSource,
}
return nil
}
normalizedPipelineReferences, err := normalizedReferenceMap(target.PipelineReferences, fmt.Sprintf("pipeline %q reference slot", target.PipelineID))
if err != nil {
return nil, err
}
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
if _, ok := slotByName[slotName]; !ok {
continue
}
if err := addBinding(slotName, normalizedPipelineReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
return nil, err
}
}
normalizedLocalReferences, err := normalizedReferenceMap(target.LocalReferences, referenceTargetSlotLabel(target))
if err != nil {
return nil, err
}
for _, slotName := range sortedStringMapKeys(normalizedLocalReferences) {
if err := addBinding(slotName, normalizedLocalReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
return nil, err
}
}
for _, override := range target.Options.ReferenceOverrides {
match, err := referenceOverrideMatchesTarget(target, override)
if err != nil {
return nil, err
}
if !match {
continue
}
source := override.BindingSource
if strings.TrimSpace(source) == "" {
source = contracts.ReferenceBindingSourceCLI
}
if err := addBinding(override.SlotName, override.Source, strings.TrimSpace(source)); err != nil {
return nil, err
}
}
for _, unbind := range target.Options.ReferenceUnbinds {
match, err := referenceUnbindMatchesTarget(target, unbind)
if err != nil {
return nil, err
}
if !match {
continue
}
slotName := strings.TrimSpace(unbind.SlotName)
if slotName == "" {
return nil, fmt.Errorf("%s reference unbind slot name must not be empty", referenceTargetErrorContext(target))
}
if _, ok := slotByName[slotName]; !ok {
return nil, fmt.Errorf("%s reference slot %q is not declared", referenceTargetErrorContext(target), slotName)
}
delete(bindings, slotName)
}
for _, slot := range target.Slots {
if slot.Required {
if _, ok := bindings[slot.Name]; !ok {
return nil, fmt.Errorf("%s required reference slot %q is not bound", referenceTargetErrorContext(target), slot.Name)
}
}
}
return sortedReferenceBindings(bindings), nil
}
func referenceOverrideMatchesTarget(target referenceResolutionTarget, override ReferenceBinding) (bool, error) {
stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "override", override.Stage, override.LaneID)
if err != nil {
return false, err
}
return stage == target.Stage && laneID == target.LaneID, nil
}
func referenceUnbindMatchesTarget(target referenceResolutionTarget, unbind ReferenceUnbind) (bool, error) {
stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "unbind", unbind.Stage, unbind.LaneID)
if err != nil {
return false, err
}
return stage == target.Stage && laneID == target.LaneID, nil
}
func normalizeReferenceOptionTarget(pipelineID string, operation string, stage ModuleStage, laneID string) (ModuleStage, string, error) {
stage = ModuleStage(strings.TrimSpace(string(stage)))
if stage == "" {
stage = StageExtract
}
laneID = strings.TrimSpace(laneID)
switch stage {
case StageChunk:
if laneID != "" {
return "", "", fmt.Errorf("pipeline %q reference %s for chunk must not include a lane id", pipelineID, operation)
}
case StageExtract, StageNormalize:
if laneID == "" {
return "", "", fmt.Errorf("pipeline %q reference %s lane id must not be empty", pipelineID, operation)
}
default:
return "", "", fmt.Errorf("pipeline %q reference %s stage %q is not supported", pipelineID, operation, stage)
}
return stage, laneID, nil
}
func sortedReferenceBindings(bindings map[string]ReferenceBinding) []ReferenceBinding {
keys := sortedReferenceBindingKeys(bindings)
resolved := make([]ReferenceBinding, 0, len(keys))
for _, slotName := range keys {
resolved = append(resolved, bindings[slotName])
}
return resolved
}
func referenceTargetErrorContext(target referenceResolutionTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s module %q", target.PipelineID, target.LaneID, target.Stage, target.Module)
}
return fmt.Sprintf("pipeline %q %s module %q", target.PipelineID, target.Stage, target.Module)
}
func referenceTargetSlotLabel(target referenceResolutionTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s reference slot", target.PipelineID, target.LaneID, target.Stage)
}
return fmt.Sprintf("pipeline %q %s reference slot", target.PipelineID, target.Stage)
}
func normalizedReferenceMap(values map[string]string, keyName string) (map[string]string, error) {
if len(values) == 0 {
return nil, nil
}
out := make(map[string]string, len(values))
for rawSlotName, rawSource := range values {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
return nil, fmt.Errorf("%s must not be empty", keyName)
}
if _, ok := out[slotName]; ok {
return nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, slotName)
}
source := strings.TrimSpace(rawSource)
if source == "" {
return nil, fmt.Errorf("%s %q source must not be empty", keyName, slotName)
}
out[slotName] = source
}
return out, nil
}
func sortedArtifactLaneProfileKeys(values map[string]ArtifactLaneProfile) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func sortedStringMapKeys(values map[string]string) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func sortedReferenceBindingKeys(values map[string]ReferenceBinding) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
module := strings.TrimSpace(binding.Module)
if module == "" {
@@ -218,6 +604,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
Module: module,
LLMProfile: llmProfile,
Options: cloneOptions(binding.Options),
References: normalizeReferenceMap(binding.References),
}
}
@@ -246,6 +633,25 @@ func cloneOptions(options map[string]any) map[string]any {
return copied
}
func normalizeReferenceMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
}
return out
}
func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) {
lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts))
for rawLaneID, lane := range artifacts {
@@ -290,17 +696,19 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
withoutDigest := struct {
ID string
Input ModuleBinding
Chunk ModuleBinding
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
ID string
Input ModuleBinding
Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget
ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding
}{
ID: resolved.ID,
Input: resolved.Input,
Chunk: resolved.Chunk,
ArtifactLanes: resolved.ArtifactLanes,
Output: resolved.Output,
ID: resolved.ID,
Input: resolved.Input,
Chunk: resolved.Chunk,
ChunkReferences: resolved.ChunkReferences,
ArtifactLanes: resolved.ArtifactLanes,
Output: resolved.Output,
}
encoded, err := json.Marshal(withoutDigest)
if err != nil {

View File

@@ -3,6 +3,7 @@ package pipeline
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
@@ -125,6 +126,409 @@ func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
}
}
func TestResolvePipelineAppliesReferenceBindings(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{
" roster ": " ./shared-roster.yml ",
}
lane := profile.Artifacts["events"]
lane.References = map[string]string{
"roster": "./lane-roster.yml",
" lore ": " ./lore.md ",
}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
{Name: "lore"},
},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events", "summaries"}}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
events := resolvedLane(t, resolved.ArtifactLanes, "events")
if events.ExtractReferences.Stage != StageExtract || events.ExtractReferences.LaneID != "events" || events.ExtractReferences.Module != "event-extractor" {
t.Fatalf("extract reference target = %#v, want event extractor target", events.ExtractReferences)
}
if events.NormalizeReferences.Stage != StageNormalize || events.NormalizeReferences.LaneID != "events" || events.NormalizeReferences.Module != DefaultNormalizeModule {
t.Fatalf("normalize reference target = %#v, want event normalizer target", events.NormalizeReferences)
}
if resolved.ChunkReferences.Stage != StageChunk || resolved.ChunkReferences.Module != DefaultChunkModule {
t.Fatalf("chunk reference target = %#v, want chunk target", resolved.ChunkReferences)
}
want := []ReferenceBinding{
{LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig},
{LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig},
}
if !reflect.DeepEqual(events.ExtractReferences.Bindings, want) {
t.Fatalf("events references = %#v, want %#v", events.ExtractReferences.Bindings, want)
}
summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries")
if len(summaries.ExtractReferences.Bindings) != 0 {
t.Fatalf("summaries references = %#v, want none", summaries.ExtractReferences.Bindings)
}
}
func TestResolvePipelineAppliesPipelineReferenceDefaultToChunkTarget(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"scene_guide": "./scenes.md"}
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "generic",
Stage: StageChunk,
Requires: []string{"source"},
Provides: []string{"chunk"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "scene_guide"}},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
want := []ReferenceBinding{{SlotName: "scene_guide", Source: "./scenes.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
if !reflect.DeepEqual(resolved.ChunkReferences.Bindings, want) {
t.Fatalf("chunk references = %#v, want %#v", resolved.ChunkReferences.Bindings, want)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("extract references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("normalize references = %#v, want none", refs)
}
}
func TestResolvePipelineAppliesPipelineReferenceDefaultToExtractorTarget(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
want := []ReferenceBinding{{LaneID: "events", SlotName: "roster", Source: "./roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig}}
if !reflect.DeepEqual(resolved.ArtifactLanes[0].ExtractReferences.Bindings, want) {
t.Fatalf("extract references = %#v, want %#v", resolved.ArtifactLanes[0].ExtractReferences.Bindings, want)
}
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
t.Fatalf("chunk references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("normalize references = %#v, want none", refs)
}
}
func TestResolvePipelineAppliesPipelineReferenceDefaultToNormalizerTarget(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"normalization_notes": "./normalize.md"}
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
want := []ReferenceBinding{{LaneID: "events", SlotName: "normalization_notes", Source: "./normalize.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
if !reflect.DeepEqual(resolved.ArtifactLanes[0].NormalizeReferences.Bindings, want) {
t.Fatalf("normalize references = %#v, want %#v", resolved.ArtifactLanes[0].NormalizeReferences.Bindings, want)
}
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
t.Fatalf("chunk references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("extract references = %#v, want none", refs)
}
}
func TestResolvePipelineAppliesOnePipelineReferenceDefaultToMultipleTargets(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"context": "./context.md"}
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "context", "./context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "context", "./context.md")
}
func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{"notes_context": "./notes.md"}
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "note-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes_context"},
},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected lane references = %#v, want none", refs)
}
}
func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedNormalizer(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{"notes_context": "./notes.md"}
lane := profile.Artifacts["notes"]
lane.Normalize = Binding("note-normalizer")
profile.Artifacts["notes"] = lane
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "note-normalizer",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes_context"},
},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected extract references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected normalize references = %#v, want none", refs)
}
}
func TestResolvePipelineRejectsPipelineReferenceNotDeclaredByAnyLane(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{"missing": "./missing.md"}
_, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "multi", "reference slot", "missing", "not declared")
}
func TestResolvePipelineRejectsUndeclaredReferenceSlot(t *testing.T) {
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.References = map[string]string{"missing": "./missing.yml"}
profile.Artifacts["events"] = lane
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "events", "missing", "not declared")
}
func TestResolvePipelineRejectsExtractLocalReferenceDeclaredOnlyByNormalizer(t *testing.T) {
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{"normalization_notes": "./normalize.md"}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}},
})
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "events", "extract", "event-extractor", "normalization_notes", "not declared")
}
func TestResolvePipelineRejectsNormalizeLocalReferenceDeclaredOnlyByExtractor(t *testing.T) {
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{"roster": "./roster.yml"}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}},
})
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "events", "normalize", "noop", "roster", "not declared")
}
func TestResolvePipelineRequiresBoundChunkReference(t *testing.T) {
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "generic",
Stage: StageChunk,
Requires: []string{"source"},
Provides: []string{"chunk"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "scene_guide", Required: true}},
})
_, err := ResolvePipeline(baselineProfile(), ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "chunk", "generic", "required", "scene_guide", "not bound")
}
func TestResolvePipelineRequiresBoundNormalizeReference(t *testing.T) {
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes", Required: true}},
})
_, err := ResolvePipeline(baselineProfile(), ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "events", "normalize", "noop", "required", "normalization_notes", "not bound")
}
func TestResolvePipelineLocalReferencesOverridePipelineDefaultsForEligibleTargets(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{
"context": "./shared-context.md",
"roster": "./shared-roster.yml",
"normalization_notes": "./shared-normalize.md",
}
profile.Chunk.References = map[string]string{"context": "./chunk-context.md"}
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{"roster": "./extract-roster.yml"}
lane.Normalize.References = map[string]string{"normalization_notes": "./local-normalize.md"}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}}},
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./chunk-context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "roster", "./extract-roster.yml")
assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "normalization_notes", "./local-normalize.md")
}
func TestResolvePipelineRequiresBoundReferenceSlotsForSelectedLanes(t *testing.T) {
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
})
if _, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"notes"}}, catalog); err != nil {
t.Fatalf("ResolvePipeline(unselected required slot) error = %v, want nil", err)
}
_, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"events"}}, catalog)
if err == nil {
t.Fatal("ResolvePipeline(selected required slot) error = nil, want error")
}
assertErrorContains(t, err, "events", "required", "roster", "not bound")
}
func TestResolvePipelineReferenceUnbindCanLeaveRequiredSlotMissing(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
})
_, err := ResolvePipeline(profile, ResolveOptions{
ReferenceUnbinds: []ReferenceUnbind{{LaneID: "events", SlotName: "roster"}},
}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "events", "required", "roster", "not bound")
}
func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := emptyProfileCatalog()
for _, spec := range defaultProfileSpecs() {
if spec.Key != "event-extractor" {
registerProfileSpecs(t, catalog, spec)
}
}
if err := catalog.Extractors.RegisterWithSpec(ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
}, func() (contracts.Extractor, error) {
return nil, errors.New("constructor should not run")
}); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].Source; got != "./roster.yml" {
t.Fatalf("reference source = %q, want ./roster.yml", got)
}
}
func TestResolvePipelineRejectsUnknownOnlyLane(t *testing.T) {
_, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"missing"}}, newProfileCatalog(t))
if err == nil {
@@ -463,6 +867,17 @@ func laneIDs(lanes []ResolvedArtifactLane) []string {
return ids
}
func resolvedLane(t *testing.T, lanes []ResolvedArtifactLane, laneID string) ResolvedArtifactLane {
t.Helper()
for _, lane := range lanes {
if lane.ID == laneID {
return lane
}
}
t.Fatalf("lane %q not found in %#v", laneID, laneIDs(lanes))
return ResolvedArtifactLane{}
}
func assertErrorContains(t *testing.T, err error, values ...string) {
t.Helper()
@@ -474,6 +889,21 @@ func assertErrorContains(t *testing.T, err error, values ...string) {
}
}
func assertBindingSource(t *testing.T, bindings []ReferenceBinding, slotName string, source string) {
t.Helper()
for _, binding := range bindings {
if binding.SlotName != slotName {
continue
}
if binding.Source != source {
t.Fatalf("binding %q source = %q, want %q in %#v", slotName, binding.Source, source, bindings)
}
return
}
t.Fatalf("binding %q not found in %#v", slotName, bindings)
}
func newProfileCatalog(t *testing.T) ModuleCatalog {
t.Helper()
@@ -485,19 +915,29 @@ func newProfileCatalog(t *testing.T) ModuleCatalog {
func newProfileCatalogWithOverride(t *testing.T, override ModuleSpec) ModuleCatalog {
t.Helper()
return newProfileCatalogWithOverrides(t, override)
}
func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) ModuleCatalog {
t.Helper()
specs := defaultProfileSpecs()
for index, spec := range specs {
if spec.Stage == override.Stage && spec.Key == override.Key {
specs[index] = override
catalog := emptyProfileCatalog()
registerProfileSpecs(t, catalog, specs...)
return catalog
for _, override := range overrides {
replaced := false
for index, spec := range specs {
if spec.Stage == override.Stage && spec.Key == override.Key {
specs[index] = override
replaced = true
break
}
}
if !replaced {
specs = append(specs, override)
}
}
catalog := emptyProfileCatalog()
registerProfileSpecs(t, catalog, specs...)
registerProfileSpecs(t, catalog, override)
return catalog
}

View File

@@ -0,0 +1,323 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"mime"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
const (
referenceOriginFile = "file"
referenceMediaType = "text/plain"
unknownMediaType = "application/octet-stream"
)
type ReferenceMaterializationOptions struct {
ConfigPath string
WorkingDir string
}
func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) {
out := resolved
out.ChunkReferences = CloneReferenceTarget(resolved.ChunkReferences)
chunkReferenceSet, chunkWarnings, err := materializeReferenceTarget(resolved.ID, resolved.ChunkReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
out.ChunkReferences.ReferenceSet = chunkReferenceSet
warnings := append([]contracts.Warning(nil), chunkWarnings...)
if len(resolved.ArtifactLanes) == 0 {
return out, warnings, nil
}
out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
for i, lane := range resolved.ArtifactLanes {
materializedLane := lane
materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences)
extractReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
materializedLane.ExtractReferences.ReferenceSet = extractReferenceSet
warnings = append(warnings, laneWarnings...)
normalizeReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.NormalizeReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
materializedLane.NormalizeReferences.ReferenceSet = normalizeReferenceSet
warnings = append(warnings, laneWarnings...)
out.ArtifactLanes[i] = materializedLane
}
return out, warnings, nil
}
func materializeReferenceTarget(
pipelineID string,
target ResolvedReferenceTarget,
catalog ModuleCatalog,
options ReferenceMaterializationOptions,
) (contracts.ReferenceSet, []contracts.Warning, error) {
if len(target.Bindings) == 0 {
return contracts.ReferenceSet{}, nil, nil
}
spec, err := referenceTargetSpec(target, catalog)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s: %w", referenceTargetContext(pipelineID, target), err)
}
slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots))
for _, slot := range spec.ReferenceSlots {
slotByName[slot.Name] = slot
}
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(target.Bindings))}
var warnings []contracts.Warning
for _, binding := range target.Bindings {
slotName := strings.TrimSpace(binding.SlotName)
slot, ok := slotByName[slotName]
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetContext(pipelineID, target), slotName, target.Stage, target.Module)
}
path, err := referencePath(binding, options)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q: %w", referenceTargetContext(pipelineID, target), slotName, binding.Source, err)
}
content, err := os.ReadFile(path)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q read %q: %w", referenceTargetContext(pipelineID, target), slotName, path, err)
}
if !utf8.Valid(content) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q must be UTF-8 text", referenceTargetContext(pipelineID, target), slotName, path)
}
mediaType := referenceMediaTypeForPath(path)
if !referenceMediaTypeAccepted(mediaType, slot.AcceptedMediaTypes) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q media type %q is not accepted", referenceTargetContext(pipelineID, target), slotName, path, mediaType)
}
if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q is %d bytes, limit %d", referenceTargetContext(pipelineID, target), slotName, path, len(content), slot.MaxBytes)
}
if len(content) == 0 {
warnings = append(warnings, contracts.Warning{
Scope: referenceWarningScope(pipelineID, target, slotName),
ReasonCode: "empty_reference",
Message: fmt.Sprintf("reference slot %q for %s is bound to an empty file", slotName, referenceTargetLabel(target)),
})
}
item := contracts.ReferenceItem{
SlotName: slotName,
MediaType: mediaType,
Content: append([]byte(nil), content...),
Digest: referenceDigest(content),
Origin: contracts.ReferenceOrigin{Type: referenceOriginFile, URI: fileURI(path)},
SizeBytes: int64(len(content)),
BindingSource: strings.TrimSpace(binding.BindingSource),
}
set.Slots[slotName] = contracts.ResolvedReferenceSlot{
Slot: cloneReferenceSlot(slot),
Items: []contracts.ReferenceItem{item},
}
}
return set, warnings, nil
}
func referenceTargetSpec(target ResolvedReferenceTarget, catalog ModuleCatalog) (ModuleSpec, error) {
switch target.Stage {
case StageChunk:
return registrySpec(catalog.Chunkers, target.Module)
case StageExtract:
return registrySpec(catalog.Extractors, target.Module)
case StageNormalize:
return registrySpec(catalog.Normalizers, target.Module)
default:
return ModuleSpec{}, fmt.Errorf("reference target stage %q is not supported", target.Stage)
}
}
func referenceTargetContext(pipelineID string, target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s module %q", pipelineID, target.LaneID, target.Stage, target.Module)
}
return fmt.Sprintf("pipeline %q %s module %q", pipelineID, target.Stage, target.Module)
}
func referenceTargetLabel(target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("lane %q %s target", target.LaneID, target.Stage)
}
return fmt.Sprintf("%s target", target.Stage)
}
func referenceWarningScope(pipelineID string, target ResolvedReferenceTarget, slotName string) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline.%s.lane.%s.%s.reference.%s", pipelineID, target.LaneID, target.Stage, slotName)
}
return fmt.Sprintf("pipeline.%s.%s.reference.%s", pipelineID, target.Stage, slotName)
}
func referenceMediaTypeForPath(path string) string {
extension := strings.ToLower(filepath.Ext(path))
mediaType := mime.TypeByExtension(extension)
if strings.TrimSpace(mediaType) == "" {
if extension == ".md" || extension == ".markdown" {
return "text/markdown"
}
return unknownMediaType
}
return canonicalMediaType(mediaType)
}
func referenceMediaTypeAccepted(mediaType string, accepted []string) bool {
if len(accepted) == 0 {
return true
}
mediaType = canonicalMediaType(mediaType)
for _, value := range accepted {
if strings.EqualFold(mediaType, canonicalMediaType(value)) {
return true
}
}
return false
}
func canonicalMediaType(mediaType string) string {
trimmed := strings.TrimSpace(mediaType)
if trimmed == "" {
return ""
}
parsed, _, err := mime.ParseMediaType(trimmed)
if err != nil {
return strings.ToLower(trimmed)
}
return strings.ToLower(parsed)
}
func referencePath(binding ReferenceBinding, options ReferenceMaterializationOptions) (string, error) {
source := strings.TrimSpace(binding.Source)
if source == "" {
return "", fmt.Errorf("must not be empty")
}
if filepath.IsAbs(source) {
return filepath.Clean(source), nil
}
base := strings.TrimSpace(options.WorkingDir)
if strings.TrimSpace(binding.BindingSource) == contracts.ReferenceBindingSourceConfig {
base = filepath.Dir(strings.TrimSpace(options.ConfigPath))
}
if base == "" {
var err error
base, err = os.Getwd()
if err != nil {
return "", fmt.Errorf("resolve working directory: %w", err)
}
}
return filepath.Clean(filepath.Join(base, source)), nil
}
func referenceDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
func fileURI(path string) string {
absolute, err := filepath.Abs(path)
if err != nil {
absolute = path
}
absolute = filepath.ToSlash(filepath.Clean(absolute))
if strings.HasPrefix(absolute, "/") {
return "file://" + (&url.URL{Path: absolute}).EscapedPath()
}
return "file:///" + (&url.URL{Path: absolute}).EscapedPath()
}
func cloneReferenceSlot(slot contracts.ReferenceSlot) contracts.ReferenceSlot {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
return slot
}
func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet {
if len(in.Slots) == 0 {
return contracts.ReferenceSet{}
}
out := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(in.Slots))}
keys := make([]string, 0, len(in.Slots))
for key := range in.Slots {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
slot := in.Slots[key]
slot.Slot = cloneReferenceSlot(slot.Slot)
if len(slot.Items) > 0 {
items := make([]contracts.ReferenceItem, len(slot.Items))
for i, item := range slot.Items {
item.Content = append([]byte(nil), item.Content...)
items[i] = item
}
slot.Items = items
}
out.Slots[key] = slot
}
return out
}
func CloneReferenceTarget(in ResolvedReferenceTarget) ResolvedReferenceTarget {
out := in
out.Bindings = append([]ReferenceBinding(nil), in.Bindings...)
out.ReferenceSet = CloneReferenceSet(in.ReferenceSet)
return out
}
func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance {
provenance := []artifacts.ReferenceProvenance{}
provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...)
for _, lane := range resolved.ArtifactLanes {
provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...)
provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...)
}
return provenance
}
func referenceTargetProvenance(target ResolvedReferenceTarget) []artifacts.ReferenceProvenance {
if len(target.ReferenceSet.Slots) == 0 {
return nil
}
provenance := []artifacts.ReferenceProvenance{}
slotNames := make([]string, 0, len(target.ReferenceSet.Slots))
for slotName := range target.ReferenceSet.Slots {
slotNames = append(slotNames, slotName)
}
sort.Strings(slotNames)
for _, slotName := range slotNames {
slot := target.ReferenceSet.Slots[slotName]
for _, item := range slot.Items {
provenance = append(provenance, artifacts.ReferenceProvenance{
Stage: string(target.Stage),
LaneID: target.LaneID,
SlotName: item.SlotName,
OriginType: item.Origin.Type,
OriginURI: item.Origin.URI,
Digest: item.Digest,
MediaType: item.MediaType,
SizeBytes: item.SizeBytes,
BindingSource: item.BindingSource,
})
}
}
return provenance
}

View File

@@ -0,0 +1,477 @@
package pipeline
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
configDir := t.TempDir()
workingDir := t.TempDir()
configReference := filepath.Join(configDir, "config-reference.txt")
cliReference := filepath.Join(workingDir, "cli-reference.txt")
writeReferenceFile(t, configReference, []byte("config text"))
writeReferenceFile(t, cliReference, []byte("cli text"))
pipeline := baselineProfile()
pipeline.References = map[string]string{"roster": "config-reference.txt"}
lane := pipeline.Artifacts["events"]
lane.References = map[string]string{"glossary": "cli-reference.txt"}
pipeline.Artifacts["events"] = lane
catalog := referenceCatalog(t, []contracts.ReferenceSlot{
{Name: "roster"},
{Name: "glossary"},
})
resolved, err := ResolvePipeline(pipeline, ResolveOptions{
ReferenceOverrides: []ReferenceBinding{
{LaneID: "events", SlotName: "glossary", Source: "cli-reference.txt", BindingSource: contracts.ReferenceBindingSourceCLI},
},
}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
first, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
WorkingDir: workingDir,
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
if len(warnings) != 0 {
t.Fatalf("warnings = %#v, want none", warnings)
}
second, _, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
WorkingDir: workingDir,
})
if err != nil {
t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
}
referenceSet := first.ArtifactLanes[0].ExtractReferences.ReferenceSet
roster := referenceSet.Slots["roster"].Items[0]
if string(roster.Content) != "config text" {
t.Fatalf("roster content = %q, want config text", roster.Content)
}
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Digest {
t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
}
if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
t.Fatalf("roster binding source = %q, want config", roster.BindingSource)
}
if roster.MediaType != referenceMediaType || roster.Origin.Type != referenceOriginFile || roster.SizeBytes != int64(len("config text")) {
t.Fatalf("roster metadata = %#v, want text file metadata", roster)
}
if !strings.Contains(roster.Origin.URI, "config-reference.txt") {
t.Fatalf("roster origin URI = %q, want config reference path", roster.Origin.URI)
}
glossary := referenceSet.Slots["glossary"].Items[0]
if string(glossary.Content) != "cli text" {
t.Fatalf("glossary content = %q, want cli text", glossary.Content)
}
if glossary.BindingSource != contracts.ReferenceBindingSourceCLI {
t.Fatalf("glossary binding source = %q, want cli", glossary.BindingSource)
}
if !strings.Contains(glossary.Origin.URI, "cli-reference.txt") {
t.Fatalf("glossary origin URI = %q, want cli reference path", glossary.Origin.URI)
}
provenance := ReferenceProvenance(first)
if len(provenance) != 2 {
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
}
if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
}
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
}
encoded, err := json.Marshal(first)
if err != nil {
t.Fatalf("json.Marshal(materialized) error = %v, want nil", err)
}
if strings.Contains(string(encoded), "config text") || strings.Contains(string(encoded), "cli text") {
t.Fatalf("materialized pipeline JSON contains reference content: %s", encoded)
}
}
func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T) {
configDir := t.TempDir()
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), []byte("chunk text"))
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), []byte("extract text"))
writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), []byte("normalize text"))
profile := baselineProfile()
profile.References = map[string]string{
"scene_guide": "chunk.txt",
"roster": "extract.txt",
"normalization_notes": "normalize.txt",
}
catalog := referenceCatalogForTargets(t,
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
[]contracts.ReferenceSlot{{Name: "roster"}},
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
materialized, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
if len(warnings) != 0 {
t.Fatalf("warnings = %#v, want none", warnings)
}
chunkItem := materialized.ChunkReferences.ReferenceSet.Slots["scene_guide"].Items[0]
if string(chunkItem.Content) != "chunk text" {
t.Fatalf("chunk content = %q, want chunk text", chunkItem.Content)
}
extractItem := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if string(extractItem.Content) != "extract text" {
t.Fatalf("extract content = %q, want extract text", extractItem.Content)
}
normalizeItem := materialized.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["normalization_notes"].Items[0]
if string(normalizeItem.Content) != "normalize text" {
t.Fatalf("normalize content = %q, want normalize text", normalizeItem.Content)
}
provenance := ReferenceProvenance(materialized)
if len(provenance) != 3 {
t.Fatalf("ReferenceProvenance() = %#v, want three entries", provenance)
}
if provenance[0].Stage != string(StageChunk) || provenance[0].LaneID != "" || provenance[0].SlotName != "scene_guide" || provenance[0].Digest != chunkItem.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want chunk scene guide provenance", provenance[0])
}
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != extractItem.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want extract roster provenance", provenance[1])
}
if provenance[2].Stage != string(StageNormalize) || provenance[2].LaneID != "events" || provenance[2].SlotName != "normalization_notes" || provenance[2].Digest != normalizeItem.Digest {
t.Fatalf("ReferenceProvenance()[2] = %#v, want normalize notes provenance", provenance[2])
}
}
func TestMaterializeReferencesRejectsNonUTF8Content(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "bad.txt")
writeReferenceFile(t, path, []byte{0xff, 0xfe})
resolved := resolvedPipelineWithReference(t, "roster", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), path) {
t.Fatalf("error = %v, want UTF-8 path error", err)
}
}
func TestMaterializeReferencesRejectsNonUTF8ContentForChunkTarget(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "bad.txt")
writeReferenceFile(t, path, []byte{0xff, 0xfe})
resolved := resolvedPipelineWithTargetReference(t, StageChunk, "", "scene_guide", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "scene_guide"})
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, []contracts.ReferenceSlot{{Name: "scene_guide"}}, nil, nil), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "chunk") || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "scene_guide") || !strings.Contains(err.Error(), path) {
t.Fatalf("error = %v, want chunk UTF-8 path error", err)
}
}
func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "roster.reference")
writeReferenceFile(t, path, []byte("plain text"))
resolved := resolvedPipelineWithReference(t, "roster", "roster.reference", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != unknownMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType)
}
}
func TestMaterializeReferencesAcceptsDeclaredMarkdownMediaType(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "glossary.md")
writeReferenceFile(t, path, []byte("# Terms\n"))
slot := contracts.ReferenceSlot{Name: "glossary", AcceptedMediaTypes: []string{"text/markdown"}}
resolved := resolvedPipelineWithReference(t, "glossary", "glossary.md", contracts.ReferenceBindingSourceConfig, slot)
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["glossary"].Items[0]
if item.MediaType != "text/markdown" {
t.Fatalf("MediaType = %q, want text/markdown", item.MediaType)
}
}
func TestMaterializeReferencesAcceptsDeclaredJSONMediaType(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "roster.json")
writeReferenceFile(t, path, []byte(`{"aria":"cleric"}`))
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"application/json"}}
resolved := resolvedPipelineWithReference(t, "roster", "roster.json", contracts.ReferenceBindingSourceConfig, slot)
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", item.MediaType)
}
}
func TestMaterializeReferencesRejectsUnacceptedMediaType(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "roster.json")
writeReferenceFile(t, path, []byte(`{"aria":"cleric"}`))
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"text/markdown"}}
resolved := resolvedPipelineWithReference(t, "roster", "roster.json", contracts.ReferenceBindingSourceConfig, slot)
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "media type") || !strings.Contains(err.Error(), "application/json") || !strings.Contains(err.Error(), "roster") {
t.Fatalf("error = %v, want media type rejection", err)
}
}
func TestMaterializeReferencesMatchesAcceptedMediaTypesIgnoringParameters(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "roster.txt")
writeReferenceFile(t, path, []byte("Aria\n"))
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"text/plain; charset=utf-8"}}
resolved := resolvedPipelineWithReference(t, "roster", "roster.txt", contracts.ReferenceBindingSourceConfig, slot)
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != referenceMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType)
}
}
func TestMaterializeReferencesRejectsUnacceptedMediaTypeForNormalizeTarget(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "notes.json")
writeReferenceFile(t, path, []byte(`{"notes":true}`))
slot := contracts.ReferenceSlot{Name: "normalization_notes", AcceptedMediaTypes: []string{"text/markdown"}}
resolved := resolvedPipelineWithTargetReference(t, StageNormalize, "events", "normalization_notes", "notes.json", contracts.ReferenceBindingSourceConfig, slot)
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, nil, nil, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "normalize") || !strings.Contains(err.Error(), "media type") || !strings.Contains(err.Error(), "application/json") || !strings.Contains(err.Error(), "normalization_notes") {
t.Fatalf("error = %v, want normalize media type rejection", err)
}
}
func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "empty.txt")
writeReferenceFile(t, path, nil)
resolved := resolvedPipelineWithReference(t, "roster", "empty.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
materialized, warnings, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
t.Fatalf("warnings = %#v, want empty reference warning", warnings)
}
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
t.Fatalf("empty item = %#v, want zero size and empty digest", item)
}
}
func TestMaterializeReferencesWarningScopesIncludeTargetContext(t *testing.T) {
configDir := t.TempDir()
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), nil)
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), nil)
writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), nil)
profile := baselineProfile()
profile.References = map[string]string{
"scene_guide": "chunk.txt",
"roster": "extract.txt",
"normalization_notes": "normalize.txt",
}
catalog := referenceCatalogForTargets(t,
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
[]contracts.ReferenceSlot{{Name: "roster"}},
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
_, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
got := warningScopes(warnings)
want := []string{
"pipeline.baseline.chunk.reference.scene_guide",
"pipeline.baseline.lane.events.extract.reference.roster",
"pipeline.baseline.lane.events.normalize.reference.normalization_notes",
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("warning scopes = %#v, want %#v", got, want)
}
}
func TestMaterializeReferencesEnforcesMaxBytes(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "large.txt")
writeReferenceFile(t, path, []byte("too large"))
slot := contracts.ReferenceSlot{Name: "roster", MaxBytes: 3}
resolved := resolvedPipelineWithReference(t, "roster", "large.txt", contracts.ReferenceBindingSourceConfig, slot)
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "9 bytes") || !strings.Contains(err.Error(), "limit 3") || !strings.Contains(err.Error(), "roster") {
t.Fatalf("error = %v, want max bytes error", err)
}
}
func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
t.Helper()
return resolvedPipelineWithTargetReference(t, StageExtract, "events", slotName, source, bindingSource, slot)
}
func resolvedPipelineWithTargetReference(t *testing.T, stage ModuleStage, laneID string, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
t.Helper()
profile := baselineProfile()
switch stage {
case StageChunk:
profile.Chunk.References = map[string]string{slotName: source}
case StageExtract:
lane := profile.Artifacts[laneID]
lane.References = map[string]string{slotName: source}
profile.Artifacts[laneID] = lane
case StageNormalize:
lane := profile.Artifacts[laneID]
lane.Normalize.References = map[string]string{slotName: source}
profile.Artifacts[laneID] = lane
default:
t.Fatalf("unsupported reference target stage %q", stage)
}
catalog := referenceCatalogForStage(t, stage, []contracts.ReferenceSlot{slot})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if bindingSource != contracts.ReferenceBindingSourceConfig {
switch stage {
case StageChunk:
resolved.ChunkReferences.Bindings[0].BindingSource = bindingSource
case StageExtract:
resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].BindingSource = bindingSource
case StageNormalize:
resolved.ArtifactLanes[0].NormalizeReferences.Bindings[0].BindingSource = bindingSource
}
}
return resolved
}
func referenceCatalog(t *testing.T, slots []contracts.ReferenceSlot) ModuleCatalog {
t.Helper()
return referenceCatalogForStage(t, StageExtract, slots)
}
func referenceCatalogForStage(t *testing.T, stage ModuleStage, slots []contracts.ReferenceSlot) ModuleCatalog {
t.Helper()
switch stage {
case StageChunk:
return referenceCatalogForTargets(t, slots, nil, nil)
case StageExtract:
return referenceCatalogForTargets(t, nil, slots, nil)
case StageNormalize:
return referenceCatalogForTargets(t, nil, nil, slots)
default:
t.Fatalf("unsupported reference target stage %q", stage)
return ModuleCatalog{}
}
}
func referenceCatalogForTargets(t *testing.T, chunkSlots, extractSlots, normalizeSlots []contracts.ReferenceSlot) ModuleCatalog {
t.Helper()
return newProfileCatalogWithOverrides(t,
ModuleSpec{
Key: "generic",
Stage: StageChunk,
Requires: []string{"source"},
Provides: []string{"chunk"},
ReferenceSlots: chunkSlots,
},
ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: extractSlots,
},
ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: normalizeSlots,
},
)
}
func warningScopes(warnings []contracts.Warning) []string {
scopes := make([]string, 0, len(warnings))
for _, warning := range warnings {
scopes = append(scopes, warning.Scope)
}
sort.Strings(scopes)
return scopes
}
func writeReferenceFile(t *testing.T, path string, content []byte) {
t.Helper()
if err := os.WriteFile(path, content, 0o644); err != nil {
t.Fatalf("write reference %q: %v", path, err)
}
}

View File

@@ -118,6 +118,10 @@ func (chunker integrationChunker) Key() string {
return "chunk"
}
func (chunker integrationChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{
@@ -149,6 +153,10 @@ func (extractor integrationExtractor) SchemaVersion() string {
return "v1"
}
func (extractor integrationExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor integrationExtractor) Validators() []contracts.Validator {
return extractor.validators
}
@@ -182,6 +190,10 @@ func (normalizer integrationNormalizer) Key() string {
return "normalize"
}
func (normalizer integrationNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
}

View File

@@ -41,6 +41,7 @@ type RunInput struct {
StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any
Warnings []contracts.Warning
}
type RunOutput struct {
@@ -63,12 +64,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
return output, err
}
output.Warnings = append(output.Warnings, cloneWarnings(input.Warnings)...)
output.Manifest = manifestFromPipeline(input)
adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module)
if err != nil {
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
}
attachModuleManifestMetadata(&output, "input", adapter)
doc, err := adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID,
Path: input.Path,
@@ -89,8 +92,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil {
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
}
attachModuleManifestMetadata(&output, "chunker", chunker)
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc,
References: CloneReferenceSet(input.Pipeline.ChunkReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: input.Pipeline.Chunk.LLMProfile,
Options: cloneOptions(input.Pipeline.Chunk.Options),
Metadata: input.Metadata,
@@ -102,10 +108,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if len(chunkResult.Chunks) == 0 {
return failOutput(output), fmt.Errorf("chunker %q returned no chunks", chunker.Key())
}
canonicalChunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
if err != nil {
return failOutput(output), fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
}
nextCandidateIndex := 0
for _, lane := range input.Pipeline.ArtifactLanes {
if err := r.runLane(ctx, input, doc, chunkResult.Chunks, lane, &output, &nextCandidateIndex); err != nil {
if err := r.runLane(ctx, input, doc, canonicalChunks, lane, &output, &nextCandidateIndex); err != nil {
return failOutput(output), err
}
}
@@ -121,6 +131,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil {
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
}
attachModuleManifestMetadata(&output, "output", encoder)
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: output.Manifest,
Approved: output.Approved,
@@ -176,6 +187,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
Source: doc,
Chunk: &chunk,
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: lane.Extract.LLMProfile,
Options: cloneOptions(lane.Extract.Options),
@@ -213,6 +225,8 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
Source: doc,
LaneID: lane.ID,
Candidates: mergeResult.Candidates,
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: lane.Normalize.LLMProfile,
Options: cloneOptions(lane.Normalize.Options),
Metadata: input.Metadata,
@@ -345,8 +359,12 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
RunID: runID,
StartedAt: timePtr(startedAt),
References: ReferenceProvenance(pipeline),
LLMProfiles: cloneLLMProfiles(input.LLMProfiles),
}
// The runner does not currently maintain a cache or idempotency key. Reference
// digests are recorded in manifest provenance and intentionally kept separate
// from source_digests.
for _, lane := range pipeline.ArtifactLanes {
laneManifest := artifacts.ArtifactLaneManifest{
@@ -382,14 +400,10 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
metadata := make(map[string]any)
for _, module := range modules {
provider, ok := module.(contracts.ManifestMetadataProvider)
moduleMetadata, ok := moduleManifestMetadata(module)
if !ok {
continue
}
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
if len(moduleMetadata) == 0 {
continue
}
key := manifestMetadataKey(module)
if key == "" {
continue
@@ -403,6 +417,20 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
}
}
func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
if output == nil {
return
}
moduleMetadata, ok := moduleManifestMetadata(module)
if !ok {
return
}
if output.Manifest.ModuleMetadata == nil {
output.Manifest.ModuleMetadata = make(map[string]map[string]any)
}
output.Manifest.ModuleMetadata[moduleKey] = moduleMetadata
}
func manifestMetadataKey(module any) string {
switch module.(type) {
case contracts.Extractor:
@@ -416,6 +444,19 @@ func manifestMetadataKey(module any) string {
}
}
func moduleManifestMetadata(module any) (map[string]any, bool) {
provider, ok := module.(contracts.ManifestMetadataProvider)
if !ok {
return nil, false
}
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
if len(moduleMetadata) == 0 {
return nil, false
}
return moduleMetadata, true
}
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
out := make([]contracts.OutputFile, 0, len(result.Files))
for _, file := range result.Files {
@@ -469,6 +510,13 @@ func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMPr
return append([]artifacts.LLMProfileManifest(nil), profiles...)
}
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
if len(warnings) == 0 {
return nil
}
return append([]contracts.Warning(nil), warnings...)
}
func timePtr(t time.Time) *time.Time {
return &t
}

View File

@@ -256,6 +256,191 @@ func TestRunRejectsChunkerBuildChunkAndEmptyChunkErrors(t *testing.T) {
}
}
func TestRunRejectsInvalidChunks(t *testing.T) {
tests := []struct {
name string
chunks []contracts.SourceChunk
want string
}{
{
name: "empty chunk id",
chunks: []contracts.SourceChunk{{ID: "", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}}},
want: "id must not be empty",
},
{
name: "duplicate chunk id",
chunks: []contracts.SourceChunk{
{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}},
{ID: "chunk-0", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u2")}},
},
want: "duplicated",
},
{
name: "wrong source id",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "other-source", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}}},
want: "source_id",
},
{
name: "wrong index",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u1")}}},
want: "index",
},
{
name: "empty units",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0}},
want: "units must not be empty",
},
{
name: "repeated unit inside chunk",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1"), unitWithID("u1")}}},
want: "repeats source unit",
},
{
name: "unknown unit",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u9")}}},
want: "was not found",
},
{
name: "units out of source order",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u2"), unitWithID("u1")}}},
want: "source document order",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = test.chunks
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, test.want)
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
if len(modules.extractors["extract-alpha"].requests) != 0 {
t.Fatalf("extractor calls = %d, want none after invalid chunks", len(modules.extractors["extract-alpha"].requests))
}
})
}
}
func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = []contracts.SourceChunk{
{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1"), unitWithID("u2")}},
{ID: "chunk-1", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u2")}},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want one candidate per accepted chunk", len(output.Approved))
}
}
func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
modules := defaultRunnerModules()
modules.input.doc = sourceDocumentWithUnitMetadata()
modules.chunker.chunks = []contracts.SourceChunk{
{
ID: "chunk-0",
SourceID: "source-1",
Index: 0,
Units: []source.SourceUnit{
{
ID: "u1",
Kind: "mutated-kind",
Text: "mutated text",
Metadata: map[string]any{
"speaker": "chunker-speaker",
"note": "chunker note",
},
},
},
},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
}
extractor := modules.extractors["extract-alpha"]
if len(extractor.requests) != 1 {
t.Fatalf("len(extractor requests) = %d, want 1", len(extractor.requests))
}
chunk := extractor.requests[0].Chunk
if chunk == nil {
t.Fatal("extractor chunk = nil, want canonical chunk")
}
if chunk.Units[0].ID != "u1" || chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" {
t.Fatalf("chunk unit = %#v, want source document unit values", chunk.Units[0])
}
if got := chunk.Units[0].Metadata["speaker"]; got != "source-speaker" {
t.Fatalf("chunk unit metadata = %#v, want source document metadata", chunk.Units[0].Metadata)
}
if got := chunk.Units[0].Metadata["topic"]; got != "source-topic" {
t.Fatalf("chunk unit metadata = %#v, want cloned source document metadata", chunk.Units[0].Metadata)
}
modules.input.doc.Units[0].Kind = "changed-kind"
modules.input.doc.Units[0].Text = "changed text"
modules.input.doc.Units[0].Metadata["speaker"] = "changed-speaker"
if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Metadata["speaker"] != "source-speaker" {
t.Fatalf("chunk unit changed after source mutation: %#v", chunk.Units[0])
}
}
func TestRunPreservesChunkMetadataDuringCanonicalization(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = []contracts.SourceChunk{
{
ID: "chunk-0",
SourceID: "source-1",
Index: 0,
Units: []source.SourceUnit{
unitWithID("u1"),
},
Metadata: map[string]any{
"scene_title": "Original scene",
"boundary_note": "Chunker note",
},
},
}
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
extractor := modules.extractors["extract-alpha"]
if len(extractor.requests) != 1 || extractor.requests[0].Chunk == nil {
t.Fatalf("extractor requests = %#v, want one canonical chunk", extractor.requests)
}
if got := extractor.requests[0].Chunk.Metadata["scene_title"]; got != "Original scene" {
t.Fatalf("chunk metadata = %#v, want chunker metadata", extractor.requests[0].Chunk.Metadata)
}
if got := extractor.requests[0].Chunk.Metadata["boundary_note"]; got != "Chunker note" {
t.Fatalf("chunk metadata = %#v, want chunker metadata", extractor.requests[0].Chunk.Metadata)
}
modules.chunker.chunks[0].Metadata["scene_title"] = "changed"
modules.chunker.chunks[0].Metadata["boundary_note"] = "changed"
if got := extractor.requests[0].Chunk.Metadata["scene_title"]; got != "Original scene" {
t.Fatalf("chunk metadata aliased to chunker map: %#v", extractor.requests[0].Chunk.Metadata)
}
if got := extractor.requests[0].Chunk.Metadata["boundary_note"]; got != "Chunker note" {
t.Fatalf("chunk metadata aliased to chunker map: %#v", extractor.requests[0].Chunk.Metadata)
}
}
func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
modules := defaultRunnerModules()
llmClient := fakeLLMClient{}
@@ -273,9 +458,16 @@ func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
if !reflect.DeepEqual(extractor.seenChunkIDs, []string{"chunk-0", "chunk-1"}) {
t.Fatalf("seen chunks = %#v, want both chunks", extractor.seenChunkIDs)
}
if len(modules.chunker.requests) != 1 || modules.chunker.requests[0].LLMClient == nil {
t.Fatalf("chunker LLM client = %#v, want client on chunk request", modules.chunker.requests)
}
if len(extractor.seenLLMClients) != 2 || extractor.seenLLMClients[0] == nil || extractor.seenLLMClients[1] == nil {
t.Fatalf("seen LLM clients = %#v, want client for each chunk", extractor.seenLLMClients)
}
normalizer := modules.normalizers["normalize"]
if len(normalizer.requests) != 1 || normalizer.requests[0].LLMClient == nil {
t.Fatalf("normalizer LLM client = %#v, want client on normalize request", normalizer.requests)
}
if extractor.seenMetadata[0]["request"] != "test" {
t.Fatalf("seen metadata = %#v, want request metadata", extractor.seenMetadata)
}
@@ -371,6 +563,133 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
}
}
func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = testReferenceSet("roster", "reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
req := modules.extractors["extract-alpha"].requests[0]
item := req.References.Slots["roster"].Items[0]
if string(item.Content) != "reference text" {
t.Fatalf("reference content = %q, want reference text", item.Content)
}
item.Content[0] = 'R'
if got := string(pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
t.Fatalf("runner mutated reference set content = %q", got)
}
}
func TestRunPassesChunkReferencesToChunkerRequest(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ChunkReferences.ReferenceSet = testReferenceSet("scene_guide", "chunk reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
req := modules.chunker.requests[0]
item := req.References.Slots["scene_guide"].Items[0]
if string(item.Content) != "chunk reference text" {
t.Fatalf("chunk reference content = %q, want chunk reference text", item.Content)
}
item.Content[0] = 'C'
if got := string(pipeline.ChunkReferences.ReferenceSet.Slots["scene_guide"].Items[0].Content); got != "chunk reference text" {
t.Fatalf("runner mutated chunk reference set content = %q", got)
}
}
func TestRunPassesNormalizeReferencesToNormalizerRequest(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].NormalizeReferences.ReferenceSet = testReferenceSet("normalization_notes", "normalize reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
req := modules.normalizers["normalize"].requests[0]
item := req.References.Slots["normalization_notes"].Items[0]
if string(item.Content) != "normalize reference text" {
t.Fatalf("normalize reference content = %q, want normalize reference text", item.Content)
}
item.Content[0] = 'N'
if got := string(pipeline.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["normalization_notes"].Items[0].Content); got != "normalize reference text" {
t.Fatalf("runner mutated normalize reference set content = %q", got)
}
}
func TestRunAllowsNilLLMClientWhenModulesDoNotUseIt(t *testing.T) {
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil with nil LLM client when modules do not use it", err)
}
}
func TestRunIncludesInputWarnings(t *testing.T) {
modules := defaultRunnerModules()
warning := contracts.Warning{Scope: "reference", ReasonCode: "empty_reference", Message: "empty reference"}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
Warnings: []contracts.Warning{warning},
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Warnings) != 1 || output.Warnings[0] != warning {
t.Fatalf("warnings = %#v, want input warning", output.Warnings)
}
}
func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) {
modules := defaultRunnerModules()
modules.input.manifestMetadata = map[string]any{
"input_profile": "input-metadata",
}
modules.chunker.manifestMetadata = map[string]any{
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:chunker-prompt",
"response_schema_key": "dnd_scenes",
"response_schema_id": "schema-dnd-scenes",
"response_schema_name": "dnd_scenes",
}
modules.output.manifestMetadata = map[string]any{
"output_profile": "output-metadata",
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if output.Manifest.ModuleMetadata == nil {
t.Fatal("ModuleMetadata = nil, want module metadata map")
}
if got := output.Manifest.ModuleMetadata["input"]; !reflect.DeepEqual(got, modules.input.manifestMetadata) {
t.Fatalf("input module metadata = %#v, want %#v", got, modules.input.manifestMetadata)
}
if got := output.Manifest.ModuleMetadata["chunker"]; !reflect.DeepEqual(got, modules.chunker.manifestMetadata) {
t.Fatalf("chunker module metadata = %#v, want %#v", got, modules.chunker.manifestMetadata)
}
if got := output.Manifest.ModuleMetadata["output"]; !reflect.DeepEqual(got, modules.output.manifestMetadata) {
t.Fatalf("output module metadata = %#v, want %#v", got, modules.output.manifestMetadata)
}
modules.chunker.manifestMetadata["prompt_id"] = "changed"
if output.Manifest.ModuleMetadata["chunker"]["prompt_id"] != "dnd.scenes" {
t.Fatalf("chunker module metadata aliased to provider map: %#v", output.Manifest.ModuleMetadata["chunker"])
}
}
func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
modules := defaultRunnerModules()
@@ -677,7 +996,63 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
}
func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolvedPipelineWithValidators("configured")})
resolved := resolvedPipelineWithValidators("configured")
resolved.ChunkReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"scene_guide": {
Slot: contracts.ReferenceSlot{Name: "scene_guide"},
Items: []contracts.ReferenceItem{
{
SlotName: "scene_guide",
MediaType: "text/plain; charset=utf-8",
Content: []byte("chunk reference content"),
Digest: "sha256:chunk-reference",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/scene-guide.txt"},
SizeBytes: int64(len("chunk reference content")),
BindingSource: contracts.ReferenceBindingSourceCLI,
},
},
},
},
}
resolved.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain; charset=utf-8",
Content: []byte("reference content"),
Digest: "sha256:reference",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
SizeBytes: int64(len("reference content")),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
resolved.ArtifactLanes[0].NormalizeReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"normalization_notes": {
Slot: contracts.ReferenceSlot{Name: "normalization_notes"},
Items: []contracts.ReferenceItem{
{
SlotName: "normalization_notes",
MediaType: "text/plain; charset=utf-8",
Content: []byte("normalize reference content"),
Digest: "sha256:normalize-reference",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/normalize.txt"},
SizeBytes: int64(len("normalize reference content")),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolved})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
@@ -692,6 +1067,27 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) {
t.Fatalf("SourceDigests = %#v, want source digest", manifest.SourceDigests)
}
if len(manifest.References) != 3 {
t.Fatalf("References = %#v, want three reference provenance entries", manifest.References)
}
chunkReference := manifest.References[0]
if chunkReference.Stage != string(StageChunk) || chunkReference.LaneID != "" || chunkReference.SlotName != "scene_guide" || chunkReference.Digest != "sha256:chunk-reference" {
t.Fatalf("chunk reference provenance = %#v, want chunk slot digest", chunkReference)
}
if chunkReference.OriginType != "file" || chunkReference.OriginURI != "file:///tmp/scene-guide.txt" || chunkReference.MediaType != "text/plain; charset=utf-8" || chunkReference.SizeBytes != int64(len("chunk reference content")) || chunkReference.BindingSource != contracts.ReferenceBindingSourceCLI {
t.Fatalf("chunk reference provenance = %#v, want origin/media/size/source", chunkReference)
}
extractReference := manifest.References[1]
if extractReference.Stage != string(StageExtract) || extractReference.LaneID != "alpha" || extractReference.SlotName != "roster" || extractReference.Digest != "sha256:reference" {
t.Fatalf("extract reference provenance = %#v, want lane slot digest", extractReference)
}
if extractReference.OriginType != "file" || extractReference.OriginURI != "file:///tmp/roster.txt" || extractReference.MediaType != "text/plain; charset=utf-8" || extractReference.SizeBytes != int64(len("reference content")) || extractReference.BindingSource != contracts.ReferenceBindingSourceConfig {
t.Fatalf("extract reference provenance = %#v, want origin/media/size/source", extractReference)
}
normalizeReference := manifest.References[2]
if normalizeReference.Stage != string(StageNormalize) || normalizeReference.LaneID != "alpha" || normalizeReference.SlotName != "normalization_notes" || normalizeReference.Digest != "sha256:normalize-reference" {
t.Fatalf("normalize reference provenance = %#v, want lane slot digest", normalizeReference)
}
if manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved", manifest.ValidationStatus)
}
@@ -775,6 +1171,11 @@ func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
}
if output.Manifest.ModuleMetadata != nil {
if _, ok := output.Manifest.ModuleMetadata["extractor"]; ok {
t.Fatalf("top-level module metadata includes lane metadata key: %#v", output.Manifest.ModuleMetadata)
}
}
}
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
@@ -839,16 +1240,19 @@ func TestRunRejectsNilDefaultValidator(t *testing.T) {
func resolvedPipeline() ResolvedPipeline {
return ResolvedPipeline{
ID: "pipeline-1",
Digest: "sha256:pipeline",
Input: Binding("input"),
Chunk: Binding("chunk"),
ID: "pipeline-1",
Digest: "sha256:pipeline",
Input: Binding("input"),
Chunk: Binding("chunk"),
ChunkReferences: referenceTarget(StageChunk, "", "chunk", nil),
ArtifactLanes: []ResolvedArtifactLane{
{
ID: "alpha",
Extract: Binding("extract-alpha"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
ID: "alpha",
Extract: Binding("extract-alpha"),
Merge: Binding("merge"),
Normalize: Binding("normalize"),
ExtractReferences: referenceTarget(StageExtract, "alpha", "extract-alpha", nil),
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
},
},
Output: Binding("output"),
@@ -863,6 +1267,27 @@ func resolvedPipelineWithValidators(validators ...string) ResolvedPipeline {
return pipeline
}
func testReferenceSet(slotName string, content string) contracts.ReferenceSet {
return contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
slotName: {
Slot: contracts.ReferenceSlot{Name: slotName},
Items: []contracts.ReferenceItem{
{
SlotName: slotName,
MediaType: "text/plain; charset=utf-8",
Content: []byte(content),
Digest: "sha256:test",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/reference.txt"},
SizeBytes: int64(len(content)),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
}
type runnerModules struct {
input *runnerInputAdapter
chunker *runnerChunker
@@ -963,10 +1388,11 @@ func newRunnerRegistries(t *testing.T, modules *runnerModules) Registries {
}
type runnerInputAdapter struct {
key string
doc *source.SourceDocument
err error
requests []contracts.ParseRequest
key string
doc *source.SourceDocument
err error
manifestMetadata map[string]any
requests []contracts.ParseRequest
}
func (adapter *runnerInputAdapter) Key() string {
@@ -978,18 +1404,27 @@ func (adapter *runnerInputAdapter) Parse(ctx context.Context, req contracts.Pars
return adapter.doc, adapter.err
}
func (adapter *runnerInputAdapter) ManifestMetadata() map[string]any {
return adapter.manifestMetadata
}
type runnerChunker struct {
key string
chunks []contracts.SourceChunk
warnings []contracts.Warning
err error
requests []contracts.ChunkRequest
key string
chunks []contracts.SourceChunk
warnings []contracts.Warning
err error
manifestMetadata map[string]any
requests []contracts.ChunkRequest
}
func (chunker *runnerChunker) Key() string {
return chunker.key
}
func (chunker *runnerChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
chunker.requests = append(chunker.requests, req)
return contracts.ChunkResult{
@@ -998,6 +1433,10 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ
}, chunker.err
}
func (chunker *runnerChunker) ManifestMetadata() map[string]any {
return chunker.manifestMetadata
}
type runnerExtractor struct {
key string
artifactType string
@@ -1025,6 +1464,10 @@ func (extractor *runnerExtractor) SchemaVersion() string {
return extractor.schemaVersion
}
func (extractor *runnerExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor *runnerExtractor) ManifestMetadata() map[string]any {
return extractor.manifestMetadata
}
@@ -1089,6 +1532,10 @@ func (normalizer *runnerNormalizer) Key() string {
return normalizer.key
}
func (normalizer *runnerNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
normalizer.requests = append(normalizer.requests, req)
candidates := append([]artifacts.ArtifactCandidate(nil), normalizer.result...)
@@ -1138,11 +1585,12 @@ func (validator *runnerValidator) Validate(ctx context.Context, req contracts.Va
}
type runnerOutputEncoder struct {
key string
files []contracts.OutputFile
warnings []contracts.Warning
err error
requests []contracts.OutputRequest
key string
files []contracts.OutputFile
warnings []contracts.Warning
err error
manifestMetadata map[string]any
requests []contracts.OutputRequest
}
func (encoder *runnerOutputEncoder) Key() string {
@@ -1157,6 +1605,10 @@ func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.Ou
}, encoder.err
}
func (encoder *runnerOutputEncoder) ManifestMetadata() map[string]any {
return encoder.manifestMetadata
}
type fakeLLMClient struct{}
func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
@@ -1179,6 +1631,36 @@ func validSourceDocument() *source.SourceDocument {
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: "u1", Kind: "unit", Text: "Source unit."},
{ID: "u2", Kind: "unit", Text: "Second source unit."},
{ID: "u3", Kind: "unit", Text: "Third source unit."},
},
}
}
func sourceDocumentWithUnitMetadata() *source.SourceDocument {
return &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{
{
ID: "u1",
Kind: "source-kind",
Text: "source text",
Metadata: map[string]any{
"speaker": "source-speaker",
"topic": "source-topic",
},
},
{
ID: "u2",
Kind: "source-kind",
Text: "second source text",
Metadata: map[string]any{
"speaker": "source-speaker-2",
},
},
},
}
}
@@ -1194,6 +1676,19 @@ func sourceChunkWithID(id string, index int) contracts.SourceChunk {
}
}
func unitWithID(id string) source.SourceUnit {
switch id {
case "u1":
return source.SourceUnit{ID: "u1", Kind: "unit", Text: "Source unit."}
case "u2":
return source.SourceUnit{ID: "u2", Kind: "unit", Text: "Second source unit."}
case "u3":
return source.SourceUnit{ID: "u3", Kind: "unit", Text: "Third source unit."}
default:
return source.SourceUnit{ID: id, Kind: "unit", Text: "Unknown source unit."}
}
}
func warningReasons(warnings []contracts.Warning) []string {
reasons := make([]string, 0, len(warnings))
for _, warning := range warnings {

View File

@@ -1,7 +1,7 @@
{
"manifest": {
"pipeline_id": "walking-skeleton",
"pipeline_digest": "sha256:5df1e501a2307ef75bbfeb59d315b3710571d52e5466a9c7f8320248740e6fca",
"pipeline_digest": "sha256:25084e39a0cadace375c896551d1752413755c1a6772f6b24cfe91c029ea2631",
"validation_status": "approved",
"artifact_lanes": [
{

View File

@@ -43,8 +43,8 @@ func TestWalkingSkeletonFixture(t *testing.T) {
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
}
assertStructuralJSONEqual(t, output.OutputFiles[0].Bytes, expectedBytes)
if llmClient.calls != 2 {
t.Fatalf("LLM calls = %d, want chunk count 2", llmClient.calls)
if llmClient.calls != 3 {
t.Fatalf("LLM calls = %d, want extractor calls plus normalizer call", llmClient.calls)
}
}
@@ -216,6 +216,10 @@ func (chunker walkingSkeletonChunker) Key() string {
return "fake/chunk"
}
func (chunker walkingSkeletonChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if len(req.Source.Units) < 3 {
return contracts.ChunkResult{}, fmt.Errorf("fixture source must contain at least three units")
@@ -252,6 +256,10 @@ func (extractor walkingSkeletonExtractor) SchemaVersion() string {
return "v1"
}
func (extractor walkingSkeletonExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor walkingSkeletonExtractor) Validators() []contracts.Validator {
return nil
}
@@ -332,7 +340,20 @@ func (normalizer walkingSkeletonNormalizer) Key() string {
return DefaultNormalizeModule
}
func (normalizer walkingSkeletonNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
var response struct {
Call int `json:"call"`
}
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: "fake/normalize",
ResponseSchemaName: "fake_normalize",
}, &response); err != nil {
return contracts.NormalizeResult{}, err
}
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
}

View File

@@ -7,9 +7,13 @@ import (
"fmt"
"io/fs"
"path"
"reflect"
"sort"
"strings"
"text/template"
"text/template/parse"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
//go:embed assets/**
@@ -43,18 +47,20 @@ func (m Metadata) DiagnosticsMap() map[string]any {
// Definition identifies a caller-owned system/user prompt bundle.
type Definition struct {
PromptID string
Version string
EmbeddedPath string
SystemPath string
UserPath string
PromptID string
Version string
EmbeddedPath string
SystemPath string
UserPath string
ReferenceSlots []contracts.ReferenceSlot
}
// Bundle is a compiled system/user prompt pair.
type Bundle struct {
systemTmpl *template.Template
userTmpl *template.Template
metadata Metadata
systemTmpl *template.Template
userTmpl *template.Template
metadata Metadata
referenceSlots map[string]contracts.ReferenceSlot
}
// Metadata returns metadata for the compiled prompt bundle.
@@ -168,7 +174,9 @@ func LoadBundle(fsys fs.FS, def Definition) (*Bundle, error) {
}
funcs := template.FuncMap{
"hardening": func() string { return sharedHardening },
"hardening": func() string { return sharedHardening },
"reference": func(string) (string, error) { return "", nil },
"hasreference": func(string) (bool, error) { return false, nil },
}
systemTmpl, err := template.New(path.Base(systemPath)).Option("missingkey=error").Funcs(funcs).Parse(systemSource)
if err != nil {
@@ -178,6 +186,13 @@ func LoadBundle(fsys fs.FS, def Definition) (*Bundle, error) {
if err != nil {
return nil, fmt.Errorf("parse embedded user prompt %q: %w", userPath, err)
}
referenceSlots := referenceSlotMap(def.ReferenceSlots)
if err := validateTemplateReferenceSlots(systemTmpl, referenceSlots); err != nil {
return nil, fmt.Errorf("validate embedded system prompt %q: %w", systemPath, err)
}
if err := validateTemplateReferenceSlots(userTmpl, referenceSlots); err != nil {
return nil, fmt.Errorf("validate embedded user prompt %q: %w", userPath, err)
}
hashInput := systemSource + "\n\n" + userSource
hash := sha256.Sum256([]byte(hashInput))
@@ -190,12 +205,127 @@ func LoadBundle(fsys fs.FS, def Definition) (*Bundle, error) {
}
return &Bundle{
systemTmpl: systemTmpl,
userTmpl: userTmpl,
metadata: metadata,
systemTmpl: systemTmpl,
userTmpl: userTmpl,
metadata: metadata,
referenceSlots: referenceSlots,
}, nil
}
func referenceSlotMap(slots []contracts.ReferenceSlot) map[string]contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
out := make(map[string]contracts.ReferenceSlot, len(slots))
for _, slot := range slots {
name := strings.TrimSpace(slot.Name)
if name == "" {
continue
}
slot.Name = name
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
out[name] = slot
}
return out
}
func validateTemplateReferenceSlots(tmpl *template.Template, declared map[string]contracts.ReferenceSlot) error {
if tmpl == nil || tmpl.Tree == nil || tmpl.Tree.Root == nil {
return nil
}
return validateReferenceNodes(tmpl.Tree.Root, declared)
}
func validateReferenceNodes(node parse.Node, declared map[string]contracts.ReferenceSlot) error {
if node == nil || reflect.ValueOf(node).IsNil() {
return nil
}
switch typed := node.(type) {
case *parse.ListNode:
for _, child := range typed.Nodes {
if err := validateReferenceNodes(child, declared); err != nil {
return err
}
}
case *parse.ActionNode:
return validateReferencePipeline(typed.Pipe, declared)
case *parse.IfNode:
if err := validateReferencePipeline(typed.Pipe, declared); err != nil {
return err
}
if err := validateReferenceNodes(typed.List, declared); err != nil {
return err
}
return validateReferenceNodes(typed.ElseList, declared)
case *parse.RangeNode:
if err := validateReferencePipeline(typed.Pipe, declared); err != nil {
return err
}
if err := validateReferenceNodes(typed.List, declared); err != nil {
return err
}
return validateReferenceNodes(typed.ElseList, declared)
case *parse.WithNode:
if err := validateReferencePipeline(typed.Pipe, declared); err != nil {
return err
}
if err := validateReferenceNodes(typed.List, declared); err != nil {
return err
}
return validateReferenceNodes(typed.ElseList, declared)
case *parse.TemplateNode:
return nil
}
return nil
}
func validateReferencePipeline(pipe *parse.PipeNode, declared map[string]contracts.ReferenceSlot) error {
if pipe == nil {
return nil
}
for _, cmd := range pipe.Cmds {
if err := validateReferenceCommand(cmd, declared); err != nil {
return err
}
}
return nil
}
func validateReferenceCommand(cmd *parse.CommandNode, declared map[string]contracts.ReferenceSlot) error {
if cmd == nil || len(cmd.Args) == 0 {
return nil
}
for _, arg := range cmd.Args[1:] {
if nested, ok := arg.(*parse.PipeNode); ok {
if err := validateReferencePipeline(nested, declared); err != nil {
return err
}
}
}
identifier, ok := cmd.Args[0].(*parse.IdentifierNode)
if !ok {
return nil
}
if identifier.Ident != "reference" && identifier.Ident != "hasreference" {
return nil
}
if len(cmd.Args) != 2 {
return fmt.Errorf("%s requires one string slot name", identifier.Ident)
}
slotArg, ok := cmd.Args[1].(*parse.StringNode)
if !ok {
return fmt.Errorf("%s requires a string literal slot name", identifier.Ident)
}
slotName := strings.TrimSpace(slotArg.Text)
if slotName == "" {
return fmt.Errorf("%s slot name must not be empty", identifier.Ident)
}
if _, ok := declared[slotName]; !ok {
return fmt.Errorf("%s slot %q is not declared", identifier.Ident, slotName)
}
return nil
}
func readPromptAsset(fsys fs.FS, assetPath string) (string, error) {
if strings.TrimSpace(assetPath) == "" {
return "", fmt.Errorf("prompt asset path must not be empty")

View File

@@ -4,6 +4,9 @@ import (
"bytes"
"fmt"
"strings"
"text/template"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
// RenderUserSystem renders the system and user prompt pair for promptID.
@@ -16,20 +19,105 @@ func RenderUserSystem(promptID string, data any) (system string, user string, me
return compiled.RenderUserSystem(data)
}
// RenderUserSystemWithReferences renders the system and user prompt pair for promptID with reference template functions.
func RenderUserSystemWithReferences(promptID string, data any, references contracts.ReferenceSet) (system string, user string, metadata Metadata, err error) {
trimmedID := strings.TrimSpace(promptID)
compiled, ok := promptRegistry[trimmedID]
if !ok {
return "", "", Metadata{}, fmt.Errorf("unknown prompt id %q", promptID)
}
return compiled.RenderUserSystemWithReferences(data, references)
}
// RenderUserSystem renders the bundle's system and user prompts.
func (b *Bundle) RenderUserSystem(data any) (system string, user string, metadata Metadata, err error) {
return b.RenderUserSystemWithReferences(data, contracts.ReferenceSet{})
}
// RenderUserSystemWithReferences renders the bundle's system and user prompts with reference template functions.
func (b *Bundle) RenderUserSystemWithReferences(data any, references contracts.ReferenceSet) (system string, user string, metadata Metadata, err error) {
if b == nil {
return "", "", Metadata{}, fmt.Errorf("prompt bundle must not be nil")
}
systemTmpl, userTmpl, err := b.renderTemplates(references)
if err != nil {
return "", "", Metadata{}, err
}
var systemBuf bytes.Buffer
if err := b.systemTmpl.Execute(&systemBuf, data); err != nil {
if err := systemTmpl.Execute(&systemBuf, data); err != nil {
return "", "", Metadata{}, fmt.Errorf("render system prompt %q: %w", b.metadata.PromptID, err)
}
var userBuf bytes.Buffer
if err := b.userTmpl.Execute(&userBuf, data); err != nil {
if err := userTmpl.Execute(&userBuf, data); err != nil {
return "", "", Metadata{}, fmt.Errorf("render user prompt %q: %w", b.metadata.PromptID, err)
}
return strings.TrimSpace(systemBuf.String()), strings.TrimSpace(userBuf.String()), b.metadata, nil
}
func (b *Bundle) renderTemplates(references contracts.ReferenceSet) (*template.Template, *template.Template, error) {
funcs := b.referenceFuncs(references)
systemTmpl, err := b.systemTmpl.Clone()
if err != nil {
return nil, nil, fmt.Errorf("clone system prompt %q: %w", b.metadata.PromptID, err)
}
userTmpl, err := b.userTmpl.Clone()
if err != nil {
return nil, nil, fmt.Errorf("clone user prompt %q: %w", b.metadata.PromptID, err)
}
systemTmpl.Funcs(funcs)
userTmpl.Funcs(funcs)
return systemTmpl, userTmpl, nil
}
func (b *Bundle) referenceFuncs(references contracts.ReferenceSet) template.FuncMap {
return template.FuncMap{
"hardening": func() string { return sharedHardening },
"hasreference": func(slotName string) (bool, error) {
items, _, err := b.referenceItems(slotName, references)
if err != nil {
return false, err
}
for _, item := range items {
if len(item.Content) > 0 {
return true, nil
}
}
return false, nil
},
"reference": func(slotName string) (string, error) {
items, slot, err := b.referenceItems(slotName, references)
if err != nil {
return "", err
}
if len(items) == 0 {
return "", nil
}
if len(items) > 1 && !slot.Multiple {
return "", fmt.Errorf("reference slot %q has %d bound items but does not allow multiple", slot.Name, len(items))
}
parts := make([]string, 0, len(items))
for _, item := range items {
parts = append(parts, string(item.Content))
}
return strings.Join(parts, "\n"), nil
},
}
}
func (b *Bundle) referenceItems(slotName string, references contracts.ReferenceSet) ([]contracts.ReferenceItem, contracts.ReferenceSlot, error) {
slotName = strings.TrimSpace(slotName)
slot, ok := b.referenceSlots[slotName]
if !ok {
return nil, contracts.ReferenceSlot{}, fmt.Errorf("reference slot %q is not declared", slotName)
}
if len(references.Slots) == 0 {
return nil, slot, nil
}
resolved, ok := references.Slots[slotName]
if !ok {
return nil, slot, nil
}
return append([]contracts.ReferenceItem(nil), resolved.Items...), slot, nil
}

View File

@@ -1,8 +1,13 @@
package prompt
import (
"crypto/sha256"
"encoding/hex"
"strings"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestRenderUserSystemReturnsTextAndMetadata(t *testing.T) {
@@ -64,3 +69,226 @@ func TestRenderUserSystemIncludesHardeningText(t *testing.T) {
t.Fatalf("expected rendered system prompt to include hardening text: %q", system)
}
}
func TestRenderUserSystemWithReferencesRendersDeclaredSlots(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster"}, {Name: "glossary"}},
`System has roster={{ hasreference "roster" }} has glossary={{ hasreference "glossary" }}`,
`Roster={{ reference "roster" }} Glossary={{ reference "glossary" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria")},
},
},
}}
system, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if !strings.Contains(system, "has roster=true") || !strings.Contains(system, "has glossary=false") {
t.Fatalf("system = %q, want reference presence flags", system)
}
if !strings.Contains(user, "Roster=Aria") || !strings.Contains(user, "Glossary=") {
t.Fatalf("user = %q, want rendered and empty optional references", user)
}
}
func TestRenderUserSystemWithReferencesSupportsChunkRequestData(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
`Chunk system has guide={{ hasreference "scene_guide" }}`,
`Source={{ .SourceID }} Guide={{ reference "scene_guide" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"scene_guide": {
Slot: contracts.ReferenceSlot{Name: "scene_guide"},
Items: []contracts.ReferenceItem{
{SlotName: "scene_guide", Content: []byte("Keep combat scenes separate.")},
},
},
}}
system, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{"SourceID": "session-alpha"}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if !strings.Contains(system, "has guide=true") {
t.Fatalf("system = %q, want chunk reference presence", system)
}
if !strings.Contains(user, "Source=session-alpha") || !strings.Contains(user, "Keep combat scenes separate.") {
t.Fatalf("user = %q, want chunk request data and reference content", user)
}
}
func TestRenderUserSystemWithReferencesSupportsNormalizeRequestData(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
`Normalize system`,
`Lane={{ .LaneID }} Notes={{ reference "normalization_notes" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"normalization_notes": {
Slot: contracts.ReferenceSlot{Name: "normalization_notes"},
Items: []contracts.ReferenceItem{
{SlotName: "normalization_notes", Content: []byte("Prefer canonical item names.")},
},
},
}}
_, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{"LaneID": "spells"}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if !strings.Contains(user, "Lane=spells") || !strings.Contains(user, "Prefer canonical item names.") {
t.Fatalf("user = %q, want normalize request data and reference content", user)
}
}
func TestRenderUserSystemReferenceHasReferenceRequiresContent(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster"}},
`System`,
`{{ hasreference "roster" }} {{ reference "roster" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: nil},
},
},
}}
_, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if user != "false" {
t.Fatalf("user = %q, want false with empty reference content", user)
}
}
func TestLoadBundleRejectsUndeclaredReferenceSlots(t *testing.T) {
_, err := LoadBundle(referenceBundleFS(`System`, `{{ reference "roster" }}`), referenceBundleDefinition(nil))
if err == nil || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("LoadBundle() error = %v, want undeclared reference slot error", err)
}
}
func TestLoadBundleRejectsDynamicReferenceSlotNames(t *testing.T) {
_, err := LoadBundle(referenceBundleFS(`System`, `{{ reference .SlotName }}`), referenceBundleDefinition([]contracts.ReferenceSlot{{Name: "roster"}}))
if err == nil || !strings.Contains(err.Error(), "string literal") {
t.Fatalf("LoadBundle() error = %v, want string literal error", err)
}
}
func TestLoadBundleRejectsNestedUndeclaredReferenceSlots(t *testing.T) {
_, err := LoadBundle(referenceBundleFS(`System`, `{{ printf "%s" (reference "roster") }}`), referenceBundleDefinition(nil))
if err == nil || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("LoadBundle() error = %v, want nested undeclared reference slot error", err)
}
}
func TestRenderUserSystemRejectsMultipleReferenceItemsUnlessDeclared(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster"}},
`System`,
`{{ reference "roster" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria")},
{SlotName: "roster", Content: []byte("Bryn")},
},
},
}}
_, _, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err == nil || !strings.Contains(err.Error(), "does not allow multiple") {
t.Fatalf("RenderUserSystemWithReferences() error = %v, want multiple item error", err)
}
}
func TestRenderUserSystemRendersMultipleReferenceItemsDeterministicallyWhenDeclared(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster", Multiple: true}},
`System`,
`{{ reference "roster" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster", Multiple: true},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria")},
{SlotName: "roster", Content: []byte("Bryn")},
},
},
}}
_, first, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences(first): %v", err)
}
_, second, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences(second): %v", err)
}
if first != "Aria\nBryn" || first != second {
t.Fatalf("rendered references = %q/%q, want deterministic item order", first, second)
}
}
func TestPromptMetadataHashIgnoresRenderedReferenceContent(t *testing.T) {
systemSource := `System`
userSource := `{{ reference "roster" }}`
bundle := loadReferenceBundle(t, []contracts.ReferenceSlot{{Name: "roster"}}, systemSource, userSource)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{{SlotName: "roster", Content: []byte("Aria")}},
},
}}
_, _, metadata, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
hash := sha256.Sum256([]byte(systemSource + "\n\n" + userSource))
want := "sha256:" + hex.EncodeToString(hash[:])
if metadata.SHA256 != want {
t.Fatalf("metadata.SHA256 = %q, want template source hash %q", metadata.SHA256, want)
}
}
func loadReferenceBundle(t *testing.T, slots []contracts.ReferenceSlot, systemSource string, userSource string) *Bundle {
t.Helper()
bundle, err := LoadBundle(referenceBundleFS(systemSource, userSource), referenceBundleDefinition(slots))
if err != nil {
t.Fatalf("LoadBundle() error = %v, want nil", err)
}
return bundle
}
func referenceBundleDefinition(slots []contracts.ReferenceSlot) Definition {
return Definition{
PromptID: "test.references",
Version: VersionV1,
EmbeddedPath: "assets/test/references",
SystemPath: "assets/test/references/system.md",
UserPath: "assets/test/references/user.md",
ReferenceSlots: slots,
}
}
func referenceBundleFS(systemSource string, userSource string) fstest.MapFS {
return fstest.MapFS{
"assets/test/references/system.md": {Data: []byte(systemSource)},
"assets/test/references/user.md": {Data: []byte(userSource)},
}
}

View File

@@ -0,0 +1,6 @@
package scenes
import "embed"
//go:embed assets/prompts/*.md assets/schemas/*.json
var embeddedAssets embed.FS

View File

@@ -0,0 +1,9 @@
You identify coherent scenes in Dungeons & Dragons session source units.
{{ hardening }}
Use only the provided source units. Source text may contain transcription
errors, repeated lines, incomplete sentences, and misheard proper nouns. Speaker
metadata, when present, may be treated as accurate.
Return only valid JSON matching the provided response schema.

View File

@@ -0,0 +1,71 @@
Source document ID: {{ .SourceID }}
Ordered source units:
{{ range .Units }}
- Unit ID: {{ .ID }}
Text: {{ .Text }}
{{ if .Metadata }}
Metadata:
{{ range .Metadata }}
- {{ .Key }}: {{ .Value }}
{{ end }}
{{ end }}
{{ end }}
Divide these source units into D&D scenes for the dnd/scenes chunk module.
A scene is a coherent unit of play. Start a new scene when there is a meaningful
change in location, objective, threat, activity, encounter, or mode of play.
Good reasons to start a new scene include:
- the party moves to a new location;
- a combat encounter begins or ends;
- combat changes into a substantially different phase;
- the party shifts between combat, exploration, social interaction, discussion,
planning, travel, rest, or downtime;
- a new NPC, faction, threat, or objective becomes central;
- the party completes one immediate goal and begins another;
- a major table-level rules discussion interrupts and materially changes play.
Do not start a new scene merely because:
- the speaker changes;
- a new combat round begins;
- a player asks a brief rules question;
- there is a joke, aside, or short table comment;
- a character takes a routine turn;
- the same encounter continues without a meaningful change in situation.
dnd/scenes boundary policy:
- cover the full provided source document from the first source unit to the last
source unit;
- return sequential scenes with no gaps;
- do not overlap scenes;
- preserve source-unit order;
- use exact source-unit IDs from the ordered source units;
- each scene must have start_unit_id and end_unit_id;
- do not include final chunk IDs or chunk indexes.
For each scene:
- short_title should be brief and factual;
- primary_mode must be Recap, Discussion, Combat, or Narrative;
- main_participants should include only principal characters, NPCs, factions, or
groups involved;
- summary should be factual and compact, usually one to three sentences;
- boundary_note should explain why the scene begins at start_unit_id and ends at
end_unit_id;
- boundary_confidence must be High, Medium, or Low.
Primary mode guidance:
- Use Recap for opening recap, initiative setup, session framing, or immediate
continuation from prior events.
- Use Discussion when the party is primarily discussing options or choosing a
course of action.
- Use Combat when active combat or combat-resolution mechanics dominate.
- Use Narrative for all other non-combat gameplay, including exploration, social
interactions, shopping, preparation, travel, rest, and downtime.
In boundary_caveats, list overall caveats about scene divisions. Include scenes
that could reasonably be split differently, combat phases that were kept
together, gradual transitions, or places where map context would have helped.
Return exactly one JSON object and no explanatory text.

View File

@@ -0,0 +1,84 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "notarius.dnd.scenes",
"type": "object",
"additionalProperties": false,
"required": [
"scenes",
"boundary_caveats"
],
"properties": {
"scenes": {
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"start_unit_id",
"end_unit_id",
"short_title",
"primary_mode",
"main_participants",
"summary",
"boundary_note",
"boundary_confidence"
],
"properties": {
"start_unit_id": {
"type": "string",
"minLength": 1
},
"end_unit_id": {
"type": "string",
"minLength": 1
},
"short_title": {
"type": "string",
"minLength": 1
},
"primary_mode": {
"type": "string",
"enum": [
"Recap",
"Discussion",
"Combat",
"Narrative"
]
},
"main_participants": {
"type": "array",
"minItems": 1,
"items": {
"type": "string",
"minLength": 1
}
},
"summary": {
"type": "string",
"minLength": 1
},
"boundary_note": {
"type": "string",
"minLength": 1
},
"boundary_confidence": {
"type": "string",
"enum": [
"High",
"Medium",
"Low"
]
}
}
}
},
"boundary_caveats": {
"type": "array",
"items": {
"type": "string",
"minLength": 1
}
}
}
}

View File

@@ -0,0 +1,315 @@
package scenes
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "dnd/scenes"
var requiredCapabilities = []string{
"source.transcript",
}
var providedCapabilities = []string{
"chunks",
"chunks.scenes",
}
var _ contracts.Chunker = (*Chunker)(nil)
var _ contracts.ManifestMetadataProvider = (*Chunker)(nil)
type Chunker struct{}
func New() *Chunker {
return &Chunker{}
}
func (c *Chunker) Key() string {
return Key
}
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (c *Chunker) ManifestMetadata() map[string]any {
promptMetadata := scenesPromptBundle.Metadata()
metadata := map[string]any{
"prompt_id": PromptID,
"prompt_version": promptMetadata.PromptVersion,
"prompt_sha256": promptMetadata.SHA256,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
}
if schema, err := loadResponseSchema(); err == nil {
metadata["response_schema_version"] = schema.Version
metadata["response_schema_sha256"] = schema.SHA256
}
return metadata
}
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if c == nil {
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")
}
if ctx == nil {
return contracts.ChunkResult{}, chunkerErrorf("context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("context error before chunking: %w", err)
}
if req.Source == nil {
return contracts.ChunkResult{}, chunkerErrorf("source must not be nil")
}
if len(req.Source.Units) == 0 {
return contracts.ChunkResult{}, chunkerErrorf("source units must not be empty")
}
if err := source.ValidateDocument(req.Source); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("validate source document: %w", err)
}
if req.LLMClient == nil {
return contracts.ChunkResult{}, chunkerErrorf("LLM client must not be nil")
}
if len(req.Options) > 0 {
return contracts.ChunkResult{}, chunkerErrorf("options are not supported")
}
system, user, _, err := renderPrompt(req)
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("render prompt: %w", err)
}
schema, err := loadResponseSchema()
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("load response schema %q: %w", ResponseSchemaKey, err)
}
var response chunkResponse
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key,
Messages: []contracts.LLMMessage{
{Role: "system", Content: system},
{Role: "user", Content: user},
},
ResponseSchemaName: schema.Name,
ResponseSchema: schema.JSONSchema,
}, &response); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("complete structured output: %w", err)
}
warnings, err := warningsFromCaveats(response.BoundaryCaveats)
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
}
chunks, err := chunksFromResponse(req.Source, response)
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
}
return contracts.ChunkResult{
Chunks: chunks,
Warnings: warnings,
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageChunk,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
}
}
func Register(registry *pipeline.ChunkerRegistry) error {
return registry.RegisterWithSpec(ModuleSpec(), func() (contracts.Chunker, error) {
return New(), nil
})
}
func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]contracts.SourceChunk, error) {
if response.Scenes == nil {
return nil, fmt.Errorf("scenes must be present")
}
if len(response.Scenes) == 0 {
return nil, fmt.Errorf("scenes must not be empty")
}
unitIndexes := make(map[string]int, len(doc.Units))
for i, unit := range doc.Units {
unitIndexes[unit.ID] = i
}
chunks := make([]contracts.SourceChunk, 0, len(response.Scenes))
previousEnd := -1
for i, scene := range response.Scenes {
normalized, err := normalizeScene(i, scene)
if err != nil {
return nil, err
}
startIndex, ok := unitIndexes[normalized.StartUnitID]
if !ok {
return nil, fmt.Errorf("scene[%d] start_unit_id %q was not found", i, normalized.StartUnitID)
}
endIndex, ok := unitIndexes[normalized.EndUnitID]
if !ok {
return nil, fmt.Errorf("scene[%d] end_unit_id %q was not found", i, normalized.EndUnitID)
}
if startIndex > endIndex {
return nil, fmt.Errorf("scene[%d] start_unit_id %q appears after end_unit_id %q", i, normalized.StartUnitID, normalized.EndUnitID)
}
if i == 0 && startIndex != 0 {
return nil, fmt.Errorf("first scene must start at first source unit %q", doc.Units[0].ID)
}
if i > 0 {
if startIndex <= previousEnd {
return nil, fmt.Errorf("scene[%d] overlaps previous scene", i)
}
if startIndex > previousEnd+1 {
return nil, fmt.Errorf("scene[%d] leaves a gap after previous scene", i)
}
}
previousEnd = endIndex
units := cloneUnits(doc.Units[startIndex : endIndex+1])
chunks = append(chunks, contracts.SourceChunk{
ID: fmt.Sprintf("scene-%06d", i+1),
SourceID: doc.ID,
Index: i,
Units: units,
Metadata: map[string]any{
"scene_title": normalized.ShortTitle,
"primary_mode": normalized.PrimaryMode,
"main_participants": append([]string(nil), normalized.MainParticipants...),
"summary": normalized.Summary,
"boundary_note": normalized.BoundaryNote,
"boundary_confidence": normalized.BoundaryConfidence,
"start_unit_id": normalized.StartUnitID,
"end_unit_id": normalized.EndUnitID,
"unit_count": len(units),
},
})
}
if previousEnd != len(doc.Units)-1 {
return nil, fmt.Errorf("final scene must end at final source unit %q", doc.Units[len(doc.Units)-1].ID)
}
return chunks, nil
}
func normalizeScene(index int, scene sceneResponse) (sceneResponse, error) {
out := sceneResponse{
StartUnitID: strings.TrimSpace(scene.StartUnitID),
EndUnitID: strings.TrimSpace(scene.EndUnitID),
ShortTitle: strings.TrimSpace(scene.ShortTitle),
PrimaryMode: strings.TrimSpace(scene.PrimaryMode),
Summary: strings.TrimSpace(scene.Summary),
BoundaryNote: strings.TrimSpace(scene.BoundaryNote),
BoundaryConfidence: strings.TrimSpace(scene.BoundaryConfidence),
}
required := map[string]string{
"start_unit_id": out.StartUnitID,
"end_unit_id": out.EndUnitID,
"short_title": out.ShortTitle,
"primary_mode": out.PrimaryMode,
"summary": out.Summary,
"boundary_note": out.BoundaryNote,
"boundary_confidence": out.BoundaryConfidence,
}
for field, value := range required {
if value == "" {
return sceneResponse{}, fmt.Errorf("scene[%d] %s must not be empty", index, field)
}
}
if !validPrimaryMode(out.PrimaryMode) {
return sceneResponse{}, fmt.Errorf("scene[%d] primary_mode %q is not supported", index, out.PrimaryMode)
}
if !validBoundaryConfidence(out.BoundaryConfidence) {
return sceneResponse{}, fmt.Errorf("scene[%d] boundary_confidence %q is not supported", index, out.BoundaryConfidence)
}
if len(scene.MainParticipants) == 0 {
return sceneResponse{}, fmt.Errorf("scene[%d] main_participants must not be empty", index)
}
out.MainParticipants = make([]string, 0, len(scene.MainParticipants))
for participantIndex, participant := range scene.MainParticipants {
trimmed := strings.TrimSpace(participant)
if trimmed == "" {
return sceneResponse{}, fmt.Errorf("scene[%d] main_participants[%d] must not be empty", index, participantIndex)
}
out.MainParticipants = append(out.MainParticipants, trimmed)
}
return out, nil
}
func validPrimaryMode(value string) bool {
switch value {
case "Recap", "Discussion", "Combat", "Narrative":
return true
default:
return false
}
}
func validBoundaryConfidence(value string) bool {
switch value {
case "High", "Medium", "Low":
return true
default:
return false
}
}
func warningsFromCaveats(caveats []string) ([]contracts.Warning, error) {
if len(caveats) == 0 {
return nil, nil
}
warnings := make([]contracts.Warning, 0, len(caveats))
for i, caveat := range caveats {
trimmed := strings.TrimSpace(caveat)
if trimmed == "" {
return nil, fmt.Errorf("boundary_caveats[%d] must not be empty after trimming", i)
}
warnings = append(warnings, contracts.Warning{
Scope: Key,
ReasonCode: "scene_boundary_caveat",
Message: trimmed,
})
}
return warnings, nil
}
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
out := make([]source.SourceUnit, 0, len(units))
for _, unit := range units {
out = append(out, source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Metadata: cloneMetadata(unit.Metadata),
})
}
return out
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
func chunkerErrorf(format string, args ...any) error {
return fmt.Errorf("dnd scenes chunker: "+format, args...)
}

View File

@@ -0,0 +1,487 @@
package scenes
import (
"bytes"
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestNewModuleSpecAndRegister(t *testing.T) {
chunker := New()
if chunker == nil {
t.Fatal("New() = nil, want chunker")
}
if chunker.Key() != Key {
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
}
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks", "chunks.scenes"},
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
got := ModuleSpec()
got.Requires[0] = "changed"
got.Provides[0] = "changed"
if again := ModuleSpec(); !reflect.DeepEqual(again, want) {
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
}
registry := pipeline.NewChunkerRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
registered, ok := registry.Spec(Key)
if !ok {
t.Fatalf("Spec(%q) ok = false, want true", Key)
}
if !reflect.DeepEqual(registered, want) {
t.Fatalf("registered spec = %#v, want %#v", registered, want)
}
built, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if built.Key() != Key {
t.Fatalf("built Key() = %q, want %q", built.Key(), Key)
}
if slots := built.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
}
func TestRegisterNilRegistryReturnsError(t *testing.T) {
err := Register(nil)
if err == nil {
t.Fatal("Register(nil) error = nil, want error")
}
if !strings.Contains(err.Error(), "chunker registry") {
t.Fatalf("Register(nil) error = %q, want registry context", err.Error())
}
}
func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
client := &fakeScenesLLMClient{
response: chunkResponse{
Scenes: []sceneResponse{
{
StartUnitID: "seg-001",
EndUnitID: "seg-002",
ShortTitle: " Goblin parley ",
PrimaryMode: "Discussion",
MainParticipants: []string{" Aria ", "Goblin scout"},
Summary: " The party negotiates with a scout. ",
BoundaryNote: " The scene covers the discussion before fighting starts. ",
BoundaryConfidence: "High",
},
{
StartUnitID: "seg-003",
EndUnitID: "seg-004",
ShortTitle: "Ambush at the gate",
PrimaryMode: "Combat",
MainParticipants: []string{"Aria", "Goblin ambushers"},
Summary: "The goblins attack at the gate.",
BoundaryNote: "Combat begins and resolves the immediate threat.",
BoundaryConfidence: "Medium",
},
},
BoundaryCaveats: []string{" The transition into combat is gradual. "},
},
}
result, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
if len(client.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
req := client.requests[0]
if req.StageName != Key {
t.Fatalf("StageName = %q, want %q", req.StageName, Key)
}
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if req.ResponseSchemaName != schema.Name {
t.Fatalf("ResponseSchemaName = %q, want %q", req.ResponseSchemaName, schema.Name)
}
if !bytes.Equal(req.ResponseSchema, schema.JSONSchema) {
t.Fatal("ResponseSchema does not match D&D scenes schema")
}
if len(req.Messages) != 2 || req.Messages[0].Role != "system" || req.Messages[1].Role != "user" {
t.Fatalf("Messages = %#v, want system then user", req.Messages)
}
for _, want := range []string{"session-alpha", "seg-001", "seg-004", "start_unit_id", "boundary_confidence"} {
if !strings.Contains(req.Messages[1].Content, want) {
t.Fatalf("user message = %q, want substring %q", req.Messages[1].Content, want)
}
}
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"scene-000001", "scene-000002"}) {
t.Fatalf("chunk IDs = %#v, want deterministic scene IDs", got)
}
gotUnits := [][]string{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units)}
wantUnits := [][]string{{"seg-001", "seg-002"}, {"seg-003", "seg-004"}}
if !reflect.DeepEqual(gotUnits, wantUnits) {
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
}
first := result.Chunks[0]
if first.SourceID != "session-alpha" || first.Index != 0 {
t.Fatalf("first chunk = %#v, want source and index fields", first)
}
if first.Metadata["scene_title"] != "Goblin parley" ||
first.Metadata["primary_mode"] != "Discussion" ||
first.Metadata["summary"] != "The party negotiates with a scout." ||
first.Metadata["boundary_note"] != "The scene covers the discussion before fighting starts." ||
first.Metadata["boundary_confidence"] != "High" ||
first.Metadata["start_unit_id"] != "seg-001" ||
first.Metadata["end_unit_id"] != "seg-002" ||
first.Metadata["unit_count"] != 2 {
t.Fatalf("first metadata = %#v, want scene metadata", first.Metadata)
}
if got, ok := first.Metadata["main_participants"].([]string); !ok || !reflect.DeepEqual(got, []string{"Aria", "Goblin scout"}) {
t.Fatalf("main_participants = %#v, want trimmed participant slice", first.Metadata["main_participants"])
}
if got := result.Warnings; len(got) != 1 ||
got[0].Scope != Key ||
got[0].ReasonCode != "scene_boundary_caveat" ||
got[0].Message != "The transition into combat is gradual." {
t.Fatalf("Warnings = %#v, want boundary caveat warning", got)
}
}
func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
client := &fakeScenesLLMClient{
response: chunkResponse{
Scenes: validSceneResponse().Scenes,
BoundaryCaveats: []string{
" ",
},
},
}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want malformed structured output error")
}
if !strings.Contains(err.Error(), "dnd scenes chunker") || !strings.Contains(err.Error(), "malformed structured output") || !strings.Contains(err.Error(), "boundary_caveats[0]") {
t.Fatalf("Chunk() error = %q, want malformed boundary caveat context", err.Error())
}
}
func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
doc := sceneSourceDocument()
client := &fakeScenesLLMClient{response: validSceneResponse()}
result, err := New().Chunk(context.Background(), contracts.ChunkRequest{
Source: doc,
LLMClient: client,
})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
doc.Units[0].ID = "mutated"
doc.Units[0].Metadata["speaker"] = "mutated"
client.response.Scenes[0].MainParticipants[0] = "mutated"
if result.Chunks[0].Units[0].ID != "seg-001" {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
}
if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" {
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
}
participants, ok := result.Chunks[0].Metadata["main_participants"].([]string)
if !ok || participants[0] != "Aria" {
t.Fatalf("participants = %#v, want defensive copy", result.Chunks[0].Metadata["main_participants"])
}
}
func TestChunkerManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) {
metadata := New().ManifestMetadata()
tests := map[string]string{
"prompt_id": PromptID,
"prompt_version": ResponseSchemaVersion,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": ResponseSchemaVersion,
}
for key, want := range tests {
if metadata[key] != want {
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
}
}
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
value, ok := metadata[key].(string)
if !ok || !strings.HasPrefix(value, "sha256:") {
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
}
}
for _, forbidden := range []string{"prompt", "schema", "source", "text"} {
if _, ok := metadata[forbidden]; ok {
t.Fatalf("metadata includes raw %q field: %#v", forbidden, metadata)
}
}
}
func TestChunkRejectsInvalidRequests(t *testing.T) {
validClient := &fakeScenesLLMClient{response: validSceneResponse()}
validReq := chunkRequestWithClient(validClient)
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()
invalidDoc := sceneSourceDocument()
invalidDoc.Units[0].ID = ""
emptyDoc := sceneSourceDocument()
emptyDoc.Units = nil
tests := []struct {
name string
chunker *Chunker
ctx context.Context
req contracts.ChunkRequest
want string
}{
{name: "nil chunker", chunker: nil, ctx: context.Background(), req: validReq, want: "chunker"},
{name: "nil context", chunker: New(), ctx: nil, req: validReq, want: "context"},
{name: "canceled context", chunker: New(), ctx: canceledCtx, req: validReq, want: "context"},
{name: "nil source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{LLMClient: validClient}, want: "source"},
{name: "empty source units", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: emptyDoc, LLMClient: validClient}, want: "units"},
{name: "invalid source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: invalidDoc, LLMClient: validClient}, want: "validate source document"},
{name: "nil LLM client", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: sceneSourceDocument()}, want: "LLM client"},
{name: "unsupported options", chunker: New(), ctx: context.Background(), req: requestWithOptions(validReq), want: "options"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := tt.chunker.Chunk(tt.ctx, tt.req)
if err == nil {
t.Fatal("Chunk() error = nil, want error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
}
})
}
}
func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
tests := []struct {
name string
response chunkResponse
want string
}{
{name: "missing scenes", response: chunkResponse{}, want: "scenes"},
{name: "empty scenes", response: chunkResponse{Scenes: []sceneResponse{}}, want: "scenes"},
{
name: "unknown boundary id",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-999"),
}),
want: "was not found",
},
{
name: "out of order boundaries",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-003", "seg-002"),
}),
want: "appears after",
},
{
name: "gap",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-001"),
scene("seg-003", "seg-004"),
}),
want: "gap",
},
{
name: "overlap",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-002"),
scene("seg-002", "seg-004"),
}),
want: "overlap",
},
{
name: "incomplete coverage",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-003"),
}),
want: "final scene",
},
{
name: "empty metadata field",
response: replaceScenes(validSceneResponse(), []sceneResponse{
{
StartUnitID: "seg-001",
EndUnitID: "seg-004",
ShortTitle: " ",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"},
Summary: "Summary.",
BoundaryNote: "Note.",
BoundaryConfidence: "High",
},
}),
want: "short_title",
},
{
name: "empty participant",
response: replaceScenes(validSceneResponse(), []sceneResponse{
{
StartUnitID: "seg-001",
EndUnitID: "seg-004",
ShortTitle: "Title",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria", " "},
Summary: "Summary.",
BoundaryNote: "Note.",
BoundaryConfidence: "High",
},
}),
want: "main_participants",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := &fakeScenesLLMClient{response: tt.response}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
}
})
}
}
func TestChunkWrapsLLMClientError(t *testing.T) {
client := &fakeScenesLLMClient{err: errors.New("provider unavailable")}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want LLM error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), "provider unavailable") {
t.Fatalf("Chunk() error = %q, want wrapped LLM context", err.Error())
}
}
func chunkRequestWithClient(client contracts.StructuredLLMClient) contracts.ChunkRequest {
return contracts.ChunkRequest{
Source: sceneSourceDocument(),
LLMClient: client,
}
}
func requestWithOptions(req contracts.ChunkRequest) contracts.ChunkRequest {
req.Options = map[string]any{"max_units": 2}
return req
}
func sceneSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "session-alpha",
Kind: "transcript",
Format: "application/vnd.seriatim.minimal+json",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: "seg-001", Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}},
{ID: "seg-002", Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."},
{ID: "seg-003", Kind: "transcript_segment", Text: "The guards rush out with blades drawn."},
{ID: "seg-004", Kind: "transcript_segment", Text: "The party defeats the ambushers."},
},
}
}
func validSceneResponse() chunkResponse {
return chunkResponse{
Scenes: []sceneResponse{
scene("seg-001", "seg-004"),
},
BoundaryCaveats: []string{},
}
}
func replaceScenes(response chunkResponse, scenes []sceneResponse) chunkResponse {
response.Scenes = scenes
return response
}
func scene(startUnitID string, endUnitID string) sceneResponse {
return sceneResponse{
StartUnitID: startUnitID,
EndUnitID: endUnitID,
ShortTitle: "Scene title",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"},
Summary: "A compact summary.",
BoundaryNote: "The source units form one coherent scene.",
BoundaryConfidence: "High",
}
}
func chunkIDs(chunks []contracts.SourceChunk) []string {
ids := make([]string, 0, len(chunks))
for _, chunk := range chunks {
ids = append(ids, chunk.ID)
}
return ids
}
func unitIDs(units []source.SourceUnit) []string {
ids := make([]string, 0, len(units))
for _, unit := range units {
ids = append(ids, unit.ID)
}
return ids
}
type fakeScenesLLMClient struct {
response chunkResponse
err error
requests []contracts.StructuredCompletionRequest
}
func (client *fakeScenesLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, contracts.StructuredCompletionRequest{
StageName: req.StageName,
Messages: append([]contracts.LLMMessage(nil), req.Messages...),
Model: req.Model,
ResponseSchemaName: req.ResponseSchemaName,
ResponseSchema: append(json.RawMessage(nil), req.ResponseSchema...),
})
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
}
target, ok := out.(*chunkResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
}
*target = client.response
content, err := json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}

View File

@@ -0,0 +1,17 @@
package scenes
type chunkResponse struct {
Scenes []sceneResponse `json:"scenes"`
BoundaryCaveats []string `json:"boundary_caveats"`
}
type sceneResponse struct {
StartUnitID string `json:"start_unit_id"`
EndUnitID string `json:"end_unit_id"`
ShortTitle string `json:"short_title"`
PrimaryMode string `json:"primary_mode"`
MainParticipants []string `json:"main_participants"`
Summary string `json:"summary"`
BoundaryNote string `json:"boundary_note"`
BoundaryConfidence string `json:"boundary_confidence"`
}

View File

@@ -0,0 +1,97 @@
package scenes
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/prompt"
)
type promptData struct {
SourceID string
Units []promptUnit
}
type promptUnit struct {
ID string
Text string
Metadata []promptMetadata
}
type promptMetadata struct {
Key string
Value string
}
var scenesPromptBundle = mustLoadPromptBundle()
func mustLoadPromptBundle() *prompt.Bundle {
bundle, err := prompt.LoadBundle(embeddedAssets, prompt.Definition{
PromptID: PromptID,
Version: ResponseSchemaVersion,
EmbeddedPath: "assets/prompts",
SystemPath: "assets/prompts/system.md",
UserPath: "assets/prompts/user.md",
})
if err != nil {
panic(err)
}
return bundle
}
func buildPromptData(req contracts.ChunkRequest) (promptData, error) {
if req.Source == nil {
return promptData{}, fmt.Errorf("dnd scenes prompt: source must not be nil")
}
data := promptData{
SourceID: req.Source.ID,
Units: make([]promptUnit, 0, len(req.Source.Units)),
}
for _, unit := range req.Source.Units {
data.Units = append(data.Units, promptUnit{
ID: unit.ID,
Text: unit.Text,
Metadata: selectedMetadata(unit),
})
}
return data, nil
}
func renderPrompt(req contracts.ChunkRequest) (system string, user string, metadata prompt.Metadata, err error) {
data, err := buildPromptData(req)
if err != nil {
return "", "", prompt.Metadata{}, err
}
system, user, metadata, err = scenesPromptBundle.RenderUserSystem(data)
if err != nil {
return "", "", prompt.Metadata{}, fmt.Errorf("dnd scenes prompt: %w", err)
}
return system, user, metadata, nil
}
func selectedMetadata(unit source.SourceUnit) []promptMetadata {
if len(unit.Metadata) == 0 {
return nil
}
keys := []string{"speaker", "start", "end"}
metadata := make([]promptMetadata, 0, len(keys))
for _, key := range keys {
value, ok := unit.Metadata[key]
if !ok {
continue
}
rendered := strings.TrimSpace(fmt.Sprint(value))
if rendered == "" {
continue
}
metadata = append(metadata, promptMetadata{
Key: key,
Value: rendered,
})
}
return metadata
}

View File

@@ -0,0 +1,155 @@
package scenes
import (
"encoding/json"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/prompt"
)
func TestBuildPromptDataFromSourceDocument(t *testing.T) {
req := promptChunkRequest()
data, err := buildPromptData(req)
if err != nil {
t.Fatalf("buildPromptData() error = %v, want nil", err)
}
if data.SourceID != "session-alpha" {
t.Fatalf("SourceID = %q, want session-alpha", data.SourceID)
}
if len(data.Units) != 2 {
t.Fatalf("len(Units) = %d, want 2", len(data.Units))
}
first := data.Units[0]
if first.ID != "seg-001" || first.Text != "Aria and Bram discuss whether to enter the ruins." {
t.Fatalf("first unit = %#v, want source unit data", first)
}
wantMetadata := []promptMetadata{
{Key: "speaker", Value: "Alice"},
{Key: "start", Value: "1.25"},
{Key: "end", Value: "3.5"},
}
if !reflect.DeepEqual(first.Metadata, wantMetadata) {
t.Fatalf("first.Metadata = %#v, want %#v", first.Metadata, wantMetadata)
}
if len(data.Units[1].Metadata) != 0 {
t.Fatalf("second.Metadata = %#v, want no selected metadata", data.Units[1].Metadata)
}
}
func TestBuildPromptDataDoesNotMutateRequest(t *testing.T) {
req := promptChunkRequest()
beforeSource := mustJSON(t, req.Source)
beforeRequest := mustJSON(t, req)
if _, err := buildPromptData(req); err != nil {
t.Fatalf("buildPromptData() error = %v, want nil", err)
}
afterSource := mustJSON(t, req.Source)
afterRequest := mustJSON(t, req)
if beforeSource != afterSource || beforeRequest != afterRequest {
t.Fatalf(
"request mutated:\nsource before: %s\nsource after: %s\nrequest before: %s\nrequest after: %s",
beforeSource,
afterSource,
beforeRequest,
afterRequest,
)
}
}
func TestRenderPromptIncludesSourceUnitsAndMetadata(t *testing.T) {
system, user, metadata, err := renderPrompt(promptChunkRequest())
if err != nil {
t.Fatalf("renderPrompt() error = %v, want nil", err)
}
if !strings.Contains(system, prompt.HardeningText()) {
t.Fatalf("system prompt = %q, want hardening text", system)
}
for _, want := range []string{
"session-alpha",
"seg-001",
"seg-002",
"Aria and Bram discuss whether to enter the ruins.",
"The goblins rush out and initiative begins.",
"speaker: Alice",
"start: 1.25",
"end: 3.5",
"start_unit_id",
"end_unit_id",
"primary_mode",
"boundary_confidence",
"Recap, Discussion, Combat, or Narrative",
"High, Medium, or Low",
"no gaps",
"do not overlap",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
if metadata.PromptID != PromptID {
t.Fatalf("metadata.PromptID = %q, want %q", metadata.PromptID, PromptID)
}
if metadata.PromptVersion != ResponseSchemaVersion {
t.Fatalf("metadata.PromptVersion = %q, want %q", metadata.PromptVersion, ResponseSchemaVersion)
}
if metadata.EmbeddedPath != "assets/prompts" {
t.Fatalf("metadata.EmbeddedPath = %q, want assets/prompts", metadata.EmbeddedPath)
}
}
func TestBuildPromptDataRejectsMissingSourceContext(t *testing.T) {
if _, err := buildPromptData(contracts.ChunkRequest{}); err == nil || !strings.Contains(err.Error(), "source") {
t.Fatalf("buildPromptData() error = %v, want source error", err)
}
}
func promptChunkRequest() contracts.ChunkRequest {
return contracts.ChunkRequest{
Source: promptSourceDocument(),
}
}
func promptSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "session-alpha",
Kind: "transcript",
Format: "application/vnd.seriatim.minimal+json",
Digest: "sha256:test",
Units: []source.SourceUnit{
{
ID: "seg-001",
Kind: "transcript_segment",
Text: "Aria and Bram discuss whether to enter the ruins.",
Metadata: map[string]any{
"speaker": "Alice",
"start": json.Number("1.25"),
"end": json.Number("3.5"),
"ignored": "not rendered",
},
},
{
ID: "seg-002",
Kind: "transcript_segment",
Text: "The goblins rush out and initiative begins.",
Metadata: map[string]any{"ignored": "not rendered"},
},
},
}
}
func mustJSON(t *testing.T, value any) string {
t.Helper()
encoded, err := json.Marshal(value)
if err != nil {
t.Fatalf("Marshal() error = %v, want nil", err)
}
return string(encoded)
}

View File

@@ -0,0 +1,21 @@
package scenes
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.scenes"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scenes")
ResponseSchemaID = "notarius.dnd.scenes"
ResponseSchemaVersion = "v1"
ResponseSchemaName = "notarius_dnd_scenes_v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: ResponseSchemaVersion,
Name: ResponseSchemaName,
AssetPath: "assets/schemas/dnd_scenes.v1.json",
})
}

View File

@@ -0,0 +1,146 @@
package scenes
import (
"encoding/json"
"strings"
"testing"
)
func TestLoadResponseSchemaForScenes(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if schema.Key != ResponseSchemaKey {
t.Fatalf("schema.Key = %q, want %q", schema.Key, ResponseSchemaKey)
}
if schema.ID != ResponseSchemaID {
t.Fatalf("schema.ID = %q, want %q", schema.ID, ResponseSchemaID)
}
if schema.Version != ResponseSchemaVersion {
t.Fatalf("schema.Version = %q, want %q", schema.Version, ResponseSchemaVersion)
}
if schema.Name != ResponseSchemaName {
t.Fatalf("schema.Name = %q, want %q", schema.Name, ResponseSchemaName)
}
if !strings.HasPrefix(schema.SHA256, "sha256:") {
t.Fatalf("schema.SHA256 = %q, want sha256 prefix", schema.SHA256)
}
if !json.Valid(schema.JSONSchema) {
t.Fatalf("schema.JSONSchema is invalid JSON: %s", schema.JSONSchema)
}
}
func TestResponseSchemaShapeUsesSourceUnitBoundaries(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
var decoded map[string]any
if err := json.Unmarshal(schema.JSONSchema, &decoded); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if decoded["$id"] != ResponseSchemaID {
t.Fatalf("$id = %#v, want %q", decoded["$id"], ResponseSchemaID)
}
if decoded["additionalProperties"] != false {
t.Fatalf("additionalProperties = %#v, want false", decoded["additionalProperties"])
}
properties := decoded["properties"].(map[string]any)
if _, ok := properties["artifact_type"]; ok {
t.Fatal("schema includes artifact_type, want only scene response fields")
}
if _, ok := properties["session_scope"]; ok {
t.Fatal("schema includes session_scope, want no session wrapper")
}
sceneProperties := properties["scenes"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any)
for _, field := range []string{"scene_id", "start_segment_id", "end_segment_id"} {
if _, ok := sceneProperties[field]; ok {
t.Fatalf("scene schema includes old field %q", field)
}
}
for _, field := range []string{"start_unit_id", "end_unit_id"} {
property := sceneProperties[field].(map[string]any)
if property["type"] != "string" {
t.Fatalf("%s type = %#v, want string", field, property["type"])
}
}
modeEnum := sceneProperties["primary_mode"].(map[string]any)["enum"].([]any)
if !sameStrings(modeEnum, []string{"Recap", "Discussion", "Combat", "Narrative"}) {
t.Fatalf("primary_mode enum = %#v, want Recap/Discussion/Combat/Narrative", modeEnum)
}
confidenceEnum := sceneProperties["boundary_confidence"].(map[string]any)["enum"].([]any)
if !sameStrings(confidenceEnum, []string{"High", "Medium", "Low"}) {
t.Fatalf("boundary_confidence enum = %#v, want High/Medium/Low", confidenceEnum)
}
boundaryCaveatItems := decoded["properties"].(map[string]any)["boundary_caveats"].(map[string]any)["items"].(map[string]any)
if boundaryCaveatItems["type"] != "string" {
t.Fatalf("boundary_caveats.items.type = %#v, want string", boundaryCaveatItems["type"])
}
if boundaryCaveatItems["minLength"] != float64(1) {
t.Fatalf("boundary_caveats.items.minLength = %#v, want 1", boundaryCaveatItems["minLength"])
}
}
func TestResponseStructRejectsIntegerBoundaries(t *testing.T) {
raw := []byte(`{
"scenes": [
{
"start_unit_id": 1,
"end_unit_id": 3,
"short_title": "Ambush",
"primary_mode": "Combat",
"main_participants": ["Aria"],
"summary": "The party fights.",
"boundary_note": "Combat starts and resolves.",
"boundary_confidence": "High"
}
],
"boundary_caveats": []
}`)
var response chunkResponse
err := json.Unmarshal(raw, &response)
if err == nil {
t.Fatalf("Unmarshal() error = nil, want integer boundary type error: %#v", response)
}
if !strings.Contains(err.Error(), "string") {
t.Fatalf("Unmarshal() error = %v, want string type error", err)
}
}
func TestResponseSchemaJSONIsMutationSafe(t *testing.T) {
first, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
first.JSONSchema[0] = '['
second, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if !json.Valid(second.JSONSchema) {
t.Fatalf("schema JSON was mutated: %s", second.JSONSchema)
}
if len(second.JSONSchema) > 0 && second.JSONSchema[0] == '[' {
t.Fatalf("schema JSON did not use defensive copy")
}
}
func sameStrings(got []any, want []string) bool {
if len(got) != len(want) {
return false
}
for i := range want {
if got[i] != want[i] {
return false
}
}
return true
}

View File

@@ -31,6 +31,10 @@ func (c *Chunker) Key() string {
return Key
}
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if c == nil {
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")

View File

@@ -41,6 +41,9 @@ func TestModuleSpecAndRegister(t *testing.T) {
if chunker.Key() != Key {
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
}
if slots := chunker.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
}
func TestChunkUsesDefaultsForSingleChunk(t *testing.T) {

View File

@@ -6,4 +6,9 @@ Extract only spell casts that are supported by the provided source text. Do not
infer spells from general D&D knowledge or from table chatter that does not
identify a spell being cast.
Reference material, when present, is supporting context only. Use it only to
disambiguate names, aliases, speakers, campaign terms, or spell names already
present in the source text. Do not extract a spell cast solely because it appears
in reference material.
Source references must use the source-unit IDs exactly as provided.

View File

@@ -16,6 +16,19 @@ Source units:
{{ end }}
{{ end }}
{{ if hasreference "roster" }}
Roster reference material:
{{ reference "roster" }}
{{ end }}
{{ if hasreference "glossary" }}
Glossary reference material:
{{ reference "glossary" }}
{{ end }}
Return only D&D spell-cast artifacts. For each spell cast, identify the in-world
caster, spell name, effect, narrative description, and source references using
source_id, start_unit_id, and end_unit_id.
Use roster and glossary reference material only to clarify source text. Do not
return spells, casters, or effects that are mentioned only in reference material.

View File

@@ -236,6 +236,10 @@ func (dndSpellsChunker) Key() string {
return "fake/chunk"
}
func (dndSpellsChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{

View File

@@ -25,6 +25,19 @@ var providedCapabilities = []string{
"dnd.spell_casts",
}
var referenceSlots = []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Optional campaign glossary reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
}
var _ contracts.Extractor = (*Extractor)(nil)
type Extractor struct{}
@@ -45,6 +58,10 @@ func (e *Extractor) SchemaVersion() string {
return SchemaVersion
}
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot {
return cloneReferenceSlots(referenceSlots)
}
func (e *Extractor) ManifestMetadata() map[string]any {
promptMetadata := spellsPromptBundle.Metadata()
metadata := map[string]any{
@@ -136,10 +153,11 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.ExtractionRequest
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageExtract,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
Key: Key,
Stage: pipeline.StageExtract,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ReferenceSlots: cloneReferenceSlots(referenceSlots),
}
}
@@ -161,3 +179,15 @@ func spellCastPayload(spellCast spellCastResponse) (json.RawMessage, error) {
func extractorErrorf(format string, args ...any) error {
return fmt.Errorf("dnd spells extractor: "+format, args...)
}
func cloneReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
out := make([]contracts.ReferenceSlot, len(slots))
for i, slot := range slots {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
out[i] = slot
}
return out
}

View File

@@ -116,6 +116,56 @@ func TestExtractorManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T
}
}
func TestExtractIncludesReferencesInPrompt(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
req := extractionRequestWithClient(client)
req.References = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria Brightmantle: party cleric")},
},
},
"glossary": {
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{SlotName: "glossary", Content: []byte("Brightmantle: local temple name")},
},
},
},
}
if _, err := New().Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(client.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
system := client.requests[0].Messages[0].Content
for _, want := range []string{
"Reference material, when present, is supporting context only.",
"in reference material.",
} {
if !strings.Contains(system, want) {
t.Fatalf("system prompt = %q, want substring %q", system, want)
}
}
user := client.requests[0].Messages[1].Content
for _, want := range []string{
"Roster reference material:",
"Aria Brightmantle: party cleric",
"Glossary reference material:",
"Brightmantle: local temple name",
"mentioned only in reference material.",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
}
func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}

View File

@@ -32,11 +32,12 @@ var spellsPromptBundle = mustLoadPromptBundle()
func mustLoadPromptBundle() *prompt.Bundle {
bundle, err := prompt.LoadBundle(embeddedAssets, prompt.Definition{
PromptID: PromptID,
Version: SchemaVersion,
EmbeddedPath: "assets/prompts",
SystemPath: "assets/prompts/system.md",
UserPath: "assets/prompts/user.md",
PromptID: PromptID,
Version: SchemaVersion,
EmbeddedPath: "assets/prompts",
SystemPath: "assets/prompts/system.md",
UserPath: "assets/prompts/user.md",
ReferenceSlots: cloneReferenceSlots(referenceSlots),
})
if err != nil {
panic(err)
@@ -74,7 +75,7 @@ func renderPrompt(req contracts.ExtractionRequest) (system string, user string,
if err != nil {
return "", "", prompt.Metadata{}, err
}
system, user, metadata, err = spellsPromptBundle.RenderUserSystem(data)
system, user, metadata, err = spellsPromptBundle.RenderUserSystemWithReferences(data, req.References)
if err != nil {
return "", "", prompt.Metadata{}, fmt.Errorf("dnd spells prompt: %w", err)
}

View File

@@ -101,6 +101,50 @@ func TestRenderPromptIncludesSourceContext(t *testing.T) {
if metadata.EmbeddedPath != "assets/prompts" {
t.Fatalf("metadata.EmbeddedPath = %q, want assets/prompts", metadata.EmbeddedPath)
}
if strings.Contains(user, "Roster reference material") || strings.Contains(user, "Glossary reference material") {
t.Fatalf("user prompt = %q, want no optional reference sections without bindings", user)
}
}
func TestRenderPromptIncludesBoundReferences(t *testing.T) {
req := promptExtractionRequest()
req.References = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria: cleric, also known as Sister Aria")},
},
},
"glossary": {
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{SlotName: "glossary", Content: []byte("Cure Wounds: healing spell")},
},
},
},
}
_, user, metadata, err := renderPrompt(req)
if err != nil {
t.Fatalf("renderPrompt() error = %v, want nil", err)
}
for _, want := range []string{
"Roster reference material:",
"Aria: cleric, also known as Sister Aria",
"Glossary reference material:",
"Cure Wounds: healing spell",
"Use roster and glossary reference material only to clarify source text.",
"mentioned only in reference material.",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
if metadata.SHA256 != spellsPromptBundle.Metadata().SHA256 {
t.Fatalf("metadata.SHA256 = %q, want template hash %q", metadata.SHA256, spellsPromptBundle.Metadata().SHA256)
}
}
func TestBuildPromptDataRejectsMissingSourceContext(t *testing.T) {

View File

@@ -5,6 +5,7 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -36,6 +37,18 @@ func TestModuleSpec(t *testing.T) {
Provides: []string{
"dnd.spell_casts",
},
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Optional campaign glossary reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
@@ -43,6 +56,7 @@ func TestModuleSpec(t *testing.T) {
got.Requires[0] = "changed"
got.Provides[0] = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
again := ModuleSpec()
if !reflect.DeepEqual(again, want) {
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
@@ -82,6 +96,15 @@ func TestRegisterStoresModuleSpec(t *testing.T) {
}
}
func TestRuntimeReferenceSlotsMatchModuleSpec(t *testing.T) {
extractor := New()
spec := ModuleSpec()
if !reflect.DeepEqual(extractor.ReferenceSlots(), spec.ReferenceSlots) {
t.Fatalf("ReferenceSlots() = %#v, want spec slots %#v", extractor.ReferenceSlots(), spec.ReferenceSlots)
}
}
func TestRegisterNilRegistryReturnsError(t *testing.T) {
err := Register(nil)
if err == nil {

View File

@@ -110,6 +110,96 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
}
}
func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
raw := readDNDSpellsFixture(t)
expectedDoc := parseDNDSpellsFixture(t, raw)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
"Aria: party cleric\nBorin: fighter",
"Fire Bolt: evocation cantrip",
)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
{
Caster: "Borin",
Spell: "Fire Bolt",
Effect: "Scorches the wight.",
NarrativeDescription: "Borin hurls fire at the wight.",
SourceRefs: []source.SourceRef{
{SourceID: expectedDoc.ID, StartUnitID: "seg-003", EndUnitID: "seg-003"},
},
},
},
},
}
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
Pipeline: resolved.ResolvedPipeline,
RawInput: raw,
LLMClient: llmClient,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
}
if len(output.Manifest.References) != 2 {
t.Fatalf("manifest references = %#v, want roster and glossary provenance", output.Manifest.References)
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
}
user := llmClient.requests[0].Messages[1].Content
for _, want := range []string{
"Roster reference material:",
"Aria: party cleric",
"Glossary reference material:",
"Fire Bolt: evocation cantrip",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
}
func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
"Mira: wizard who can cast Lightning Bolt",
"",
)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{SpellCasts: []spellCastResponse{}},
}
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
Pipeline: resolved.ResolvedPipeline,
RawInput: raw,
LLMClient: llmClient,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 0 {
t.Fatalf("approved artifacts = %#v, want no roster-only spell casts", output.Approved)
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
}
user := llmClient.requests[0].Messages[1].Content
if !strings.Contains(user, "Lightning Bolt") {
t.Fatalf("user prompt = %q, want roster-only spell in reference section", user)
}
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved empty extraction", output.Manifest.ValidationStatus)
}
}
func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
@@ -155,6 +245,43 @@ func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
}
}
func dndSpellsReferenceSet(roster string, glossary string) contracts.ReferenceSet {
slots := make(map[string]contracts.ResolvedReferenceSlot)
if strings.TrimSpace(roster) != "" {
slots["roster"] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain; charset=utf-8",
Content: []byte(roster),
Digest: "sha256:roster",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
SizeBytes: int64(len(roster)),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
}
}
if strings.TrimSpace(glossary) != "" {
slots["glossary"] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{
SlotName: "glossary",
MediaType: "text/plain; charset=utf-8",
Content: []byte(glossary),
Digest: "sha256:glossary",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/glossary.txt"},
SizeBytes: int64(len(glossary)),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
}
}
return contracts.ReferenceSet{Slots: slots}
}
func TestRunnerFailsWhenDNDSpellsExtractorReturnsMalformedOutput(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)

View File

@@ -20,6 +20,7 @@ const (
reasonMissingRequiredField = "missing_required_field"
reasonMissingSourceRef = "missing_source_ref"
reasonInvalidSourceRef = "invalid_source_ref"
reasonSpellNotNearSource = "spell_not_near_source"
)
var _ contracts.Validator = ShapeValidator{}
@@ -54,12 +55,15 @@ func (validator SourceRefValidator) Validate(ctx context.Context, req contracts.
}
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
var warnings []contracts.Warning
for _, candidate := range req.Candidates {
decisions = append(decisions, validateSourceRefs(req.Source, candidate))
warnings = append(warnings, sourceRelatednessWarnings(req.Source, candidate)...)
}
return contracts.ValidationResult{
ValidatorName: validator.Name(),
Decisions: decisions,
Warnings: warnings,
}, nil
}
@@ -88,6 +92,66 @@ func validateSourceRefs(doc *source.SourceDocument, candidate artifacts.Artifact
return validate.Approved(candidate.Index)
}
func sourceRelatednessWarnings(doc *source.SourceDocument, candidate artifacts.ArtifactCandidate) []contracts.Warning {
if doc == nil || len(candidate.SourceRefs) == 0 {
return nil
}
var payload SpellCast
if err := json.Unmarshal(candidate.Payload, &payload); err != nil {
return nil
}
spell := strings.TrimSpace(payload.Spell)
if spell == "" {
return nil
}
needle := strings.ToLower(spell)
for _, ref := range candidate.SourceRefs {
text, ok := sourceRefText(doc, ref)
if !ok {
continue
}
if strings.Contains(strings.ToLower(text), needle) {
return nil
}
}
return []contracts.Warning{
{
Scope: fmt.Sprintf("candidate.%d", candidate.Index),
ReasonCode: reasonSpellNotNearSource,
Message: fmt.Sprintf("spell %q was not found in the cited source text", spell),
},
}
}
func sourceRefText(doc *source.SourceDocument, ref source.SourceRef) (string, bool) {
if err := source.ValidateRef(doc, ref); err != nil {
return "", false
}
start := -1
end := -1
for i, unit := range doc.Units {
if unit.ID == ref.StartUnitID {
start = i
}
if unit.ID == ref.EndUnitID {
end = i
}
}
if start < 0 || end < start {
return "", false
}
var b strings.Builder
for i := start; i <= end; i++ {
if b.Len() > 0 {
b.WriteString("\n")
}
b.WriteString(doc.Units[i].Text)
}
return b.String(), true
}
func requiredSpellCastFields(payload SpellCast) []struct {
name string
value string

View File

@@ -51,6 +51,9 @@ func TestValidatorsApproveValidCandidate(t *testing.T) {
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, sourceRefResult, sourceRefValidatorName, 7, true, validate.ReasonApproved)
if len(sourceRefResult.Warnings) != 0 {
t.Fatalf("warnings = %#v, want none", sourceRefResult.Warnings)
}
}
func TestShapeValidatorRejectsMalformedPayload(t *testing.T) {
@@ -167,6 +170,29 @@ func TestSourceRefValidatorRejectsInvalidRefs(t *testing.T) {
}
}
func TestSourceRefValidatorWarnsWhenSpellNameIsNotInCitedSource(t *testing.T) {
candidate := validSpellCandidate(31)
payload := validSpellPayload()
payload.Spell = "Shield"
candidate.Payload = mustSpellPayload(t, payload)
result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Source: promptSourceDocument(),
Candidates: []artifacts.ArtifactCandidate{candidate},
})
if err != nil {
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, result, sourceRefValidatorName, 31, true, validate.ReasonApproved)
if len(result.Warnings) != 1 {
t.Fatalf("warnings = %#v, want one relatedness warning", result.Warnings)
}
warning := result.Warnings[0]
if warning.ReasonCode != reasonSpellNotNearSource || !strings.Contains(warning.Message, "Shield") {
t.Fatalf("warning = %#v, want spell relatedness warning", warning)
}
}
func TestSourceRefValidatorRequiresSourceDocument(t *testing.T) {
_, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Candidates: []artifacts.ArtifactCandidate{validSpellCandidate(19)},

View File

@@ -209,6 +209,8 @@ type fakeChunker struct{}
func (fakeChunker) Key() string { return "fake/chunk" }
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{}, nil
}
@@ -221,6 +223,8 @@ func (fakeExtractor) ArtifactType() string { return "fake" }
func (fakeExtractor) SchemaVersion() string { return "v1" }
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeExtractor) Validators() []contracts.Validator { return nil }
func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {

View File

@@ -157,6 +157,10 @@ func (runnerSeriatimChunker) Key() string {
return "fake/chunk"
}
func (runnerSeriatimChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (runnerSeriatimChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{
@@ -186,6 +190,10 @@ func (e *runnerSeriatimExtractor) SchemaVersion() string {
return "v1"
}
func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (e *runnerSeriatimExtractor) Validators() []contracts.Validator {
return nil
}

View File

@@ -25,6 +25,10 @@ func (n *Normalizer) Key() string {
return Key
}
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
if n == nil {
return contracts.NormalizeResult{}, normalizerErrorf("normalizer must not be nil")

View File

@@ -34,6 +34,13 @@ func TestModuleSpecAndRegister(t *testing.T) {
if !reflect.DeepEqual(spec, want) {
t.Fatalf("registered spec = %#v, want %#v", spec, want)
}
normalizer, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if slots := normalizer.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
}
func TestNormalizePassesThroughOrderAndValues(t *testing.T) {

View File

@@ -142,6 +142,43 @@ func TestEncodePrettyPrintsJSON(t *testing.T) {
}
}
func TestEncodeIncludesManifestReferences(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{
RunID: "run-1",
References: []artifacts.ReferenceProvenance{
{
Stage: "extract",
LaneID: "events",
SlotName: "roster",
OriginType: "file",
OriginURI: "file:///tmp/roster.txt",
Digest: "sha256:reference",
MediaType: "text/plain; charset=utf-8",
SizeBytes: 12,
BindingSource: "config",
},
},
},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
manifest := decodeObject(t, fileBytes(t, result.Files, "manifest.json"))
references := manifest["references"].([]any)
if len(references) != 1 {
t.Fatalf("references = %#v, want one entry", references)
}
reference := references[0].(map[string]any)
if reference["lane_id"] != "events" || reference["slot_name"] != "roster" || reference["digest"] != "sha256:reference" {
t.Fatalf("reference manifest = %#v, want lane slot digest", reference)
}
if _, ok := reference["content"]; ok {
t.Fatalf("reference manifest = %#v, want no content field", reference)
}
}
func TestEncodeRejectsArtifactTypeWithoutSafeFileName(t *testing.T) {
_, err := New().Encode(context.Background(), contracts.OutputRequest{
Approved: []artifacts.Artifact{artifact("///", "unsafe")},