37 Commits

Author SHA1 Message Date
9532ae8121 Add a feature roadmap and implementation plan to import the scriptorium LLM library 2026-07-05 12:42:15 -05:00
7601731a2c Clean up completed reference roadmap docs 2026-07-05 11:51:36 -05:00
3aa88ab9d3 Document completed LLM and reference expansion 2026-07-05 16:42:15 +00:00
c1ba94192d Cover reference prompts for chunk and normalize usage 2026-07-05 16:39:41 +00:00
22032dfd6d Deliver references to eligible runtime targets 2026-07-05 16:38:17 +00:00
4cafde2502 Materialize references for all eligible targets 2026-07-05 16:35:24 +00:00
8c623b7ad8 Support target-aware reference selectors 2026-07-05 16:31:23 +00:00
43dc954440 Resolve references across eligible pipeline targets 2026-07-05 16:24:49 +00:00
51053d390d Add stage-local reference config bindings 2026-07-05 16:18:00 +00:00
9278797aa9 Introduce target-aware resolved reference storage 2026-07-05 16:12:38 +00:00
39e49d7f77 Expand module contracts for references and normalizer LLM access 2026-07-05 16:06:47 +00:00
84c4c06712 Add a staged implementation plan to extend first-class LLM access and reference support across all chunk, extract, and normalize stages 2026-07-05 11:01:32 -05:00
eab640aa21 Update .gitignore to ignore build and testing output artifacts 2026-07-05 10:54:16 -05:00
a516944086 Finish the references implementation for the extraction module and update roadmap documentation 2026-07-05 10:53:16 -05:00
be6803ffa1 Document extraction reference support 2026-07-05 14:52:52 +00:00
ef4bdd4f9f Use references in D&D spell extraction 2026-07-05 14:49:17 +00:00
2f97895732 Record reference provenance in manifests 2026-07-05 14:44:44 +00:00
9e89b88efc Add prompt reference template functions 2026-07-05 14:37:39 +00:00
a57c6397e3 Materialize extraction reference files 2026-07-05 14:32:35 +00:00
39e071f5ca Add CLI reference binding flags 2026-07-05 14:27:23 +00:00
70d733edaf Resolve extraction reference bindings from config 2026-07-05 14:21:36 +00:00
1c31f56af1 Add reference contracts to extractor metadata 2026-07-05 14:13:48 +00:00
f9999a73df Add a staged implementation plan for background context references 2026-07-05 09:08:31 -05:00
11d8187052 Update documentation to reflect the implemented chunking module 2026-07-05 08:40:24 -05:00
86bff552c1 Update .gitgnore to ignore build artifacts 2026-07-05 08:39:34 -05:00
d3f790095e Trim and validate scene caveats 2026-07-05 13:30:32 +00:00
95218218e2 Canonicalize chunk units before extraction 2026-07-05 13:28:09 +00:00
e700df82d8 Record top-level module metadata in run manifests 2026-07-05 13:23:56 +00:00
e19cc02c4d Add a staged implementation plan to address gaps from the initial implementation of the scene chunking module 2026-07-05 08:15:46 -05:00
8a5419448f Update feature roadmaps to reflect future work 2026-07-05 08:11:54 -05:00
c8217549a8 Verify D&D scene chunker run output 2026-07-04 13:08:45 +00:00
2130414899 Register D&D scene chunker 2026-07-04 13:05:39 +00:00
7f83a20fa6 Implement D&D scene chunker 2026-07-04 13:02:02 +00:00
317ab0472d Add D&D scene chunking assets 2026-07-04 12:57:04 +00:00
e5eb0ba5c8 Make chunk validation a framework contract 2026-07-04 12:52:16 +00:00
b95af4f87d Add a roadmap to implement a D&D-specific chunk module 2026-07-04 07:48:36 -05:00
11073b613c Update AGENTS.md 2026-07-04 07:05:06 -05:00
86 changed files with 8891 additions and 190 deletions

5
.gitignore vendored
View File

@@ -1,3 +1,7 @@
# build and testing artifacts
notarius
notarius-output
# ---> Go # ---> Go
# If you prefer the allow list template instead of the deny list, see community template: # If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
@@ -49,6 +53,7 @@ go.work.sum
# Icon must end with two \r # Icon must end with two \r
Icon Icon
# Thumbnails # Thumbnails
._* ._*

View File

@@ -1,3 +1,6 @@
Please carefully review the documents in `docs/policy` before making any changes to this repository. Please review `docs/internal/overview.md` for initial orientation in this repository.
- `architecture.md` provides the canonical high-level architecture policy for this repository.
- `documentation.md` provides the canonical documentation policy for this repository. Additionally, please carefully review the relevant documents in `docs/policy` before making any changes to this repository.
- `development.md` defines the contributor workflow for this application.
- `architecture.md` provides the canonical high-level architecture policy for this repository, and should be reviewed before writing or changing any code.
- `documentation.md` provides the canonical documentation policy for this repository, and should be reviewed before writing or changing any documentation.

View File

@@ -20,7 +20,7 @@ a bearer token.
```text ```text
notarius help notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference selector=path] [--without-reference selector]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json] notarius pipelines list --config path/to/config.yml [--json]
``` ```
@@ -46,11 +46,82 @@ Flags:
invocation. invocation.
- `--llm-profile id`: override every effective module binding to use one LLM - `--llm-profile id`: override every effective module binding to use one LLM
profile. profile.
- `--reference selector=path`: bind a reference path to a chunk, extractor, or
normalizer reference slot. Repeatable.
- `--without-reference selector`: remove a configured optional reference binding.
Repeatable. It accepts the same selector forms as `--reference`, without
`=path`.
On success, the command prints the completed pipeline ID, approved and rejected On success, the command prints the completed pipeline ID, approved and rejected
artifact counts, and the output directory. If the run completes with warnings, artifact counts, and the output directory. If the run completes with warnings,
the warning count is printed to stderr. the warning count is printed to stderr.
Reference flags are resolved against selected chunk, extractor, and normalizer
targets before the run starts. Flat slot names are accepted only when exactly
one selected target declares that slot. Bound reference files are read before
pipeline work starts, validated as UTF-8 text, and recorded as provenance for
the target that declares the slot. Runtime reference content is passed to the
chunker, extractor, or normalizer target that declares the slot. Notarius infers
reference media types from file extensions for provenance and for optional slot
checks. Reference content is not written to diagnostics, logs, errors, or
manifests.
Reference binding precedence is:
1. pipeline-level config `references`;
2. target-local config references, including legacy lane-level extractor
`references`;
3. `--reference` run flags;
4. `--without-reference` run flags.
`--reference` binds or replaces one slot for one selected target. Selectors are:
- `slot=path`: valid when exactly one selected target declares `slot`;
- `chunk.slot=path`: target the chunker;
- `lane.slot=path`: valid when exactly one selected extractor or normalizer in
that lane declares `slot`;
- `lane.extract.slot=path`: target a lane extractor;
- `lane.normalize.slot=path`: target a lane normalizer.
Use `slot=path` when the selected targets declare the slot unambiguously:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference roster=./campaign-roster.txt
```
Use an explicit selector when multiple selected targets declare the same slot or
when you want to target a specific target:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference spells.extract.glossary=./campaign-glossary.txt
```
The same grammar can target chunk and normalize slots when the configured
modules declare them:
```sh
go run ./cmd/notarius run dnd-session \
--config path/to/config.yml \
--input examples/seriatim-minimal-transcript.json \
--reference chunk.scene_guide=./campaign-scenes.txt \
--reference spells.normalize.normalization_notes=./normalization-notes.txt
```
Use `--without-reference` to remove a configured optional binding for a run:
```sh
go run ./cmd/notarius run dnd-session \
--config examples/dnd-spells.config.yml \
--input examples/seriatim-minimal-transcript.json \
--without-reference glossary
```
For durable output, diagnostics, retention, and failure inspection, see For durable output, diagnostics, retention, and failure inspection, see
[Operations](operations.md). [Operations](operations.md).
@@ -117,7 +188,7 @@ go run ./cmd/notarius pipelines list \
The production CLI currently registers these module keys: The production CLI currently registers these module keys:
- input: `seriatim` - input: `seriatim`
- chunk: `generic` - chunk: `generic`, `dnd/scenes`
- extract: `dnd/spells` - extract: `dnd/spells`
- merge: `appendorder` - merge: `appendorder`
- normalize: `noop` - normalize: `noop`

View File

@@ -27,6 +27,9 @@ llm_profiles:
pipelines: pipelines:
dnd-session: dnd-session:
input: seriatim input: seriatim
references:
roster: ./dnd-spells-roster.txt
glossary: ./dnd-spells-glossary.txt
chunk: chunk:
module: generic module: generic
options: options:
@@ -121,6 +124,9 @@ Pipeline fields:
- `artifacts`: required for pipeline resolution. It maps artifact lane IDs to - `artifacts`: required for pipeline resolution. It maps artifact lane IDs to
lane definitions. lane definitions.
- `output`: optional module binding. Default module is `json`. - `output`: optional module binding. Default module is `json`.
- `references`: optional map of reference slot names to reference paths. These
bindings are defaults for eligible pipeline targets that declare the matching
slot.
Artifact lane fields: Artifact lane fields:
@@ -129,11 +135,78 @@ Artifact lane fields:
- `normalize`: optional module binding. Default module is `noop`. - `normalize`: optional module binding. Default module is `noop`.
- `validators`: optional list of module bindings. The production CLI currently - `validators`: optional list of module bindings. The production CLI currently
does not register validator modules. does not register validator modules.
- `references`: optional compatibility alias for extractor reference bindings.
Lane bindings override pipeline-level bindings for the same slot.
`notarius run` and `notarius config validate --pipeline` resolve the pipeline `notarius run` and `notarius config validate --pipeline` resolve the pipeline
against the production module catalog and fail fast for unknown or incompatible against the production module catalog and fail fast for unknown or incompatible
module keys. module keys.
Reference bindings are validated against reference slots declared by eligible
chunk, extract, and normalize targets during pipeline resolution. Required slots
must be bound after config defaults, target-local references, lane-level
compatibility bindings, and run-time `--reference` or `--without-reference`
overrides are applied. Config-relative paths are resolved relative to the config
file; CLI reference paths are resolved relative to the current working
directory. Materialized bound files must be UTF-8 text. Materialized reference
provenance is recorded for chunk, extractor, and normalizer targets, and runtime
reference content is passed to the target that declares the slot. Reference
media types are inferred from file extensions, recorded as canonical base media
types, and checked only when a module declares
`AcceptedMediaTypes`; unknown extensions are recorded as
`application/octet-stream`. Reference content is not written to diagnostics,
logs, errors, or manifests.
Pipeline-level `references` are defaults. They are valid when at least one
eligible target in the full configured pipeline declares the slot, including
chunk, extractor, and normalizer targets. During a run, they apply only to the
selected targets that declare the slot:
```yaml
pipelines:
dnd-session:
input: seriatim
references:
roster: ./campaign/party-roster.txt
glossary: ./campaign/glossary.txt
artifacts:
spells:
extract: dnd/spells
```
Extractor binding `references` are the canonical lane-local location. The
legacy lane-level `references` field remains supported as an alias; when both
bind the same slot, `extract.references` wins:
```yaml
pipelines:
dnd-session:
input: seriatim
references:
glossary: ./campaign/glossary.txt
artifacts:
spells:
references:
roster: ./campaign/legacy-roster.txt
extract:
module: dnd/spells
references:
roster: ./campaign/session-roster.txt
```
`chunk.references` and `normalize.references` are accepted in object-form
bindings. They override pipeline-level defaults for slots declared by the chunk
or normalizer module. Extractor-local references apply only to the extractor,
and normalizer-local references apply only to the normalizer.
Stage-local reference fields use the same map shape at:
- `pipelines.<id>.chunk.references`
- `pipelines.<id>.artifacts.<lane>.extract.references`
- `pipelines.<id>.artifacts.<lane>.normalize.references`
Each binding is valid only when that target module declares the slot.
## Module Bindings ## Module Bindings
Every module binding may use shorthand: Every module binding may use shorthand:
@@ -157,6 +230,9 @@ Binding fields:
- `module`: module key. - `module`: module key.
- `llm_profile`: optional LLM profile ID. Empty means `default`. - `llm_profile`: optional LLM profile ID. Empty means `default`.
- `options`: optional module-specific settings. - `options`: optional module-specific settings.
- `references`: optional reference bindings. Supported only for `chunk`,
`extract`, and `normalize` bindings. `input`, `merge`, validator, and
`output` bindings reject this field during validation.
The `--llm-profile` run flag overrides every effective module binding to use The `--llm-profile` run flag overrides every effective module binding to use
one configured profile. one configured profile.
@@ -167,6 +243,7 @@ one configured profile.
| --- | --- | --- | | --- | --- | --- |
| input | `seriatim` | Reads Seriatim transcript JSON. | | input | `seriatim` | Reads Seriatim transcript JSON. |
| chunk | `generic` | Splits source units into ordered chunks. | | chunk | `generic` | Splits source units into ordered chunks. |
| chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. |
| extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. | | extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. |
| merge | `appendorder` | Keeps candidates in append order. | | merge | `appendorder` | Keeps candidates in append order. |
| normalize | `noop` | Passes merged artifacts through unchanged. | | normalize | `noop` | Passes merged artifacts through unchanged. |
@@ -178,6 +255,17 @@ The `generic` chunker accepts:
- `overlap_units`: non-negative integer, default `0`, and must be less than - `overlap_units`: non-negative integer, default `0`, and must be less than
`max_units`. `max_units`.
The `dnd/scenes` chunker requires transcript source capabilities, calls the
configured structured LLM provider, and does not accept module options.
The `dnd/spells` extractor declares optional text reference slots:
- `roster`
- `glossary`
The extractor uses these references only as supporting disambiguation material;
spell casts still must be present in the source transcript.
## Diagnostics ## Diagnostics
`diagnostics` fields: `diagnostics` fields:
@@ -211,3 +299,6 @@ Pipeline resolution additionally checks:
- required module keys are present; - required module keys are present;
- module keys are registered for the expected slot; - module keys are registered for the expected slot;
- module capability requirements are satisfied. - module capability requirements are satisfied.
- bound reference slots are declared by selected chunk, extractor, or
normalizer targets;
- required reference slots are bound for selected targets.

View File

@@ -17,6 +17,14 @@ The extractor requires source chunks and transcript source capability. It
returns generic artifact candidates that are serialized by the JSON output returns generic artifact candidates that are serialized by the JSON output
module. module.
The extractor accepts optional UTF-8 text references:
- `roster`: campaign roster or player-character notes.
- `glossary`: campaign glossary or spell/name notes.
References are supporting disambiguation material only. They are not source
evidence and are not addressable through `source_refs`.
## Artifact Envelope ## Artifact Envelope
Approved artifacts use the generic artifact envelope documented in Approved artifacts use the generic artifact envelope documented in
@@ -120,6 +128,11 @@ Rejection reason codes:
- `invalid_source_ref`: at least one source reference fails generic source - `invalid_source_ref`: at least one source reference fails generic source
reference validation. reference validation.
Warning reason codes:
- `spell_not_near_source`: the extracted spell name was not found in the cited
source text.
Rejected candidates are written to `rejected.json` by the JSON output module. Rejected candidates are written to `rejected.json` by the JSON output module.
## Manifest Metadata ## Manifest Metadata

View File

@@ -57,7 +57,19 @@ approved.
"pipeline_id": "dnd-session", "pipeline_id": "dnd-session",
"pipeline_digest": "sha256:...", "pipeline_digest": "sha256:...",
"input_module": "seriatim", "input_module": "seriatim",
"chunker": "generic", "chunker": "dnd/scenes",
"module_metadata": {
"chunker": {
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:...",
"response_schema_key": "dnd_scenes",
"response_schema_id": "notarius.dnd.scenes",
"response_schema_name": "notarius_dnd_scenes_v1",
"response_schema_version": "v1",
"response_schema_sha256": "sha256:..."
}
},
"source_digests": ["sha256:..."], "source_digests": ["sha256:..."],
"extractors": ["dnd/spells"], "extractors": ["dnd/spells"],
"merger": "appendorder", "merger": "appendorder",
@@ -86,9 +98,48 @@ approved.
Fields with empty values may be omitted by JSON encoding. Fields with empty values may be omitted by JSON encoding.
`source_digests` contains source document digests only. Bound references are
recorded separately under `references`, which contains provenance only:
target stage, lane ID when present, slot name, origin type and URI, digest,
media type, byte size, and binding source. Reference content is not written to
durable output.
Reference `stage` is `chunk`, `extract`, or `normalize`. `lane_id` is omitted
for chunk references and present for extract and normalize references.
When references are bound, the manifest section has this shape:
```json
{
"references": [
{
"stage": "extract",
"lane_id": "spells",
"slot_name": "roster",
"origin_type": "file",
"origin_uri": "file:///absolute/path/roster.txt",
"digest": "sha256:...",
"media_type": "text/plain",
"size_bytes": 123,
"binding_source": "config"
}
]
}
```
Reference media types are inferred from file extensions and recorded as
canonical base media types. Unknown extensions are recorded as
`application/octet-stream`.
`module_metadata` is omitted when no singleton module provides metadata.
`validation_status` is `approved` when no candidates were rejected and `validation_status` is `approved` when no candidates were rejected and
`rejected` when one or more candidates were rejected. `rejected` when one or more candidates were rejected.
Top-level `module_metadata` is reserved for singleton pipeline modules
(`input`, `chunker`, and `output`). Lane-owned module metadata remains under
`artifact_lanes[].metadata`.
## Artifact Files ## Artifact Files
Each artifact file has this shape: Each artifact file has this shape:

View File

@@ -33,6 +33,7 @@ Implemented artifact names:
- `invocation.json` - `invocation.json`
- `effective-config.json` - `effective-config.json`
- `resolved-pipeline.json` - `resolved-pipeline.json`
- `resolved-references.json`
- `source-document.json` - `source-document.json`
- `run-manifest.json` - `run-manifest.json`
- `run-report.json` - `run-report.json`

View File

@@ -17,8 +17,8 @@ The request contains messages, optional model override, response schema name,
and response schema JSON. The caller supplies a pointer target for decoded and response schema JSON. The caller supplies a pointer target for decoded
structured output. structured output.
Extractors own prompts and schemas. Provider adapters should not contain Modules that call the LLM own their prompts and schemas. Provider adapters
domain-specific prompt logic. should not contain domain-specific prompt logic.
## Production Client Construction ## Production Client Construction

View File

@@ -20,6 +20,36 @@ A production module package should provide:
Module specs should describe capabilities accurately. Resolution uses specs to Module specs should describe capabilities accurately. Resolution uses specs to
reject incompatible pipelines before execution. reject incompatible pipelines before execution.
Chunk, extract, and normalize modules that accept auxiliary reference material
must declare slots through both `ReferenceSlots()` and
`ModuleSpec().ReferenceSlots`. The runtime slot list and registry metadata
should match so config validation can inspect slots without constructing module
instances. A slot declaration names the slot, whether it is required, accepted
media types, whether multiple items are allowed, and any byte limit. Empty
`AcceptedMediaTypes` means any inferred media type is accepted, though the file
must still be UTF-8 text. When a slot declares accepted media types, Notarius
compares the canonical base media type inferred from the file extension,
case-insensitively and without parameters.
The resolver materializes reference content for chunk, extractor, and
normalizer targets. Runtime delivery uses `contracts.ChunkRequest.References`,
`contracts.ExtractionRequest.References`, and
`contracts.NormalizeRequest.References`. Reference material is not source
evidence and must not be converted into `SourceRef` values. If a module prompt
uses references, load the prompt bundle with the same declared slots and render
with `RenderUserSystemWithReferences`. Prompt templates may use the `reference`
function for content and the `hasreference` function for conditional sections.
Prompt metadata hashes remain based on template source, not rendered reference
bytes.
Chunk modules receive the structured LLM client through `contracts.ChunkRequest`
when they need model-backed chunking. The pipeline runner validates generic
chunk result invariants before extraction; module-owned policies may be stricter
but must stay within the module package.
Normalize modules receive the structured LLM client through
`contracts.NormalizeRequest` when they need model-backed reconciliation.
## `seriatim` Input ## `seriatim` Input
Package: `internal/modules/input/seriatim` Package: `internal/modules/input/seriatim`
@@ -44,6 +74,10 @@ The `generic` chunker splits source units into ordered chunks. It validates the
source document, clones source units, assigns chunk IDs such as `chunk-000001`, source document, clones source units, assigns chunk IDs such as `chunk-000001`,
and records chunk metadata for start unit, end unit, and unit count. and records chunk metadata for start unit, end unit, and unit count.
The pipeline runner canonicalizes chunk units from the source document by ID
before extractors and mergers run. Chunker-owned context should stay in
`SourceChunk.Metadata`.
Options: Options:
- `max_units`: positive integer, default `50`; - `max_units`: positive integer, default `50`;
@@ -54,6 +88,39 @@ Provides:
- `chunks` - `chunks`
## `dnd/scenes` Chunker
Package: `internal/modules/chunk/dnd/scenes`
The `dnd/scenes` chunker uses the structured LLM client to divide transcript
source units into coherent D&D scenes. It renders embedded prompts, loads the
embedded structured response schema, validates model-authored source-unit
boundaries, and converts each scene into a deterministic source chunk.
Requires:
- `source.transcript`
Provides:
- `chunks`
- `chunks.scenes`
Options: none. Non-empty options are rejected.
The chunker enforces full source-unit coverage from the first source unit to the
last, exact source-unit IDs, sequential contiguous scenes, and no overlap. It
assigns chunk IDs such as `scene-000001` and stores scene metadata including
title, primary mode, participants, summary, boundary note, confidence, boundary
unit IDs, and unit count. Boundary caveats become warnings with reason code
`scene_boundary_caveat`. Whitespace-only caveats are treated as malformed
structured output rather than silently dropped.
Malformed model output fails explicitly rather than falling back to another
chunker. The chunker exposes prompt and response-schema provenance through
top-level `module_metadata.chunker` without raw prompts, raw schemas, source
text, or secrets.
## `dnd/spells` Extractor ## `dnd/spells` Extractor
Package: `internal/modules/extract/dnd/spells` Package: `internal/modules/extract/dnd/spells`
@@ -78,15 +145,23 @@ Artifact type and schema version:
- schema version: `v1` - schema version: `v1`
The extractor adds prompt and response-schema provenance to lane manifest The extractor adds prompt and response-schema provenance to lane manifest
metadata. Durable artifact payload details belong in the metadata under `artifact_lanes[].metadata.extractor`. Durable artifact payload
details belong in the
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md). [D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
The extractor declares optional `roster` and `glossary` reference slots
accepting UTF-8 text without narrowing accepted media types. Its prompt frames
references as supporting disambiguation material only; spell-cast artifacts must
still be grounded in the source transcript.
## D&D Spell Validators ## D&D Spell Validators
The spell extractor returns two built-in validators: The spell extractor returns two built-in validators:
- `dnd/spells/shape`: rejects malformed payloads and missing required fields. - `dnd/spells/shape`: rejects malformed payloads and missing required fields.
- `dnd/spells/source_refs`: rejects candidates without valid source references. - `dnd/spells/source_refs`: rejects candidates without valid source references.
It also emits a warning when the extracted spell name is not found in the
cited source text.
Reason codes include: Reason codes include:
@@ -94,6 +169,7 @@ Reason codes include:
- `missing_required_field` - `missing_required_field`
- `missing_source_ref` - `missing_source_ref`
- `invalid_source_ref` - `invalid_source_ref`
- `spell_not_near_source`
These validators are supplied by the extractor when no validators are configured These validators are supplied by the extractor when no validators are configured
for the lane. for the lane.

View File

@@ -30,6 +30,38 @@ before execution:
The CLI writes the resolved pipeline and digest to diagnostics. The CLI writes the resolved pipeline and digest to diagnostics.
Pipeline profiles and artifact lanes may include reference binding maps keyed by
reference slot name. During resolution, pipeline-level bindings act as defaults
for selected chunk, extractor, and normalizer targets that declare the slot;
target-local bindings override or add bindings for that target. Runtime
`--reference` requests override target config bindings, and runtime unbinds
remove optional target bindings. Flat runtime slot names are resolved only when
exactly one selected target declares the slot; otherwise the CLI requires a more
specific selector such as `chunk.slot`, `lane.extract.slot`, or
`lane.normalize.slot`. Resolution validates bindings against the declaring
target specs and stores the bindings in target-aware resolved reference holders.
It does not read reference files or include reference bytes in source digests.
During run preparation, resolved file references for chunk, extractor, and
normalizer targets are materialized before any LLM-backed pipeline work. Config
bindings resolve relative to the config file, and CLI bindings resolve relative
to the current working directory. Materialization accepts UTF-8 text files,
computes `sha256:` content digests, records file origins, infers canonical base
media types from file extensions, enforces declared byte limits, and warns for
empty bound files. Media-type acceptance is checked only when a slot declares
`AcceptedMediaTypes`; unknown extensions are recorded as
`application/octet-stream`. Reference content is omitted from diagnostics and
manifests. The CLI writes provenance-only resolved reference diagnostics, and
the run manifest records target-stage reference provenance separately from
source digests. Runtime reference content is passed to the matching chunker,
extractor, or normalizer request.
Prompt bundles can declare reference slots and use `reference` and
`hasreference` template functions. Bundle loading validates string-literal slot
names against the declaration. Rendering receives a target reference set from the
caller; unbound optional slots render as empty strings, and `hasreference`
returns true only when at least one bound item has content.
## Registries And Module Specs ## Registries And Module Specs
`pipeline.Registries` holds concrete constructors for execution. A `pipeline.Registries` holds concrete constructors for execution. A
@@ -44,6 +76,11 @@ Every production module registers a `ModuleSpec` with:
- `Provides`: capabilities added after that module runs; - `Provides`: capabilities added after that module runs;
- `Requires`: capabilities that must already be available. - `Requires`: capabilities that must already be available.
Chunk, extract, and normalize specs may also declare reference slots. Slot
declarations are available from registry metadata without constructing module
instances. Input, merge, validate, and output specs must not declare reference
slots.
Capability checks prevent incompatible pipeline composition before a run starts. Capability checks prevent incompatible pipeline composition before a run starts.
## Runner Input And Output ## Runner Input And Output
@@ -73,8 +110,39 @@ The runner:
2. builds the input adapter and parses the raw input into a source document; 2. builds the input adapter and parses the raw input into a source document;
3. validates the source document; 3. validates the source document;
4. builds the chunker and produces source chunks; 4. builds the chunker and produces source chunks;
5. runs each selected artifact lane in sorted resolved order; 5. validates source chunks against framework invariants;
6. builds the output encoder and validates logical output file names. 6. runs each selected artifact lane in sorted resolved order;
7. builds the output encoder and validates logical output file names.
## Chunk Results
Chunkers implement `contracts.Chunker` and receive a `contracts.ChunkRequest`
with the validated source document, reference set, structured LLM client, the
configured LLM profile, module options, and run metadata. Deterministic and
LLM-backed chunkers use the same contract; provider construction stays outside
chunk modules.
After `Chunk` returns, the runner appends chunker warnings before returning any
chunker error. When chunking succeeds, the runner validates generic chunk
invariants before running extractors:
- chunk IDs must be non-empty and unique in the chunk result;
- each chunk `SourceID` must match the source document ID;
- each chunk `Index` must match its zero-based returned order;
- each chunk must contain at least one source unit;
- a chunk must not repeat a source unit;
- every chunk source unit must exist in the source document;
- source units inside each chunk must appear in source-document order.
After validation, the runner rebuilds each chunk from source-document units by
ID, preserving the chunk boundary order and cloning chunk metadata. Extractors
and downstream stages therefore see canonical source units, while
`SourceChunk.Metadata` remains the supported place for chunker-owned context.
The framework does not require complete source-unit coverage and does not reject
overlap between different chunks. Stricter policies, such as full coverage or
non-overlap, belong to individual chunk modules when they are part of that
module's contract.
Within an artifact lane, the runner: Within an artifact lane, the runner:
@@ -119,9 +187,15 @@ On successful execution, the manifest validation status is:
## Manifest Population ## Manifest Population
The manifest records run ID, pipeline ID, pipeline digest, module keys, artifact The manifest records run ID, pipeline ID, pipeline digest, module keys, top-level
lanes, LLM profile metadata, source digest, validation status, and timing. module metadata, artifact lanes, LLM profile metadata, source digest,
reference provenance, validation status, and timing.
Modules can add non-secret manifest metadata by implementing Singleton pipeline modules may add non-secret metadata by implementing
`contracts.ManifestMetadataProvider`. The D&D spell extractor uses this for `contracts.ManifestMetadataProvider`. The runner records that metadata under
`module_metadata` with stable keys for `input`, `chunker`, and `output`.
Lane-owned modules may add non-secret metadata through
`artifact_lanes[].metadata`. The runner records extractor, merger, and
normalizer metadata there. The D&D spell extractor uses lane metadata for
prompt and response-schema provenance. prompt and response-schema provenance.

View File

@@ -34,8 +34,9 @@ The `json` output module writes these files:
- `index.json`: file index with paths to the manifest, artifact files, - `index.json`: file index with paths to the manifest, artifact files,
rejected artifacts, and warnings. rejected artifacts, and warnings.
- `manifest.json`: run manifest with resolved pipeline provenance, module keys, - `manifest.json`: run manifest with resolved pipeline provenance, top-level
validation status, and timing. module metadata, module keys, reference provenance, validation status, and
timing.
- `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact - `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact
type. For the current D&D spell extractor, this includes type. For the current D&D spell extractor, this includes
`artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved. `artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved.
@@ -62,8 +63,11 @@ Implemented diagnostics artifacts:
path, selected lanes, run ID, and pipeline digest when available. path, selected lanes, run ID, and pipeline digest when available.
- `effective-config.json`: resolved config with API keys redacted. - `effective-config.json`: resolved config with API keys redacted.
- `resolved-pipeline.json`: resolved module bindings and pipeline digest. - `resolved-pipeline.json`: resolved module bindings and pipeline digest.
- `resolved-references.json`: resolved reference provenance, including target
stage, lane ID when present, origin, digest, media type, byte size, and
binding source, without reference content.
- `run-manifest.json`: the same run manifest written to durable output when it - `run-manifest.json`: the same run manifest written to durable output when it
is available. is available, including top-level module metadata when present.
- `warnings.json`: warning list. - `warnings.json`: warning list.
- `run-report.json`: counts, status, output path, diagnostics path, and run ID. - `run-report.json`: counts, status, output path, diagnostics path, and run ID.
- `error.log`: failure message, written after diagnostics directory creation - `error.log`: failure message, written after diagnostics directory creation
@@ -107,6 +111,10 @@ stderr, and writes warnings to durable output and diagnostics when retained.
The run manifest `validation_status` indicates whether final artifacts were The run manifest `validation_status` indicates whether final artifacts were
approved or rejected after validation. approved or rejected after validation.
Reference-related warnings include empty bound reference files and D&D spell
relatedness warnings such as `spell_not_near_source`. Empty references are still
passed to extractors so optional slots can be intentionally blank.
## Cleanup ## Cleanup
It is safe to remove specific old run directories after their output and It is safe to remove specific old run directories after their output and

View File

@@ -8,6 +8,16 @@ future work only.
- Additional input adapters, such as Markdown or note-export formats. - Additional input adapters, such as Markdown or note-export formats.
- Additional D&D extractors beyond spell casts. - Additional D&D extractors beyond spell casts.
- Add non-file reference producers, such as prior-run artifacts, derived
summaries, or entity registries, without changing module-facing reference
item contracts.
- Add token budgeting and model context-window management for reference content.
- Add per-slot or per-chunk inclusion policies so modules can avoid repeating
large reference content in every prompt when that becomes important.
- Add structured or parsed references, such as typed roster schemas, when a
module has a clear need for more than opaque UTF-8 text.
- Add reference caching, preprocessing, summarization, embedding, or retrieval
if references become large enough to require preprocessing.
- Cross-lane entity normalization. - Cross-lane entity normalization.
- Cross-chunk semantic deduplication. - Cross-chunk semantic deduplication.
- Configurable validator chains with production validator modules. - Configurable validator chains with production validator modules.
@@ -15,6 +25,13 @@ future work only.
- Parallel execution where it preserves deterministic manifests and diagnostics. - Parallel execution where it preserves deterministic manifests and diagnostics.
- Additional output encoders. - Additional output encoders.
## Candidate Architecture Work
- Evaluate replacing the local LLM adapter with an import from
`gitea.maximumdirect.net/eric/scriptorium`, provided it preserves Notarius
boundaries around provider plumbing, prompt ownership, diagnostics, and secret
handling.
## Candidate Operational Work ## Candidate Operational Work
- Packaged release artifacts for alpha distribution. - Packaged release artifacts for alpha distribution.

View File

@@ -0,0 +1,474 @@
# Scriptorium Cutover Implementation Plan
This plan implements the target state in
[scriptorium.md](scriptorium.md): a hard cutover from Notarius' local
OpenAI-compatible adapter, local LLM profile schema, and local system/user
prompt renderer to Scriptorium-backed prompt execution.
The target implementer is an LLM coding agent. Complete each stage in order.
Do not preserve backward compatibility for removed local LLM-profile or
prompt-rendering behavior unless a stage explicitly says to keep a temporary
test seam.
## Global Constraints
- Keep Scriptorium types out of chunk, extract, and normalize module contracts.
Module-facing requests should use Notarius-owned contract types.
- Keep provider plumbing behind `internal/framework/llm` and CLI construction.
- Keep module-owned prompt intent, prompt IDs, response schemas, validators, and
source-reference validation with the relevant modules.
- Keep raw input material, raw references, raw prompts, raw schemas, and API
keys out of manifests and default diagnostics.
- Treat the original source input and large references as prompt input
materials. They may be passed to Scriptorium as inline/file inputs, but should
not be copied into durable provenance.
- Use Scriptorium profile configuration directly. Do not keep Notarius'
`llm_profiles` schema as a second profile system.
- Use Scriptorium structured-output execution and validation behavior for
production calls. Notarius should still unmarshal successful structured JSON
into module-owned response structs and run module-owned deterministic
validation afterward.
## Stage 1: Dependency And API Grounding
Goal: add the Scriptorium dependency and establish the exact public API surface
used by the rest of the implementation.
Work:
- Add `gitea.maximumdirect.net/eric/scriptorium` to `go.mod` with `go get`.
- Inspect the installed package with `go doc` or source reads before coding the
adapter. Confirm names and fields for:
- `NewEngine`;
- `Config`;
- `WithPromptFS`, `WithPromptFile`, or equivalent prompt source options;
- `WithProfileFile`, `WithProfileFS`, profile directory config, and built-in
profile behavior;
- `WithSchemaFS` or equivalent schema source options;
- `RunRequest`, including prompt ID, prompt version, profile ID, inputs,
variables, metadata, direct API key, and validation override fields;
- `RunResult`, including raw output, output artifact, validation status,
prompt/profile/model metadata, usage, and timings;
- public sentinel errors.
- Add a short internal implementation note as a code comment only where needed;
do not add user-facing docs in this stage except if a package-level test
helper needs explanation.
Acceptance checks:
- `go test ./internal/framework/llm`
- `go test ./internal/cli`
- `go test ./...`
## Stage 2: Prompt Input Materials And Session IDs
Goal: make raw source input and references available to module prompt execution
without storing large content in manifests or default diagnostics.
Work:
- Add Notarius-owned contract types under `internal/framework/contracts`:
- `LLMInputMaterial` with at least `Name`, `MediaType`, `Content`, `Digest`,
`OriginURI`, and `SizeBytes`;
- a helper-friendly collection type, such as `LLMInputSet`, if useful.
- Add prompt-execution fields to `StructuredCompletionRequest`:
- `PromptID`;
- `PromptVersion`;
- `ProfileID` or reuse the request `LLMProfile` value when called from stage
requests;
- `SessionID`;
- `Inputs map[string]LLMInputMaterial`;
- `Vars map[string]any`.
- Keep or remove the old `Messages`, `Model`, `ResponseSchemaName`, and
`ResponseSchema` fields according to what makes the cutover cleanest. The
final production path must not require modules to pass rendered messages or
raw schema JSON directly to the provider adapter.
- Add `SourceInput contracts.LLMInputMaterial` and `SessionID string` to
`ChunkRequest`, `ExtractionRequest`, and `NormalizeRequest`.
- Add `SessionID string` to `pipeline.RunInput`.
- In `pipeline.Run`, build a source input material from `RunInput.RawInput` and
`RunInput.Path`:
- preserve the bytes exactly;
- infer media type from the input path extension, using `application/json`
for `.json` and a deterministic fallback for unknown extensions;
- compute a SHA-256 digest;
- use a file URI or path-derived origin URI;
- do not put the bytes in manifest metadata.
- Pass the same source input material and session ID to chunk, extract, and
normalize requests.
- Reuse existing materialized `ReferenceSet` content for reference prompt
inputs. Do not introduce a second file-reading path for references.
- Add CLI support for `--session-id <id>` on `notarius run`.
- Empty means use a deterministic default derived from the parsed source
document ID.
- The deterministic default must be stable across runs over the same parsed
source document.
- The explicit value should be trimmed and rejected if empty after trimming.
- Because the default depends on the parsed source document, compute and attach
the final session ID inside the runner after input parsing, or return enough
information from parsing for the CLI to resolve it before LLM calls.
- Record the non-secret session ID in run metadata or manifest metadata, but
never record raw prompt content.
Acceptance tests:
- Contract tests proving `LLMInputMaterial.Content` is defensively copied where
relevant and omitted from JSON.
- Pipeline tests proving chunk, extract, and normalize requests receive the same
source input bytes and session ID.
- CLI tests for `--session-id`, including explicit value, missing value,
trimming, and default behavior.
- Redaction/diagnostics tests proving raw source bytes and reference content do
not appear in manifests or default diagnostics.
Focused checks:
- `go test ./internal/framework/contracts`
- `go test ./internal/framework/pipeline`
- `go test ./internal/cli`
## Stage 3: Configuration Hard Cutover To Scriptorium Profiles
Goal: replace Notarius' local LLM profile schema with Scriptorium profile
selection and profile sources.
Work:
- Bump the Notarius file config version because this is an incompatible config
schema change.
- Remove the top-level `llm_profiles` file-config schema and the local
`config.LLMProfile` model.
- Add a top-level Scriptorium config block. Use this shape unless Scriptorium's
installed API requires a small naming adjustment:
```yaml
scriptorium:
profile_dir: ./profiles
profile_file: ./profiles.yml
```
- Treat `profile_dir` and `profile_file` as mutually exclusive in Notarius
config validation. Built-in Scriptorium profiles remain available when neither
is set.
- Keep existing module binding field name `llm_profile`; it now names a
Scriptorium profile ID.
- Stop forcing empty module `llm_profile` bindings to
`pipeline.DefaultLLMProfile`. Empty means "use the Scriptorium prompt's
`default_profile`."
- Keep `--llm-profile` as a run-level operational override. It should set the
same explicit Scriptorium profile ID for every LLM-eligible selected target.
- Remove `OpenAICompatibleClientConfig` construction from `internal/core/config`.
- Remove local OpenAI-compatible provider validation from config validation.
- Add config validation for:
- config version;
- mutually exclusive `scriptorium.profile_dir` and
`scriptorium.profile_file`;
- non-empty profile source paths when fields are present;
- non-empty explicit `llm_profile` strings after trimming.
- Add CLI/config validation that uses Scriptorium profile loading to reject
unknown explicit profile IDs when possible. Prompt-default profile failures
may surface during Scriptorium prepare/run if Scriptorium owns that lookup.
- Remove or replace environment override behavior tied to
`NOTARIUS_LLM_DEFAULT_*`. Secrets should come from Scriptorium profile
`api_key_env` values or direct request-scoped Scriptorium behavior.
- Update redacted effective config behavior for the new Scriptorium config
shape.
Acceptance tests:
- Config parsing accepts `scriptorium.profile_dir`.
- Config parsing accepts `scriptorium.profile_file`.
- Config validation rejects both fields set at once.
- Config validation rejects stale `llm_profiles`.
- Existing pipeline bindings with explicit `llm_profile` resolve to trimmed
Scriptorium profile IDs.
- Empty `llm_profile` remains empty through resolution unless `--llm-profile`
is supplied.
- CLI config validation fails cleanly for an unknown explicit profile ID.
- Redacted config diagnostics do not contain raw API keys.
Focused checks:
- `go test ./internal/core/config`
- `go test ./internal/cli`
## Stage 4: Scriptorium Prompt And Schema Assets
Goal: move production LLM prompts and schemas to Scriptorium-compatible assets
while preserving module ownership.
Work:
- Create a prompt/schema asset registration mechanism that does not put
D&D-specific prompt content in framework packages.
- Recommended shape: add a small prompt-asset registry in
`internal/framework/llm` or a sibling framework package that can collect
`fs.FS` roots for Scriptorium prompt and schema sources.
- Production module packages should register their own Scriptorium prompt and
schema assets through production catalog/registry wiring in `internal/cli`.
- Shared D&D prompt assets may live in a D&D-specific module package such as
`internal/modules/dnd/promptassets`; they must not live in `internal/core`
or source-agnostic framework packages.
- Add shared D&D prompt assets:
- stable shared system message, if needed;
- stable cacheable transcript user message:
```text
A transcript of a Dungeons & Dragons gameplay session is provided below.
{{ input "transcript" }}
```
- stable cacheable reference/context message templates for roster, glossary,
previous recap, or other large references used by current D&D modules.
- Convert `dnd/scenes` to a Scriptorium prompt definition:
- prompt ID: `dnd.scenes`;
- prompt version: current module prompt version;
- input `transcript`, required, `application/json`;
- messages ordered for cache reuse: shared system, shared transcript user
message with cache control, scene task, scene instructions;
- output JSON schema path pointing at the existing scene schema asset;
- schema IDs/names/versions remain module-owned and manifest-safe.
- Convert `dnd/spells` to a Scriptorium prompt definition:
- prompt ID: `dnd.spells`;
- prompt version: current module prompt version;
- input `transcript`, required, `application/json`;
- optional inputs for `roster` and `glossary`;
- messages ordered for cache reuse: shared system, shared transcript user
message with cache control, optional/reference context message with cache
control, spell task, spell instructions;
- output JSON schema path pointing at the existing spell schema asset.
- For optional references, pass empty inline input material when the slot is
unbound unless Scriptorium's template/input semantics support missing
optional inputs cleanly. Do not let optional missing references make prompt
rendering fail.
- For multiple reference items in one slot, concatenate deterministically with
stable headings that include only non-secret provenance, then pass the result
as that prompt input. Existing single-item slots should keep their current
behavior.
- Replace current prompt hash metadata with hashes derived from the
Scriptorium prompt definition plus message assets, or with Scriptorium
prepared-run prompt metadata if it is available without raw prompt content.
- Preserve existing manifest metadata keys where practical:
- `prompt_id`;
- `prompt_version`;
- `prompt_sha256`;
- `response_schema_key`;
- `response_schema_id`;
- `response_schema_name`;
- `response_schema_version`;
- `response_schema_sha256`.
Acceptance tests:
- Prompt asset loading fails fast for missing prompt files or schemas.
- `dnd/scenes` prepared prompt contains separate transcript and task messages.
- `dnd/spells` prepared prompt contains separate transcript, reference, and
task messages.
- The transcript message body is byte-identical to the expected shared template
plus original Seriatim JSON bytes.
- Reference prompt input rendering is deterministic.
- Prompt/schema diagnostics omit raw prompt text, raw transcript bytes, raw
reference content, and raw schema JSON.
Focused checks:
- `go test ./internal/framework/llm`
- `go test ./internal/modules/chunk/dnd/scenes`
- `go test ./internal/modules/extract/dnd/spells`
## Stage 5: Scriptorium-Backed LLM Runtime
Goal: implement the production `StructuredLLMClient` using Scriptorium.
Work:
- Add a Scriptorium-backed client under `internal/framework/llm`.
- Its constructor should accept:
- Scriptorium profile source settings from effective Notarius config;
- registered prompt/schema assets;
- request timeout or HTTP client settings only if still owned by Notarius
after the profile cutover;
- optional Scriptorium engine options for tests.
- Implement `CompleteStructured(ctx, req, out)` by:
- validating `out` is a non-nil pointer;
- validating `req.PromptID` is non-empty;
- converting Notarius `LLMInputMaterial` values to Scriptorium artifact refs;
- adding `session_id` to Scriptorium request vars when non-empty;
- passing explicit profile ID only when the module binding or CLI override
supplied one;
- passing no raw API key unless a deliberate request-scoped secret path is
implemented;
- calling Scriptorium `Run`;
- converting final validation failure into a Notarius error;
- unmarshaling successful structured JSON into `out`;
- returning `StructuredCompletionResponse` with raw JSON content, provider,
model, profile ID when available, token usage, and non-secret metadata.
- Add a profile recorder or response accumulator so `RunManifest.LLMProfiles`
records the actual Scriptorium profile/provider/model values used during the
run.
- Do not rely on one precomputed profile ID before pipeline execution.
- Deduplicate profile manifest entries deterministically.
- Keep the existing `Scheduler` and scheduled client wrapper unless Scriptorium
provides an equivalent Notarius-approved concurrency mechanism.
- Ensure all Scriptorium errors are wrapped with context and converted to
concise CLI-facing errors. Preserve `errors.Is` checks internally when
practical.
- Apply Notarius secret redaction to errors before writing diagnostics.
- Remove production construction of `OpenAICompatibleClient`.
Acceptance tests:
- Scriptorium adapter maps Notarius prompt request fields into the expected
Scriptorium run request using an injected fake Scriptorium LLM client.
- Successful structured output unmarshals into the caller target.
- Scriptorium validation failure returns an error.
- Provider/runtime failure returns an error with operation context.
- Context cancellation is respected.
- Token usage maps into `StructuredCompletionResponse`.
- Used Scriptorium profile/provider/model metadata appears in the run manifest
without secrets.
- API keys or bearer tokens in synthetic errors are redacted.
- Scheduler still bounds concurrent Scriptorium-backed calls.
Focused checks:
- `go test ./internal/framework/llm`
- `go test ./internal/framework/pipeline`
- `go test ./internal/cli`
## Stage 6: Module Cutover And Legacy Runtime Removal
Goal: update production modules to call the prompt-based contract and remove
obsolete local prompt/runtime code.
Work:
- Update `dnd/scenes`:
- stop rendering local system/user prompt strings;
- call `CompleteStructured` with `PromptID`, `PromptVersion`, `SessionID`,
`ProfileID` or request LLM profile, `transcript` input material, and any
required vars;
- keep existing response validation, chunk canonicalization, caveat handling,
and manifest metadata policy.
- Update `dnd/spells`:
- stop rendering local system/user prompt strings;
- call `CompleteStructured` with `transcript`, optional `roster`, optional
`glossary`, session ID, profile ID, and vars;
- keep existing spell response validation, source-reference validation, and
manifest metadata policy.
- Update any LLM-backed normalize modules if present. If only noop normalize is
present, ensure the contract and tests prove normalizers can receive the same
Scriptorium-capable client and prompt inputs.
- Remove the old `internal/framework/prompt` renderer if no remaining code uses
it. If generic tests still need prompt rendering, replace them with
Scriptorium prompt asset tests or delete obsolete tests.
- Remove `internal/framework/llm/openai_compatible_client.go` and its tests
after the Scriptorium adapter tests cover replacement behavior.
- Remove local OpenAI-compatible integration docs after current-behavior docs
are updated in Stage 7.
- Remove stale schema registry helpers only if they are no longer needed for
module-owned schema metadata. Keep lightweight schema hashing/loading helpers
if modules still use them for manifest metadata.
Acceptance tests:
- `dnd/scenes` fake-client tests assert the module sends prompt ID,
transcript input, session ID, and schema/prompt metadata rather than rendered
message text.
- `dnd/spells` fake-client tests assert roster/glossary inputs are passed as
inputs and not interpolated locally.
- Existing malformed LLM response tests still fail as malformed structured
output.
- Existing source-reference validation tests still pass.
- No production code imports the old prompt renderer or local
OpenAI-compatible client.
Focused checks:
- `go test ./internal/modules/chunk/dnd/scenes`
- `go test ./internal/modules/extract/dnd/spells`
- If `internal/framework/prompt` is deleted, do not run a package-specific test
for it; instead verify with `rg -n "internal/framework/prompt|RenderUserSystem" internal`
that no production code still depends on it.
## Stage 7: CLI, Examples, Docs, And Full Validation
Goal: finish user-facing behavior, examples, and canonical docs for the new
runtime.
Work:
- Update `examples/dnd-spells.config.yml` and any maintained test configs to
the new config version and Scriptorium profile source behavior.
- Update `docs/config.md`:
- new config version;
- `scriptorium.profile_dir` and `scriptorium.profile_file`;
- `llm_profile` now means Scriptorium profile ID;
- empty `llm_profile` behavior;
- removed `llm_profiles`;
- removed `NOTARIUS_LLM_DEFAULT_*` env behavior if removed;
- secret-handling policy through Scriptorium profile `api_key_env`.
- Update `docs/cli.md`:
- add `--session-id`;
- update `--llm-profile` wording to Scriptorium profile override;
- update run examples if needed.
- Update `docs/internal/llm.md`:
- Scriptorium-backed runtime;
- prompt asset loading;
- input material handling;
- scheduling;
- structured-output validation;
- profile manifest recording;
- secret redaction.
- Update `docs/internal/modules.md`:
- module prompt ownership through Scriptorium prompt definitions;
- chunk/extract/normalize request input materials and session ID;
- D&D scenes/spells prompt metadata.
- Update `docs/internal/pipeline.md`:
- raw input material lifecycle;
- reference material lifecycle into prompt inputs;
- session ID lifecycle;
- actual LLM profile provenance.
- Update `docs/integrations/json-output.md` for any manifest changes.
- Remove or rewrite `docs/integrations/openai-compatible.md`. If no local
OpenAI-compatible adapter remains, do not document it as current Notarius
behavior.
- Update `docs/troubleshooting.md` for:
- Scriptorium profile-not-found;
- prompt-not-found;
- schema/validation failures;
- missing API key env values;
- session ID usage if relevant.
- Update `docs/operations.md` if diagnostics or retention behavior changes.
- Remove the Scriptorium migration item from `docs/roadmap/future.md` once the
feature is implemented.
- Replace `docs/roadmap/implementation.md` with a completed note or remove it
after implementation is complete, according to the repository's current
roadmap cleanup pattern.
Acceptance tests and inspections:
- `rg -n "llm_profiles|OpenAICompatibleClient|openai-compatible|RenderUserSystem|NOTARIUS_LLM_DEFAULT" internal docs examples`
should return only intentional historical/deferred references, if any.
- `rg -n "scriptorium|session-id|profile_dir|profile_file" docs examples`
should show current-behavior docs and examples are updated.
- `go test ./...`
- `go vet ./...`
- `go build ./cmd/notarius`
## Cross-Stage Review Checklist
Before considering the implementation complete, verify:
- No Scriptorium public types appear in chunk, extract, or normalize contracts.
- No raw source input, reference content, prompt text, schema JSON, API key, or
bearer token appears in manifests or default diagnostics.
- The shared transcript message uses original Seriatim JSON bytes exactly.
- Large reference content can be supplied as Scriptorium prompt inputs without
changing module-facing reference slot contracts.
- `--session-id` is easy for an external orchestrator to pass.
- Empty `llm_profile` lets prompt defaults work; explicit `llm_profile` and
`--llm-profile` select Scriptorium profile IDs.
- Production Notarius LLM execution goes through Scriptorium.
- Current-behavior documentation describes only implemented behavior.

295
docs/roadmap/scriptorium.md Normal file
View File

@@ -0,0 +1,295 @@
# Scriptorium LLM Runtime Roadmap
This roadmap describes the target state for replacing Notarius' local
OpenAI-compatible LLM adapter with an integration based on
`gitea.maximumdirect.net/eric/scriptorium`.
The upgrade is worthwhile only if it preserves Notarius' architectural
boundaries:
- modules own prompt intent, prompt variables, response schemas, validation, and
domain interpretation;
- provider plumbing stays behind framework LLM contracts;
- diagnostics and manifests remain Notarius-owned and secret-free;
- raw API keys are resolved operationally and must not be stored in config,
diagnostics, manifests, examples, or prompt/profile assets;
- Scriptorium types do not leak into chunk, extract, or normalize module
contracts unless explicitly chosen as a future public contract.
## Motivation
Notarius currently has a local structured-output OpenAI-compatible adapter. That
adapter is intentionally narrow, but it makes Notarius responsible for provider
request shape, profile support, structured-output retries, model-specific
extensions, and future cache-related request fields.
Scriptorium adds capabilities that are directly useful for Notarius:
- built-in profiles for many providers and model families, including OpenRouter
and local OpenAI-compatible inference endpoints;
- prompt definitions with multiple system and user messages;
- per-prompt `session_id` support for sticky provider routing and input cache
affinity;
- per-message `cache_control` support for providers that understand it;
- a simple structured-output path where a prompt supplies a schema and
Scriptorium sends it upstream and retries non-compliant responses.
The most important near-term motivation is input-cache reuse. Existing D&D
recap prompts already use a cacheable user message shaped as:
```text
A transcript of a Dungeons & Dragons gameplay session is provided below.
{{ input "transcript" }}
```
The `transcript` input is the same Seriatim JSON transcript that Notarius
already accepts as MVP input, including stable numbered segments. Notarius
should align chunk, extract, and normalize LLM calls so this transcript message
can be byte-identical across recap generation and structured extraction work.
For large transcripts, this can materially reduce provider cost when cached
input reads are available.
## Target State
Notarius uses Scriptorium as the production LLM execution engine behind the
existing module-facing LLM boundary.
Chunk, extract, and normalize modules continue to receive a Notarius
`StructuredLLMClient` through their existing request contracts. They do not
construct provider clients, read secrets, or depend directly on Scriptorium
request/result types.
The production CLI constructs a Scriptorium-backed client from the effective
Notarius configuration, wraps it in Notarius scheduling and diagnostics policy
where needed, and returns Notarius-owned non-secret LLM profile manifest
metadata.
Module-owned prompts are rendered as ordered chat messages, not only as one
system message and one user message. Prompt rendering must support:
- repeated roles;
- shared input messages;
- module-specific task and instruction messages;
- optional cache-control metadata;
- a stable session ID for calls that should share provider routing/cache
affinity;
- structured-output schema metadata owned by the module.
The shared D&D transcript message should be rendered from the original Seriatim
input bytes, not reconstructed from normalized source units. The parsed source
document remains the canonical source-reference model for validation and
artifact grounding, but the cacheable transcript prompt message should preserve
the exact transcript payload supplied to the run.
Other large text inputs, including references such as campaign glossaries,
rosters, previous recaps, and future reference producers, should follow the same
input-material model. They should be available to prompt rendering as stable
input artifacts that can be placed in cacheable messages without being copied
into manifests or diagnostics.
## Boundary Requirements
### Provider Plumbing
Provider-specific request fields, profile expansion, and OpenAI-compatible wire
details should be isolated in the framework LLM runtime or in the
Scriptorium-backed adapter. Stage modules should request structured completion
through Notarius contracts and should not know whether the backing client is the
legacy local adapter or Scriptorium.
### Prompt Ownership
Modules remain responsible for prompt intent, prompt versioning, response schema
selection, and semantic validation. Scriptorium provides the prompt definition,
rendering, execution, and structured-output workflow, but it should not become
the owner of D&D-specific semantics.
Shared prompt assets are allowed when they express generic reusable context,
such as the D&D transcript input message. Module-specific task prompts should
remain owned by the relevant module.
### Diagnostics And Manifests
Notarius should continue to decide what appears in diagnostics and run
manifests. Scriptorium prepared/run metadata may be useful input, but Notarius
must filter it through existing policy:
- no raw prompt payloads by default;
- no raw response schema content in manifests;
- no source transcript payloads in manifests;
- no API keys or bearer tokens;
- enough prompt/profile/schema hashes and IDs to audit a run later.
### Secret Handling
Raw API keys should stay out of durable config and prompt/profile assets.
Notarius may continue resolving API keys from configured environment-variable
names, then pass the secret request-scope to Scriptorium. Redacted diagnostics
must continue to prove that resolved secrets are not serialized.
## Prompt And Cache Strategy
The desired D&D LLM prompt shape is:
1. stable shared system message;
2. stable shared transcript user message containing the original Seriatim JSON;
3. optional stable cacheable context messages, such as previous recap,
glossary, roster, or other references;
4. module-specific task message;
5. module-specific instruction message;
6. module-owned structured-output schema, when the module expects JSON.
The transcript message should be byte-identical whenever the same transcript
input bytes are used. Avoid reconstructing JSON from parsed source units because
formatting, key order, whitespace, or escaping changes would defeat cache reuse.
The transcript and large reference messages should be cacheable when the active
provider path supports cache-control metadata. Providers that ignore cache
controls should still receive a valid prompt.
The session ID should be stable for all LLM calls that operate on the same
session transcript and should be explicitly visible in diagnostics or manifest
metadata only as a non-secret identifier.
Notarius should provide an easy CLI UX for setting the session ID. This lets an
external D&D pipeline orchestrator pass the same session ID used by recap or
other LLM steps, maximizing provider routing affinity and cached input reuse.
## Configuration Intent
Notarius configuration should remain the user-facing source of pipeline
composition and operational settings. For LLM execution profiles, Notarius
should cut over to Scriptorium profile configuration instead of maintaining a
separate Notarius-specific profile schema.
Notarius should allow configuration to select Scriptorium built-in profiles and
to point at Scriptorium profile files or directories. This avoids evolving two
nearly identical profile systems and gives users immediate access to the
provider and model catalog that motivated the migration.
Notarius still owns validation, redaction, and manifest provenance for the
effective pipeline. The integration should wrap Scriptorium profile loading so
that errors are actionable, secrets remain environment-based, and emitted
metadata stays non-secret.
## Compatibility Policy
This feature should be a hard cutover to Scriptorium-backed prompt execution and
profile loading. Backward compatibility with the local Notarius prompt renderer,
local LLM profile schema, or local OpenAI-compatible adapter is not a product
requirement for this migration.
The final production runtime should have one documented LLM execution path.
Module behavior may still be tested with fake Notarius `StructuredLLMClient`
implementations, but production runtime behavior should be Scriptorium-backed.
## Documentation Outcomes
When this feature is implemented, current-behavior docs should be updated to
describe:
- the implemented production LLM runtime;
- supported LLM profile fields and provider/profile selection behavior;
- cacheable transcript prompt behavior, if exposed to users;
- session ID behavior;
- diagnostics and manifest provenance;
- troubleshooting for Scriptorium profile, prompt, schema, and validation
failures.
The OpenAI-compatible integration doc should either be retired, narrowed to the
legacy fallback, or reframed as an upstream provider contract delegated through
Scriptorium, depending on the final runtime shape.
## Deferred Work
The Scriptorium migration does not itself require:
- a general workflow language;
- arbitrary per-stage prompt authoring by end users;
- token budgeting or context-window planning;
- non-file reference producers;
- semantic retrieval over transcript or reference content;
- multiple effective LLM profiles in one Notarius run.
Those remain separate future features.
## Resolved Design Choices
### Prompt Asset Ownership And Format
Production LLM prompts should cut over to Scriptorium-compatible prompt
definitions without preserving backward compatibility for the current Notarius
system/user prompt renderer.
Module-owned prompt definitions should keep task prompts and schemas near the
module that owns the semantic behavior. Shared prompt messages, such as the D&D
transcript input message and shared reference/context messages, should live in a
shared prompt asset area and be referenced by module-owned prompt definitions.
This uses Scriptorium's native strengths: ordered messages, cache-control
metadata, session IDs, input helpers, profile selection, and schema-backed
execution. It also gives Notarius a direct way to reuse the exact transcript and
reference messages across D&D recap, chunking, extraction, and normalization
work without each module hand-rolling message assembly.
### Raw Transcript Preservation Location
Original input bytes should be preserved as run-scoped input material and made
available to prompt rendering without storing them in run manifests. The parsed
`SourceDocument` should continue to carry normalized source units for framework
logic and source-reference validation.
The same model should apply to all large text inputs, including references.
Glossaries, campaign rosters, previous recaps, and other future reference
documents may be large enough to deserve their own cacheable messages. They
should be treated as prompt input artifacts rather than being copied into source
document metadata.
This keeps the cacheable transcript message byte-identical to the original
Seriatim JSON and avoids source-format leakage into chunk, extract, and
normalize contracts. It also reduces the risk that raw inputs or references are
accidentally serialized into diagnostics or manifests.
### Session ID Source
Notarius should add an explicit session ID concept for LLM calls, with a
deterministic default derived from the source document identity when no
operator-provided value is configured.
The CLI should provide an easy way to supply this session ID. This lets an
external D&D pipeline orchestrator call Notarius with the same session ID used
by recap generation or other LLM steps, maximizing provider routing affinity and
cached input savings.
OpenRouter sticky routing and provider cache behavior are most useful when
every related call for the same session shares a stable identifier. An explicit
concept makes the behavior auditable and avoids each module inventing its own
ID. A deterministic default keeps simple local runs ergonomic.
### Scriptorium Profile Exposure
Notarius should cut over to Scriptorium profile configuration instead of
maintaining a separate Notarius LLM profile schema. Configuration should be able
to select Scriptorium built-in profiles and point at Scriptorium profile files
or directories.
The Scriptorium profile format is already close to the desired Notarius target.
Maintaining a separate Notarius profile schema would likely create duplicate
configuration that eventually converges back toward Scriptorium's model. A hard
cutover avoids that churn and gives users immediate access to Scriptorium's
provider and model catalog.
Notarius still needs to wrap this profile loading with Notarius-owned
validation, diagnostics, manifest provenance, and secret-redaction policy.
### Structured-Output Retry Ownership
Notarius should use Scriptorium's structured-output execution and retry behavior
for production calls, while translating results and errors back into Notarius'
`StructuredLLMClient` response and error expectations.
This avoids duplicating structured-output enforcement in Notarius and lets
Scriptorium own provider-specific request and repair mechanics. Notarius still
retains module-level validation and source-reference checks after decoded
structured output is returned.

View File

@@ -102,6 +102,45 @@ go run ./cmd/notarius run dnd-session \
- For `config validate`, include `--pipeline` when using `--only`. - For `config validate`, include `--pipeline` when using `--only`.
- Confirm the lane ID exists under `pipelines.<id>.artifacts`. - Confirm the lane ID exists under `pipelines.<id>.artifacts`.
## Reference Binding Failure
Symptoms include:
- `reference slot "..." is not declared`
- `reference slot "..." is declared by multiple selected targets`
- `required reference slot "..." is not bound`
- `--reference must use slot=path`
- `--without-reference must use a reference selector without =path`
- `read "...": no such file`
- `must be UTF-8 text`
- `media type "..." is not accepted`
- `is ... bytes, limit ...`
Fix:
- Confirm the selected chunker, extractor, or normalizer declares the slot. The
implemented `dnd/spells` extractor declares optional `roster` and `glossary`
slots.
- Use a specific selector when more than one selected target declares the same
slot. Examples include `chunk.context=./context.txt`,
`spells.extract.context=./extract-context.txt`, and
`spells.normalize.context=./normalize-context.txt`.
- `lane.slot=path` is valid only when exactly one selected extractor or
normalizer in that lane declares the slot. If both do, use
`lane.extract.slot=path` or `lane.normalize.slot=path`.
- Use `--without-reference selector` to remove optional config bindings; do not
pass an empty `--reference selector=`.
- Check whether a path came from config or CLI. Config paths are relative to
the config file. CLI reference paths are relative to the current working
directory.
- Ensure the file is readable UTF-8 text and within any byte limit declared by
the declaring module.
- If the declaring module narrows accepted media types, use a file extension that
infers an accepted type such as `text/markdown` or `application/json`.
Unknown extensions infer `application/octet-stream`.
- If diagnostics are retained, inspect `resolved-pipeline.json`,
`resolved-references.json`, and `error.log`.
## Seriatim Input Validation Failure ## Seriatim Input Validation Failure
Symptoms include `seriatim input`, `parse JSON`, `segments must not be empty`, Symptoms include `seriatim input`, `parse JSON`, `segments must not be empty`,
@@ -177,6 +216,34 @@ Fix:
Provider error messages are redacted for configured API key values. Provider error messages are redacted for configured API key values.
## Scene Chunking Failure
Symptoms include:
- `dnd scenes chunker`
- `malformed structured output`
- `boundary_caveats`
- `start_unit_id`
- `end_unit_id`
- `gap`
- `overlap`
- `final scene`
- `complete structured output`
Fix:
- Validate the pipeline configuration and confirm the input module provides a
transcript source when using `chunk: dnd/scenes`.
- Confirm the LLM profile has a working OpenAI-compatible `base_url`, `model`,
and credentials.
- Inspect retained diagnostics for the run error and resolved pipeline.
- If the error names malformed structured output, retry with a model that
follows structured response schemas reliably.
- If the error names `boundary_caveats`, check for blank or whitespace-only
caveat text in the scene response.
- Scene boundaries must use exact source-unit IDs, cover the full source
document, be contiguous, and not overlap.
## Output Write Failure ## Output Write Failure
Symptoms include: Symptoms include:

View File

@@ -0,0 +1,2 @@
Cure Wounds: healing spell cast by touch.
Shield: defensive reaction spell.

View File

@@ -0,0 +1,3 @@
Aria: party cleric and recurring healer.
Borin: fighter ally.
Bandit mage: hostile spellcaster.

View File

@@ -7,6 +7,9 @@ llm_profiles:
pipelines: pipelines:
dnd-session: dnd-session:
input: seriatim input: seriatim
references:
roster: ./dnd-spells-roster.txt
glossary: ./dnd-spells-glossary.txt
chunk: chunk:
module: generic module: generic
options: options:

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic" "gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim" "gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
@@ -34,6 +35,9 @@ func productionRegistries() (pipeline.Registries, error) {
if err := generic.Register(registries.Chunkers); err != nil { if err := generic.Register(registries.Chunkers); err != nil {
return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err) return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err)
} }
if err := scenes.Register(registries.Chunkers); err != nil {
return pipeline.Registries{}, fmt.Errorf("register dnd scenes chunker: %w", err)
}
if err := spells.Register(registries.Extractors); err != nil { if err := spells.Register(registries.Extractors); err != nil {
return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err) return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err)
} }

View File

@@ -25,7 +25,7 @@ const defaultOutputRoot = "./notarius-output"
const usage = `Usage: const usage = `Usage:
notarius help notarius help
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--reference selector=path] [--without-reference selector]
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list --config path/to/config.yml [--json] notarius pipelines list --config path/to/config.yml [--json]
` `
@@ -95,6 +95,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
outputDir := fs.String("output-dir", "", "output directory") outputDir := fs.String("output-dir", "", "output directory")
diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory") diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory")
llmProfile := fs.String("llm-profile", "", "LLM profile override") llmProfile := fs.String("llm-profile", "", "LLM profile override")
referenceFlags := stringListFlag{}
withoutReferenceFlags := stringListFlag{}
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, lane.slot=path, lane.extract.slot=path, or lane.normalize.slot=path")
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
if err := fs.Parse(reorderRunArgs(args)); err != nil { if err := fs.Parse(reorderRunArgs(args)); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err) fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2 return 2
@@ -121,6 +125,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stderr, "notarius: %v\n", err) fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2 return 2
} }
referenceRequests, err := parseReferenceFlags(referenceFlags)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
referenceUnbindRequests, err := parseReferenceUnbindFlags(withoutReferenceFlags)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
cfg, loadedConfigPath, err := loadConfig(*configPath, opts) cfg, loadedConfigPath, err := loadConfig(*configPath, opts)
if err != nil { if err != nil {
@@ -155,15 +169,33 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil { if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err) return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
} }
referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests)
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
effective, err := cfg.Resolve(config.ResolveInput{ effective, err := cfg.Resolve(config.ResolveInput{
PipelineID: pipelineID, PipelineID: pipelineID,
Only: only, Only: only,
Catalog: catalog, Catalog: catalog,
LLMProfileOverride: *llmProfile, LLMProfileOverride: *llmProfile,
ReferenceOverrides: referenceOverrides,
ReferenceUnbinds: referenceUnbinds,
}) })
if err != nil { if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err) return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
} }
workingDir, err := os.Getwd()
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("resolve working directory: %w", err))
}
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
ConfigPath: loadedConfigPath,
WorkingDir: workingDir,
})
if err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
}
effective.ResolvedPipeline = materialized
invocation.PipelineDigest = effective.ResolvedPipeline.Digest invocation.PipelineDigest = effective.ResolvedPipeline.Digest
if err := runDir.WriteInvocationMetadata(invocation); err != nil { if err := runDir.WriteInvocationMetadata(invocation); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err)) return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
@@ -174,6 +206,9 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err := runDir.WriteResolvedPipeline(effective.ResolvedPipeline); err != nil { if err := runDir.WriteResolvedPipeline(effective.ResolvedPipeline); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved pipeline: %w", err)) return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved pipeline: %w", err))
} }
if err := runDir.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline)); err != nil {
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved references: %w", err))
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline) profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if len(profileIDs) != 1 { if len(profileIDs) != 1 {
@@ -205,6 +240,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
StartedAt: startedAt, StartedAt: startedAt,
LLMProfiles: llmProfiles, LLMProfiles: llmProfiles,
Metadata: runMetadata(*outputDir, *diagnosticsDir), Metadata: runMetadata(*outputDir, *diagnosticsDir),
Warnings: referenceWarnings,
}) })
if err != nil { if err != nil {
if output.Manifest.PipelineID != "" { if output.Manifest.PipelineID != "" {
@@ -412,7 +448,7 @@ func reorderRunArgs(args []string) []string {
func runFlagTakesValue(arg string) bool { func runFlagTakesValue(arg string) bool {
switch arg { switch arg {
case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile": case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile", "--reference", "--without-reference":
return true return true
default: default:
return false return false
@@ -661,6 +697,417 @@ func parseOnly(raw string) ([]string, error) {
return result, nil return result, nil
} }
type stringListFlag []string
func (flag *stringListFlag) String() string {
if flag == nil {
return ""
}
return strings.Join(*flag, ",")
}
func (flag *stringListFlag) Set(value string) error {
*flag = append(*flag, value)
return nil
}
type cliReferenceRequest struct {
Selector cliReferenceSelector
Source string
}
type cliReferenceUnbindRequest struct {
Selector cliReferenceSelector
}
type cliReferenceSelector struct {
LaneID string
Stage pipeline.ModuleStage
SlotName string
}
func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
if len(values) == 0 {
return nil, nil
}
requests := make([]cliReferenceRequest, 0, len(values))
for _, raw := range values {
name, source, ok := strings.Cut(raw, "=")
if !ok {
return nil, fmt.Errorf("--reference must use slot=path or lane.slot=path")
}
if strings.TrimSpace(source) == "" {
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
}
selector, err := parseReferenceSelector(name, "--reference")
if err != nil {
return nil, err
}
requests = append(requests, cliReferenceRequest{
Selector: selector,
Source: strings.TrimSpace(source),
})
}
return requests, nil
}
func parseReferenceUnbindFlags(values []string) ([]cliReferenceUnbindRequest, error) {
if len(values) == 0 {
return nil, nil
}
requests := make([]cliReferenceUnbindRequest, 0, len(values))
for _, raw := range values {
if strings.Contains(raw, "=") {
return nil, fmt.Errorf("--without-reference must use a reference selector without =path")
}
selector, err := parseReferenceSelector(raw, "--without-reference")
if err != nil {
return nil, err
}
requests = append(requests, cliReferenceUnbindRequest{
Selector: selector,
})
}
return requests, nil
}
func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector, error) {
selector := strings.TrimSpace(raw)
if selector == "" {
return cliReferenceSelector{}, fmt.Errorf("%s reference slot must not be empty", flagName)
}
parts := strings.Split(selector, ".")
for _, part := range parts {
if strings.TrimSpace(part) == "" {
return cliReferenceSelector{}, fmt.Errorf("%s must use non-empty reference selector values", flagName)
}
}
switch len(parts) {
case 1:
return cliReferenceSelector{SlotName: strings.TrimSpace(parts[0])}, nil
case 2:
first := strings.TrimSpace(parts[0])
slotName := strings.TrimSpace(parts[1])
if first == string(pipeline.StageChunk) {
return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil
}
return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
case 3:
laneID := strings.TrimSpace(parts[0])
stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
slotName := strings.TrimSpace(parts[2])
if stage != pipeline.StageExtract && stage != pipeline.StageNormalize {
return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot or lane.normalize.slot", flagName)
}
return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil
default:
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, lane.slot, lane.extract.slot, or lane.normalize.slot", flagName)
}
}
func resolveCLIReferenceRequests(
cfg config.Config,
pipelineID string,
only []string,
catalog pipeline.ModuleCatalog,
referenceRequests []cliReferenceRequest,
unbindRequests []cliReferenceUnbindRequest,
) ([]pipeline.ReferenceBinding, []pipeline.ReferenceUnbind, error) {
if len(referenceRequests) == 0 && len(unbindRequests) == 0 {
return nil, nil, nil
}
targets, err := selectedReferenceTargets(cfg, pipelineID, only, catalog)
if err != nil {
return nil, nil, err
}
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
for _, request := range referenceRequests {
target, err := resolveCLIReferenceTarget(targets, request.Selector)
if err != nil {
return nil, nil, err
}
overrides = append(overrides, pipeline.ReferenceBinding{
Stage: target.stage,
LaneID: target.laneID,
SlotName: request.Selector.SlotName,
Source: request.Source,
BindingSource: contracts.ReferenceBindingSourceCLI,
})
}
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
for _, request := range unbindRequests {
target, err := resolveCLIReferenceTarget(targets, request.Selector)
if err != nil {
return nil, nil, err
}
unbinds = append(unbinds, pipeline.ReferenceUnbind{
Stage: target.stage,
LaneID: target.laneID,
SlotName: request.Selector.SlotName,
})
}
return overrides, unbinds, nil
}
type selectedReferenceTarget struct {
laneID string
stage pipeline.ModuleStage
module string
slots map[string]struct{}
}
func selectedReferenceTargets(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceTarget, error) {
profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID)
if !ok {
return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID))
}
lanesByID := make(map[string]pipeline.ArtifactLaneProfile, len(profile.Artifacts))
for rawLaneID, lane := range profile.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
return nil, fmt.Errorf("pipeline %q artifact lane id must not be empty", strings.TrimSpace(pipelineID))
}
if _, ok := lanesByID[laneID]; ok {
return nil, fmt.Errorf("pipeline %q artifact lane %q is duplicated after trimming", strings.TrimSpace(pipelineID), laneID)
}
lanesByID[laneID] = lane
}
selectedIDs := make([]string, 0, len(lanesByID))
if len(only) == 0 {
for laneID := range lanesByID {
selectedIDs = append(selectedIDs, laneID)
}
} else {
seen := make(map[string]struct{}, len(only))
for _, rawLaneID := range only {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
return nil, fmt.Errorf("pipeline %q selected artifact lane id must not be empty", strings.TrimSpace(pipelineID))
}
if _, ok := lanesByID[laneID]; !ok {
return nil, fmt.Errorf("pipeline %q selected artifact lane %q is not declared", strings.TrimSpace(pipelineID), laneID)
}
if _, ok := seen[laneID]; !ok {
selectedIDs = append(selectedIDs, laneID)
seen[laneID] = struct{}{}
}
}
}
sort.Strings(selectedIDs)
targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*2)
chunk := pipeline.Binding(profile.Chunk.Module)
chunk.Module = strings.TrimSpace(profile.Chunk.Module)
if chunk.Module == "" {
chunk.Module = pipeline.DefaultChunkModule
}
chunkSpec, err := cliReferenceChunkerSpec(catalog, chunk.Module)
if err != nil {
return nil, fmt.Errorf("pipeline %q chunk module %q: %w", strings.TrimSpace(pipelineID), chunk.Module, err)
}
targets = append(targets, selectedReferenceTarget{
stage: pipeline.StageChunk,
module: chunk.Module,
slots: referenceSlotSet(chunkSpec.ReferenceSlots),
})
for _, laneID := range selectedIDs {
lane := lanesByID[laneID]
extractModule := strings.TrimSpace(lane.Extract.Module)
if extractModule == "" {
return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID)
}
extractSpec, err := cliReferenceExtractorSpec(catalog, extractModule)
if err != nil {
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: %w", strings.TrimSpace(pipelineID), laneID, extractModule, err)
}
targets = append(targets, selectedReferenceTarget{
laneID: laneID,
stage: pipeline.StageExtract,
module: extractModule,
slots: referenceSlotSet(extractSpec.ReferenceSlots),
})
normalizeModule := strings.TrimSpace(lane.Normalize.Module)
if normalizeModule == "" {
normalizeModule = pipeline.DefaultNormalizeModule
}
normalizeSpec, err := cliReferenceNormalizerSpec(catalog, normalizeModule)
if err != nil {
return nil, fmt.Errorf("pipeline %q lane %q normalize module %q: %w", strings.TrimSpace(pipelineID), laneID, normalizeModule, err)
}
targets = append(targets, selectedReferenceTarget{
laneID: laneID,
stage: pipeline.StageNormalize,
module: normalizeModule,
slots: referenceSlotSet(normalizeSpec.ReferenceSlots),
})
}
return targets, nil
}
func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
pipelineID = strings.TrimSpace(pipelineID)
for rawID, profile := range profiles {
if strings.TrimSpace(rawID) == pipelineID {
return profile, true
}
}
return pipeline.PipelineProfile{}, false
}
func cliReferenceChunkerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Chunkers == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Chunkers.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func cliReferenceExtractorSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Extractors == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Extractors.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
if catalog.Normalizers == nil {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
spec, ok := catalog.Normalizers.Spec(module)
if !ok {
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
}
return spec, nil
}
func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} {
slotSet := make(map[string]struct{}, len(slots))
for _, slot := range slots {
slotSet[slot.Name] = struct{}{}
}
return slotSet
}
func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliReferenceSelector) (selectedReferenceTarget, error) {
slotName := strings.TrimSpace(selector.SlotName)
if slotName == "" {
return selectedReferenceTarget{}, fmt.Errorf("reference slot must not be empty")
}
if selector.Stage == pipeline.StageChunk {
for _, target := range targets {
if target.stage != pipeline.StageChunk {
continue
}
if _, ok := target.slots[slotName]; !ok {
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
}
return target, nil
}
return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
}
if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageNormalize {
for _, target := range targets {
if target.laneID == selector.LaneID && target.stage == selector.Stage {
if _, ok := target.slots[slotName]; !ok {
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
}
return target, nil
}
}
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
}
if strings.TrimSpace(selector.LaneID) != "" {
return resolveCLIReferenceLaneTarget(targets, strings.TrimSpace(selector.LaneID), slotName)
}
return resolveCLIReferenceFlatTarget(targets, slotName)
}
func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
laneSelected := false
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if target.laneID != laneID {
continue
}
laneSelected = true
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
if !laneSelected {
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", laneID)
}
switch len(matches) {
case 0:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, laneID)
case 1:
return matches[0], nil
default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use %s.extract.%s or %s.normalize.%s", slotName, laneID, targetList(matches), laneID, slotName, laneID, slotName)
}
}
func resolveCLIReferenceFlatTarget(targets []selectedReferenceTarget, slotName string) (selectedReferenceTarget, error) {
matches := make([]selectedReferenceTarget, 0, 2)
for _, target := range targets {
if _, ok := target.slots[slotName]; ok {
matches = append(matches, target)
}
}
switch len(matches) {
case 0:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected reference target", slotName)
case 1:
return matches[0], nil
default:
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets (%s); use a more specific selector such as %s", slotName, targetList(matches), selectorSuggestions(matches, slotName))
}
}
func targetList(targets []selectedReferenceTarget) string {
labels := make([]string, 0, len(targets))
for _, target := range targets {
labels = append(labels, targetLabel(target))
}
sort.Strings(labels)
return strings.Join(labels, ", ")
}
func targetLabel(target selectedReferenceTarget) string {
if target.stage == pipeline.StageChunk {
return "chunk"
}
return target.laneID + "." + string(target.stage)
}
func selectorSuggestions(targets []selectedReferenceTarget, slotName string) string {
suggestions := make([]string, 0, len(targets))
for _, target := range targets {
if target.stage == pipeline.StageChunk {
suggestions = append(suggestions, "chunk."+slotName)
continue
}
suggestions = append(suggestions, target.laneID+"."+string(target.stage)+"."+slotName)
}
sort.Strings(suggestions)
return strings.Join(suggestions, " or ")
}
func sortedPipelineIDs(cfg config.Config) []string { func sortedPipelineIDs(cfg config.Config) []string {
ids := make([]string, 0, len(cfg.Pipelines)) ids := make([]string, 0, len(cfg.Pipelines))
for id := range cfg.Pipelines { for id := range cfg.Pipelines {

File diff suppressed because it is too large Load Diff

View File

@@ -48,6 +48,18 @@ type LLMProfileManifest struct {
Model string `json:"model,omitempty"` Model string `json:"model,omitempty"`
} }
type ReferenceProvenance struct {
Stage string `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
OriginType string `json:"origin_type"`
OriginURI string `json:"origin_uri,omitempty"`
Digest string `json:"digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
SizeBytes int64 `json:"size_bytes,omitempty"`
BindingSource string `json:"binding_source,omitempty"`
}
type RunManifest struct { type RunManifest struct {
RunID string `json:"run_id,omitempty"` RunID string `json:"run_id,omitempty"`
PipelineID string `json:"pipeline_id,omitempty"` PipelineID string `json:"pipeline_id,omitempty"`
@@ -59,7 +71,9 @@ type RunManifest struct {
Merger string `json:"merger,omitempty"` Merger string `json:"merger,omitempty"`
Normalizer string `json:"normalizer,omitempty"` Normalizer string `json:"normalizer,omitempty"`
OutputEncoder string `json:"output_encoder,omitempty"` OutputEncoder string `json:"output_encoder,omitempty"`
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"` ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
References []ReferenceProvenance `json:"references,omitempty"`
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"` LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
SchemaVersion string `json:"schema_version,omitempty"` SchemaVersion string `json:"schema_version,omitempty"`
ValidationStatus string `json:"validation_status,omitempty"` ValidationStatus string `json:"validation_status,omitempty"`

View File

@@ -183,6 +183,80 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata") assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
} }
func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
manifest := RunManifest{
References: []ReferenceProvenance{
{
Stage: "extract",
LaneID: "events",
SlotName: "roster",
OriginType: "file",
OriginURI: "file:///tmp/roster.txt",
Digest: "sha256:reference",
MediaType: "text/plain; charset=utf-8",
SizeBytes: 12,
BindingSource: "config",
},
},
}
gotJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var got RunManifest
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
if len(got.References) != 1 {
t.Fatalf("len(References) = %d, want 1", len(got.References))
}
reference := got.References[0]
if reference.Stage != "extract" || reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference)
}
if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" {
t.Fatalf("reference provenance = %#v, want digest/media/size/source details", reference)
}
}
func TestRunManifestIncludesTopLevelModuleMetadata(t *testing.T) {
manifest := RunManifest{
ModuleMetadata: map[string]map[string]any{
"chunker": {
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:abc123",
"response_schema_key": "dnd_scenes",
"response_schema_name": "dnd_scenes",
},
},
}
gotJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var got map[string]any
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
moduleMetadata, ok := got["module_metadata"].(map[string]any)
if !ok {
t.Fatalf("module_metadata = %#v, want object", got["module_metadata"])
}
assertHasKeys(t, moduleMetadata, "chunker")
chunkerMetadata, ok := moduleMetadata["chunker"].(map[string]any)
if !ok {
t.Fatalf("module_metadata.chunker = %#v, want object", moduleMetadata["chunker"])
}
assertHasKeys(t, chunkerMetadata, "prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_name")
}
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) { func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
t.Helper() t.Helper()

View File

@@ -73,6 +73,7 @@ func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile
out.Input = cloneModuleBinding(in.Input) out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk) out.Chunk = cloneModuleBinding(in.Chunk)
out.Output = cloneModuleBinding(in.Output) out.Output = cloneModuleBinding(in.Output)
out.References = cloneStringMap(in.References)
if len(in.Artifacts) > 0 { if len(in.Artifacts) > 0 {
out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts)) out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts))
for key, lane := range in.Artifacts { for key, lane := range in.Artifacts {
@@ -87,6 +88,7 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
out.Extract = cloneModuleBinding(in.Extract) out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge) out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize) out.Normalize = cloneModuleBinding(in.Normalize)
out.References = cloneStringMap(in.References)
if len(in.Validators) > 0 { if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators)) out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators { for i, binding := range in.Validators {
@@ -96,11 +98,23 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
return out return out
} }
func cloneStringMap(in map[string]string) map[string]string {
if len(in) == 0 {
return nil
}
out := make(map[string]string, len(in))
for key, value := range in {
out[key] = value
}
return out
}
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding { func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out := in out := in
if len(in.Options) > 0 { if len(in.Options) > 0 {
out.Options = cloneOptions(in.Options) out.Options = cloneOptions(in.Options)
} }
out.References = cloneStringMap(in.References)
return out return out
} }

View File

@@ -14,12 +14,16 @@ type ResolveInput struct {
Only []string Only []string
Catalog pipeline.ModuleCatalog Catalog pipeline.ModuleCatalog
LLMProfileOverride string LLMProfileOverride string
ReferenceOverrides []pipeline.ReferenceBinding
ReferenceUnbinds []pipeline.ReferenceUnbind
} }
type EffectiveConfig struct { type EffectiveConfig struct {
Config Config Config Config
PipelineID string PipelineID string
Only []string Only []string
ReferenceOverrides []pipeline.ReferenceBinding
ReferenceUnbinds []pipeline.ReferenceUnbind
ResolvedPipeline pipeline.ResolvedPipeline ResolvedPipeline pipeline.ResolvedPipeline
} }
@@ -46,7 +50,11 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
applyLLMProfileOverride(&profile, override) applyLLMProfileOverride(&profile, override)
} }
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{Only: input.Only}, input.Catalog) resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{
Only: input.Only,
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
}, input.Catalog)
if err != nil { if err != nil {
return EffectiveConfig{}, fmt.Errorf("resolve pipeline %q: %w", pipelineID, err) return EffectiveConfig{}, fmt.Errorf("resolve pipeline %q: %w", pipelineID, err)
} }
@@ -55,6 +63,8 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
Config: cloneConfig(c), Config: cloneConfig(c),
PipelineID: pipelineID, PipelineID: pipelineID,
Only: append([]string(nil), input.Only...), Only: append([]string(nil), input.Only...),
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
ResolvedPipeline: resolved, ResolvedPipeline: resolved,
}, nil }, nil
} }

View File

@@ -98,6 +98,43 @@ func TestResolveSurfacesMissingCapabilityThroughCatalog(t *testing.T) {
} }
} }
func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
profile.Chunk = pipeline.Binding("dnd/scenes")
lane := profile.Artifacts["events"]
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"events": lane}
cfg.Pipelines["example"] = profile
catalog := fakeCatalog(t,
pipeline.ModuleSpec{
Key: "fake/input",
Stage: pipeline.StageInput,
Provides: []string{"source.transcript"},
},
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"artifact"},
},
)
mustRegisterChunker(t, catalog.Chunkers, pipeline.ModuleSpec{
Key: "dnd/scenes",
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks", "chunks.scenes"},
})
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
if got := effective.ResolvedPipeline.Chunk.Module; got != "dnd/scenes" {
t.Fatalf("Chunk.Module = %q, want dnd/scenes", got)
}
}
func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) { func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
cfg := validConfig() cfg := validConfig()
first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)}) first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})

View File

@@ -39,6 +39,7 @@ type FilePipelineProfile struct {
Chunk *fileModuleBinding `yaml:"chunk,omitempty"` Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"` Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
Output *fileModuleBinding `yaml:"output,omitempty"` Output *fileModuleBinding `yaml:"output,omitempty"`
References map[string]string `yaml:"references,omitempty"`
} }
type FileArtifactLaneProfile struct { type FileArtifactLaneProfile struct {
@@ -46,6 +47,7 @@ type FileArtifactLaneProfile struct {
Merge *fileModuleBinding `yaml:"merge,omitempty"` Merge *fileModuleBinding `yaml:"merge,omitempty"`
Normalize *fileModuleBinding `yaml:"normalize,omitempty"` Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
Validators []fileModuleBinding `yaml:"validators,omitempty"` Validators []fileModuleBinding `yaml:"validators,omitempty"`
References map[string]string `yaml:"references,omitempty"`
} }
type FileConcurrencyConfig struct { type FileConcurrencyConfig struct {
@@ -97,6 +99,7 @@ type fileModuleBinding struct {
Module string Module string
LLMProfile string LLMProfile string
Options map[string]any Options map[string]any
References map[string]string
} }
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error { func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
@@ -131,6 +134,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err return err
} }
b.Options = normalizeOptions(options) b.Options = normalizeOptions(options)
case "references":
var references map[string]string
if err := valueNode.Decode(&references); err != nil {
return err
}
b.References = references
default: default:
return fmt.Errorf("field %s not found in module binding", keyNode.Value) return fmt.Errorf("field %s not found in module binding", keyNode.Value)
} }
@@ -146,6 +155,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
Module: strings.TrimSpace(b.Module), Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile), LLMProfile: strings.TrimSpace(b.LLMProfile),
Options: cloneOptions(b.Options), Options: cloneOptions(b.Options),
References: normalizedStringMap(b.References),
} }
} }
@@ -212,6 +222,49 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil { if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
return err return err
} }
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Chunk != nil {
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
return err
}
}
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
return err
}
if filePipeline.Output != nil {
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
return err
}
}
for rawLaneID, fileLane := range filePipeline.Artifacts {
laneID := strings.TrimSpace(rawLaneID)
if laneID == "" {
continue
}
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
return err
}
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
return err
}
if fileLane.Merge != nil {
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
return err
}
}
if fileLane.Normalize != nil {
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
return err
}
}
for i, validator := range fileLane.Validators {
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
return err
}
}
}
} }
for _, profileID := range profileIDs { for _, profileID := range profileIDs {
@@ -256,6 +309,7 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
ID: pipelineID, ID: pipelineID,
Input: filePipeline.Input.toPipelineBinding(), Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)), Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: normalizedStringMap(filePipeline.References),
} }
if filePipeline.Chunk != nil { if filePipeline.Chunk != nil {
profile.Chunk = filePipeline.Chunk.toPipelineBinding() profile.Chunk = filePipeline.Chunk.toPipelineBinding()
@@ -265,8 +319,11 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
} }
for _, laneID := range laneIDs { for _, laneID := range laneIDs {
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]] fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
extract := fileLane.Extract.toPipelineBinding()
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
lane := pipeline.ArtifactLaneProfile{ lane := pipeline.ArtifactLaneProfile{
Extract: fileLane.Extract.toPipelineBinding(), Extract: extract,
References: normalizedStringMap(fileLane.References),
} }
if fileLane.Merge != nil { if fileLane.Merge != nil {
lane.Merge = fileLane.Merge.toPipelineBinding() lane.Merge = fileLane.Merge.toPipelineBinding()
@@ -318,6 +375,39 @@ func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, ma
return keys, rawByNormalized, nil return keys, rawByNormalized, nil
} }
func normalizedStringMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
}
return out
}
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
for key, value := range base {
out[key] = value
}
for key, value := range override {
out[key] = value
}
return out
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) { func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName) name := strings.TrimSpace(envName)
if name == "" { if name == "" {

View File

@@ -144,6 +144,78 @@ pipelines:
} }
} }
func TestParseFileConfigReferenceMaps(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
pipelines:
example:
input: fake/input
references:
" roster ": " ./shared-roster.yml "
artifacts:
events:
extract: fake/extract
references:
" lore ": " ./lore.md "
`)
profile := cfg.Pipelines["example"]
if !reflect.DeepEqual(profile.References, map[string]string{"roster": "./shared-roster.yml"}) {
t.Fatalf("pipeline references = %#v, want trimmed map", profile.References)
}
gotLaneRefs := profile.Artifacts["events"].References
if !reflect.DeepEqual(gotLaneRefs, map[string]string{"lore": "./lore.md"}) {
t.Fatalf("lane references = %#v, want trimmed map", gotLaneRefs)
}
}
func TestParseFileConfigStageLocalReferenceMaps(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 1
pipelines:
example:
input: fake/input
chunk:
module: generic
references:
" scene_guide ": " ./scenes.md "
artifacts:
events:
extract:
module: fake/extract
references:
" glossary ": " ./glossary.md "
" roster ": " ./extract-roster.yml "
references:
roster: ./legacy-roster.yml
lore: ./lore.md
normalize:
module: noop
references:
" normalization_notes ": " ./normalization.md "
`)
profile := cfg.Pipelines["example"]
if !reflect.DeepEqual(profile.Chunk.References, map[string]string{"scene_guide": "./scenes.md"}) {
t.Fatalf("chunk references = %#v, want trimmed map", profile.Chunk.References)
}
lane := profile.Artifacts["events"]
if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) {
t.Fatalf("legacy lane references = %#v, want trimmed map", lane.References)
}
wantExtract := map[string]string{
"glossary": "./glossary.md",
"lore": "./lore.md",
"roster": "./extract-roster.yml",
}
if !reflect.DeepEqual(lane.Extract.References, wantExtract) {
t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract)
}
if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) {
t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References)
}
}
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) { func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
cfg := parseAndApplyConfig(t, ` cfg := parseAndApplyConfig(t, `
version: 1 version: 1
@@ -297,6 +369,108 @@ pipelines:
} }
} }
func TestApplyFileConfigRejectsDuplicateTrimmedReferenceSlots(t *testing.T) {
tests := []struct {
name string
raw string
want string
}{
{
name: "pipeline",
raw: `
version: 1
pipelines:
example:
input: fake/input
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" reference slot`,
},
{
name: "lane",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract: fake/extract
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" reference slot`,
},
{
name: "chunk",
raw: `
version: 1
pipelines:
example:
input: fake/input
chunk:
module: generic
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" chunk reference slot`,
},
{
name: "extract",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract:
module: fake/extract
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" extract reference slot`,
},
{
name: "normalize",
raw: `
version: 1
pipelines:
example:
input: fake/input
artifacts:
events:
extract: fake/extract
normalize:
module: noop
references:
roster: ./first.yml
" roster ": ./second.yml
`,
want: `pipeline "example" lane "events" normalize reference slot`,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(tc.raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML: %v", err)
}
cfg := Default()
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "duplicated") {
t.Fatalf("expected duplicate reference slot error, got %v", err)
}
})
}
}
func TestApplyFileConfigAllowsRetryOnlyLLMProfile(t *testing.T) { func TestApplyFileConfigAllowsRetryOnlyLLMProfile(t *testing.T) {
cfg := parseAndApplyConfig(t, ` cfg := parseAndApplyConfig(t, `
version: 1 version: 1

View File

@@ -24,6 +24,8 @@ func (e EffectiveConfig) RedactedDiagnosticsPayload() any {
Config: e.Config.Redacted(), Config: e.Config.Redacted(),
PipelineID: e.PipelineID, PipelineID: e.PipelineID,
Only: append([]string(nil), e.Only...), Only: append([]string(nil), e.Only...),
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), e.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), e.ReferenceUnbinds...),
ResolvedPipeline: cloneResolvedPipeline(e.ResolvedPipeline), ResolvedPipeline: cloneResolvedPipeline(e.ResolvedPipeline),
} }
} }
@@ -32,6 +34,7 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
out := in out := in
out.Input = cloneModuleBinding(in.Input) out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk) out.Chunk = cloneModuleBinding(in.Chunk)
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
out.Output = cloneModuleBinding(in.Output) out.Output = cloneModuleBinding(in.Output)
if len(in.ArtifactLanes) > 0 { if len(in.ArtifactLanes) > 0 {
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes)) out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
@@ -47,6 +50,8 @@ func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.Resolv
out.Extract = cloneModuleBinding(in.Extract) out.Extract = cloneModuleBinding(in.Extract)
out.Merge = cloneModuleBinding(in.Merge) out.Merge = cloneModuleBinding(in.Merge)
out.Normalize = cloneModuleBinding(in.Normalize) out.Normalize = cloneModuleBinding(in.Normalize)
out.ExtractReferences = pipeline.CloneReferenceTarget(in.ExtractReferences)
out.NormalizeReferences = pipeline.CloneReferenceTarget(in.NormalizeReferences)
if len(in.Validators) > 0 { if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators)) out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators { for i, binding := range in.Validators {

View File

@@ -3,6 +3,7 @@ package config
import ( import (
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
@@ -65,12 +66,47 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile
lane := cfg.Pipelines["example"].Artifacts["events"] lane := cfg.Pipelines["example"].Artifacts["events"]
lane.Extract.Options = map[string]any{"temperature": 0.2} lane.Extract.Options = map[string]any{"temperature": 0.2}
lane.References = map[string]string{"roster": "./roster.yml"}
lane.Extract.References = map[string]string{"glossary": "./glossary.md"}
lane.Normalize.References = map[string]string{"notes": "./normalize.md"}
cfg.Pipelines["example"].Artifacts["events"] = lane cfg.Pipelines["example"].Artifacts["events"] = lane
pipelineProfile := cfg.Pipelines["example"]
pipelineProfile.Chunk.References = map[string]string{"scene_guide": "./scene.md"}
cfg.Pipelines["example"] = pipelineProfile
effective, err := cfg.Resolve(ResolveInput{ effective, err := cfg.Resolve(ResolveInput{
PipelineID: "example", PipelineID: "example",
Only: []string{"events"}, Only: []string{"events"},
Catalog: fakeCatalog(t), Catalog: fakeCatalog(t,
pipeline.ModuleSpec{
Key: "generic",
Stage: pipeline.StageChunk,
Requires: []string{"source"},
Provides: []string{"chunks"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "scene_guide"},
},
},
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks"},
Provides: []string{"artifact"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "glossary"},
{Name: "roster"},
},
},
pipeline.ModuleSpec{
Key: "noop",
Stage: pipeline.StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes"},
},
},
),
}) })
if err != nil { if err != nil {
t.Fatalf("Resolve: %v", err) t.Fatalf("Resolve: %v", err)
@@ -98,4 +134,52 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 { if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 {
t.Fatalf("expected resolved pipeline options to be copied") t.Fatalf("expected resolved pipeline options to be copied")
} }
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml"
if referenceBindingSource(effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings, "roster") != "./roster.yml" {
t.Fatalf("expected resolved pipeline references to be copied")
}
payload.ResolvedPipeline.Chunk.References["scene_guide"] = "./changed-scene.md"
if effective.ResolvedPipeline.Chunk.References["scene_guide"] != "./scene.md" {
t.Fatalf("expected chunk references to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] = "./changed-glossary.md"
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] != "./glossary.md" {
t.Fatalf("expected extract references to be copied")
}
payload.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] = "./changed-normalize.md"
if effective.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] != "./normalize.md" {
t.Fatalf("expected normalize references to be copied")
}
effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
Content: []byte("reference content"),
},
},
},
},
}
payload, ok = effective.RedactedDiagnosticsPayload().(EffectiveConfig)
if !ok {
t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload())
}
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content[0] = 'X'
got := effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content
if string(got) != "reference content" {
t.Fatalf("expected materialized reference content to be copied, got %q", got)
}
}
func referenceBindingSource(bindings []pipeline.ReferenceBinding, slotName string) string {
for _, binding := range bindings {
if binding.SlotName == slotName {
return binding.Source
}
}
return ""
} }

View File

@@ -89,13 +89,16 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if profile.ID != "" && strings.TrimSpace(profile.ID) != id { if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID) return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
} }
if err := validateBindingLLMProfile(id, "", "input", profile.Input, llmProfiles); err != nil { if err := validateBinding(id, "", "input", profile.Input, llmProfiles, false); err != nil {
return err return err
} }
if err := validateBindingLLMProfile(id, "", "chunk", profile.Chunk, llmProfiles); err != nil { if err := validateBinding(id, "", "chunk", profile.Chunk, llmProfiles, true); err != nil {
return err return err
} }
if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil { if err := validateBinding(id, "", "output", profile.Output, llmProfiles, false); err != nil {
return err
}
if err := validateReferenceMap(id, "", profile.References); err != nil {
return err return err
} }
for rawLaneID, lane := range profile.Artifacts { for rawLaneID, lane := range profile.Artifacts {
@@ -103,17 +106,20 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
if laneID == "" { if laneID == "" {
return fmt.Errorf("pipeline %q artifact lane id must not be empty", id) return fmt.Errorf("pipeline %q artifact lane id must not be empty", id)
} }
if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil { if err := validateReferenceMap(id, laneID, lane.References); err != nil {
return err return err
} }
if err := validateBindingLLMProfile(id, laneID, "merge", lane.Merge, llmProfiles); err != nil { if err := validateBinding(id, laneID, "extract", lane.Extract, llmProfiles, true); err != nil {
return err return err
} }
if err := validateBindingLLMProfile(id, laneID, "normalize", lane.Normalize, llmProfiles); err != nil { if err := validateBinding(id, laneID, "merge", lane.Merge, llmProfiles, false); err != nil {
return err
}
if err := validateBinding(id, laneID, "normalize", lane.Normalize, llmProfiles, true); err != nil {
return err return err
} }
for i, validator := range lane.Validators { for i, validator := range lane.Validators {
if err := validateBindingLLMProfile(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles); err != nil { if err := validateBinding(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles, false); err != nil {
return err return err
} }
} }
@@ -122,6 +128,64 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
return nil return nil
} }
func validateBinding(
pipelineID string,
laneID string,
slot string,
binding pipeline.ModuleBinding,
profiles map[string]LLMProfile,
referencesAllowed bool,
) error {
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding, profiles); err != nil {
return err
}
if len(binding.References) == 0 {
return nil
}
if !referencesAllowed {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q %s references are not supported", pipelineID, laneID, slot)
}
return fmt.Errorf("pipeline %q %s references are not supported", pipelineID, slot)
}
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
}
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
return validateReferenceMapForContext(pipelineID, laneID, "", references)
}
func validateReferenceMapForContext(pipelineID string, laneID string, slot string, references map[string]string) error {
seen := make(map[string]struct{}, len(references))
for rawSlotName, rawSource := range references {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
return fmt.Errorf("%s reference slot name must not be empty", referenceContext(pipelineID, laneID, slot))
}
if _, ok := seen[slotName]; ok {
return fmt.Errorf("%s reference slot %q is duplicated after trimming", referenceContext(pipelineID, laneID, slot), slotName)
}
seen[slotName] = struct{}{}
if strings.TrimSpace(rawSource) == "" {
return fmt.Errorf("%s reference slot %q source must not be empty", referenceContext(pipelineID, laneID, slot), slotName)
}
}
return nil
}
func referenceContext(pipelineID string, laneID string, slot string) string {
if laneID != "" && slot != "" {
return fmt.Sprintf("pipeline %q lane %q %s", pipelineID, laneID, slot)
}
if laneID != "" {
return fmt.Sprintf("pipeline %q lane %q", pipelineID, laneID)
}
if slot != "" {
return fmt.Sprintf("pipeline %q %s", pipelineID, slot)
}
return fmt.Sprintf("pipeline %q", pipelineID)
}
func validateBindingLLMProfile( func validateBindingLLMProfile(
pipelineID string, pipelineID string,
laneID string, laneID string,

View File

@@ -116,6 +116,211 @@ func TestValidateRejectsInvalidDiagnosticsRetention(t *testing.T) {
} }
} }
func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
tests := []struct {
name string
mutate func(Config) Config
want []string
}{
{
name: "empty chunk slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Chunk.References = map[string]string{" ": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "chunk", "reference slot", "empty"},
},
{
name: "empty chunk source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Chunk.References = map[string]string{"roster": " "}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "chunk", "roster", "source", "empty"},
},
{
name: "empty extract slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "extract", "reference slot", "empty"},
},
{
name: "empty extract source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "extract", "roster", "source", "empty"},
},
{
name: "empty normalize slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "normalize", "reference slot", "empty"},
},
{
name: "empty normalize source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "normalize", "roster", "source", "empty"},
},
{
name: "empty pipeline slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.References = map[string]string{" ": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "reference slot", "empty"},
},
{
name: "empty pipeline source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.References = map[string]string{"roster": " "}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "roster", "source", "empty"},
},
{
name: "empty lane slot",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.References = map[string]string{" ": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "reference slot", "empty"},
},
{
name: "empty lane source",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.References = map[string]string{"roster": " "}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "roster", "source", "empty"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := tc.mutate(validConfig()).Validate()
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
})
}
}
func TestValidateRejectsReferencesOnUnsupportedBindings(t *testing.T) {
tests := []struct {
name string
mutate func(Config) Config
want []string
}{
{
name: "input",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Input.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "input", "references", "not supported"},
},
{
name: "merge",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Merge.References = map[string]string{"roster": "./roster.yml"}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "merge", "references", "not supported"},
},
{
name: "validator",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Validators = []pipeline.ModuleBinding{{
Module: "fake/validator",
References: map[string]string{"roster": "./roster.yml"},
}}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "events", "validator[0]", "references", "not supported"},
},
{
name: "output",
mutate: func(cfg Config) Config {
profile := cfg.Pipelines["example"]
profile.Output.References = map[string]string{"roster": "./roster.yml"}
cfg.Pipelines["example"] = profile
return cfg
},
want: []string{"example", "output", "references", "not supported"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := tc.mutate(validConfig()).Validate()
if err == nil {
t.Fatal("Validate() error = nil, want error")
}
for _, want := range tc.want {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
})
}
}
func TestValidateRejectsEmptyIDs(t *testing.T) { func TestValidateRejectsEmptyIDs(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -4,6 +4,7 @@ const (
ArtifactInvocationMetadata = "invocation.json" ArtifactInvocationMetadata = "invocation.json"
ArtifactEffectiveConfig = "effective-config.json" ArtifactEffectiveConfig = "effective-config.json"
ArtifactResolvedPipeline = "resolved-pipeline.json" ArtifactResolvedPipeline = "resolved-pipeline.json"
ArtifactResolvedReferences = "resolved-references.json"
ArtifactSourceDocument = "source-document.json" ArtifactSourceDocument = "source-document.json"
ArtifactRunManifest = "run-manifest.json" ArtifactRunManifest = "run-manifest.json"
ArtifactRunReport = "run-report.json" ArtifactRunReport = "run-report.json"

View File

@@ -7,6 +7,7 @@ func TestArtifactNamesUseExtractionOrientedNames(t *testing.T) {
ArtifactInvocationMetadata, ArtifactInvocationMetadata,
ArtifactEffectiveConfig, ArtifactEffectiveConfig,
ArtifactResolvedPipeline, ArtifactResolvedPipeline,
ArtifactResolvedReferences,
ArtifactSourceDocument, ArtifactSourceDocument,
ArtifactRunManifest, ArtifactRunManifest,
ArtifactRunReport, ArtifactRunReport,

View File

@@ -149,6 +149,10 @@ func (r *RunDirectory) WriteResolvedPipeline(payload any) error {
return r.WriteJSONArtifact(ArtifactResolvedPipeline, payload) return r.WriteJSONArtifact(ArtifactResolvedPipeline, payload)
} }
func (r *RunDirectory) WriteResolvedReferences(payload any) error {
return r.WriteJSONArtifact(ArtifactResolvedReferences, payload)
}
func (r *RunDirectory) WriteSourceDocument(payload any) error { func (r *RunDirectory) WriteSourceDocument(payload any) error {
return r.WriteJSONArtifact(ArtifactSourceDocument, payload) return r.WriteJSONArtifact(ArtifactSourceDocument, payload)
} }

View File

@@ -191,6 +191,9 @@ func TestWriteTypedArtifacts(t *testing.T) {
if err := runDir.WriteResolvedPipeline(map[string]any{"pipeline": "test"}); err != nil { if err := runDir.WriteResolvedPipeline(map[string]any{"pipeline": "test"}); err != nil {
t.Fatalf("WriteResolvedPipeline: %v", err) t.Fatalf("WriteResolvedPipeline: %v", err)
} }
if err := runDir.WriteResolvedReferences([]artifacts.ReferenceProvenance{{LaneID: "events", SlotName: "roster"}}); err != nil {
t.Fatalf("WriteResolvedReferences: %v", err)
}
if err := runDir.WriteSourceDocument(map[string]any{"source_id": "source-1"}); err != nil { if err := runDir.WriteSourceDocument(map[string]any{"source_id": "source-1"}); err != nil {
t.Fatalf("WriteSourceDocument: %v", err) t.Fatalf("WriteSourceDocument: %v", err)
} }
@@ -207,6 +210,7 @@ func TestWriteTypedArtifacts(t *testing.T) {
for _, name := range []string{ for _, name := range []string{
ArtifactEffectiveConfig, ArtifactEffectiveConfig,
ArtifactResolvedPipeline, ArtifactResolvedPipeline,
ArtifactResolvedReferences,
ArtifactSourceDocument, ArtifactSourceDocument,
ArtifactRunManifest, ArtifactRunManifest,
ArtifactRunReport, ArtifactRunReport,

View File

@@ -17,6 +17,7 @@ var _ contracts.Extractor = compositionExtractor{}
var _ contracts.Merger = compositionMerger{} var _ contracts.Merger = compositionMerger{}
var _ contracts.Normalizer = compositionNormalizer{} var _ contracts.Normalizer = compositionNormalizer{}
var _ contracts.Validator = compositionValidator{} var _ contracts.Validator = compositionValidator{}
var _ contracts.StructuredLLMClient = compositionLLMClient{}
var _ contracts.OutputEncoder = compositionOutputEncoder{} var _ contracts.OutputEncoder = compositionOutputEncoder{}
func TestContractsComposeAcrossPackages(t *testing.T) { func TestContractsComposeAcrossPackages(t *testing.T) {
@@ -39,6 +40,7 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
chunking, err := chunker.Chunk(ctx, contracts.ChunkRequest{ chunking, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc, Source: doc,
LLMClient: compositionLLMClient{},
Metadata: map[string]any{"max_units": 2}, Metadata: map[string]any{"max_units": 2},
}) })
if err != nil { if err != nil {
@@ -160,10 +162,17 @@ func (chunker compositionChunker) Key() string {
return "generic-chunker" return "generic-chunker"
} }
func (chunker compositionChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) { func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if req.Source == nil { if req.Source == nil {
return contracts.ChunkResult{}, errors.New("source document is required") return contracts.ChunkResult{}, errors.New("source document is required")
} }
if req.LLMClient == nil {
return contracts.ChunkResult{}, errors.New("structured llm client is required")
}
return contracts.ChunkResult{ return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{ Chunks: []contracts.SourceChunk{
@@ -178,6 +187,12 @@ func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.Chunk
}, nil }, nil
} }
type compositionLLMClient struct{}
func (client compositionLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
return contracts.StructuredCompletionResponse{}, nil
}
type compositionExtractor struct{} type compositionExtractor struct{}
func (extractor compositionExtractor) Key() string { func (extractor compositionExtractor) Key() string {
@@ -192,6 +207,10 @@ func (extractor compositionExtractor) SchemaVersion() string {
return "v1" return "v1"
} }
func (extractor compositionExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor compositionExtractor) Validators() []contracts.Validator { func (extractor compositionExtractor) Validators() []contracts.Validator {
return []contracts.Validator{compositionValidator{}} return []contracts.Validator{compositionValidator{}}
} }
@@ -249,6 +268,10 @@ func (normalizer compositionNormalizer) Key() string {
return "generic-normalizer" return "generic-normalizer"
} }
func (normalizer compositionNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) { func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: req.Candidates}, nil return contracts.NormalizeResult{Candidates: req.Candidates}, nil
} }

View File

@@ -58,6 +58,8 @@ type SourceChunk struct {
type ChunkRequest struct { type ChunkRequest struct {
Source *source.SourceDocument `json:"-"` Source *source.SourceDocument `json:"-"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
@@ -70,13 +72,53 @@ type ChunkResult struct {
type Chunker interface { type Chunker interface {
Key() string Key() string
ReferenceSlots() []ReferenceSlot
Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
} }
const (
ReferenceBindingSourceConfig = "config"
ReferenceBindingSourceCLI = "cli"
)
type ReferenceSlot struct {
Name string `json:"name"`
Description string `json:"description,omitempty"`
Required bool `json:"required,omitempty"`
AcceptedMediaTypes []string `json:"accepted_media_types,omitempty"`
Multiple bool `json:"multiple,omitempty"`
MaxBytes int64 `json:"max_bytes,omitempty"`
}
type ReferenceOrigin struct {
Type string `json:"type"`
URI string `json:"uri,omitempty"`
}
type ReferenceItem struct {
SlotName string `json:"slot_name"`
MediaType string `json:"media_type,omitempty"`
Content []byte `json:"-"`
Digest string `json:"digest,omitempty"`
Origin ReferenceOrigin `json:"origin"`
SizeBytes int64 `json:"size_bytes,omitempty"`
BindingSource string `json:"binding_source,omitempty"`
}
type ResolvedReferenceSlot struct {
Slot ReferenceSlot `json:"slot"`
Items []ReferenceItem `json:"items,omitempty"`
}
type ReferenceSet struct {
Slots map[string]ResolvedReferenceSlot `json:"slots,omitempty"`
}
type ExtractionRequest struct { type ExtractionRequest struct {
Source *source.SourceDocument `json:"-"` Source *source.SourceDocument `json:"-"`
Chunk *SourceChunk `json:"chunk,omitempty"` Chunk *SourceChunk `json:"chunk,omitempty"`
AmbientContext map[string]any `json:"ambient_context,omitempty"` AmbientContext map[string]any `json:"ambient_context,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"` LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
@@ -92,6 +134,7 @@ type Extractor interface {
Key() string Key() string
ArtifactType() string ArtifactType() string
SchemaVersion() string SchemaVersion() string
ReferenceSlots() []ReferenceSlot
Validators() []Validator Validators() []Validator
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
} }
@@ -124,6 +167,8 @@ type NormalizeRequest struct {
Source *source.SourceDocument `json:"-"` Source *source.SourceDocument `json:"-"`
LaneID string `json:"lane_id"` LaneID string `json:"lane_id"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"` Candidates []artifacts.ArtifactCandidate `json:"candidates"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
@@ -136,6 +181,7 @@ type NormalizeResult struct {
type Normalizer interface { type Normalizer interface {
Key() string Key() string
ReferenceSlots() []ReferenceSlot
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
} }

View File

@@ -117,6 +117,27 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
} }
} }
func TestFakeChunkerReceivesLLMClient(t *testing.T) {
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: "u1", Kind: "section", Text: "Source text."},
},
}
client := fakeLLMClient{}
chunker := &recordingChunker{key: "llm-chunker"}
if _, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc, LLMClient: client}); err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
if chunker.request.LLMClient == nil {
t.Fatal("ChunkRequest.LLMClient = nil, want structured LLM client")
}
}
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) { func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
extractor := fakeExtractor{ extractor := fakeExtractor{
key: "generic-extractor", key: "generic-extractor",
@@ -165,6 +186,71 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
} }
} }
func TestReferenceSetDataTypes(t *testing.T) {
references := ReferenceSet{
Slots: map[string]ResolvedReferenceSlot{
"roster": {
Slot: ReferenceSlot{
Name: "roster",
Description: "Known characters",
Required: true,
AcceptedMediaTypes: []string{"text/plain"},
Multiple: true,
MaxBytes: 4096,
},
Items: []ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain",
Content: []byte("Aria\nBryn\n"),
Digest: "sha256:reference",
Origin: ReferenceOrigin{
Type: "file",
URI: "file:///tmp/roster.txt",
},
SizeBytes: 10,
BindingSource: ReferenceBindingSourceConfig,
},
},
},
},
}
item := references.Slots["roster"].Items[0]
if item.SlotName != "roster" || item.MediaType != "text/plain" || string(item.Content) != "Aria\nBryn\n" {
t.Fatalf("reference item = %#v, want constructed item fields", item)
}
if item.BindingSource != ReferenceBindingSourceConfig {
t.Fatalf("BindingSource = %q, want %q", item.BindingSource, ReferenceBindingSourceConfig)
}
}
func TestReferenceItemJSONOmitsContent(t *testing.T) {
item := ReferenceItem{
SlotName: "roster",
MediaType: "text/plain",
Content: []byte("reference content"),
Digest: "sha256:reference",
Origin: ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
}
encoded, err := json.Marshal(item)
if err != nil {
t.Fatalf("json.Marshal() error = %v, want nil", err)
}
var got map[string]any
if err := json.Unmarshal(encoded, &got); err != nil {
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
}
if _, ok := got["content"]; ok {
t.Fatalf("encoded reference item leaked content: %s", encoded)
}
if _, ok := got["Content"]; ok {
t.Fatalf("encoded reference item leaked Content: %s", encoded)
}
}
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) { func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
candidate := artifacts.ArtifactCandidate{ candidate := artifacts.ArtifactCandidate{
Index: 0, Index: 0,
@@ -292,6 +378,10 @@ func (chunker fakeChunker) Key() string {
return chunker.key return chunker.key
} }
func (chunker fakeChunker) ReferenceSlots() []ReferenceSlot {
return nil
}
func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) { func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
return ChunkResult{ return ChunkResult{
Chunks: []SourceChunk{ Chunks: []SourceChunk{
@@ -305,6 +395,24 @@ func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkRe
}, nil }, nil
} }
type recordingChunker struct {
key string
request ChunkRequest
}
func (chunker *recordingChunker) Key() string {
return chunker.key
}
func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
return nil
}
func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
chunker.request = req
return fakeChunker{key: chunker.key}.Chunk(ctx, req)
}
type fakeExtractor struct { type fakeExtractor struct {
key string key string
artifactType string artifactType string
@@ -324,6 +432,10 @@ func (extractor fakeExtractor) SchemaVersion() string {
return extractor.schemaVersion return extractor.schemaVersion
} }
func (extractor fakeExtractor) ReferenceSlots() []ReferenceSlot {
return nil
}
func (extractor fakeExtractor) Validators() []Validator { func (extractor fakeExtractor) Validators() []Validator {
return extractor.validators return extractor.validators
} }
@@ -383,6 +495,10 @@ func (normalizer fakeNormalizer) Key() string {
return normalizer.key return normalizer.key
} }
func (normalizer fakeNormalizer) ReferenceSlots() []ReferenceSlot {
return nil
}
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) { func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) {
return NormalizeResult{Candidates: req.Candidates}, nil return NormalizeResult{Candidates: req.Candidates}, nil
} }

View File

@@ -0,0 +1,82 @@
package pipeline
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
sourceUnitIndexes := make(map[string]int, len(doc.Units))
sourceUnits := make(map[string]source.SourceUnit, len(doc.Units))
for index, unit := range doc.Units {
sourceUnitIndexes[unit.ID] = index
sourceUnits[unit.ID] = unit
}
canonicalChunks := make([]contracts.SourceChunk, 0, len(chunks))
seenChunkIDs := make(map[string]struct{}, len(chunks))
for chunkIndex, chunk := range chunks {
if strings.TrimSpace(chunk.ID) == "" {
return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
}
if _, ok := seenChunkIDs[chunk.ID]; ok {
return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
}
seenChunkIDs[chunk.ID] = struct{}{}
if chunk.SourceID != doc.ID {
return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
}
if chunk.Index != chunkIndex {
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
}
if len(chunk.Units) == 0 {
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
}
seenUnitIDs := make(map[string]struct{}, len(chunk.Units))
previousSourceIndex := -1
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
for unitIndex, unit := range chunk.Units {
if strings.TrimSpace(unit.ID) == "" {
return nil, fmt.Errorf("chunk %q unit[%d].id must not be empty", chunk.ID, unitIndex)
}
if _, ok := seenUnitIDs[unit.ID]; ok {
return nil, fmt.Errorf("chunk %q repeats source unit %q", chunk.ID, unit.ID)
}
seenUnitIDs[unit.ID] = struct{}{}
sourceIndex, ok := sourceUnitIndexes[unit.ID]
if !ok {
return nil, fmt.Errorf("chunk %q source unit %q was not found in source document %q", chunk.ID, unit.ID, doc.ID)
}
if sourceIndex <= previousSourceIndex {
return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
}
previousSourceIndex = sourceIndex
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
}
canonicalChunks = append(canonicalChunks, contracts.SourceChunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Units: canonicalUnits,
Metadata: cloneMetadata(chunk.Metadata),
})
}
return canonicalChunks, nil
}
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
return source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Metadata: cloneMetadata(unit.Metadata),
}
}

View File

@@ -330,6 +330,10 @@ func (chunker registryChunker) Key() string {
return chunker.key return chunker.key
} }
func (chunker registryChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) { func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{}, nil return contracts.ChunkResult{}, nil
} }
@@ -354,6 +358,10 @@ func (normalizer registryNormalizer) Key() string {
return normalizer.key return normalizer.key
} }
func (normalizer registryNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) { func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{}, nil return contracts.NormalizeResult{}, nil
} }

View File

@@ -117,6 +117,8 @@ func (defaultExtractor) ArtifactType() string { return "record" }
func (defaultExtractor) SchemaVersion() string { return "v1" } func (defaultExtractor) SchemaVersion() string { return "v1" }
func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (defaultExtractor) Validators() []contracts.Validator { return nil } func (defaultExtractor) Validators() []contracts.Validator { return nil }
func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) { func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {

View File

@@ -49,6 +49,20 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
Stage: StageExtract, Stage: StageExtract,
Provides: []string{" generic-artifact ", "source-citations", "generic-artifact", ""}, Provides: []string{" generic-artifact ", "source-citations", "generic-artifact", ""},
Requires: []string{" source-document ", "source-document", ""}, Requires: []string{" source-document ", "source-document", ""},
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: " glossary ",
Description: " Supporting terms ",
AcceptedMediaTypes: []string{" text/plain ", "text/markdown", "text/plain", ""},
MaxBytes: 1024,
},
{
Name: " roster ",
Description: " Characters ",
Required: true,
Multiple: true,
},
},
} }
if err := registry.RegisterWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil { if err := registry.RegisterWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil {
@@ -64,12 +78,28 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
Stage: StageExtract, Stage: StageExtract,
Provides: []string{"generic-artifact", "source-citations"}, Provides: []string{"generic-artifact", "source-citations"},
Requires: []string{"source-document"}, Requires: []string{"source-document"},
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Supporting terms",
AcceptedMediaTypes: []string{"text/markdown", "text/plain"},
MaxBytes: 1024,
},
{
Name: "roster",
Description: "Characters",
Required: true,
Multiple: true,
},
},
} }
if !reflect.DeepEqual(got, want) { if !reflect.DeepEqual(got, want) {
t.Fatalf("Spec() = %#v, want %#v", got, want) t.Fatalf("Spec() = %#v, want %#v", got, want)
} }
got.Provides[0] = "changed" got.Provides[0] = "changed"
got.ReferenceSlots[0].Name = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
again, ok := registry.Spec("generic-extractor") again, ok := registry.Spec("generic-extractor")
if !ok { if !ok {
t.Fatal("Spec() after caller mutation ok = false, want true") t.Fatal("Spec() after caller mutation ok = false, want true")
@@ -109,6 +139,50 @@ func TestExtractorRegistryRegisterWithSpecRejectsWrongStage(t *testing.T) {
} }
} }
func TestExtractorRegistryRejectsInvalidReferenceSlots(t *testing.T) {
tests := []struct {
name string
slots []contracts.ReferenceSlot
want string
}{
{
name: "empty name",
slots: []contracts.ReferenceSlot{{Name: " "}},
want: "name",
},
{
name: "duplicate name after trim",
slots: []contracts.ReferenceSlot{
{Name: "roster"},
{Name: " roster "},
},
want: "duplicated",
},
{
name: "negative max bytes",
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
want: "max_bytes",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
registry := NewExtractorRegistry()
err := registry.RegisterWithSpec(ModuleSpec{
Key: "generic-extractor",
Stage: StageExtract,
ReferenceSlots: test.slots,
}, fakeExtractorConstructor("generic-extractor"))
if err == nil {
t.Fatal("RegisterWithSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), test.want) {
t.Fatalf("RegisterWithSpec() error = %q, want %q", err.Error(), test.want)
}
})
}
}
func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) { func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) {
registry := NewExtractorRegistry() registry := NewExtractorRegistry()
@@ -301,6 +375,10 @@ func (extractor registryFakeExtractor) SchemaVersion() string {
return "v1" return "v1"
} }
func (extractor registryFakeExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor registryFakeExtractor) Validators() []contracts.Validator { func (extractor registryFakeExtractor) Validators() []contracts.Validator {
return nil return nil
} }

View File

@@ -4,6 +4,8 @@ import (
"fmt" "fmt"
"sort" "sort"
"strings" "strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
type ModuleStage string type ModuleStage string
@@ -23,6 +25,7 @@ type ModuleSpec struct {
Stage ModuleStage Stage ModuleStage
Provides []string Provides []string
Requires []string Requires []string
ReferenceSlots []contracts.ReferenceSlot
} }
func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec { func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec {
@@ -38,6 +41,7 @@ func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
Stage: spec.Stage, Stage: spec.Stage,
Provides: normalizeCapabilities(spec.Provides), Provides: normalizeCapabilities(spec.Provides),
Requires: normalizeCapabilities(spec.Requires), Requires: normalizeCapabilities(spec.Requires),
ReferenceSlots: normalizeReferenceSlots(spec.ReferenceSlots),
} }
} }
@@ -72,6 +76,7 @@ func cloneModuleSpec(spec ModuleSpec) ModuleSpec {
Stage: spec.Stage, Stage: spec.Stage,
Provides: append([]string(nil), spec.Provides...), Provides: append([]string(nil), spec.Provides...),
Requires: append([]string(nil), spec.Requires...), Requires: append([]string(nil), spec.Requires...),
ReferenceSlots: cloneReferenceSlots(spec.ReferenceSlots),
} }
} }
@@ -82,9 +87,19 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
if spec.Stage != expectedStage { if spec.Stage != expectedStage {
return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage) return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage)
} }
if !referenceSlotStage(spec.Stage) && len(spec.ReferenceSlots) > 0 {
return fmt.Errorf("%s %q must not declare reference slots", kind, spec.Key)
}
if err := validateReferenceSlots(spec.ReferenceSlots); err != nil {
return fmt.Errorf("%s %q reference slots: %w", kind, spec.Key, err)
}
return nil return nil
} }
func referenceSlotStage(stage ModuleStage) bool {
return stage == StageChunk || stage == StageExtract || stage == StageNormalize
}
func sortedRegistryKeys[C any](constructors map[string]C) []string { func sortedRegistryKeys[C any](constructors map[string]C) []string {
if len(constructors) == 0 { if len(constructors) == 0 {
return nil return nil
@@ -97,3 +112,74 @@ func sortedRegistryKeys[C any](constructors map[string]C) []string {
sort.Strings(keys) sort.Strings(keys)
return keys return keys
} }
func normalizeReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
normalized := make([]contracts.ReferenceSlot, 0, len(slots))
for _, slot := range slots {
slot.Name = strings.TrimSpace(slot.Name)
slot.Description = strings.TrimSpace(slot.Description)
slot.AcceptedMediaTypes = normalizeStringSet(slot.AcceptedMediaTypes)
normalized = append(normalized, slot)
}
sort.SliceStable(normalized, func(i, j int) bool {
return normalized[i].Name < normalized[j].Name
})
return normalized
}
func normalizeStringSet(values []string) []string {
if len(values) == 0 {
return nil
}
seen := make(map[string]struct{}, len(values))
for _, value := range values {
normalized := strings.TrimSpace(value)
if normalized == "" {
continue
}
seen[normalized] = struct{}{}
}
if len(seen) == 0 {
return nil
}
out := make([]string, 0, len(seen))
for value := range seen {
out = append(out, value)
}
sort.Strings(out)
return out
}
func validateReferenceSlots(slots []contracts.ReferenceSlot) error {
seen := make(map[string]struct{}, len(slots))
for i, slot := range slots {
if slot.Name == "" {
return fmt.Errorf("slot[%d].name must not be empty", i)
}
if _, ok := seen[slot.Name]; ok {
return fmt.Errorf("slot name %q is duplicated", slot.Name)
}
seen[slot.Name] = struct{}{}
if slot.MaxBytes < 0 {
return fmt.Errorf("slot %q max_bytes must not be negative", slot.Name)
}
}
return nil
}
func cloneReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
out := make([]contracts.ReferenceSlot, 0, len(slots))
for _, slot := range slots {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
out = append(out, slot)
}
return out
}

View File

@@ -0,0 +1,123 @@
package pipeline
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestValidateModuleSpecAllowsReferenceSlotsForEligibleStages(t *testing.T) {
tests := []struct {
name string
kind string
stage ModuleStage
}{
{name: "chunker", kind: "chunker", stage: StageChunk},
{name: "extractor", kind: "extractor", stage: StageExtract},
{name: "normalizer", kind: "normalizer", stage: StageNormalize},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
spec := normalizeModuleSpec(ModuleSpec{
Key: "module",
Stage: test.stage,
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Description: "Character roster", MaxBytes: 1024},
},
})
err := validateModuleSpec(test.kind, test.stage, spec)
if err != nil {
t.Fatalf("validateModuleSpec() error = %v, want nil", err)
}
})
}
}
func TestValidateModuleSpecRejectsReferenceSlotsForIneligibleStages(t *testing.T) {
tests := []struct {
name string
kind string
stage ModuleStage
}{
{name: "input", kind: "input adapter", stage: StageInput},
{name: "merge", kind: "merger", stage: StageMerge},
{name: "validate", kind: "validator", stage: StageValidate},
{name: "output", kind: "output encoder", stage: StageOutput},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
spec := normalizeModuleSpec(ModuleSpec{
Key: "module",
Stage: test.stage,
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster"},
},
})
err := validateModuleSpec(test.kind, test.stage, spec)
if err == nil {
t.Fatal("validateModuleSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), "reference slots") {
t.Fatalf("validateModuleSpec() error = %q, want reference slots context", err.Error())
}
})
}
}
func TestValidateModuleSpecRejectsInvalidReferenceSlotsForEligibleStages(t *testing.T) {
invalidSlots := []struct {
name string
slots []contracts.ReferenceSlot
want string
}{
{
name: "empty name",
slots: []contracts.ReferenceSlot{{Name: " "}},
want: "name",
},
{
name: "duplicate name after trim",
slots: []contracts.ReferenceSlot{
{Name: "roster"},
{Name: " roster "},
},
want: "duplicated",
},
{
name: "negative max bytes",
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
want: "max_bytes",
},
}
eligibleStages := []struct {
name string
kind string
stage ModuleStage
}{
{name: "chunk", kind: "chunker", stage: StageChunk},
{name: "extract", kind: "extractor", stage: StageExtract},
{name: "normalize", kind: "normalizer", stage: StageNormalize},
}
for _, stage := range eligibleStages {
for _, invalid := range invalidSlots {
t.Run(stage.name+"/"+invalid.name, func(t *testing.T) {
spec := normalizeModuleSpec(ModuleSpec{
Key: "module",
Stage: stage.stage,
ReferenceSlots: invalid.slots,
})
err := validateModuleSpec(stage.kind, stage.stage, spec)
if err == nil {
t.Fatal("validateModuleSpec() error = nil, want error")
}
if !strings.Contains(err.Error(), invalid.want) {
t.Fatalf("validateModuleSpec() error = %q, want %q", err.Error(), invalid.want)
}
})
}
}
}

View File

@@ -7,6 +7,8 @@ import (
"fmt" "fmt"
"sort" "sort"
"strings" "strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
const ( const (
@@ -21,6 +23,7 @@ type ModuleBinding struct {
Module string `json:"module"` Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"`
} }
type ArtifactLaneProfile struct { type ArtifactLaneProfile struct {
@@ -28,6 +31,7 @@ type ArtifactLaneProfile struct {
Merge ModuleBinding `json:"merge,omitempty"` Merge ModuleBinding `json:"merge,omitempty"`
Normalize ModuleBinding `json:"normalize,omitempty"` Normalize ModuleBinding `json:"normalize,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"` Validators []ModuleBinding `json:"validators,omitempty"`
References map[string]string `json:"references,omitempty"`
} }
type PipelineProfile struct { type PipelineProfile struct {
@@ -36,10 +40,35 @@ type PipelineProfile struct {
Chunk ModuleBinding `json:"chunk,omitempty"` Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"` Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
Output ModuleBinding `json:"output,omitempty"` Output ModuleBinding `json:"output,omitempty"`
References map[string]string `json:"references,omitempty"`
} }
type ResolveOptions struct { type ResolveOptions struct {
Only []string Only []string
ReferenceOverrides []ReferenceBinding
ReferenceUnbinds []ReferenceUnbind
}
type ReferenceBinding struct {
Stage ModuleStage `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
Source string `json:"source"`
BindingSource string `json:"binding_source,omitempty"`
}
type ReferenceUnbind struct {
Stage ModuleStage `json:"stage,omitempty"`
LaneID string `json:"lane_id,omitempty"`
SlotName string `json:"slot_name"`
}
type ResolvedReferenceTarget struct {
Stage ModuleStage `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
Module string `json:"module"`
Bindings []ReferenceBinding `json:"bindings,omitempty"`
ReferenceSet contracts.ReferenceSet `json:"-"`
} }
type ResolvedArtifactLane struct { type ResolvedArtifactLane struct {
@@ -48,6 +77,8 @@ type ResolvedArtifactLane struct {
Merge ModuleBinding Merge ModuleBinding
Normalize ModuleBinding Normalize ModuleBinding
Validators []ModuleBinding Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
} }
type ResolvedPipeline struct { type ResolvedPipeline struct {
@@ -55,6 +86,7 @@ type ResolvedPipeline struct {
Digest string Digest string
Input ModuleBinding Input ModuleBinding
Chunk ModuleBinding Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
ArtifactLanes []ResolvedArtifactLane ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding Output ModuleBinding
} }
@@ -115,18 +147,34 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
if len(selectedLaneIDs) == 0 { if len(selectedLaneIDs) == 0 {
return ResolvedPipeline{}, fmt.Errorf("pipeline %q must select at least one artifact lane", pipelineID) return ResolvedPipeline{}, fmt.Errorf("pipeline %q must select at least one artifact lane", pipelineID)
} }
if err := validatePipelineReferenceDefaults(pipelineID, profile.References, chunkSpec, lanesByID, catalog); err != nil {
return ResolvedPipeline{}, err
}
chunkReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
Stage: StageChunk,
Module: chunk.Module,
Slots: chunkSpec.ReferenceSlots,
PipelineReferences: profile.References,
LocalReferences: chunk.References,
Options: options,
})
if err != nil {
return ResolvedPipeline{}, err
}
resolved := ResolvedPipeline{ resolved := ResolvedPipeline{
ID: pipelineID, ID: pipelineID,
Input: input, Input: input,
Chunk: chunk, Chunk: chunk,
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: resolveBinding(profile.Output, DefaultOutputModule), Output: resolveBinding(profile.Output, DefaultOutputModule),
} }
outputCapabilities := capabilities.clone() outputCapabilities := capabilities.clone()
for _, laneID := range selectedLaneIDs { for _, laneID := range selectedLaneIDs {
laneProfile := lanesByID[laneID] laneProfile := lanesByID[laneID]
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, capabilities, catalog) lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
if err != nil { if err != nil {
return ResolvedPipeline{}, err return ResolvedPipeline{}, err
} }
@@ -150,7 +198,15 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
return resolved, nil return resolved, nil
} }
func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile, inherited capabilitySet, catalog ModuleCatalog) (ResolvedArtifactLane, capabilitySet, error) { func resolveArtifactLane(
pipelineID string,
laneID string,
profile ArtifactLaneProfile,
pipelineReferences map[string]string,
options ResolveOptions,
inherited capabilitySet,
catalog ModuleCatalog,
) (ResolvedArtifactLane, capabilitySet, error) {
lane := ResolvedArtifactLane{ lane := ResolvedArtifactLane{
ID: laneID, ID: laneID,
Extract: resolveBinding(profile.Extract, ""), Extract: resolveBinding(profile.Extract, ""),
@@ -171,6 +227,21 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
if missing, ok := capabilities.missing(extractSpec.Requires); ok { if missing, ok := capabilities.missing(extractSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing) return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
} }
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
LaneID: laneID,
Stage: StageExtract,
Module: lane.Extract.Module,
Slots: extractSpec.ReferenceSlots,
PipelineReferences: pipelineReferences,
LocalReferences: extractReferences,
Options: options,
})
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
capabilities.add(extractSpec.Provides...) capabilities.add(extractSpec.Provides...)
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module) mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
@@ -189,6 +260,20 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok { if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing) return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
} }
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
PipelineID: pipelineID,
LaneID: laneID,
Stage: StageNormalize,
Module: lane.Normalize.Module,
Slots: normalizeSpec.ReferenceSlots,
PipelineReferences: pipelineReferences,
LocalReferences: lane.Normalize.References,
Options: options,
})
if err != nil {
return ResolvedArtifactLane{}, nil, err
}
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
capabilities.add(normalizeSpec.Provides...) capabilities.add(normalizeSpec.Provides...)
for _, validator := range lane.Validators { for _, validator := range lane.Validators {
@@ -205,6 +290,307 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
return lane, capabilities, nil return lane, capabilities, nil
} }
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
return ResolvedReferenceTarget{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
Module: strings.TrimSpace(module),
Bindings: append([]ReferenceBinding(nil), bindings...),
}
}
func mergeReferenceMaps(base map[string]string, override map[string]string) map[string]string {
if len(base) == 0 && len(override) == 0 {
return nil
}
out := make(map[string]string, len(base)+len(override))
for key, value := range base {
out[key] = value
}
for key, value := range override {
out[key] = value
}
return out
}
func validatePipelineReferenceDefaults(
pipelineID string,
pipelineReferences map[string]string,
chunkSpec ModuleSpec,
lanesByID map[string]ArtifactLaneProfile,
catalog ModuleCatalog,
) error {
normalizedPipelineReferences, err := normalizedReferenceMap(pipelineReferences, fmt.Sprintf("pipeline %q reference slot", pipelineID))
if err != nil {
return err
}
if len(normalizedPipelineReferences) == 0 {
return nil
}
declaredByAnyTarget := make(map[string]struct{}, len(normalizedPipelineReferences))
for _, slot := range chunkSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
}
for _, laneID := range sortedArtifactLaneProfileKeys(lanesByID) {
laneProfile := lanesByID[laneID]
extract := resolveBinding(laneProfile.Extract, "")
if extract.Module == "" {
return fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
}
extractSpec, err := extractorSpec(catalog, extract.Module)
if err != nil {
return moduleLookupError(pipelineID, laneID, StageExtract, extract.Module, err)
}
for _, slot := range extractSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
}
normalize := resolveBinding(laneProfile.Normalize, DefaultNormalizeModule)
normalizeSpec, err := normalizerSpec(catalog, normalize.Module)
if err != nil {
return moduleLookupError(pipelineID, laneID, StageNormalize, normalize.Module, err)
}
for _, slot := range normalizeSpec.ReferenceSlots {
declaredByAnyTarget[slot.Name] = struct{}{}
}
}
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
if _, ok := declaredByAnyTarget[slotName]; !ok {
return fmt.Errorf("pipeline %q reference slot %q is not declared by any eligible reference target", pipelineID, slotName)
}
}
return nil
}
type referenceResolutionTarget struct {
PipelineID string
LaneID string
Stage ModuleStage
Module string
Slots []contracts.ReferenceSlot
PipelineReferences map[string]string
LocalReferences map[string]string
Options ResolveOptions
}
func resolveReferenceTargetBindings(target referenceResolutionTarget) ([]ReferenceBinding, error) {
slotByName := make(map[string]contracts.ReferenceSlot, len(target.Slots))
for _, slot := range target.Slots {
slotByName[slot.Name] = slot
}
bindings := make(map[string]ReferenceBinding)
addBinding := func(slotName, source, bindingSource string) error {
slotName = strings.TrimSpace(slotName)
source = strings.TrimSpace(source)
if slotName == "" {
return fmt.Errorf("%s reference slot name must not be empty", referenceTargetErrorContext(target))
}
if source == "" {
return fmt.Errorf("%s reference slot %q source must not be empty", referenceTargetErrorContext(target), slotName)
}
if _, ok := slotByName[slotName]; !ok {
return fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetErrorContext(target), slotName, target.Stage, target.Module)
}
bindings[slotName] = ReferenceBinding{
LaneID: target.LaneID,
SlotName: slotName,
Source: source,
BindingSource: bindingSource,
}
return nil
}
normalizedPipelineReferences, err := normalizedReferenceMap(target.PipelineReferences, fmt.Sprintf("pipeline %q reference slot", target.PipelineID))
if err != nil {
return nil, err
}
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
if _, ok := slotByName[slotName]; !ok {
continue
}
if err := addBinding(slotName, normalizedPipelineReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
return nil, err
}
}
normalizedLocalReferences, err := normalizedReferenceMap(target.LocalReferences, referenceTargetSlotLabel(target))
if err != nil {
return nil, err
}
for _, slotName := range sortedStringMapKeys(normalizedLocalReferences) {
if err := addBinding(slotName, normalizedLocalReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
return nil, err
}
}
for _, override := range target.Options.ReferenceOverrides {
match, err := referenceOverrideMatchesTarget(target, override)
if err != nil {
return nil, err
}
if !match {
continue
}
source := override.BindingSource
if strings.TrimSpace(source) == "" {
source = contracts.ReferenceBindingSourceCLI
}
if err := addBinding(override.SlotName, override.Source, strings.TrimSpace(source)); err != nil {
return nil, err
}
}
for _, unbind := range target.Options.ReferenceUnbinds {
match, err := referenceUnbindMatchesTarget(target, unbind)
if err != nil {
return nil, err
}
if !match {
continue
}
slotName := strings.TrimSpace(unbind.SlotName)
if slotName == "" {
return nil, fmt.Errorf("%s reference unbind slot name must not be empty", referenceTargetErrorContext(target))
}
if _, ok := slotByName[slotName]; !ok {
return nil, fmt.Errorf("%s reference slot %q is not declared", referenceTargetErrorContext(target), slotName)
}
delete(bindings, slotName)
}
for _, slot := range target.Slots {
if slot.Required {
if _, ok := bindings[slot.Name]; !ok {
return nil, fmt.Errorf("%s required reference slot %q is not bound", referenceTargetErrorContext(target), slot.Name)
}
}
}
return sortedReferenceBindings(bindings), nil
}
func referenceOverrideMatchesTarget(target referenceResolutionTarget, override ReferenceBinding) (bool, error) {
stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "override", override.Stage, override.LaneID)
if err != nil {
return false, err
}
return stage == target.Stage && laneID == target.LaneID, nil
}
func referenceUnbindMatchesTarget(target referenceResolutionTarget, unbind ReferenceUnbind) (bool, error) {
stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "unbind", unbind.Stage, unbind.LaneID)
if err != nil {
return false, err
}
return stage == target.Stage && laneID == target.LaneID, nil
}
func normalizeReferenceOptionTarget(pipelineID string, operation string, stage ModuleStage, laneID string) (ModuleStage, string, error) {
stage = ModuleStage(strings.TrimSpace(string(stage)))
if stage == "" {
stage = StageExtract
}
laneID = strings.TrimSpace(laneID)
switch stage {
case StageChunk:
if laneID != "" {
return "", "", fmt.Errorf("pipeline %q reference %s for chunk must not include a lane id", pipelineID, operation)
}
case StageExtract, StageNormalize:
if laneID == "" {
return "", "", fmt.Errorf("pipeline %q reference %s lane id must not be empty", pipelineID, operation)
}
default:
return "", "", fmt.Errorf("pipeline %q reference %s stage %q is not supported", pipelineID, operation, stage)
}
return stage, laneID, nil
}
func sortedReferenceBindings(bindings map[string]ReferenceBinding) []ReferenceBinding {
keys := sortedReferenceBindingKeys(bindings)
resolved := make([]ReferenceBinding, 0, len(keys))
for _, slotName := range keys {
resolved = append(resolved, bindings[slotName])
}
return resolved
}
func referenceTargetErrorContext(target referenceResolutionTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s module %q", target.PipelineID, target.LaneID, target.Stage, target.Module)
}
return fmt.Sprintf("pipeline %q %s module %q", target.PipelineID, target.Stage, target.Module)
}
func referenceTargetSlotLabel(target referenceResolutionTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s reference slot", target.PipelineID, target.LaneID, target.Stage)
}
return fmt.Sprintf("pipeline %q %s reference slot", target.PipelineID, target.Stage)
}
func normalizedReferenceMap(values map[string]string, keyName string) (map[string]string, error) {
if len(values) == 0 {
return nil, nil
}
out := make(map[string]string, len(values))
for rawSlotName, rawSource := range values {
slotName := strings.TrimSpace(rawSlotName)
if slotName == "" {
return nil, fmt.Errorf("%s must not be empty", keyName)
}
if _, ok := out[slotName]; ok {
return nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, slotName)
}
source := strings.TrimSpace(rawSource)
if source == "" {
return nil, fmt.Errorf("%s %q source must not be empty", keyName, slotName)
}
out[slotName] = source
}
return out, nil
}
func sortedArtifactLaneProfileKeys(values map[string]ArtifactLaneProfile) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func sortedStringMapKeys(values map[string]string) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func sortedReferenceBindingKeys(values map[string]ReferenceBinding) []string {
if len(values) == 0 {
return nil
}
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding { func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
module := strings.TrimSpace(binding.Module) module := strings.TrimSpace(binding.Module)
if module == "" { if module == "" {
@@ -218,6 +604,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
Module: module, Module: module,
LLMProfile: llmProfile, LLMProfile: llmProfile,
Options: cloneOptions(binding.Options), Options: cloneOptions(binding.Options),
References: normalizeReferenceMap(binding.References),
} }
} }
@@ -246,6 +633,25 @@ func cloneOptions(options map[string]any) map[string]any {
return copied return copied
} }
func normalizeReferenceMap(values map[string]string) map[string]string {
if len(values) == 0 {
return nil
}
out := make(map[string]string, len(values))
keys := make([]string, 0, len(values))
rawByNormalized := make(map[string]string, len(values))
for rawKey := range values {
key := strings.TrimSpace(rawKey)
rawByNormalized[key] = rawKey
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
}
return out
}
func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) { func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) {
lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts)) lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts))
for rawLaneID, lane := range artifacts { for rawLaneID, lane := range artifacts {
@@ -293,12 +699,14 @@ func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
ID string ID string
Input ModuleBinding Input ModuleBinding
Chunk ModuleBinding Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget
ArtifactLanes []ResolvedArtifactLane ArtifactLanes []ResolvedArtifactLane
Output ModuleBinding Output ModuleBinding
}{ }{
ID: resolved.ID, ID: resolved.ID,
Input: resolved.Input, Input: resolved.Input,
Chunk: resolved.Chunk, Chunk: resolved.Chunk,
ChunkReferences: resolved.ChunkReferences,
ArtifactLanes: resolved.ArtifactLanes, ArtifactLanes: resolved.ArtifactLanes,
Output: resolved.Output, Output: resolved.Output,
} }

View File

@@ -3,6 +3,7 @@ package pipeline
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
@@ -125,6 +126,409 @@ func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
} }
} }
func TestResolvePipelineAppliesReferenceBindings(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{
" roster ": " ./shared-roster.yml ",
}
lane := profile.Artifacts["events"]
lane.References = map[string]string{
"roster": "./lane-roster.yml",
" lore ": " ./lore.md ",
}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
{Name: "lore"},
},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events", "summaries"}}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
events := resolvedLane(t, resolved.ArtifactLanes, "events")
if events.ExtractReferences.Stage != StageExtract || events.ExtractReferences.LaneID != "events" || events.ExtractReferences.Module != "event-extractor" {
t.Fatalf("extract reference target = %#v, want event extractor target", events.ExtractReferences)
}
if events.NormalizeReferences.Stage != StageNormalize || events.NormalizeReferences.LaneID != "events" || events.NormalizeReferences.Module != DefaultNormalizeModule {
t.Fatalf("normalize reference target = %#v, want event normalizer target", events.NormalizeReferences)
}
if resolved.ChunkReferences.Stage != StageChunk || resolved.ChunkReferences.Module != DefaultChunkModule {
t.Fatalf("chunk reference target = %#v, want chunk target", resolved.ChunkReferences)
}
want := []ReferenceBinding{
{LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig},
{LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig},
}
if !reflect.DeepEqual(events.ExtractReferences.Bindings, want) {
t.Fatalf("events references = %#v, want %#v", events.ExtractReferences.Bindings, want)
}
summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries")
if len(summaries.ExtractReferences.Bindings) != 0 {
t.Fatalf("summaries references = %#v, want none", summaries.ExtractReferences.Bindings)
}
}
func TestResolvePipelineAppliesPipelineReferenceDefaultToChunkTarget(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"scene_guide": "./scenes.md"}
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "generic",
Stage: StageChunk,
Requires: []string{"source"},
Provides: []string{"chunk"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "scene_guide"}},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
want := []ReferenceBinding{{SlotName: "scene_guide", Source: "./scenes.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
if !reflect.DeepEqual(resolved.ChunkReferences.Bindings, want) {
t.Fatalf("chunk references = %#v, want %#v", resolved.ChunkReferences.Bindings, want)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("extract references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("normalize references = %#v, want none", refs)
}
}
func TestResolvePipelineAppliesPipelineReferenceDefaultToExtractorTarget(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
want := []ReferenceBinding{{LaneID: "events", SlotName: "roster", Source: "./roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig}}
if !reflect.DeepEqual(resolved.ArtifactLanes[0].ExtractReferences.Bindings, want) {
t.Fatalf("extract references = %#v, want %#v", resolved.ArtifactLanes[0].ExtractReferences.Bindings, want)
}
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
t.Fatalf("chunk references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("normalize references = %#v, want none", refs)
}
}
func TestResolvePipelineAppliesPipelineReferenceDefaultToNormalizerTarget(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"normalization_notes": "./normalize.md"}
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
want := []ReferenceBinding{{LaneID: "events", SlotName: "normalization_notes", Source: "./normalize.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
if !reflect.DeepEqual(resolved.ArtifactLanes[0].NormalizeReferences.Bindings, want) {
t.Fatalf("normalize references = %#v, want %#v", resolved.ArtifactLanes[0].NormalizeReferences.Bindings, want)
}
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
t.Fatalf("chunk references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("extract references = %#v, want none", refs)
}
}
func TestResolvePipelineAppliesOnePipelineReferenceDefaultToMultipleTargets(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"context": "./context.md"}
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "context", "./context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "context", "./context.md")
}
func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{"notes_context": "./notes.md"}
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "note-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes_context"},
},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected lane references = %#v, want none", refs)
}
}
func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedNormalizer(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{"notes_context": "./notes.md"}
lane := profile.Artifacts["notes"]
lane.Normalize = Binding("note-normalizer")
profile.Artifacts["notes"] = lane
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "note-normalizer",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "notes_context"},
},
})
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected extract references = %#v, want none", refs)
}
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
t.Fatalf("selected normalize references = %#v, want none", refs)
}
}
func TestResolvePipelineRejectsPipelineReferenceNotDeclaredByAnyLane(t *testing.T) {
profile := multiLaneProfile()
profile.References = map[string]string{"missing": "./missing.md"}
_, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "multi", "reference slot", "missing", "not declared")
}
func TestResolvePipelineRejectsUndeclaredReferenceSlot(t *testing.T) {
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.References = map[string]string{"missing": "./missing.yml"}
profile.Artifacts["events"] = lane
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "events", "missing", "not declared")
}
func TestResolvePipelineRejectsExtractLocalReferenceDeclaredOnlyByNormalizer(t *testing.T) {
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{"normalization_notes": "./normalize.md"}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}},
})
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "events", "extract", "event-extractor", "normalization_notes", "not declared")
}
func TestResolvePipelineRejectsNormalizeLocalReferenceDeclaredOnlyByExtractor(t *testing.T) {
profile := baselineProfile()
lane := profile.Artifacts["events"]
lane.Normalize.References = map[string]string{"roster": "./roster.yml"}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}},
})
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "events", "normalize", "noop", "roster", "not declared")
}
func TestResolvePipelineRequiresBoundChunkReference(t *testing.T) {
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "generic",
Stage: StageChunk,
Requires: []string{"source"},
Provides: []string{"chunk"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "scene_guide", Required: true}},
})
_, err := ResolvePipeline(baselineProfile(), ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "chunk", "generic", "required", "scene_guide", "not bound")
}
func TestResolvePipelineRequiresBoundNormalizeReference(t *testing.T) {
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes", Required: true}},
})
_, err := ResolvePipeline(baselineProfile(), ResolveOptions{}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "events", "normalize", "noop", "required", "normalization_notes", "not bound")
}
func TestResolvePipelineLocalReferencesOverridePipelineDefaultsForEligibleTargets(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{
"context": "./shared-context.md",
"roster": "./shared-roster.yml",
"normalization_notes": "./shared-normalize.md",
}
profile.Chunk.References = map[string]string{"context": "./chunk-context.md"}
lane := profile.Artifacts["events"]
lane.Extract.References = map[string]string{"roster": "./extract-roster.yml"}
lane.Normalize.References = map[string]string{"normalization_notes": "./local-normalize.md"}
profile.Artifacts["events"] = lane
catalog := newProfileCatalogWithOverrides(t,
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}}},
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./chunk-context.md")
assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "roster", "./extract-roster.yml")
assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "normalization_notes", "./local-normalize.md")
}
func TestResolvePipelineRequiresBoundReferenceSlotsForSelectedLanes(t *testing.T) {
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
})
if _, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"notes"}}, catalog); err != nil {
t.Fatalf("ResolvePipeline(unselected required slot) error = %v, want nil", err)
}
_, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"events"}}, catalog)
if err == nil {
t.Fatal("ResolvePipeline(selected required slot) error = nil, want error")
}
assertErrorContains(t, err, "events", "required", "roster", "not bound")
}
func TestResolvePipelineReferenceUnbindCanLeaveRequiredSlotMissing(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
})
_, err := ResolvePipeline(profile, ResolveOptions{
ReferenceUnbinds: []ReferenceUnbind{{LaneID: "events", SlotName: "roster"}},
}, catalog)
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "events", "required", "roster", "not bound")
}
func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t *testing.T) {
profile := baselineProfile()
profile.References = map[string]string{"roster": "./roster.yml"}
catalog := emptyProfileCatalog()
for _, spec := range defaultProfileSpecs() {
if spec.Key != "event-extractor" {
registerProfileSpecs(t, catalog, spec)
}
}
if err := catalog.Extractors.RegisterWithSpec(ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "roster", Required: true},
},
}, func() (contracts.Extractor, error) {
return nil, errors.New("constructor should not run")
}); err != nil {
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if got := resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].Source; got != "./roster.yml" {
t.Fatalf("reference source = %q, want ./roster.yml", got)
}
}
func TestResolvePipelineRejectsUnknownOnlyLane(t *testing.T) { func TestResolvePipelineRejectsUnknownOnlyLane(t *testing.T) {
_, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"missing"}}, newProfileCatalog(t)) _, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"missing"}}, newProfileCatalog(t))
if err == nil { if err == nil {
@@ -463,6 +867,17 @@ func laneIDs(lanes []ResolvedArtifactLane) []string {
return ids return ids
} }
func resolvedLane(t *testing.T, lanes []ResolvedArtifactLane, laneID string) ResolvedArtifactLane {
t.Helper()
for _, lane := range lanes {
if lane.ID == laneID {
return lane
}
}
t.Fatalf("lane %q not found in %#v", laneID, laneIDs(lanes))
return ResolvedArtifactLane{}
}
func assertErrorContains(t *testing.T, err error, values ...string) { func assertErrorContains(t *testing.T, err error, values ...string) {
t.Helper() t.Helper()
@@ -474,6 +889,21 @@ func assertErrorContains(t *testing.T, err error, values ...string) {
} }
} }
func assertBindingSource(t *testing.T, bindings []ReferenceBinding, slotName string, source string) {
t.Helper()
for _, binding := range bindings {
if binding.SlotName != slotName {
continue
}
if binding.Source != source {
t.Fatalf("binding %q source = %q, want %q in %#v", slotName, binding.Source, source, bindings)
}
return
}
t.Fatalf("binding %q not found in %#v", slotName, bindings)
}
func newProfileCatalog(t *testing.T) ModuleCatalog { func newProfileCatalog(t *testing.T) ModuleCatalog {
t.Helper() t.Helper()
@@ -485,19 +915,29 @@ func newProfileCatalog(t *testing.T) ModuleCatalog {
func newProfileCatalogWithOverride(t *testing.T, override ModuleSpec) ModuleCatalog { func newProfileCatalogWithOverride(t *testing.T, override ModuleSpec) ModuleCatalog {
t.Helper() t.Helper()
return newProfileCatalogWithOverrides(t, override)
}
func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) ModuleCatalog {
t.Helper()
specs := defaultProfileSpecs() specs := defaultProfileSpecs()
for _, override := range overrides {
replaced := false
for index, spec := range specs { for index, spec := range specs {
if spec.Stage == override.Stage && spec.Key == override.Key { if spec.Stage == override.Stage && spec.Key == override.Key {
specs[index] = override specs[index] = override
catalog := emptyProfileCatalog() replaced = true
registerProfileSpecs(t, catalog, specs...) break
return catalog }
}
if !replaced {
specs = append(specs, override)
} }
} }
catalog := emptyProfileCatalog() catalog := emptyProfileCatalog()
registerProfileSpecs(t, catalog, specs...) registerProfileSpecs(t, catalog, specs...)
registerProfileSpecs(t, catalog, override)
return catalog return catalog
} }

View File

@@ -0,0 +1,323 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"mime"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
const (
referenceOriginFile = "file"
referenceMediaType = "text/plain"
unknownMediaType = "application/octet-stream"
)
type ReferenceMaterializationOptions struct {
ConfigPath string
WorkingDir string
}
func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) {
out := resolved
out.ChunkReferences = CloneReferenceTarget(resolved.ChunkReferences)
chunkReferenceSet, chunkWarnings, err := materializeReferenceTarget(resolved.ID, resolved.ChunkReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
out.ChunkReferences.ReferenceSet = chunkReferenceSet
warnings := append([]contracts.Warning(nil), chunkWarnings...)
if len(resolved.ArtifactLanes) == 0 {
return out, warnings, nil
}
out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
for i, lane := range resolved.ArtifactLanes {
materializedLane := lane
materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences)
extractReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
materializedLane.ExtractReferences.ReferenceSet = extractReferenceSet
warnings = append(warnings, laneWarnings...)
normalizeReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.NormalizeReferences, catalog, options)
if err != nil {
return ResolvedPipeline{}, nil, err
}
materializedLane.NormalizeReferences.ReferenceSet = normalizeReferenceSet
warnings = append(warnings, laneWarnings...)
out.ArtifactLanes[i] = materializedLane
}
return out, warnings, nil
}
func materializeReferenceTarget(
pipelineID string,
target ResolvedReferenceTarget,
catalog ModuleCatalog,
options ReferenceMaterializationOptions,
) (contracts.ReferenceSet, []contracts.Warning, error) {
if len(target.Bindings) == 0 {
return contracts.ReferenceSet{}, nil, nil
}
spec, err := referenceTargetSpec(target, catalog)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s: %w", referenceTargetContext(pipelineID, target), err)
}
slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots))
for _, slot := range spec.ReferenceSlots {
slotByName[slot.Name] = slot
}
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(target.Bindings))}
var warnings []contracts.Warning
for _, binding := range target.Bindings {
slotName := strings.TrimSpace(binding.SlotName)
slot, ok := slotByName[slotName]
if !ok {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetContext(pipelineID, target), slotName, target.Stage, target.Module)
}
path, err := referencePath(binding, options)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q: %w", referenceTargetContext(pipelineID, target), slotName, binding.Source, err)
}
content, err := os.ReadFile(path)
if err != nil {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q read %q: %w", referenceTargetContext(pipelineID, target), slotName, path, err)
}
if !utf8.Valid(content) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q must be UTF-8 text", referenceTargetContext(pipelineID, target), slotName, path)
}
mediaType := referenceMediaTypeForPath(path)
if !referenceMediaTypeAccepted(mediaType, slot.AcceptedMediaTypes) {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q media type %q is not accepted", referenceTargetContext(pipelineID, target), slotName, path, mediaType)
}
if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes {
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q is %d bytes, limit %d", referenceTargetContext(pipelineID, target), slotName, path, len(content), slot.MaxBytes)
}
if len(content) == 0 {
warnings = append(warnings, contracts.Warning{
Scope: referenceWarningScope(pipelineID, target, slotName),
ReasonCode: "empty_reference",
Message: fmt.Sprintf("reference slot %q for %s is bound to an empty file", slotName, referenceTargetLabel(target)),
})
}
item := contracts.ReferenceItem{
SlotName: slotName,
MediaType: mediaType,
Content: append([]byte(nil), content...),
Digest: referenceDigest(content),
Origin: contracts.ReferenceOrigin{Type: referenceOriginFile, URI: fileURI(path)},
SizeBytes: int64(len(content)),
BindingSource: strings.TrimSpace(binding.BindingSource),
}
set.Slots[slotName] = contracts.ResolvedReferenceSlot{
Slot: cloneReferenceSlot(slot),
Items: []contracts.ReferenceItem{item},
}
}
return set, warnings, nil
}
func referenceTargetSpec(target ResolvedReferenceTarget, catalog ModuleCatalog) (ModuleSpec, error) {
switch target.Stage {
case StageChunk:
return registrySpec(catalog.Chunkers, target.Module)
case StageExtract:
return registrySpec(catalog.Extractors, target.Module)
case StageNormalize:
return registrySpec(catalog.Normalizers, target.Module)
default:
return ModuleSpec{}, fmt.Errorf("reference target stage %q is not supported", target.Stage)
}
}
func referenceTargetContext(pipelineID string, target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline %q lane %q %s module %q", pipelineID, target.LaneID, target.Stage, target.Module)
}
return fmt.Sprintf("pipeline %q %s module %q", pipelineID, target.Stage, target.Module)
}
func referenceTargetLabel(target ResolvedReferenceTarget) string {
if target.LaneID != "" {
return fmt.Sprintf("lane %q %s target", target.LaneID, target.Stage)
}
return fmt.Sprintf("%s target", target.Stage)
}
func referenceWarningScope(pipelineID string, target ResolvedReferenceTarget, slotName string) string {
if target.LaneID != "" {
return fmt.Sprintf("pipeline.%s.lane.%s.%s.reference.%s", pipelineID, target.LaneID, target.Stage, slotName)
}
return fmt.Sprintf("pipeline.%s.%s.reference.%s", pipelineID, target.Stage, slotName)
}
func referenceMediaTypeForPath(path string) string {
extension := strings.ToLower(filepath.Ext(path))
mediaType := mime.TypeByExtension(extension)
if strings.TrimSpace(mediaType) == "" {
if extension == ".md" || extension == ".markdown" {
return "text/markdown"
}
return unknownMediaType
}
return canonicalMediaType(mediaType)
}
func referenceMediaTypeAccepted(mediaType string, accepted []string) bool {
if len(accepted) == 0 {
return true
}
mediaType = canonicalMediaType(mediaType)
for _, value := range accepted {
if strings.EqualFold(mediaType, canonicalMediaType(value)) {
return true
}
}
return false
}
func canonicalMediaType(mediaType string) string {
trimmed := strings.TrimSpace(mediaType)
if trimmed == "" {
return ""
}
parsed, _, err := mime.ParseMediaType(trimmed)
if err != nil {
return strings.ToLower(trimmed)
}
return strings.ToLower(parsed)
}
func referencePath(binding ReferenceBinding, options ReferenceMaterializationOptions) (string, error) {
source := strings.TrimSpace(binding.Source)
if source == "" {
return "", fmt.Errorf("must not be empty")
}
if filepath.IsAbs(source) {
return filepath.Clean(source), nil
}
base := strings.TrimSpace(options.WorkingDir)
if strings.TrimSpace(binding.BindingSource) == contracts.ReferenceBindingSourceConfig {
base = filepath.Dir(strings.TrimSpace(options.ConfigPath))
}
if base == "" {
var err error
base, err = os.Getwd()
if err != nil {
return "", fmt.Errorf("resolve working directory: %w", err)
}
}
return filepath.Clean(filepath.Join(base, source)), nil
}
func referenceDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
func fileURI(path string) string {
absolute, err := filepath.Abs(path)
if err != nil {
absolute = path
}
absolute = filepath.ToSlash(filepath.Clean(absolute))
if strings.HasPrefix(absolute, "/") {
return "file://" + (&url.URL{Path: absolute}).EscapedPath()
}
return "file:///" + (&url.URL{Path: absolute}).EscapedPath()
}
func cloneReferenceSlot(slot contracts.ReferenceSlot) contracts.ReferenceSlot {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
return slot
}
func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet {
if len(in.Slots) == 0 {
return contracts.ReferenceSet{}
}
out := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(in.Slots))}
keys := make([]string, 0, len(in.Slots))
for key := range in.Slots {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
slot := in.Slots[key]
slot.Slot = cloneReferenceSlot(slot.Slot)
if len(slot.Items) > 0 {
items := make([]contracts.ReferenceItem, len(slot.Items))
for i, item := range slot.Items {
item.Content = append([]byte(nil), item.Content...)
items[i] = item
}
slot.Items = items
}
out.Slots[key] = slot
}
return out
}
func CloneReferenceTarget(in ResolvedReferenceTarget) ResolvedReferenceTarget {
out := in
out.Bindings = append([]ReferenceBinding(nil), in.Bindings...)
out.ReferenceSet = CloneReferenceSet(in.ReferenceSet)
return out
}
func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance {
provenance := []artifacts.ReferenceProvenance{}
provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...)
for _, lane := range resolved.ArtifactLanes {
provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...)
provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...)
}
return provenance
}
func referenceTargetProvenance(target ResolvedReferenceTarget) []artifacts.ReferenceProvenance {
if len(target.ReferenceSet.Slots) == 0 {
return nil
}
provenance := []artifacts.ReferenceProvenance{}
slotNames := make([]string, 0, len(target.ReferenceSet.Slots))
for slotName := range target.ReferenceSet.Slots {
slotNames = append(slotNames, slotName)
}
sort.Strings(slotNames)
for _, slotName := range slotNames {
slot := target.ReferenceSet.Slots[slotName]
for _, item := range slot.Items {
provenance = append(provenance, artifacts.ReferenceProvenance{
Stage: string(target.Stage),
LaneID: target.LaneID,
SlotName: item.SlotName,
OriginType: item.Origin.Type,
OriginURI: item.Origin.URI,
Digest: item.Digest,
MediaType: item.MediaType,
SizeBytes: item.SizeBytes,
BindingSource: item.BindingSource,
})
}
}
return provenance
}

View File

@@ -0,0 +1,477 @@
package pipeline
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
configDir := t.TempDir()
workingDir := t.TempDir()
configReference := filepath.Join(configDir, "config-reference.txt")
cliReference := filepath.Join(workingDir, "cli-reference.txt")
writeReferenceFile(t, configReference, []byte("config text"))
writeReferenceFile(t, cliReference, []byte("cli text"))
pipeline := baselineProfile()
pipeline.References = map[string]string{"roster": "config-reference.txt"}
lane := pipeline.Artifacts["events"]
lane.References = map[string]string{"glossary": "cli-reference.txt"}
pipeline.Artifacts["events"] = lane
catalog := referenceCatalog(t, []contracts.ReferenceSlot{
{Name: "roster"},
{Name: "glossary"},
})
resolved, err := ResolvePipeline(pipeline, ResolveOptions{
ReferenceOverrides: []ReferenceBinding{
{LaneID: "events", SlotName: "glossary", Source: "cli-reference.txt", BindingSource: contracts.ReferenceBindingSourceCLI},
},
}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
first, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
WorkingDir: workingDir,
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
if len(warnings) != 0 {
t.Fatalf("warnings = %#v, want none", warnings)
}
second, _, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
WorkingDir: workingDir,
})
if err != nil {
t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
}
referenceSet := first.ArtifactLanes[0].ExtractReferences.ReferenceSet
roster := referenceSet.Slots["roster"].Items[0]
if string(roster.Content) != "config text" {
t.Fatalf("roster content = %q, want config text", roster.Content)
}
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Digest {
t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
}
if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
t.Fatalf("roster binding source = %q, want config", roster.BindingSource)
}
if roster.MediaType != referenceMediaType || roster.Origin.Type != referenceOriginFile || roster.SizeBytes != int64(len("config text")) {
t.Fatalf("roster metadata = %#v, want text file metadata", roster)
}
if !strings.Contains(roster.Origin.URI, "config-reference.txt") {
t.Fatalf("roster origin URI = %q, want config reference path", roster.Origin.URI)
}
glossary := referenceSet.Slots["glossary"].Items[0]
if string(glossary.Content) != "cli text" {
t.Fatalf("glossary content = %q, want cli text", glossary.Content)
}
if glossary.BindingSource != contracts.ReferenceBindingSourceCLI {
t.Fatalf("glossary binding source = %q, want cli", glossary.BindingSource)
}
if !strings.Contains(glossary.Origin.URI, "cli-reference.txt") {
t.Fatalf("glossary origin URI = %q, want cli reference path", glossary.Origin.URI)
}
provenance := ReferenceProvenance(first)
if len(provenance) != 2 {
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
}
if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
}
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
}
encoded, err := json.Marshal(first)
if err != nil {
t.Fatalf("json.Marshal(materialized) error = %v, want nil", err)
}
if strings.Contains(string(encoded), "config text") || strings.Contains(string(encoded), "cli text") {
t.Fatalf("materialized pipeline JSON contains reference content: %s", encoded)
}
}
func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T) {
configDir := t.TempDir()
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), []byte("chunk text"))
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), []byte("extract text"))
writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), []byte("normalize text"))
profile := baselineProfile()
profile.References = map[string]string{
"scene_guide": "chunk.txt",
"roster": "extract.txt",
"normalization_notes": "normalize.txt",
}
catalog := referenceCatalogForTargets(t,
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
[]contracts.ReferenceSlot{{Name: "roster"}},
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
materialized, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
if len(warnings) != 0 {
t.Fatalf("warnings = %#v, want none", warnings)
}
chunkItem := materialized.ChunkReferences.ReferenceSet.Slots["scene_guide"].Items[0]
if string(chunkItem.Content) != "chunk text" {
t.Fatalf("chunk content = %q, want chunk text", chunkItem.Content)
}
extractItem := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if string(extractItem.Content) != "extract text" {
t.Fatalf("extract content = %q, want extract text", extractItem.Content)
}
normalizeItem := materialized.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["normalization_notes"].Items[0]
if string(normalizeItem.Content) != "normalize text" {
t.Fatalf("normalize content = %q, want normalize text", normalizeItem.Content)
}
provenance := ReferenceProvenance(materialized)
if len(provenance) != 3 {
t.Fatalf("ReferenceProvenance() = %#v, want three entries", provenance)
}
if provenance[0].Stage != string(StageChunk) || provenance[0].LaneID != "" || provenance[0].SlotName != "scene_guide" || provenance[0].Digest != chunkItem.Digest {
t.Fatalf("ReferenceProvenance()[0] = %#v, want chunk scene guide provenance", provenance[0])
}
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != extractItem.Digest {
t.Fatalf("ReferenceProvenance()[1] = %#v, want extract roster provenance", provenance[1])
}
if provenance[2].Stage != string(StageNormalize) || provenance[2].LaneID != "events" || provenance[2].SlotName != "normalization_notes" || provenance[2].Digest != normalizeItem.Digest {
t.Fatalf("ReferenceProvenance()[2] = %#v, want normalize notes provenance", provenance[2])
}
}
func TestMaterializeReferencesRejectsNonUTF8Content(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "bad.txt")
writeReferenceFile(t, path, []byte{0xff, 0xfe})
resolved := resolvedPipelineWithReference(t, "roster", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), path) {
t.Fatalf("error = %v, want UTF-8 path error", err)
}
}
func TestMaterializeReferencesRejectsNonUTF8ContentForChunkTarget(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "bad.txt")
writeReferenceFile(t, path, []byte{0xff, 0xfe})
resolved := resolvedPipelineWithTargetReference(t, StageChunk, "", "scene_guide", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "scene_guide"})
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, []contracts.ReferenceSlot{{Name: "scene_guide"}}, nil, nil), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "chunk") || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "scene_guide") || !strings.Contains(err.Error(), path) {
t.Fatalf("error = %v, want chunk UTF-8 path error", err)
}
}
func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "roster.reference")
writeReferenceFile(t, path, []byte("plain text"))
resolved := resolvedPipelineWithReference(t, "roster", "roster.reference", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != unknownMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType)
}
}
func TestMaterializeReferencesAcceptsDeclaredMarkdownMediaType(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "glossary.md")
writeReferenceFile(t, path, []byte("# Terms\n"))
slot := contracts.ReferenceSlot{Name: "glossary", AcceptedMediaTypes: []string{"text/markdown"}}
resolved := resolvedPipelineWithReference(t, "glossary", "glossary.md", contracts.ReferenceBindingSourceConfig, slot)
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["glossary"].Items[0]
if item.MediaType != "text/markdown" {
t.Fatalf("MediaType = %q, want text/markdown", item.MediaType)
}
}
func TestMaterializeReferencesAcceptsDeclaredJSONMediaType(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "roster.json")
writeReferenceFile(t, path, []byte(`{"aria":"cleric"}`))
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"application/json"}}
resolved := resolvedPipelineWithReference(t, "roster", "roster.json", contracts.ReferenceBindingSourceConfig, slot)
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != "application/json" {
t.Fatalf("MediaType = %q, want application/json", item.MediaType)
}
}
func TestMaterializeReferencesRejectsUnacceptedMediaType(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "roster.json")
writeReferenceFile(t, path, []byte(`{"aria":"cleric"}`))
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"text/markdown"}}
resolved := resolvedPipelineWithReference(t, "roster", "roster.json", contracts.ReferenceBindingSourceConfig, slot)
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "media type") || !strings.Contains(err.Error(), "application/json") || !strings.Contains(err.Error(), "roster") {
t.Fatalf("error = %v, want media type rejection", err)
}
}
func TestMaterializeReferencesMatchesAcceptedMediaTypesIgnoringParameters(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "roster.txt")
writeReferenceFile(t, path, []byte("Aria\n"))
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"text/plain; charset=utf-8"}}
resolved := resolvedPipelineWithReference(t, "roster", "roster.txt", contracts.ReferenceBindingSourceConfig, slot)
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != referenceMediaType {
t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType)
}
}
func TestMaterializeReferencesRejectsUnacceptedMediaTypeForNormalizeTarget(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "notes.json")
writeReferenceFile(t, path, []byte(`{"notes":true}`))
slot := contracts.ReferenceSlot{Name: "normalization_notes", AcceptedMediaTypes: []string{"text/markdown"}}
resolved := resolvedPipelineWithTargetReference(t, StageNormalize, "events", "normalization_notes", "notes.json", contracts.ReferenceBindingSourceConfig, slot)
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, nil, nil, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "normalize") || !strings.Contains(err.Error(), "media type") || !strings.Contains(err.Error(), "application/json") || !strings.Contains(err.Error(), "normalization_notes") {
t.Fatalf("error = %v, want normalize media type rejection", err)
}
}
func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "empty.txt")
writeReferenceFile(t, path, nil)
resolved := resolvedPipelineWithReference(t, "roster", "empty.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
materialized, warnings, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
t.Fatalf("warnings = %#v, want empty reference warning", warnings)
}
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
t.Fatalf("empty item = %#v, want zero size and empty digest", item)
}
}
func TestMaterializeReferencesWarningScopesIncludeTargetContext(t *testing.T) {
configDir := t.TempDir()
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), nil)
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), nil)
writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), nil)
profile := baselineProfile()
profile.References = map[string]string{
"scene_guide": "chunk.txt",
"roster": "extract.txt",
"normalization_notes": "normalize.txt",
}
catalog := referenceCatalogForTargets(t,
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
[]contracts.ReferenceSlot{{Name: "roster"}},
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
)
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
_, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
got := warningScopes(warnings)
want := []string{
"pipeline.baseline.chunk.reference.scene_guide",
"pipeline.baseline.lane.events.extract.reference.roster",
"pipeline.baseline.lane.events.normalize.reference.normalization_notes",
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("warning scopes = %#v, want %#v", got, want)
}
}
func TestMaterializeReferencesEnforcesMaxBytes(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "large.txt")
writeReferenceFile(t, path, []byte("too large"))
slot := contracts.ReferenceSlot{Name: "roster", MaxBytes: 3}
resolved := resolvedPipelineWithReference(t, "roster", "large.txt", contracts.ReferenceBindingSourceConfig, slot)
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err == nil || !strings.Contains(err.Error(), "9 bytes") || !strings.Contains(err.Error(), "limit 3") || !strings.Contains(err.Error(), "roster") {
t.Fatalf("error = %v, want max bytes error", err)
}
}
func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
t.Helper()
return resolvedPipelineWithTargetReference(t, StageExtract, "events", slotName, source, bindingSource, slot)
}
func resolvedPipelineWithTargetReference(t *testing.T, stage ModuleStage, laneID string, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
t.Helper()
profile := baselineProfile()
switch stage {
case StageChunk:
profile.Chunk.References = map[string]string{slotName: source}
case StageExtract:
lane := profile.Artifacts[laneID]
lane.References = map[string]string{slotName: source}
profile.Artifacts[laneID] = lane
case StageNormalize:
lane := profile.Artifacts[laneID]
lane.Normalize.References = map[string]string{slotName: source}
profile.Artifacts[laneID] = lane
default:
t.Fatalf("unsupported reference target stage %q", stage)
}
catalog := referenceCatalogForStage(t, stage, []contracts.ReferenceSlot{slot})
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
if bindingSource != contracts.ReferenceBindingSourceConfig {
switch stage {
case StageChunk:
resolved.ChunkReferences.Bindings[0].BindingSource = bindingSource
case StageExtract:
resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].BindingSource = bindingSource
case StageNormalize:
resolved.ArtifactLanes[0].NormalizeReferences.Bindings[0].BindingSource = bindingSource
}
}
return resolved
}
func referenceCatalog(t *testing.T, slots []contracts.ReferenceSlot) ModuleCatalog {
t.Helper()
return referenceCatalogForStage(t, StageExtract, slots)
}
func referenceCatalogForStage(t *testing.T, stage ModuleStage, slots []contracts.ReferenceSlot) ModuleCatalog {
t.Helper()
switch stage {
case StageChunk:
return referenceCatalogForTargets(t, slots, nil, nil)
case StageExtract:
return referenceCatalogForTargets(t, nil, slots, nil)
case StageNormalize:
return referenceCatalogForTargets(t, nil, nil, slots)
default:
t.Fatalf("unsupported reference target stage %q", stage)
return ModuleCatalog{}
}
}
func referenceCatalogForTargets(t *testing.T, chunkSlots, extractSlots, normalizeSlots []contracts.ReferenceSlot) ModuleCatalog {
t.Helper()
return newProfileCatalogWithOverrides(t,
ModuleSpec{
Key: "generic",
Stage: StageChunk,
Requires: []string{"source"},
Provides: []string{"chunk"},
ReferenceSlots: chunkSlots,
},
ModuleSpec{
Key: "event-extractor",
Stage: StageExtract,
Requires: []string{"chunk"},
Provides: []string{"candidate"},
ReferenceSlots: extractSlots,
},
ModuleSpec{
Key: "noop",
Stage: StageNormalize,
Requires: []string{"merged"},
Provides: []string{"normalized"},
ReferenceSlots: normalizeSlots,
},
)
}
func warningScopes(warnings []contracts.Warning) []string {
scopes := make([]string, 0, len(warnings))
for _, warning := range warnings {
scopes = append(scopes, warning.Scope)
}
sort.Strings(scopes)
return scopes
}
func writeReferenceFile(t *testing.T, path string, content []byte) {
t.Helper()
if err := os.WriteFile(path, content, 0o644); err != nil {
t.Fatalf("write reference %q: %v", path, err)
}
}

View File

@@ -118,6 +118,10 @@ func (chunker integrationChunker) Key() string {
return "chunk" return "chunk"
} }
func (chunker integrationChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) { func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{ return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{ Chunks: []contracts.SourceChunk{
@@ -149,6 +153,10 @@ func (extractor integrationExtractor) SchemaVersion() string {
return "v1" return "v1"
} }
func (extractor integrationExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor integrationExtractor) Validators() []contracts.Validator { func (extractor integrationExtractor) Validators() []contracts.Validator {
return extractor.validators return extractor.validators
} }
@@ -182,6 +190,10 @@ func (normalizer integrationNormalizer) Key() string {
return "normalize" return "normalize"
} }
func (normalizer integrationNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) { func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
return contracts.NormalizeResult{Candidates: req.Candidates}, nil return contracts.NormalizeResult{Candidates: req.Candidates}, nil
} }

View File

@@ -41,6 +41,7 @@ type RunInput struct {
StartedAt time.Time StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any Metadata map[string]any
Warnings []contracts.Warning
} }
type RunOutput struct { type RunOutput struct {
@@ -63,12 +64,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
return output, err return output, err
} }
output.Warnings = append(output.Warnings, cloneWarnings(input.Warnings)...)
output.Manifest = manifestFromPipeline(input) output.Manifest = manifestFromPipeline(input)
adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module) adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module)
if err != nil { if err != nil {
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err) return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
} }
attachModuleManifestMetadata(&output, "input", adapter)
doc, err := adapter.Parse(ctx, contracts.ParseRequest{ doc, err := adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID, SourceID: input.SourceID,
Path: input.Path, Path: input.Path,
@@ -89,8 +92,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil { if err != nil {
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err) return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
} }
attachModuleManifestMetadata(&output, "chunker", chunker)
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{ chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
Source: doc, Source: doc,
References: CloneReferenceSet(input.Pipeline.ChunkReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: input.Pipeline.Chunk.LLMProfile, LLMProfile: input.Pipeline.Chunk.LLMProfile,
Options: cloneOptions(input.Pipeline.Chunk.Options), Options: cloneOptions(input.Pipeline.Chunk.Options),
Metadata: input.Metadata, Metadata: input.Metadata,
@@ -102,10 +108,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if len(chunkResult.Chunks) == 0 { if len(chunkResult.Chunks) == 0 {
return failOutput(output), fmt.Errorf("chunker %q returned no chunks", chunker.Key()) return failOutput(output), fmt.Errorf("chunker %q returned no chunks", chunker.Key())
} }
canonicalChunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
if err != nil {
return failOutput(output), fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
}
nextCandidateIndex := 0 nextCandidateIndex := 0
for _, lane := range input.Pipeline.ArtifactLanes { for _, lane := range input.Pipeline.ArtifactLanes {
if err := r.runLane(ctx, input, doc, chunkResult.Chunks, lane, &output, &nextCandidateIndex); err != nil { if err := r.runLane(ctx, input, doc, canonicalChunks, lane, &output, &nextCandidateIndex); err != nil {
return failOutput(output), err return failOutput(output), err
} }
} }
@@ -121,6 +131,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil { if err != nil {
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err) return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
} }
attachModuleManifestMetadata(&output, "output", encoder)
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{ encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
Manifest: output.Manifest, Manifest: output.Manifest,
Approved: output.Approved, Approved: output.Approved,
@@ -176,6 +187,7 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{ result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
Source: doc, Source: doc,
Chunk: &chunk, Chunk: &chunk,
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
LLMClient: input.LLMClient, LLMClient: input.LLMClient,
LLMProfile: lane.Extract.LLMProfile, LLMProfile: lane.Extract.LLMProfile,
Options: cloneOptions(lane.Extract.Options), Options: cloneOptions(lane.Extract.Options),
@@ -213,6 +225,8 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
Source: doc, Source: doc,
LaneID: lane.ID, LaneID: lane.ID,
Candidates: mergeResult.Candidates, Candidates: mergeResult.Candidates,
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
LLMClient: input.LLMClient,
LLMProfile: lane.Normalize.LLMProfile, LLMProfile: lane.Normalize.LLMProfile,
Options: cloneOptions(lane.Normalize.Options), Options: cloneOptions(lane.Normalize.Options),
Metadata: input.Metadata, Metadata: input.Metadata,
@@ -345,8 +359,12 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)), ArtifactLanes: make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
RunID: runID, RunID: runID,
StartedAt: timePtr(startedAt), StartedAt: timePtr(startedAt),
References: ReferenceProvenance(pipeline),
LLMProfiles: cloneLLMProfiles(input.LLMProfiles), LLMProfiles: cloneLLMProfiles(input.LLMProfiles),
} }
// The runner does not currently maintain a cache or idempotency key. Reference
// digests are recorded in manifest provenance and intentionally kept separate
// from source_digests.
for _, lane := range pipeline.ArtifactLanes { for _, lane := range pipeline.ArtifactLanes {
laneManifest := artifacts.ArtifactLaneManifest{ laneManifest := artifacts.ArtifactLaneManifest{
@@ -382,14 +400,10 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
metadata := make(map[string]any) metadata := make(map[string]any)
for _, module := range modules { for _, module := range modules {
provider, ok := module.(contracts.ManifestMetadataProvider) moduleMetadata, ok := moduleManifestMetadata(module)
if !ok { if !ok {
continue continue
} }
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
if len(moduleMetadata) == 0 {
continue
}
key := manifestMetadataKey(module) key := manifestMetadataKey(module)
if key == "" { if key == "" {
continue continue
@@ -403,6 +417,20 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
} }
} }
func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
if output == nil {
return
}
moduleMetadata, ok := moduleManifestMetadata(module)
if !ok {
return
}
if output.Manifest.ModuleMetadata == nil {
output.Manifest.ModuleMetadata = make(map[string]map[string]any)
}
output.Manifest.ModuleMetadata[moduleKey] = moduleMetadata
}
func manifestMetadataKey(module any) string { func manifestMetadataKey(module any) string {
switch module.(type) { switch module.(type) {
case contracts.Extractor: case contracts.Extractor:
@@ -416,6 +444,19 @@ func manifestMetadataKey(module any) string {
} }
} }
func moduleManifestMetadata(module any) (map[string]any, bool) {
provider, ok := module.(contracts.ManifestMetadataProvider)
if !ok {
return nil, false
}
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
if len(moduleMetadata) == 0 {
return nil, false
}
return moduleMetadata, true
}
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) { func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
out := make([]contracts.OutputFile, 0, len(result.Files)) out := make([]contracts.OutputFile, 0, len(result.Files))
for _, file := range result.Files { for _, file := range result.Files {
@@ -469,6 +510,13 @@ func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMPr
return append([]artifacts.LLMProfileManifest(nil), profiles...) return append([]artifacts.LLMProfileManifest(nil), profiles...)
} }
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
if len(warnings) == 0 {
return nil
}
return append([]contracts.Warning(nil), warnings...)
}
func timePtr(t time.Time) *time.Time { func timePtr(t time.Time) *time.Time {
return &t return &t
} }

View File

@@ -256,6 +256,191 @@ func TestRunRejectsChunkerBuildChunkAndEmptyChunkErrors(t *testing.T) {
} }
} }
func TestRunRejectsInvalidChunks(t *testing.T) {
tests := []struct {
name string
chunks []contracts.SourceChunk
want string
}{
{
name: "empty chunk id",
chunks: []contracts.SourceChunk{{ID: "", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}}},
want: "id must not be empty",
},
{
name: "duplicate chunk id",
chunks: []contracts.SourceChunk{
{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}},
{ID: "chunk-0", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u2")}},
},
want: "duplicated",
},
{
name: "wrong source id",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "other-source", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}}},
want: "source_id",
},
{
name: "wrong index",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u1")}}},
want: "index",
},
{
name: "empty units",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0}},
want: "units must not be empty",
},
{
name: "repeated unit inside chunk",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1"), unitWithID("u1")}}},
want: "repeats source unit",
},
{
name: "unknown unit",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u9")}}},
want: "was not found",
},
{
name: "units out of source order",
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u2"), unitWithID("u1")}}},
want: "source document order",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = test.chunks
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
assertRunError(t, err, test.want)
if output.Manifest.ValidationStatus != "failed" {
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
}
if len(modules.extractors["extract-alpha"].requests) != 0 {
t.Fatalf("extractor calls = %d, want none after invalid chunks", len(modules.extractors["extract-alpha"].requests))
}
})
}
}
func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = []contracts.SourceChunk{
{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1"), unitWithID("u2")}},
{ID: "chunk-1", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u2")}},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 2 {
t.Fatalf("len(Approved) = %d, want one candidate per accepted chunk", len(output.Approved))
}
}
func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
modules := defaultRunnerModules()
modules.input.doc = sourceDocumentWithUnitMetadata()
modules.chunker.chunks = []contracts.SourceChunk{
{
ID: "chunk-0",
SourceID: "source-1",
Index: 0,
Units: []source.SourceUnit{
{
ID: "u1",
Kind: "mutated-kind",
Text: "mutated text",
Metadata: map[string]any{
"speaker": "chunker-speaker",
"note": "chunker note",
},
},
},
},
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
}
extractor := modules.extractors["extract-alpha"]
if len(extractor.requests) != 1 {
t.Fatalf("len(extractor requests) = %d, want 1", len(extractor.requests))
}
chunk := extractor.requests[0].Chunk
if chunk == nil {
t.Fatal("extractor chunk = nil, want canonical chunk")
}
if chunk.Units[0].ID != "u1" || chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" {
t.Fatalf("chunk unit = %#v, want source document unit values", chunk.Units[0])
}
if got := chunk.Units[0].Metadata["speaker"]; got != "source-speaker" {
t.Fatalf("chunk unit metadata = %#v, want source document metadata", chunk.Units[0].Metadata)
}
if got := chunk.Units[0].Metadata["topic"]; got != "source-topic" {
t.Fatalf("chunk unit metadata = %#v, want cloned source document metadata", chunk.Units[0].Metadata)
}
modules.input.doc.Units[0].Kind = "changed-kind"
modules.input.doc.Units[0].Text = "changed text"
modules.input.doc.Units[0].Metadata["speaker"] = "changed-speaker"
if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Metadata["speaker"] != "source-speaker" {
t.Fatalf("chunk unit changed after source mutation: %#v", chunk.Units[0])
}
}
func TestRunPreservesChunkMetadataDuringCanonicalization(t *testing.T) {
modules := defaultRunnerModules()
modules.chunker.chunks = []contracts.SourceChunk{
{
ID: "chunk-0",
SourceID: "source-1",
Index: 0,
Units: []source.SourceUnit{
unitWithID("u1"),
},
Metadata: map[string]any{
"scene_title": "Original scene",
"boundary_note": "Chunker note",
},
},
}
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
extractor := modules.extractors["extract-alpha"]
if len(extractor.requests) != 1 || extractor.requests[0].Chunk == nil {
t.Fatalf("extractor requests = %#v, want one canonical chunk", extractor.requests)
}
if got := extractor.requests[0].Chunk.Metadata["scene_title"]; got != "Original scene" {
t.Fatalf("chunk metadata = %#v, want chunker metadata", extractor.requests[0].Chunk.Metadata)
}
if got := extractor.requests[0].Chunk.Metadata["boundary_note"]; got != "Chunker note" {
t.Fatalf("chunk metadata = %#v, want chunker metadata", extractor.requests[0].Chunk.Metadata)
}
modules.chunker.chunks[0].Metadata["scene_title"] = "changed"
modules.chunker.chunks[0].Metadata["boundary_note"] = "changed"
if got := extractor.requests[0].Chunk.Metadata["scene_title"]; got != "Original scene" {
t.Fatalf("chunk metadata aliased to chunker map: %#v", extractor.requests[0].Chunk.Metadata)
}
if got := extractor.requests[0].Chunk.Metadata["boundary_note"]; got != "Chunker note" {
t.Fatalf("chunk metadata aliased to chunker map: %#v", extractor.requests[0].Chunk.Metadata)
}
}
func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) { func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
llmClient := fakeLLMClient{} llmClient := fakeLLMClient{}
@@ -273,9 +458,16 @@ func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
if !reflect.DeepEqual(extractor.seenChunkIDs, []string{"chunk-0", "chunk-1"}) { if !reflect.DeepEqual(extractor.seenChunkIDs, []string{"chunk-0", "chunk-1"}) {
t.Fatalf("seen chunks = %#v, want both chunks", extractor.seenChunkIDs) t.Fatalf("seen chunks = %#v, want both chunks", extractor.seenChunkIDs)
} }
if len(modules.chunker.requests) != 1 || modules.chunker.requests[0].LLMClient == nil {
t.Fatalf("chunker LLM client = %#v, want client on chunk request", modules.chunker.requests)
}
if len(extractor.seenLLMClients) != 2 || extractor.seenLLMClients[0] == nil || extractor.seenLLMClients[1] == nil { if len(extractor.seenLLMClients) != 2 || extractor.seenLLMClients[0] == nil || extractor.seenLLMClients[1] == nil {
t.Fatalf("seen LLM clients = %#v, want client for each chunk", extractor.seenLLMClients) t.Fatalf("seen LLM clients = %#v, want client for each chunk", extractor.seenLLMClients)
} }
normalizer := modules.normalizers["normalize"]
if len(normalizer.requests) != 1 || normalizer.requests[0].LLMClient == nil {
t.Fatalf("normalizer LLM client = %#v, want client on normalize request", normalizer.requests)
}
if extractor.seenMetadata[0]["request"] != "test" { if extractor.seenMetadata[0]["request"] != "test" {
t.Fatalf("seen metadata = %#v, want request metadata", extractor.seenMetadata) t.Fatalf("seen metadata = %#v, want request metadata", extractor.seenMetadata)
} }
@@ -371,6 +563,133 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
} }
} }
func TestRunPassesLaneReferencesToExtractorRequests(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = testReferenceSet("roster", "reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
req := modules.extractors["extract-alpha"].requests[0]
item := req.References.Slots["roster"].Items[0]
if string(item.Content) != "reference text" {
t.Fatalf("reference content = %q, want reference text", item.Content)
}
item.Content[0] = 'R'
if got := string(pipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content); got != "reference text" {
t.Fatalf("runner mutated reference set content = %q", got)
}
}
func TestRunPassesChunkReferencesToChunkerRequest(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ChunkReferences.ReferenceSet = testReferenceSet("scene_guide", "chunk reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
req := modules.chunker.requests[0]
item := req.References.Slots["scene_guide"].Items[0]
if string(item.Content) != "chunk reference text" {
t.Fatalf("chunk reference content = %q, want chunk reference text", item.Content)
}
item.Content[0] = 'C'
if got := string(pipeline.ChunkReferences.ReferenceSet.Slots["scene_guide"].Items[0].Content); got != "chunk reference text" {
t.Fatalf("runner mutated chunk reference set content = %q", got)
}
}
func TestRunPassesNormalizeReferencesToNormalizerRequest(t *testing.T) {
modules := defaultRunnerModules()
pipeline := resolvedPipeline()
pipeline.ArtifactLanes[0].NormalizeReferences.ReferenceSet = testReferenceSet("normalization_notes", "normalize reference text")
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: pipeline})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
req := modules.normalizers["normalize"].requests[0]
item := req.References.Slots["normalization_notes"].Items[0]
if string(item.Content) != "normalize reference text" {
t.Fatalf("normalize reference content = %q, want normalize reference text", item.Content)
}
item.Content[0] = 'N'
if got := string(pipeline.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["normalization_notes"].Items[0].Content); got != "normalize reference text" {
t.Fatalf("runner mutated normalize reference set content = %q", got)
}
}
func TestRunAllowsNilLLMClientWhenModulesDoNotUseIt(t *testing.T) {
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil with nil LLM client when modules do not use it", err)
}
}
func TestRunIncludesInputWarnings(t *testing.T) {
modules := defaultRunnerModules()
warning := contracts.Warning{Scope: "reference", ReasonCode: "empty_reference", Message: "empty reference"}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
Pipeline: resolvedPipeline(),
Warnings: []contracts.Warning{warning},
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Warnings) != 1 || output.Warnings[0] != warning {
t.Fatalf("warnings = %#v, want input warning", output.Warnings)
}
}
func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) {
modules := defaultRunnerModules()
modules.input.manifestMetadata = map[string]any{
"input_profile": "input-metadata",
}
modules.chunker.manifestMetadata = map[string]any{
"prompt_id": "dnd.scenes",
"prompt_version": "v1",
"prompt_sha256": "sha256:chunker-prompt",
"response_schema_key": "dnd_scenes",
"response_schema_id": "schema-dnd-scenes",
"response_schema_name": "dnd_scenes",
}
modules.output.manifestMetadata = map[string]any{
"output_profile": "output-metadata",
}
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if output.Manifest.ModuleMetadata == nil {
t.Fatal("ModuleMetadata = nil, want module metadata map")
}
if got := output.Manifest.ModuleMetadata["input"]; !reflect.DeepEqual(got, modules.input.manifestMetadata) {
t.Fatalf("input module metadata = %#v, want %#v", got, modules.input.manifestMetadata)
}
if got := output.Manifest.ModuleMetadata["chunker"]; !reflect.DeepEqual(got, modules.chunker.manifestMetadata) {
t.Fatalf("chunker module metadata = %#v, want %#v", got, modules.chunker.manifestMetadata)
}
if got := output.Manifest.ModuleMetadata["output"]; !reflect.DeepEqual(got, modules.output.manifestMetadata) {
t.Fatalf("output module metadata = %#v, want %#v", got, modules.output.manifestMetadata)
}
modules.chunker.manifestMetadata["prompt_id"] = "changed"
if output.Manifest.ModuleMetadata["chunker"]["prompt_id"] != "dnd.scenes" {
t.Fatalf("chunker module metadata aliased to provider map: %#v", output.Manifest.ModuleMetadata["chunker"])
}
}
func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) { func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
modules := defaultRunnerModules() modules := defaultRunnerModules()
@@ -677,7 +996,63 @@ func TestRunReturnsFailedManifestWhenOutputEncoderFails(t *testing.T) {
} }
func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) { func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolvedPipelineWithValidators("configured")}) resolved := resolvedPipelineWithValidators("configured")
resolved.ChunkReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"scene_guide": {
Slot: contracts.ReferenceSlot{Name: "scene_guide"},
Items: []contracts.ReferenceItem{
{
SlotName: "scene_guide",
MediaType: "text/plain; charset=utf-8",
Content: []byte("chunk reference content"),
Digest: "sha256:chunk-reference",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/scene-guide.txt"},
SizeBytes: int64(len("chunk reference content")),
BindingSource: contracts.ReferenceBindingSourceCLI,
},
},
},
},
}
resolved.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain; charset=utf-8",
Content: []byte("reference content"),
Digest: "sha256:reference",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
SizeBytes: int64(len("reference content")),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
resolved.ArtifactLanes[0].NormalizeReferences.ReferenceSet = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"normalization_notes": {
Slot: contracts.ReferenceSlot{Name: "normalization_notes"},
Items: []contracts.ReferenceItem{
{
SlotName: "normalization_notes",
MediaType: "text/plain; charset=utf-8",
Content: []byte("normalize reference content"),
Digest: "sha256:normalize-reference",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/normalize.txt"},
SizeBytes: int64(len("normalize reference content")),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
output, err := New(newRunnerRegistries(t, nil)).Run(context.Background(), RunInput{Pipeline: resolved})
if err != nil { if err != nil {
t.Fatalf("Run() error = %v, want nil", err) t.Fatalf("Run() error = %v, want nil", err)
} }
@@ -692,6 +1067,27 @@ func TestRunManifestIncludesPipelineAndLaneDetails(t *testing.T) {
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) { if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) {
t.Fatalf("SourceDigests = %#v, want source digest", manifest.SourceDigests) t.Fatalf("SourceDigests = %#v, want source digest", manifest.SourceDigests)
} }
if len(manifest.References) != 3 {
t.Fatalf("References = %#v, want three reference provenance entries", manifest.References)
}
chunkReference := manifest.References[0]
if chunkReference.Stage != string(StageChunk) || chunkReference.LaneID != "" || chunkReference.SlotName != "scene_guide" || chunkReference.Digest != "sha256:chunk-reference" {
t.Fatalf("chunk reference provenance = %#v, want chunk slot digest", chunkReference)
}
if chunkReference.OriginType != "file" || chunkReference.OriginURI != "file:///tmp/scene-guide.txt" || chunkReference.MediaType != "text/plain; charset=utf-8" || chunkReference.SizeBytes != int64(len("chunk reference content")) || chunkReference.BindingSource != contracts.ReferenceBindingSourceCLI {
t.Fatalf("chunk reference provenance = %#v, want origin/media/size/source", chunkReference)
}
extractReference := manifest.References[1]
if extractReference.Stage != string(StageExtract) || extractReference.LaneID != "alpha" || extractReference.SlotName != "roster" || extractReference.Digest != "sha256:reference" {
t.Fatalf("extract reference provenance = %#v, want lane slot digest", extractReference)
}
if extractReference.OriginType != "file" || extractReference.OriginURI != "file:///tmp/roster.txt" || extractReference.MediaType != "text/plain; charset=utf-8" || extractReference.SizeBytes != int64(len("reference content")) || extractReference.BindingSource != contracts.ReferenceBindingSourceConfig {
t.Fatalf("extract reference provenance = %#v, want origin/media/size/source", extractReference)
}
normalizeReference := manifest.References[2]
if normalizeReference.Stage != string(StageNormalize) || normalizeReference.LaneID != "alpha" || normalizeReference.SlotName != "normalization_notes" || normalizeReference.Digest != "sha256:normalize-reference" {
t.Fatalf("normalize reference provenance = %#v, want lane slot digest", normalizeReference)
}
if manifest.ValidationStatus != "approved" { if manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved", manifest.ValidationStatus) t.Fatalf("ValidationStatus = %q, want approved", manifest.ValidationStatus)
} }
@@ -775,6 +1171,11 @@ func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" { if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata) t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
} }
if output.Manifest.ModuleMetadata != nil {
if _, ok := output.Manifest.ModuleMetadata["extractor"]; ok {
t.Fatalf("top-level module metadata includes lane metadata key: %#v", output.Manifest.ModuleMetadata)
}
}
} }
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) { func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
@@ -843,12 +1244,15 @@ func resolvedPipeline() ResolvedPipeline {
Digest: "sha256:pipeline", Digest: "sha256:pipeline",
Input: Binding("input"), Input: Binding("input"),
Chunk: Binding("chunk"), Chunk: Binding("chunk"),
ChunkReferences: referenceTarget(StageChunk, "", "chunk", nil),
ArtifactLanes: []ResolvedArtifactLane{ ArtifactLanes: []ResolvedArtifactLane{
{ {
ID: "alpha", ID: "alpha",
Extract: Binding("extract-alpha"), Extract: Binding("extract-alpha"),
Merge: Binding("merge"), Merge: Binding("merge"),
Normalize: Binding("normalize"), Normalize: Binding("normalize"),
ExtractReferences: referenceTarget(StageExtract, "alpha", "extract-alpha", nil),
NormalizeReferences: referenceTarget(StageNormalize, "alpha", "normalize", nil),
}, },
}, },
Output: Binding("output"), Output: Binding("output"),
@@ -863,6 +1267,27 @@ func resolvedPipelineWithValidators(validators ...string) ResolvedPipeline {
return pipeline return pipeline
} }
func testReferenceSet(slotName string, content string) contracts.ReferenceSet {
return contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
slotName: {
Slot: contracts.ReferenceSlot{Name: slotName},
Items: []contracts.ReferenceItem{
{
SlotName: slotName,
MediaType: "text/plain; charset=utf-8",
Content: []byte(content),
Digest: "sha256:test",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/reference.txt"},
SizeBytes: int64(len(content)),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
},
},
}
}
type runnerModules struct { type runnerModules struct {
input *runnerInputAdapter input *runnerInputAdapter
chunker *runnerChunker chunker *runnerChunker
@@ -966,6 +1391,7 @@ type runnerInputAdapter struct {
key string key string
doc *source.SourceDocument doc *source.SourceDocument
err error err error
manifestMetadata map[string]any
requests []contracts.ParseRequest requests []contracts.ParseRequest
} }
@@ -978,11 +1404,16 @@ func (adapter *runnerInputAdapter) Parse(ctx context.Context, req contracts.Pars
return adapter.doc, adapter.err return adapter.doc, adapter.err
} }
func (adapter *runnerInputAdapter) ManifestMetadata() map[string]any {
return adapter.manifestMetadata
}
type runnerChunker struct { type runnerChunker struct {
key string key string
chunks []contracts.SourceChunk chunks []contracts.SourceChunk
warnings []contracts.Warning warnings []contracts.Warning
err error err error
manifestMetadata map[string]any
requests []contracts.ChunkRequest requests []contracts.ChunkRequest
} }
@@ -990,6 +1421,10 @@ func (chunker *runnerChunker) Key() string {
return chunker.key return chunker.key
} }
func (chunker *runnerChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) { func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
chunker.requests = append(chunker.requests, req) chunker.requests = append(chunker.requests, req)
return contracts.ChunkResult{ return contracts.ChunkResult{
@@ -998,6 +1433,10 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ
}, chunker.err }, chunker.err
} }
func (chunker *runnerChunker) ManifestMetadata() map[string]any {
return chunker.manifestMetadata
}
type runnerExtractor struct { type runnerExtractor struct {
key string key string
artifactType string artifactType string
@@ -1025,6 +1464,10 @@ func (extractor *runnerExtractor) SchemaVersion() string {
return extractor.schemaVersion return extractor.schemaVersion
} }
func (extractor *runnerExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor *runnerExtractor) ManifestMetadata() map[string]any { func (extractor *runnerExtractor) ManifestMetadata() map[string]any {
return extractor.manifestMetadata return extractor.manifestMetadata
} }
@@ -1089,6 +1532,10 @@ func (normalizer *runnerNormalizer) Key() string {
return normalizer.key return normalizer.key
} }
func (normalizer *runnerNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) { func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
normalizer.requests = append(normalizer.requests, req) normalizer.requests = append(normalizer.requests, req)
candidates := append([]artifacts.ArtifactCandidate(nil), normalizer.result...) candidates := append([]artifacts.ArtifactCandidate(nil), normalizer.result...)
@@ -1142,6 +1589,7 @@ type runnerOutputEncoder struct {
files []contracts.OutputFile files []contracts.OutputFile
warnings []contracts.Warning warnings []contracts.Warning
err error err error
manifestMetadata map[string]any
requests []contracts.OutputRequest requests []contracts.OutputRequest
} }
@@ -1157,6 +1605,10 @@ func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.Ou
}, encoder.err }, encoder.err
} }
func (encoder *runnerOutputEncoder) ManifestMetadata() map[string]any {
return encoder.manifestMetadata
}
type fakeLLMClient struct{} type fakeLLMClient struct{}
func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
@@ -1179,6 +1631,36 @@ func validSourceDocument() *source.SourceDocument {
Digest: "sha256:source", Digest: "sha256:source",
Units: []source.SourceUnit{ Units: []source.SourceUnit{
{ID: "u1", Kind: "unit", Text: "Source unit."}, {ID: "u1", Kind: "unit", Text: "Source unit."},
{ID: "u2", Kind: "unit", Text: "Second source unit."},
{ID: "u3", Kind: "unit", Text: "Third source unit."},
},
}
}
func sourceDocumentWithUnitMetadata() *source.SourceDocument {
return &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:source",
Units: []source.SourceUnit{
{
ID: "u1",
Kind: "source-kind",
Text: "source text",
Metadata: map[string]any{
"speaker": "source-speaker",
"topic": "source-topic",
},
},
{
ID: "u2",
Kind: "source-kind",
Text: "second source text",
Metadata: map[string]any{
"speaker": "source-speaker-2",
},
},
}, },
} }
} }
@@ -1194,6 +1676,19 @@ func sourceChunkWithID(id string, index int) contracts.SourceChunk {
} }
} }
func unitWithID(id string) source.SourceUnit {
switch id {
case "u1":
return source.SourceUnit{ID: "u1", Kind: "unit", Text: "Source unit."}
case "u2":
return source.SourceUnit{ID: "u2", Kind: "unit", Text: "Second source unit."}
case "u3":
return source.SourceUnit{ID: "u3", Kind: "unit", Text: "Third source unit."}
default:
return source.SourceUnit{ID: id, Kind: "unit", Text: "Unknown source unit."}
}
}
func warningReasons(warnings []contracts.Warning) []string { func warningReasons(warnings []contracts.Warning) []string {
reasons := make([]string, 0, len(warnings)) reasons := make([]string, 0, len(warnings))
for _, warning := range warnings { for _, warning := range warnings {

View File

@@ -1,7 +1,7 @@
{ {
"manifest": { "manifest": {
"pipeline_id": "walking-skeleton", "pipeline_id": "walking-skeleton",
"pipeline_digest": "sha256:5df1e501a2307ef75bbfeb59d315b3710571d52e5466a9c7f8320248740e6fca", "pipeline_digest": "sha256:25084e39a0cadace375c896551d1752413755c1a6772f6b24cfe91c029ea2631",
"validation_status": "approved", "validation_status": "approved",
"artifact_lanes": [ "artifact_lanes": [
{ {

View File

@@ -43,8 +43,8 @@ func TestWalkingSkeletonFixture(t *testing.T) {
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType) t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
} }
assertStructuralJSONEqual(t, output.OutputFiles[0].Bytes, expectedBytes) assertStructuralJSONEqual(t, output.OutputFiles[0].Bytes, expectedBytes)
if llmClient.calls != 2 { if llmClient.calls != 3 {
t.Fatalf("LLM calls = %d, want chunk count 2", llmClient.calls) t.Fatalf("LLM calls = %d, want extractor calls plus normalizer call", llmClient.calls)
} }
} }
@@ -216,6 +216,10 @@ func (chunker walkingSkeletonChunker) Key() string {
return "fake/chunk" return "fake/chunk"
} }
func (chunker walkingSkeletonChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) { func (chunker walkingSkeletonChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if len(req.Source.Units) < 3 { if len(req.Source.Units) < 3 {
return contracts.ChunkResult{}, fmt.Errorf("fixture source must contain at least three units") return contracts.ChunkResult{}, fmt.Errorf("fixture source must contain at least three units")
@@ -252,6 +256,10 @@ func (extractor walkingSkeletonExtractor) SchemaVersion() string {
return "v1" return "v1"
} }
func (extractor walkingSkeletonExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (extractor walkingSkeletonExtractor) Validators() []contracts.Validator { func (extractor walkingSkeletonExtractor) Validators() []contracts.Validator {
return nil return nil
} }
@@ -332,7 +340,20 @@ func (normalizer walkingSkeletonNormalizer) Key() string {
return DefaultNormalizeModule return DefaultNormalizeModule
} }
func (normalizer walkingSkeletonNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) { func (normalizer walkingSkeletonNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
var response struct {
Call int `json:"call"`
}
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: "fake/normalize",
ResponseSchemaName: "fake_normalize",
}, &response); err != nil {
return contracts.NormalizeResult{}, err
}
return contracts.NormalizeResult{Candidates: req.Candidates}, nil return contracts.NormalizeResult{Candidates: req.Candidates}, nil
} }

View File

@@ -7,9 +7,13 @@ import (
"fmt" "fmt"
"io/fs" "io/fs"
"path" "path"
"reflect"
"sort" "sort"
"strings" "strings"
"text/template" "text/template"
"text/template/parse"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
//go:embed assets/** //go:embed assets/**
@@ -48,6 +52,7 @@ type Definition struct {
EmbeddedPath string EmbeddedPath string
SystemPath string SystemPath string
UserPath string UserPath string
ReferenceSlots []contracts.ReferenceSlot
} }
// Bundle is a compiled system/user prompt pair. // Bundle is a compiled system/user prompt pair.
@@ -55,6 +60,7 @@ type Bundle struct {
systemTmpl *template.Template systemTmpl *template.Template
userTmpl *template.Template userTmpl *template.Template
metadata Metadata metadata Metadata
referenceSlots map[string]contracts.ReferenceSlot
} }
// Metadata returns metadata for the compiled prompt bundle. // Metadata returns metadata for the compiled prompt bundle.
@@ -169,6 +175,8 @@ func LoadBundle(fsys fs.FS, def Definition) (*Bundle, error) {
funcs := template.FuncMap{ funcs := template.FuncMap{
"hardening": func() string { return sharedHardening }, "hardening": func() string { return sharedHardening },
"reference": func(string) (string, error) { return "", nil },
"hasreference": func(string) (bool, error) { return false, nil },
} }
systemTmpl, err := template.New(path.Base(systemPath)).Option("missingkey=error").Funcs(funcs).Parse(systemSource) systemTmpl, err := template.New(path.Base(systemPath)).Option("missingkey=error").Funcs(funcs).Parse(systemSource)
if err != nil { if err != nil {
@@ -178,6 +186,13 @@ func LoadBundle(fsys fs.FS, def Definition) (*Bundle, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("parse embedded user prompt %q: %w", userPath, err) return nil, fmt.Errorf("parse embedded user prompt %q: %w", userPath, err)
} }
referenceSlots := referenceSlotMap(def.ReferenceSlots)
if err := validateTemplateReferenceSlots(systemTmpl, referenceSlots); err != nil {
return nil, fmt.Errorf("validate embedded system prompt %q: %w", systemPath, err)
}
if err := validateTemplateReferenceSlots(userTmpl, referenceSlots); err != nil {
return nil, fmt.Errorf("validate embedded user prompt %q: %w", userPath, err)
}
hashInput := systemSource + "\n\n" + userSource hashInput := systemSource + "\n\n" + userSource
hash := sha256.Sum256([]byte(hashInput)) hash := sha256.Sum256([]byte(hashInput))
@@ -193,9 +208,124 @@ func LoadBundle(fsys fs.FS, def Definition) (*Bundle, error) {
systemTmpl: systemTmpl, systemTmpl: systemTmpl,
userTmpl: userTmpl, userTmpl: userTmpl,
metadata: metadata, metadata: metadata,
referenceSlots: referenceSlots,
}, nil }, nil
} }
func referenceSlotMap(slots []contracts.ReferenceSlot) map[string]contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
out := make(map[string]contracts.ReferenceSlot, len(slots))
for _, slot := range slots {
name := strings.TrimSpace(slot.Name)
if name == "" {
continue
}
slot.Name = name
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
out[name] = slot
}
return out
}
func validateTemplateReferenceSlots(tmpl *template.Template, declared map[string]contracts.ReferenceSlot) error {
if tmpl == nil || tmpl.Tree == nil || tmpl.Tree.Root == nil {
return nil
}
return validateReferenceNodes(tmpl.Tree.Root, declared)
}
func validateReferenceNodes(node parse.Node, declared map[string]contracts.ReferenceSlot) error {
if node == nil || reflect.ValueOf(node).IsNil() {
return nil
}
switch typed := node.(type) {
case *parse.ListNode:
for _, child := range typed.Nodes {
if err := validateReferenceNodes(child, declared); err != nil {
return err
}
}
case *parse.ActionNode:
return validateReferencePipeline(typed.Pipe, declared)
case *parse.IfNode:
if err := validateReferencePipeline(typed.Pipe, declared); err != nil {
return err
}
if err := validateReferenceNodes(typed.List, declared); err != nil {
return err
}
return validateReferenceNodes(typed.ElseList, declared)
case *parse.RangeNode:
if err := validateReferencePipeline(typed.Pipe, declared); err != nil {
return err
}
if err := validateReferenceNodes(typed.List, declared); err != nil {
return err
}
return validateReferenceNodes(typed.ElseList, declared)
case *parse.WithNode:
if err := validateReferencePipeline(typed.Pipe, declared); err != nil {
return err
}
if err := validateReferenceNodes(typed.List, declared); err != nil {
return err
}
return validateReferenceNodes(typed.ElseList, declared)
case *parse.TemplateNode:
return nil
}
return nil
}
func validateReferencePipeline(pipe *parse.PipeNode, declared map[string]contracts.ReferenceSlot) error {
if pipe == nil {
return nil
}
for _, cmd := range pipe.Cmds {
if err := validateReferenceCommand(cmd, declared); err != nil {
return err
}
}
return nil
}
func validateReferenceCommand(cmd *parse.CommandNode, declared map[string]contracts.ReferenceSlot) error {
if cmd == nil || len(cmd.Args) == 0 {
return nil
}
for _, arg := range cmd.Args[1:] {
if nested, ok := arg.(*parse.PipeNode); ok {
if err := validateReferencePipeline(nested, declared); err != nil {
return err
}
}
}
identifier, ok := cmd.Args[0].(*parse.IdentifierNode)
if !ok {
return nil
}
if identifier.Ident != "reference" && identifier.Ident != "hasreference" {
return nil
}
if len(cmd.Args) != 2 {
return fmt.Errorf("%s requires one string slot name", identifier.Ident)
}
slotArg, ok := cmd.Args[1].(*parse.StringNode)
if !ok {
return fmt.Errorf("%s requires a string literal slot name", identifier.Ident)
}
slotName := strings.TrimSpace(slotArg.Text)
if slotName == "" {
return fmt.Errorf("%s slot name must not be empty", identifier.Ident)
}
if _, ok := declared[slotName]; !ok {
return fmt.Errorf("%s slot %q is not declared", identifier.Ident, slotName)
}
return nil
}
func readPromptAsset(fsys fs.FS, assetPath string) (string, error) { func readPromptAsset(fsys fs.FS, assetPath string) (string, error) {
if strings.TrimSpace(assetPath) == "" { if strings.TrimSpace(assetPath) == "" {
return "", fmt.Errorf("prompt asset path must not be empty") return "", fmt.Errorf("prompt asset path must not be empty")

View File

@@ -4,6 +4,9 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"strings" "strings"
"text/template"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
// RenderUserSystem renders the system and user prompt pair for promptID. // RenderUserSystem renders the system and user prompt pair for promptID.
@@ -16,20 +19,105 @@ func RenderUserSystem(promptID string, data any) (system string, user string, me
return compiled.RenderUserSystem(data) return compiled.RenderUserSystem(data)
} }
// RenderUserSystemWithReferences renders the system and user prompt pair for promptID with reference template functions.
func RenderUserSystemWithReferences(promptID string, data any, references contracts.ReferenceSet) (system string, user string, metadata Metadata, err error) {
trimmedID := strings.TrimSpace(promptID)
compiled, ok := promptRegistry[trimmedID]
if !ok {
return "", "", Metadata{}, fmt.Errorf("unknown prompt id %q", promptID)
}
return compiled.RenderUserSystemWithReferences(data, references)
}
// RenderUserSystem renders the bundle's system and user prompts. // RenderUserSystem renders the bundle's system and user prompts.
func (b *Bundle) RenderUserSystem(data any) (system string, user string, metadata Metadata, err error) { func (b *Bundle) RenderUserSystem(data any) (system string, user string, metadata Metadata, err error) {
return b.RenderUserSystemWithReferences(data, contracts.ReferenceSet{})
}
// RenderUserSystemWithReferences renders the bundle's system and user prompts with reference template functions.
func (b *Bundle) RenderUserSystemWithReferences(data any, references contracts.ReferenceSet) (system string, user string, metadata Metadata, err error) {
if b == nil { if b == nil {
return "", "", Metadata{}, fmt.Errorf("prompt bundle must not be nil") return "", "", Metadata{}, fmt.Errorf("prompt bundle must not be nil")
} }
systemTmpl, userTmpl, err := b.renderTemplates(references)
if err != nil {
return "", "", Metadata{}, err
}
var systemBuf bytes.Buffer var systemBuf bytes.Buffer
if err := b.systemTmpl.Execute(&systemBuf, data); err != nil { if err := systemTmpl.Execute(&systemBuf, data); err != nil {
return "", "", Metadata{}, fmt.Errorf("render system prompt %q: %w", b.metadata.PromptID, err) return "", "", Metadata{}, fmt.Errorf("render system prompt %q: %w", b.metadata.PromptID, err)
} }
var userBuf bytes.Buffer var userBuf bytes.Buffer
if err := b.userTmpl.Execute(&userBuf, data); err != nil { if err := userTmpl.Execute(&userBuf, data); err != nil {
return "", "", Metadata{}, fmt.Errorf("render user prompt %q: %w", b.metadata.PromptID, err) return "", "", Metadata{}, fmt.Errorf("render user prompt %q: %w", b.metadata.PromptID, err)
} }
return strings.TrimSpace(systemBuf.String()), strings.TrimSpace(userBuf.String()), b.metadata, nil return strings.TrimSpace(systemBuf.String()), strings.TrimSpace(userBuf.String()), b.metadata, nil
} }
func (b *Bundle) renderTemplates(references contracts.ReferenceSet) (*template.Template, *template.Template, error) {
funcs := b.referenceFuncs(references)
systemTmpl, err := b.systemTmpl.Clone()
if err != nil {
return nil, nil, fmt.Errorf("clone system prompt %q: %w", b.metadata.PromptID, err)
}
userTmpl, err := b.userTmpl.Clone()
if err != nil {
return nil, nil, fmt.Errorf("clone user prompt %q: %w", b.metadata.PromptID, err)
}
systemTmpl.Funcs(funcs)
userTmpl.Funcs(funcs)
return systemTmpl, userTmpl, nil
}
func (b *Bundle) referenceFuncs(references contracts.ReferenceSet) template.FuncMap {
return template.FuncMap{
"hardening": func() string { return sharedHardening },
"hasreference": func(slotName string) (bool, error) {
items, _, err := b.referenceItems(slotName, references)
if err != nil {
return false, err
}
for _, item := range items {
if len(item.Content) > 0 {
return true, nil
}
}
return false, nil
},
"reference": func(slotName string) (string, error) {
items, slot, err := b.referenceItems(slotName, references)
if err != nil {
return "", err
}
if len(items) == 0 {
return "", nil
}
if len(items) > 1 && !slot.Multiple {
return "", fmt.Errorf("reference slot %q has %d bound items but does not allow multiple", slot.Name, len(items))
}
parts := make([]string, 0, len(items))
for _, item := range items {
parts = append(parts, string(item.Content))
}
return strings.Join(parts, "\n"), nil
},
}
}
func (b *Bundle) referenceItems(slotName string, references contracts.ReferenceSet) ([]contracts.ReferenceItem, contracts.ReferenceSlot, error) {
slotName = strings.TrimSpace(slotName)
slot, ok := b.referenceSlots[slotName]
if !ok {
return nil, contracts.ReferenceSlot{}, fmt.Errorf("reference slot %q is not declared", slotName)
}
if len(references.Slots) == 0 {
return nil, slot, nil
}
resolved, ok := references.Slots[slotName]
if !ok {
return nil, slot, nil
}
return append([]contracts.ReferenceItem(nil), resolved.Items...), slot, nil
}

View File

@@ -1,8 +1,13 @@
package prompt package prompt
import ( import (
"crypto/sha256"
"encoding/hex"
"strings" "strings"
"testing" "testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
) )
func TestRenderUserSystemReturnsTextAndMetadata(t *testing.T) { func TestRenderUserSystemReturnsTextAndMetadata(t *testing.T) {
@@ -64,3 +69,226 @@ func TestRenderUserSystemIncludesHardeningText(t *testing.T) {
t.Fatalf("expected rendered system prompt to include hardening text: %q", system) t.Fatalf("expected rendered system prompt to include hardening text: %q", system)
} }
} }
func TestRenderUserSystemWithReferencesRendersDeclaredSlots(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster"}, {Name: "glossary"}},
`System has roster={{ hasreference "roster" }} has glossary={{ hasreference "glossary" }}`,
`Roster={{ reference "roster" }} Glossary={{ reference "glossary" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria")},
},
},
}}
system, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if !strings.Contains(system, "has roster=true") || !strings.Contains(system, "has glossary=false") {
t.Fatalf("system = %q, want reference presence flags", system)
}
if !strings.Contains(user, "Roster=Aria") || !strings.Contains(user, "Glossary=") {
t.Fatalf("user = %q, want rendered and empty optional references", user)
}
}
func TestRenderUserSystemWithReferencesSupportsChunkRequestData(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
`Chunk system has guide={{ hasreference "scene_guide" }}`,
`Source={{ .SourceID }} Guide={{ reference "scene_guide" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"scene_guide": {
Slot: contracts.ReferenceSlot{Name: "scene_guide"},
Items: []contracts.ReferenceItem{
{SlotName: "scene_guide", Content: []byte("Keep combat scenes separate.")},
},
},
}}
system, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{"SourceID": "session-alpha"}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if !strings.Contains(system, "has guide=true") {
t.Fatalf("system = %q, want chunk reference presence", system)
}
if !strings.Contains(user, "Source=session-alpha") || !strings.Contains(user, "Keep combat scenes separate.") {
t.Fatalf("user = %q, want chunk request data and reference content", user)
}
}
func TestRenderUserSystemWithReferencesSupportsNormalizeRequestData(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
`Normalize system`,
`Lane={{ .LaneID }} Notes={{ reference "normalization_notes" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"normalization_notes": {
Slot: contracts.ReferenceSlot{Name: "normalization_notes"},
Items: []contracts.ReferenceItem{
{SlotName: "normalization_notes", Content: []byte("Prefer canonical item names.")},
},
},
}}
_, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{"LaneID": "spells"}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if !strings.Contains(user, "Lane=spells") || !strings.Contains(user, "Prefer canonical item names.") {
t.Fatalf("user = %q, want normalize request data and reference content", user)
}
}
func TestRenderUserSystemReferenceHasReferenceRequiresContent(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster"}},
`System`,
`{{ hasreference "roster" }} {{ reference "roster" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: nil},
},
},
}}
_, user, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
if user != "false" {
t.Fatalf("user = %q, want false with empty reference content", user)
}
}
func TestLoadBundleRejectsUndeclaredReferenceSlots(t *testing.T) {
_, err := LoadBundle(referenceBundleFS(`System`, `{{ reference "roster" }}`), referenceBundleDefinition(nil))
if err == nil || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("LoadBundle() error = %v, want undeclared reference slot error", err)
}
}
func TestLoadBundleRejectsDynamicReferenceSlotNames(t *testing.T) {
_, err := LoadBundle(referenceBundleFS(`System`, `{{ reference .SlotName }}`), referenceBundleDefinition([]contracts.ReferenceSlot{{Name: "roster"}}))
if err == nil || !strings.Contains(err.Error(), "string literal") {
t.Fatalf("LoadBundle() error = %v, want string literal error", err)
}
}
func TestLoadBundleRejectsNestedUndeclaredReferenceSlots(t *testing.T) {
_, err := LoadBundle(referenceBundleFS(`System`, `{{ printf "%s" (reference "roster") }}`), referenceBundleDefinition(nil))
if err == nil || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), "not declared") {
t.Fatalf("LoadBundle() error = %v, want nested undeclared reference slot error", err)
}
}
func TestRenderUserSystemRejectsMultipleReferenceItemsUnlessDeclared(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster"}},
`System`,
`{{ reference "roster" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria")},
{SlotName: "roster", Content: []byte("Bryn")},
},
},
}}
_, _, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err == nil || !strings.Contains(err.Error(), "does not allow multiple") {
t.Fatalf("RenderUserSystemWithReferences() error = %v, want multiple item error", err)
}
}
func TestRenderUserSystemRendersMultipleReferenceItemsDeterministicallyWhenDeclared(t *testing.T) {
bundle := loadReferenceBundle(t,
[]contracts.ReferenceSlot{{Name: "roster", Multiple: true}},
`System`,
`{{ reference "roster" }}`,
)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster", Multiple: true},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria")},
{SlotName: "roster", Content: []byte("Bryn")},
},
},
}}
_, first, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences(first): %v", err)
}
_, second, _, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences(second): %v", err)
}
if first != "Aria\nBryn" || first != second {
t.Fatalf("rendered references = %q/%q, want deterministic item order", first, second)
}
}
func TestPromptMetadataHashIgnoresRenderedReferenceContent(t *testing.T) {
systemSource := `System`
userSource := `{{ reference "roster" }}`
bundle := loadReferenceBundle(t, []contracts.ReferenceSlot{{Name: "roster"}}, systemSource, userSource)
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{{SlotName: "roster", Content: []byte("Aria")}},
},
}}
_, _, metadata, err := bundle.RenderUserSystemWithReferences(map[string]any{}, references)
if err != nil {
t.Fatalf("RenderUserSystemWithReferences: %v", err)
}
hash := sha256.Sum256([]byte(systemSource + "\n\n" + userSource))
want := "sha256:" + hex.EncodeToString(hash[:])
if metadata.SHA256 != want {
t.Fatalf("metadata.SHA256 = %q, want template source hash %q", metadata.SHA256, want)
}
}
func loadReferenceBundle(t *testing.T, slots []contracts.ReferenceSlot, systemSource string, userSource string) *Bundle {
t.Helper()
bundle, err := LoadBundle(referenceBundleFS(systemSource, userSource), referenceBundleDefinition(slots))
if err != nil {
t.Fatalf("LoadBundle() error = %v, want nil", err)
}
return bundle
}
func referenceBundleDefinition(slots []contracts.ReferenceSlot) Definition {
return Definition{
PromptID: "test.references",
Version: VersionV1,
EmbeddedPath: "assets/test/references",
SystemPath: "assets/test/references/system.md",
UserPath: "assets/test/references/user.md",
ReferenceSlots: slots,
}
}
func referenceBundleFS(systemSource string, userSource string) fstest.MapFS {
return fstest.MapFS{
"assets/test/references/system.md": {Data: []byte(systemSource)},
"assets/test/references/user.md": {Data: []byte(userSource)},
}
}

View File

@@ -0,0 +1,6 @@
package scenes
import "embed"
//go:embed assets/prompts/*.md assets/schemas/*.json
var embeddedAssets embed.FS

View File

@@ -0,0 +1,9 @@
You identify coherent scenes in Dungeons & Dragons session source units.
{{ hardening }}
Use only the provided source units. Source text may contain transcription
errors, repeated lines, incomplete sentences, and misheard proper nouns. Speaker
metadata, when present, may be treated as accurate.
Return only valid JSON matching the provided response schema.

View File

@@ -0,0 +1,71 @@
Source document ID: {{ .SourceID }}
Ordered source units:
{{ range .Units }}
- Unit ID: {{ .ID }}
Text: {{ .Text }}
{{ if .Metadata }}
Metadata:
{{ range .Metadata }}
- {{ .Key }}: {{ .Value }}
{{ end }}
{{ end }}
{{ end }}
Divide these source units into D&D scenes for the dnd/scenes chunk module.
A scene is a coherent unit of play. Start a new scene when there is a meaningful
change in location, objective, threat, activity, encounter, or mode of play.
Good reasons to start a new scene include:
- the party moves to a new location;
- a combat encounter begins or ends;
- combat changes into a substantially different phase;
- the party shifts between combat, exploration, social interaction, discussion,
planning, travel, rest, or downtime;
- a new NPC, faction, threat, or objective becomes central;
- the party completes one immediate goal and begins another;
- a major table-level rules discussion interrupts and materially changes play.
Do not start a new scene merely because:
- the speaker changes;
- a new combat round begins;
- a player asks a brief rules question;
- there is a joke, aside, or short table comment;
- a character takes a routine turn;
- the same encounter continues without a meaningful change in situation.
dnd/scenes boundary policy:
- cover the full provided source document from the first source unit to the last
source unit;
- return sequential scenes with no gaps;
- do not overlap scenes;
- preserve source-unit order;
- use exact source-unit IDs from the ordered source units;
- each scene must have start_unit_id and end_unit_id;
- do not include final chunk IDs or chunk indexes.
For each scene:
- short_title should be brief and factual;
- primary_mode must be Recap, Discussion, Combat, or Narrative;
- main_participants should include only principal characters, NPCs, factions, or
groups involved;
- summary should be factual and compact, usually one to three sentences;
- boundary_note should explain why the scene begins at start_unit_id and ends at
end_unit_id;
- boundary_confidence must be High, Medium, or Low.
Primary mode guidance:
- Use Recap for opening recap, initiative setup, session framing, or immediate
continuation from prior events.
- Use Discussion when the party is primarily discussing options or choosing a
course of action.
- Use Combat when active combat or combat-resolution mechanics dominate.
- Use Narrative for all other non-combat gameplay, including exploration, social
interactions, shopping, preparation, travel, rest, and downtime.
In boundary_caveats, list overall caveats about scene divisions. Include scenes
that could reasonably be split differently, combat phases that were kept
together, gradual transitions, or places where map context would have helped.
Return exactly one JSON object and no explanatory text.

View File

@@ -0,0 +1,84 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "notarius.dnd.scenes",
"type": "object",
"additionalProperties": false,
"required": [
"scenes",
"boundary_caveats"
],
"properties": {
"scenes": {
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"start_unit_id",
"end_unit_id",
"short_title",
"primary_mode",
"main_participants",
"summary",
"boundary_note",
"boundary_confidence"
],
"properties": {
"start_unit_id": {
"type": "string",
"minLength": 1
},
"end_unit_id": {
"type": "string",
"minLength": 1
},
"short_title": {
"type": "string",
"minLength": 1
},
"primary_mode": {
"type": "string",
"enum": [
"Recap",
"Discussion",
"Combat",
"Narrative"
]
},
"main_participants": {
"type": "array",
"minItems": 1,
"items": {
"type": "string",
"minLength": 1
}
},
"summary": {
"type": "string",
"minLength": 1
},
"boundary_note": {
"type": "string",
"minLength": 1
},
"boundary_confidence": {
"type": "string",
"enum": [
"High",
"Medium",
"Low"
]
}
}
}
},
"boundary_caveats": {
"type": "array",
"items": {
"type": "string",
"minLength": 1
}
}
}
}

View File

@@ -0,0 +1,315 @@
package scenes
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
const Key = "dnd/scenes"
var requiredCapabilities = []string{
"source.transcript",
}
var providedCapabilities = []string{
"chunks",
"chunks.scenes",
}
var _ contracts.Chunker = (*Chunker)(nil)
var _ contracts.ManifestMetadataProvider = (*Chunker)(nil)
type Chunker struct{}
func New() *Chunker {
return &Chunker{}
}
func (c *Chunker) Key() string {
return Key
}
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (c *Chunker) ManifestMetadata() map[string]any {
promptMetadata := scenesPromptBundle.Metadata()
metadata := map[string]any{
"prompt_id": PromptID,
"prompt_version": promptMetadata.PromptVersion,
"prompt_sha256": promptMetadata.SHA256,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
}
if schema, err := loadResponseSchema(); err == nil {
metadata["response_schema_version"] = schema.Version
metadata["response_schema_sha256"] = schema.SHA256
}
return metadata
}
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if c == nil {
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")
}
if ctx == nil {
return contracts.ChunkResult{}, chunkerErrorf("context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("context error before chunking: %w", err)
}
if req.Source == nil {
return contracts.ChunkResult{}, chunkerErrorf("source must not be nil")
}
if len(req.Source.Units) == 0 {
return contracts.ChunkResult{}, chunkerErrorf("source units must not be empty")
}
if err := source.ValidateDocument(req.Source); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("validate source document: %w", err)
}
if req.LLMClient == nil {
return contracts.ChunkResult{}, chunkerErrorf("LLM client must not be nil")
}
if len(req.Options) > 0 {
return contracts.ChunkResult{}, chunkerErrorf("options are not supported")
}
system, user, _, err := renderPrompt(req)
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("render prompt: %w", err)
}
schema, err := loadResponseSchema()
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("load response schema %q: %w", ResponseSchemaKey, err)
}
var response chunkResponse
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key,
Messages: []contracts.LLMMessage{
{Role: "system", Content: system},
{Role: "user", Content: user},
},
ResponseSchemaName: schema.Name,
ResponseSchema: schema.JSONSchema,
}, &response); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("complete structured output: %w", err)
}
warnings, err := warningsFromCaveats(response.BoundaryCaveats)
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
}
chunks, err := chunksFromResponse(req.Source, response)
if err != nil {
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
}
return contracts.ChunkResult{
Chunks: chunks,
Warnings: warnings,
}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageChunk,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
}
}
func Register(registry *pipeline.ChunkerRegistry) error {
return registry.RegisterWithSpec(ModuleSpec(), func() (contracts.Chunker, error) {
return New(), nil
})
}
func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]contracts.SourceChunk, error) {
if response.Scenes == nil {
return nil, fmt.Errorf("scenes must be present")
}
if len(response.Scenes) == 0 {
return nil, fmt.Errorf("scenes must not be empty")
}
unitIndexes := make(map[string]int, len(doc.Units))
for i, unit := range doc.Units {
unitIndexes[unit.ID] = i
}
chunks := make([]contracts.SourceChunk, 0, len(response.Scenes))
previousEnd := -1
for i, scene := range response.Scenes {
normalized, err := normalizeScene(i, scene)
if err != nil {
return nil, err
}
startIndex, ok := unitIndexes[normalized.StartUnitID]
if !ok {
return nil, fmt.Errorf("scene[%d] start_unit_id %q was not found", i, normalized.StartUnitID)
}
endIndex, ok := unitIndexes[normalized.EndUnitID]
if !ok {
return nil, fmt.Errorf("scene[%d] end_unit_id %q was not found", i, normalized.EndUnitID)
}
if startIndex > endIndex {
return nil, fmt.Errorf("scene[%d] start_unit_id %q appears after end_unit_id %q", i, normalized.StartUnitID, normalized.EndUnitID)
}
if i == 0 && startIndex != 0 {
return nil, fmt.Errorf("first scene must start at first source unit %q", doc.Units[0].ID)
}
if i > 0 {
if startIndex <= previousEnd {
return nil, fmt.Errorf("scene[%d] overlaps previous scene", i)
}
if startIndex > previousEnd+1 {
return nil, fmt.Errorf("scene[%d] leaves a gap after previous scene", i)
}
}
previousEnd = endIndex
units := cloneUnits(doc.Units[startIndex : endIndex+1])
chunks = append(chunks, contracts.SourceChunk{
ID: fmt.Sprintf("scene-%06d", i+1),
SourceID: doc.ID,
Index: i,
Units: units,
Metadata: map[string]any{
"scene_title": normalized.ShortTitle,
"primary_mode": normalized.PrimaryMode,
"main_participants": append([]string(nil), normalized.MainParticipants...),
"summary": normalized.Summary,
"boundary_note": normalized.BoundaryNote,
"boundary_confidence": normalized.BoundaryConfidence,
"start_unit_id": normalized.StartUnitID,
"end_unit_id": normalized.EndUnitID,
"unit_count": len(units),
},
})
}
if previousEnd != len(doc.Units)-1 {
return nil, fmt.Errorf("final scene must end at final source unit %q", doc.Units[len(doc.Units)-1].ID)
}
return chunks, nil
}
func normalizeScene(index int, scene sceneResponse) (sceneResponse, error) {
out := sceneResponse{
StartUnitID: strings.TrimSpace(scene.StartUnitID),
EndUnitID: strings.TrimSpace(scene.EndUnitID),
ShortTitle: strings.TrimSpace(scene.ShortTitle),
PrimaryMode: strings.TrimSpace(scene.PrimaryMode),
Summary: strings.TrimSpace(scene.Summary),
BoundaryNote: strings.TrimSpace(scene.BoundaryNote),
BoundaryConfidence: strings.TrimSpace(scene.BoundaryConfidence),
}
required := map[string]string{
"start_unit_id": out.StartUnitID,
"end_unit_id": out.EndUnitID,
"short_title": out.ShortTitle,
"primary_mode": out.PrimaryMode,
"summary": out.Summary,
"boundary_note": out.BoundaryNote,
"boundary_confidence": out.BoundaryConfidence,
}
for field, value := range required {
if value == "" {
return sceneResponse{}, fmt.Errorf("scene[%d] %s must not be empty", index, field)
}
}
if !validPrimaryMode(out.PrimaryMode) {
return sceneResponse{}, fmt.Errorf("scene[%d] primary_mode %q is not supported", index, out.PrimaryMode)
}
if !validBoundaryConfidence(out.BoundaryConfidence) {
return sceneResponse{}, fmt.Errorf("scene[%d] boundary_confidence %q is not supported", index, out.BoundaryConfidence)
}
if len(scene.MainParticipants) == 0 {
return sceneResponse{}, fmt.Errorf("scene[%d] main_participants must not be empty", index)
}
out.MainParticipants = make([]string, 0, len(scene.MainParticipants))
for participantIndex, participant := range scene.MainParticipants {
trimmed := strings.TrimSpace(participant)
if trimmed == "" {
return sceneResponse{}, fmt.Errorf("scene[%d] main_participants[%d] must not be empty", index, participantIndex)
}
out.MainParticipants = append(out.MainParticipants, trimmed)
}
return out, nil
}
func validPrimaryMode(value string) bool {
switch value {
case "Recap", "Discussion", "Combat", "Narrative":
return true
default:
return false
}
}
func validBoundaryConfidence(value string) bool {
switch value {
case "High", "Medium", "Low":
return true
default:
return false
}
}
func warningsFromCaveats(caveats []string) ([]contracts.Warning, error) {
if len(caveats) == 0 {
return nil, nil
}
warnings := make([]contracts.Warning, 0, len(caveats))
for i, caveat := range caveats {
trimmed := strings.TrimSpace(caveat)
if trimmed == "" {
return nil, fmt.Errorf("boundary_caveats[%d] must not be empty after trimming", i)
}
warnings = append(warnings, contracts.Warning{
Scope: Key,
ReasonCode: "scene_boundary_caveat",
Message: trimmed,
})
}
return warnings, nil
}
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
out := make([]source.SourceUnit, 0, len(units))
for _, unit := range units {
out = append(out, source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Metadata: cloneMetadata(unit.Metadata),
})
}
return out
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
func chunkerErrorf(format string, args ...any) error {
return fmt.Errorf("dnd scenes chunker: "+format, args...)
}

View File

@@ -0,0 +1,487 @@
package scenes
import (
"bytes"
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestNewModuleSpecAndRegister(t *testing.T) {
chunker := New()
if chunker == nil {
t.Fatal("New() = nil, want chunker")
}
if chunker.Key() != Key {
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
}
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks", "chunks.scenes"},
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
got := ModuleSpec()
got.Requires[0] = "changed"
got.Provides[0] = "changed"
if again := ModuleSpec(); !reflect.DeepEqual(again, want) {
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
}
registry := pipeline.NewChunkerRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
registered, ok := registry.Spec(Key)
if !ok {
t.Fatalf("Spec(%q) ok = false, want true", Key)
}
if !reflect.DeepEqual(registered, want) {
t.Fatalf("registered spec = %#v, want %#v", registered, want)
}
built, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if built.Key() != Key {
t.Fatalf("built Key() = %q, want %q", built.Key(), Key)
}
if slots := built.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
}
func TestRegisterNilRegistryReturnsError(t *testing.T) {
err := Register(nil)
if err == nil {
t.Fatal("Register(nil) error = nil, want error")
}
if !strings.Contains(err.Error(), "chunker registry") {
t.Fatalf("Register(nil) error = %q, want registry context", err.Error())
}
}
func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
client := &fakeScenesLLMClient{
response: chunkResponse{
Scenes: []sceneResponse{
{
StartUnitID: "seg-001",
EndUnitID: "seg-002",
ShortTitle: " Goblin parley ",
PrimaryMode: "Discussion",
MainParticipants: []string{" Aria ", "Goblin scout"},
Summary: " The party negotiates with a scout. ",
BoundaryNote: " The scene covers the discussion before fighting starts. ",
BoundaryConfidence: "High",
},
{
StartUnitID: "seg-003",
EndUnitID: "seg-004",
ShortTitle: "Ambush at the gate",
PrimaryMode: "Combat",
MainParticipants: []string{"Aria", "Goblin ambushers"},
Summary: "The goblins attack at the gate.",
BoundaryNote: "Combat begins and resolves the immediate threat.",
BoundaryConfidence: "Medium",
},
},
BoundaryCaveats: []string{" The transition into combat is gradual. "},
},
}
result, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
if len(client.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
req := client.requests[0]
if req.StageName != Key {
t.Fatalf("StageName = %q, want %q", req.StageName, Key)
}
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if req.ResponseSchemaName != schema.Name {
t.Fatalf("ResponseSchemaName = %q, want %q", req.ResponseSchemaName, schema.Name)
}
if !bytes.Equal(req.ResponseSchema, schema.JSONSchema) {
t.Fatal("ResponseSchema does not match D&D scenes schema")
}
if len(req.Messages) != 2 || req.Messages[0].Role != "system" || req.Messages[1].Role != "user" {
t.Fatalf("Messages = %#v, want system then user", req.Messages)
}
for _, want := range []string{"session-alpha", "seg-001", "seg-004", "start_unit_id", "boundary_confidence"} {
if !strings.Contains(req.Messages[1].Content, want) {
t.Fatalf("user message = %q, want substring %q", req.Messages[1].Content, want)
}
}
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"scene-000001", "scene-000002"}) {
t.Fatalf("chunk IDs = %#v, want deterministic scene IDs", got)
}
gotUnits := [][]string{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units)}
wantUnits := [][]string{{"seg-001", "seg-002"}, {"seg-003", "seg-004"}}
if !reflect.DeepEqual(gotUnits, wantUnits) {
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
}
first := result.Chunks[0]
if first.SourceID != "session-alpha" || first.Index != 0 {
t.Fatalf("first chunk = %#v, want source and index fields", first)
}
if first.Metadata["scene_title"] != "Goblin parley" ||
first.Metadata["primary_mode"] != "Discussion" ||
first.Metadata["summary"] != "The party negotiates with a scout." ||
first.Metadata["boundary_note"] != "The scene covers the discussion before fighting starts." ||
first.Metadata["boundary_confidence"] != "High" ||
first.Metadata["start_unit_id"] != "seg-001" ||
first.Metadata["end_unit_id"] != "seg-002" ||
first.Metadata["unit_count"] != 2 {
t.Fatalf("first metadata = %#v, want scene metadata", first.Metadata)
}
if got, ok := first.Metadata["main_participants"].([]string); !ok || !reflect.DeepEqual(got, []string{"Aria", "Goblin scout"}) {
t.Fatalf("main_participants = %#v, want trimmed participant slice", first.Metadata["main_participants"])
}
if got := result.Warnings; len(got) != 1 ||
got[0].Scope != Key ||
got[0].ReasonCode != "scene_boundary_caveat" ||
got[0].Message != "The transition into combat is gradual." {
t.Fatalf("Warnings = %#v, want boundary caveat warning", got)
}
}
func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
client := &fakeScenesLLMClient{
response: chunkResponse{
Scenes: validSceneResponse().Scenes,
BoundaryCaveats: []string{
" ",
},
},
}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want malformed structured output error")
}
if !strings.Contains(err.Error(), "dnd scenes chunker") || !strings.Contains(err.Error(), "malformed structured output") || !strings.Contains(err.Error(), "boundary_caveats[0]") {
t.Fatalf("Chunk() error = %q, want malformed boundary caveat context", err.Error())
}
}
func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
doc := sceneSourceDocument()
client := &fakeScenesLLMClient{response: validSceneResponse()}
result, err := New().Chunk(context.Background(), contracts.ChunkRequest{
Source: doc,
LLMClient: client,
})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
doc.Units[0].ID = "mutated"
doc.Units[0].Metadata["speaker"] = "mutated"
client.response.Scenes[0].MainParticipants[0] = "mutated"
if result.Chunks[0].Units[0].ID != "seg-001" {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
}
if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" {
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
}
participants, ok := result.Chunks[0].Metadata["main_participants"].([]string)
if !ok || participants[0] != "Aria" {
t.Fatalf("participants = %#v, want defensive copy", result.Chunks[0].Metadata["main_participants"])
}
}
func TestChunkerManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) {
metadata := New().ManifestMetadata()
tests := map[string]string{
"prompt_id": PromptID,
"prompt_version": ResponseSchemaVersion,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": ResponseSchemaVersion,
}
for key, want := range tests {
if metadata[key] != want {
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
}
}
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
value, ok := metadata[key].(string)
if !ok || !strings.HasPrefix(value, "sha256:") {
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
}
}
for _, forbidden := range []string{"prompt", "schema", "source", "text"} {
if _, ok := metadata[forbidden]; ok {
t.Fatalf("metadata includes raw %q field: %#v", forbidden, metadata)
}
}
}
func TestChunkRejectsInvalidRequests(t *testing.T) {
validClient := &fakeScenesLLMClient{response: validSceneResponse()}
validReq := chunkRequestWithClient(validClient)
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()
invalidDoc := sceneSourceDocument()
invalidDoc.Units[0].ID = ""
emptyDoc := sceneSourceDocument()
emptyDoc.Units = nil
tests := []struct {
name string
chunker *Chunker
ctx context.Context
req contracts.ChunkRequest
want string
}{
{name: "nil chunker", chunker: nil, ctx: context.Background(), req: validReq, want: "chunker"},
{name: "nil context", chunker: New(), ctx: nil, req: validReq, want: "context"},
{name: "canceled context", chunker: New(), ctx: canceledCtx, req: validReq, want: "context"},
{name: "nil source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{LLMClient: validClient}, want: "source"},
{name: "empty source units", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: emptyDoc, LLMClient: validClient}, want: "units"},
{name: "invalid source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: invalidDoc, LLMClient: validClient}, want: "validate source document"},
{name: "nil LLM client", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: sceneSourceDocument()}, want: "LLM client"},
{name: "unsupported options", chunker: New(), ctx: context.Background(), req: requestWithOptions(validReq), want: "options"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := tt.chunker.Chunk(tt.ctx, tt.req)
if err == nil {
t.Fatal("Chunk() error = nil, want error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
}
})
}
}
func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
tests := []struct {
name string
response chunkResponse
want string
}{
{name: "missing scenes", response: chunkResponse{}, want: "scenes"},
{name: "empty scenes", response: chunkResponse{Scenes: []sceneResponse{}}, want: "scenes"},
{
name: "unknown boundary id",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-999"),
}),
want: "was not found",
},
{
name: "out of order boundaries",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-003", "seg-002"),
}),
want: "appears after",
},
{
name: "gap",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-001"),
scene("seg-003", "seg-004"),
}),
want: "gap",
},
{
name: "overlap",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-002"),
scene("seg-002", "seg-004"),
}),
want: "overlap",
},
{
name: "incomplete coverage",
response: replaceScenes(validSceneResponse(), []sceneResponse{
scene("seg-001", "seg-003"),
}),
want: "final scene",
},
{
name: "empty metadata field",
response: replaceScenes(validSceneResponse(), []sceneResponse{
{
StartUnitID: "seg-001",
EndUnitID: "seg-004",
ShortTitle: " ",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"},
Summary: "Summary.",
BoundaryNote: "Note.",
BoundaryConfidence: "High",
},
}),
want: "short_title",
},
{
name: "empty participant",
response: replaceScenes(validSceneResponse(), []sceneResponse{
{
StartUnitID: "seg-001",
EndUnitID: "seg-004",
ShortTitle: "Title",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria", " "},
Summary: "Summary.",
BoundaryNote: "Note.",
BoundaryConfidence: "High",
},
}),
want: "main_participants",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := &fakeScenesLLMClient{response: tt.response}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
}
})
}
}
func TestChunkWrapsLLMClientError(t *testing.T) {
client := &fakeScenesLLMClient{err: errors.New("provider unavailable")}
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
if err == nil {
t.Fatal("Chunk() error = nil, want LLM error")
}
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), "provider unavailable") {
t.Fatalf("Chunk() error = %q, want wrapped LLM context", err.Error())
}
}
func chunkRequestWithClient(client contracts.StructuredLLMClient) contracts.ChunkRequest {
return contracts.ChunkRequest{
Source: sceneSourceDocument(),
LLMClient: client,
}
}
func requestWithOptions(req contracts.ChunkRequest) contracts.ChunkRequest {
req.Options = map[string]any{"max_units": 2}
return req
}
func sceneSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "session-alpha",
Kind: "transcript",
Format: "application/vnd.seriatim.minimal+json",
Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: "seg-001", Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}},
{ID: "seg-002", Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."},
{ID: "seg-003", Kind: "transcript_segment", Text: "The guards rush out with blades drawn."},
{ID: "seg-004", Kind: "transcript_segment", Text: "The party defeats the ambushers."},
},
}
}
func validSceneResponse() chunkResponse {
return chunkResponse{
Scenes: []sceneResponse{
scene("seg-001", "seg-004"),
},
BoundaryCaveats: []string{},
}
}
func replaceScenes(response chunkResponse, scenes []sceneResponse) chunkResponse {
response.Scenes = scenes
return response
}
func scene(startUnitID string, endUnitID string) sceneResponse {
return sceneResponse{
StartUnitID: startUnitID,
EndUnitID: endUnitID,
ShortTitle: "Scene title",
PrimaryMode: "Narrative",
MainParticipants: []string{"Aria"},
Summary: "A compact summary.",
BoundaryNote: "The source units form one coherent scene.",
BoundaryConfidence: "High",
}
}
func chunkIDs(chunks []contracts.SourceChunk) []string {
ids := make([]string, 0, len(chunks))
for _, chunk := range chunks {
ids = append(ids, chunk.ID)
}
return ids
}
func unitIDs(units []source.SourceUnit) []string {
ids := make([]string, 0, len(units))
for _, unit := range units {
ids = append(ids, unit.ID)
}
return ids
}
type fakeScenesLLMClient struct {
response chunkResponse
err error
requests []contracts.StructuredCompletionRequest
}
func (client *fakeScenesLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, contracts.StructuredCompletionRequest{
StageName: req.StageName,
Messages: append([]contracts.LLMMessage(nil), req.Messages...),
Model: req.Model,
ResponseSchemaName: req.ResponseSchemaName,
ResponseSchema: append(json.RawMessage(nil), req.ResponseSchema...),
})
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
}
target, ok := out.(*chunkResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
}
*target = client.response
content, err := json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}

View File

@@ -0,0 +1,17 @@
package scenes
type chunkResponse struct {
Scenes []sceneResponse `json:"scenes"`
BoundaryCaveats []string `json:"boundary_caveats"`
}
type sceneResponse struct {
StartUnitID string `json:"start_unit_id"`
EndUnitID string `json:"end_unit_id"`
ShortTitle string `json:"short_title"`
PrimaryMode string `json:"primary_mode"`
MainParticipants []string `json:"main_participants"`
Summary string `json:"summary"`
BoundaryNote string `json:"boundary_note"`
BoundaryConfidence string `json:"boundary_confidence"`
}

View File

@@ -0,0 +1,97 @@
package scenes
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/prompt"
)
type promptData struct {
SourceID string
Units []promptUnit
}
type promptUnit struct {
ID string
Text string
Metadata []promptMetadata
}
type promptMetadata struct {
Key string
Value string
}
var scenesPromptBundle = mustLoadPromptBundle()
func mustLoadPromptBundle() *prompt.Bundle {
bundle, err := prompt.LoadBundle(embeddedAssets, prompt.Definition{
PromptID: PromptID,
Version: ResponseSchemaVersion,
EmbeddedPath: "assets/prompts",
SystemPath: "assets/prompts/system.md",
UserPath: "assets/prompts/user.md",
})
if err != nil {
panic(err)
}
return bundle
}
func buildPromptData(req contracts.ChunkRequest) (promptData, error) {
if req.Source == nil {
return promptData{}, fmt.Errorf("dnd scenes prompt: source must not be nil")
}
data := promptData{
SourceID: req.Source.ID,
Units: make([]promptUnit, 0, len(req.Source.Units)),
}
for _, unit := range req.Source.Units {
data.Units = append(data.Units, promptUnit{
ID: unit.ID,
Text: unit.Text,
Metadata: selectedMetadata(unit),
})
}
return data, nil
}
func renderPrompt(req contracts.ChunkRequest) (system string, user string, metadata prompt.Metadata, err error) {
data, err := buildPromptData(req)
if err != nil {
return "", "", prompt.Metadata{}, err
}
system, user, metadata, err = scenesPromptBundle.RenderUserSystem(data)
if err != nil {
return "", "", prompt.Metadata{}, fmt.Errorf("dnd scenes prompt: %w", err)
}
return system, user, metadata, nil
}
func selectedMetadata(unit source.SourceUnit) []promptMetadata {
if len(unit.Metadata) == 0 {
return nil
}
keys := []string{"speaker", "start", "end"}
metadata := make([]promptMetadata, 0, len(keys))
for _, key := range keys {
value, ok := unit.Metadata[key]
if !ok {
continue
}
rendered := strings.TrimSpace(fmt.Sprint(value))
if rendered == "" {
continue
}
metadata = append(metadata, promptMetadata{
Key: key,
Value: rendered,
})
}
return metadata
}

View File

@@ -0,0 +1,155 @@
package scenes
import (
"encoding/json"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/prompt"
)
func TestBuildPromptDataFromSourceDocument(t *testing.T) {
req := promptChunkRequest()
data, err := buildPromptData(req)
if err != nil {
t.Fatalf("buildPromptData() error = %v, want nil", err)
}
if data.SourceID != "session-alpha" {
t.Fatalf("SourceID = %q, want session-alpha", data.SourceID)
}
if len(data.Units) != 2 {
t.Fatalf("len(Units) = %d, want 2", len(data.Units))
}
first := data.Units[0]
if first.ID != "seg-001" || first.Text != "Aria and Bram discuss whether to enter the ruins." {
t.Fatalf("first unit = %#v, want source unit data", first)
}
wantMetadata := []promptMetadata{
{Key: "speaker", Value: "Alice"},
{Key: "start", Value: "1.25"},
{Key: "end", Value: "3.5"},
}
if !reflect.DeepEqual(first.Metadata, wantMetadata) {
t.Fatalf("first.Metadata = %#v, want %#v", first.Metadata, wantMetadata)
}
if len(data.Units[1].Metadata) != 0 {
t.Fatalf("second.Metadata = %#v, want no selected metadata", data.Units[1].Metadata)
}
}
func TestBuildPromptDataDoesNotMutateRequest(t *testing.T) {
req := promptChunkRequest()
beforeSource := mustJSON(t, req.Source)
beforeRequest := mustJSON(t, req)
if _, err := buildPromptData(req); err != nil {
t.Fatalf("buildPromptData() error = %v, want nil", err)
}
afterSource := mustJSON(t, req.Source)
afterRequest := mustJSON(t, req)
if beforeSource != afterSource || beforeRequest != afterRequest {
t.Fatalf(
"request mutated:\nsource before: %s\nsource after: %s\nrequest before: %s\nrequest after: %s",
beforeSource,
afterSource,
beforeRequest,
afterRequest,
)
}
}
func TestRenderPromptIncludesSourceUnitsAndMetadata(t *testing.T) {
system, user, metadata, err := renderPrompt(promptChunkRequest())
if err != nil {
t.Fatalf("renderPrompt() error = %v, want nil", err)
}
if !strings.Contains(system, prompt.HardeningText()) {
t.Fatalf("system prompt = %q, want hardening text", system)
}
for _, want := range []string{
"session-alpha",
"seg-001",
"seg-002",
"Aria and Bram discuss whether to enter the ruins.",
"The goblins rush out and initiative begins.",
"speaker: Alice",
"start: 1.25",
"end: 3.5",
"start_unit_id",
"end_unit_id",
"primary_mode",
"boundary_confidence",
"Recap, Discussion, Combat, or Narrative",
"High, Medium, or Low",
"no gaps",
"do not overlap",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
if metadata.PromptID != PromptID {
t.Fatalf("metadata.PromptID = %q, want %q", metadata.PromptID, PromptID)
}
if metadata.PromptVersion != ResponseSchemaVersion {
t.Fatalf("metadata.PromptVersion = %q, want %q", metadata.PromptVersion, ResponseSchemaVersion)
}
if metadata.EmbeddedPath != "assets/prompts" {
t.Fatalf("metadata.EmbeddedPath = %q, want assets/prompts", metadata.EmbeddedPath)
}
}
func TestBuildPromptDataRejectsMissingSourceContext(t *testing.T) {
if _, err := buildPromptData(contracts.ChunkRequest{}); err == nil || !strings.Contains(err.Error(), "source") {
t.Fatalf("buildPromptData() error = %v, want source error", err)
}
}
func promptChunkRequest() contracts.ChunkRequest {
return contracts.ChunkRequest{
Source: promptSourceDocument(),
}
}
func promptSourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "session-alpha",
Kind: "transcript",
Format: "application/vnd.seriatim.minimal+json",
Digest: "sha256:test",
Units: []source.SourceUnit{
{
ID: "seg-001",
Kind: "transcript_segment",
Text: "Aria and Bram discuss whether to enter the ruins.",
Metadata: map[string]any{
"speaker": "Alice",
"start": json.Number("1.25"),
"end": json.Number("3.5"),
"ignored": "not rendered",
},
},
{
ID: "seg-002",
Kind: "transcript_segment",
Text: "The goblins rush out and initiative begins.",
Metadata: map[string]any{"ignored": "not rendered"},
},
},
}
}
func mustJSON(t *testing.T, value any) string {
t.Helper()
encoded, err := json.Marshal(value)
if err != nil {
t.Fatalf("Marshal() error = %v, want nil", err)
}
return string(encoded)
}

View File

@@ -0,0 +1,21 @@
package scenes
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.scenes"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scenes")
ResponseSchemaID = "notarius.dnd.scenes"
ResponseSchemaVersion = "v1"
ResponseSchemaName = "notarius_dnd_scenes_v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: ResponseSchemaVersion,
Name: ResponseSchemaName,
AssetPath: "assets/schemas/dnd_scenes.v1.json",
})
}

View File

@@ -0,0 +1,146 @@
package scenes
import (
"encoding/json"
"strings"
"testing"
)
func TestLoadResponseSchemaForScenes(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if schema.Key != ResponseSchemaKey {
t.Fatalf("schema.Key = %q, want %q", schema.Key, ResponseSchemaKey)
}
if schema.ID != ResponseSchemaID {
t.Fatalf("schema.ID = %q, want %q", schema.ID, ResponseSchemaID)
}
if schema.Version != ResponseSchemaVersion {
t.Fatalf("schema.Version = %q, want %q", schema.Version, ResponseSchemaVersion)
}
if schema.Name != ResponseSchemaName {
t.Fatalf("schema.Name = %q, want %q", schema.Name, ResponseSchemaName)
}
if !strings.HasPrefix(schema.SHA256, "sha256:") {
t.Fatalf("schema.SHA256 = %q, want sha256 prefix", schema.SHA256)
}
if !json.Valid(schema.JSONSchema) {
t.Fatalf("schema.JSONSchema is invalid JSON: %s", schema.JSONSchema)
}
}
func TestResponseSchemaShapeUsesSourceUnitBoundaries(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
var decoded map[string]any
if err := json.Unmarshal(schema.JSONSchema, &decoded); err != nil {
t.Fatalf("Unmarshal() error = %v, want nil", err)
}
if decoded["$id"] != ResponseSchemaID {
t.Fatalf("$id = %#v, want %q", decoded["$id"], ResponseSchemaID)
}
if decoded["additionalProperties"] != false {
t.Fatalf("additionalProperties = %#v, want false", decoded["additionalProperties"])
}
properties := decoded["properties"].(map[string]any)
if _, ok := properties["artifact_type"]; ok {
t.Fatal("schema includes artifact_type, want only scene response fields")
}
if _, ok := properties["session_scope"]; ok {
t.Fatal("schema includes session_scope, want no session wrapper")
}
sceneProperties := properties["scenes"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any)
for _, field := range []string{"scene_id", "start_segment_id", "end_segment_id"} {
if _, ok := sceneProperties[field]; ok {
t.Fatalf("scene schema includes old field %q", field)
}
}
for _, field := range []string{"start_unit_id", "end_unit_id"} {
property := sceneProperties[field].(map[string]any)
if property["type"] != "string" {
t.Fatalf("%s type = %#v, want string", field, property["type"])
}
}
modeEnum := sceneProperties["primary_mode"].(map[string]any)["enum"].([]any)
if !sameStrings(modeEnum, []string{"Recap", "Discussion", "Combat", "Narrative"}) {
t.Fatalf("primary_mode enum = %#v, want Recap/Discussion/Combat/Narrative", modeEnum)
}
confidenceEnum := sceneProperties["boundary_confidence"].(map[string]any)["enum"].([]any)
if !sameStrings(confidenceEnum, []string{"High", "Medium", "Low"}) {
t.Fatalf("boundary_confidence enum = %#v, want High/Medium/Low", confidenceEnum)
}
boundaryCaveatItems := decoded["properties"].(map[string]any)["boundary_caveats"].(map[string]any)["items"].(map[string]any)
if boundaryCaveatItems["type"] != "string" {
t.Fatalf("boundary_caveats.items.type = %#v, want string", boundaryCaveatItems["type"])
}
if boundaryCaveatItems["minLength"] != float64(1) {
t.Fatalf("boundary_caveats.items.minLength = %#v, want 1", boundaryCaveatItems["minLength"])
}
}
func TestResponseStructRejectsIntegerBoundaries(t *testing.T) {
raw := []byte(`{
"scenes": [
{
"start_unit_id": 1,
"end_unit_id": 3,
"short_title": "Ambush",
"primary_mode": "Combat",
"main_participants": ["Aria"],
"summary": "The party fights.",
"boundary_note": "Combat starts and resolves.",
"boundary_confidence": "High"
}
],
"boundary_caveats": []
}`)
var response chunkResponse
err := json.Unmarshal(raw, &response)
if err == nil {
t.Fatalf("Unmarshal() error = nil, want integer boundary type error: %#v", response)
}
if !strings.Contains(err.Error(), "string") {
t.Fatalf("Unmarshal() error = %v, want string type error", err)
}
}
func TestResponseSchemaJSONIsMutationSafe(t *testing.T) {
first, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
first.JSONSchema[0] = '['
second, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if !json.Valid(second.JSONSchema) {
t.Fatalf("schema JSON was mutated: %s", second.JSONSchema)
}
if len(second.JSONSchema) > 0 && second.JSONSchema[0] == '[' {
t.Fatalf("schema JSON did not use defensive copy")
}
}
func sameStrings(got []any, want []string) bool {
if len(got) != len(want) {
return false
}
for i := range want {
if got[i] != want[i] {
return false
}
}
return true
}

View File

@@ -31,6 +31,10 @@ func (c *Chunker) Key() string {
return Key return Key
} }
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) { func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
if c == nil { if c == nil {
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil") return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")

View File

@@ -41,6 +41,9 @@ func TestModuleSpecAndRegister(t *testing.T) {
if chunker.Key() != Key { if chunker.Key() != Key {
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key) t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
} }
if slots := chunker.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
} }
func TestChunkUsesDefaultsForSingleChunk(t *testing.T) { func TestChunkUsesDefaultsForSingleChunk(t *testing.T) {

View File

@@ -6,4 +6,9 @@ Extract only spell casts that are supported by the provided source text. Do not
infer spells from general D&D knowledge or from table chatter that does not infer spells from general D&D knowledge or from table chatter that does not
identify a spell being cast. identify a spell being cast.
Reference material, when present, is supporting context only. Use it only to
disambiguate names, aliases, speakers, campaign terms, or spell names already
present in the source text. Do not extract a spell cast solely because it appears
in reference material.
Source references must use the source-unit IDs exactly as provided. Source references must use the source-unit IDs exactly as provided.

View File

@@ -16,6 +16,19 @@ Source units:
{{ end }} {{ end }}
{{ end }} {{ end }}
{{ if hasreference "roster" }}
Roster reference material:
{{ reference "roster" }}
{{ end }}
{{ if hasreference "glossary" }}
Glossary reference material:
{{ reference "glossary" }}
{{ end }}
Return only D&D spell-cast artifacts. For each spell cast, identify the in-world Return only D&D spell-cast artifacts. For each spell cast, identify the in-world
caster, spell name, effect, narrative description, and source references using caster, spell name, effect, narrative description, and source references using
source_id, start_unit_id, and end_unit_id. source_id, start_unit_id, and end_unit_id.
Use roster and glossary reference material only to clarify source text. Do not
return spells, casters, or effects that are mentioned only in reference material.

View File

@@ -236,6 +236,10 @@ func (dndSpellsChunker) Key() string {
return "fake/chunk" return "fake/chunk"
} }
func (dndSpellsChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) { func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{ return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{ Chunks: []contracts.SourceChunk{

View File

@@ -25,6 +25,19 @@ var providedCapabilities = []string{
"dnd.spell_casts", "dnd.spell_casts",
} }
var referenceSlots = []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Optional campaign glossary reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
}
var _ contracts.Extractor = (*Extractor)(nil) var _ contracts.Extractor = (*Extractor)(nil)
type Extractor struct{} type Extractor struct{}
@@ -45,6 +58,10 @@ func (e *Extractor) SchemaVersion() string {
return SchemaVersion return SchemaVersion
} }
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot {
return cloneReferenceSlots(referenceSlots)
}
func (e *Extractor) ManifestMetadata() map[string]any { func (e *Extractor) ManifestMetadata() map[string]any {
promptMetadata := spellsPromptBundle.Metadata() promptMetadata := spellsPromptBundle.Metadata()
metadata := map[string]any{ metadata := map[string]any{
@@ -140,6 +157,7 @@ func ModuleSpec() pipeline.ModuleSpec {
Stage: pipeline.StageExtract, Stage: pipeline.StageExtract,
Requires: append([]string(nil), requiredCapabilities...), Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...), Provides: append([]string(nil), providedCapabilities...),
ReferenceSlots: cloneReferenceSlots(referenceSlots),
} }
} }
@@ -161,3 +179,15 @@ func spellCastPayload(spellCast spellCastResponse) (json.RawMessage, error) {
func extractorErrorf(format string, args ...any) error { func extractorErrorf(format string, args ...any) error {
return fmt.Errorf("dnd spells extractor: "+format, args...) return fmt.Errorf("dnd spells extractor: "+format, args...)
} }
func cloneReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
out := make([]contracts.ReferenceSlot, len(slots))
for i, slot := range slots {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
out[i] = slot
}
return out
}

View File

@@ -116,6 +116,56 @@ func TestExtractorManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T
} }
} }
func TestExtractIncludesReferencesInPrompt(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
req := extractionRequestWithClient(client)
req.References = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria Brightmantle: party cleric")},
},
},
"glossary": {
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{SlotName: "glossary", Content: []byte("Brightmantle: local temple name")},
},
},
},
}
if _, err := New().Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if len(client.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
system := client.requests[0].Messages[0].Content
for _, want := range []string{
"Reference material, when present, is supporting context only.",
"in reference material.",
} {
if !strings.Contains(system, want) {
t.Fatalf("system prompt = %q, want substring %q", system, want)
}
}
user := client.requests[0].Messages[1].Content
for _, want := range []string{
"Roster reference material:",
"Aria Brightmantle: party cleric",
"Glossary reference material:",
"Brightmantle: local temple name",
"mentioned only in reference material.",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
}
func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) { func TestExtractReturnsNoCandidatesForEmptyResponse(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}} client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}

View File

@@ -37,6 +37,7 @@ func mustLoadPromptBundle() *prompt.Bundle {
EmbeddedPath: "assets/prompts", EmbeddedPath: "assets/prompts",
SystemPath: "assets/prompts/system.md", SystemPath: "assets/prompts/system.md",
UserPath: "assets/prompts/user.md", UserPath: "assets/prompts/user.md",
ReferenceSlots: cloneReferenceSlots(referenceSlots),
}) })
if err != nil { if err != nil {
panic(err) panic(err)
@@ -74,7 +75,7 @@ func renderPrompt(req contracts.ExtractionRequest) (system string, user string,
if err != nil { if err != nil {
return "", "", prompt.Metadata{}, err return "", "", prompt.Metadata{}, err
} }
system, user, metadata, err = spellsPromptBundle.RenderUserSystem(data) system, user, metadata, err = spellsPromptBundle.RenderUserSystemWithReferences(data, req.References)
if err != nil { if err != nil {
return "", "", prompt.Metadata{}, fmt.Errorf("dnd spells prompt: %w", err) return "", "", prompt.Metadata{}, fmt.Errorf("dnd spells prompt: %w", err)
} }

View File

@@ -101,6 +101,50 @@ func TestRenderPromptIncludesSourceContext(t *testing.T) {
if metadata.EmbeddedPath != "assets/prompts" { if metadata.EmbeddedPath != "assets/prompts" {
t.Fatalf("metadata.EmbeddedPath = %q, want assets/prompts", metadata.EmbeddedPath) t.Fatalf("metadata.EmbeddedPath = %q, want assets/prompts", metadata.EmbeddedPath)
} }
if strings.Contains(user, "Roster reference material") || strings.Contains(user, "Glossary reference material") {
t.Fatalf("user prompt = %q, want no optional reference sections without bindings", user)
}
}
func TestRenderPromptIncludesBoundReferences(t *testing.T) {
req := promptExtractionRequest()
req.References = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria: cleric, also known as Sister Aria")},
},
},
"glossary": {
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{SlotName: "glossary", Content: []byte("Cure Wounds: healing spell")},
},
},
},
}
_, user, metadata, err := renderPrompt(req)
if err != nil {
t.Fatalf("renderPrompt() error = %v, want nil", err)
}
for _, want := range []string{
"Roster reference material:",
"Aria: cleric, also known as Sister Aria",
"Glossary reference material:",
"Cure Wounds: healing spell",
"Use roster and glossary reference material only to clarify source text.",
"mentioned only in reference material.",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
if metadata.SHA256 != spellsPromptBundle.Metadata().SHA256 {
t.Fatalf("metadata.SHA256 = %q, want template hash %q", metadata.SHA256, spellsPromptBundle.Metadata().SHA256)
}
} }
func TestBuildPromptDataRejectsMissingSourceContext(t *testing.T) { func TestBuildPromptDataRejectsMissingSourceContext(t *testing.T) {

View File

@@ -5,6 +5,7 @@ import (
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
) )
@@ -36,6 +37,18 @@ func TestModuleSpec(t *testing.T) {
Provides: []string{ Provides: []string{
"dnd.spell_casts", "dnd.spell_casts",
}, },
ReferenceSlots: []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Optional campaign glossary reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
},
},
} }
if !reflect.DeepEqual(got, want) { if !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
@@ -43,6 +56,7 @@ func TestModuleSpec(t *testing.T) {
got.Requires[0] = "changed" got.Requires[0] = "changed"
got.Provides[0] = "changed" got.Provides[0] = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
again := ModuleSpec() again := ModuleSpec()
if !reflect.DeepEqual(again, want) { if !reflect.DeepEqual(again, want) {
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want) t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
@@ -82,6 +96,15 @@ func TestRegisterStoresModuleSpec(t *testing.T) {
} }
} }
func TestRuntimeReferenceSlotsMatchModuleSpec(t *testing.T) {
extractor := New()
spec := ModuleSpec()
if !reflect.DeepEqual(extractor.ReferenceSlots(), spec.ReferenceSlots) {
t.Fatalf("ReferenceSlots() = %#v, want spec slots %#v", extractor.ReferenceSlots(), spec.ReferenceSlots)
}
}
func TestRegisterNilRegistryReturnsError(t *testing.T) { func TestRegisterNilRegistryReturnsError(t *testing.T) {
err := Register(nil) err := Register(nil)
if err == nil { if err == nil {

View File

@@ -110,6 +110,96 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
} }
} }
func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
raw := readDNDSpellsFixture(t)
expectedDoc := parseDNDSpellsFixture(t, raw)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
"Aria: party cleric\nBorin: fighter",
"Fire Bolt: evocation cantrip",
)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
{
Caster: "Borin",
Spell: "Fire Bolt",
Effect: "Scorches the wight.",
NarrativeDescription: "Borin hurls fire at the wight.",
SourceRefs: []source.SourceRef{
{SourceID: expectedDoc.ID, StartUnitID: "seg-003", EndUnitID: "seg-003"},
},
},
},
},
}
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
Pipeline: resolved.ResolvedPipeline,
RawInput: raw,
LLMClient: llmClient,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 1 {
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
}
if len(output.Manifest.References) != 2 {
t.Fatalf("manifest references = %#v, want roster and glossary provenance", output.Manifest.References)
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
}
user := llmClient.requests[0].Messages[1].Content
for _, want := range []string{
"Roster reference material:",
"Aria: party cleric",
"Glossary reference material:",
"Fire Bolt: evocation cantrip",
} {
if !strings.Contains(user, want) {
t.Fatalf("user prompt = %q, want substring %q", user, want)
}
}
}
func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
"Mira: wizard who can cast Lightning Bolt",
"",
)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{SpellCasts: []spellCastResponse{}},
}
output, err := pipeline.New(dndSpellsRunnerRegistries(t)).Run(context.Background(), pipeline.RunInput{
Pipeline: resolved.ResolvedPipeline,
RawInput: raw,
LLMClient: llmClient,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.Approved) != 0 {
t.Fatalf("approved artifacts = %#v, want no roster-only spell casts", output.Approved)
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
}
user := llmClient.requests[0].Messages[1].Content
if !strings.Contains(user, "Lightning Bolt") {
t.Fatalf("user prompt = %q, want roster-only spell in reference section", user)
}
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved empty extraction", output.Manifest.ValidationStatus)
}
}
func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) { func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
raw := readDNDSpellsFixture(t) raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t) resolved := resolveDNDSpellsPipeline(t)
@@ -155,6 +245,43 @@ func TestRunnerRejectsDNDSpellCastWithInvalidSourceRef(t *testing.T) {
} }
} }
func dndSpellsReferenceSet(roster string, glossary string) contracts.ReferenceSet {
slots := make(map[string]contracts.ResolvedReferenceSlot)
if strings.TrimSpace(roster) != "" {
slots["roster"] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain; charset=utf-8",
Content: []byte(roster),
Digest: "sha256:roster",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
SizeBytes: int64(len(roster)),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
}
}
if strings.TrimSpace(glossary) != "" {
slots["glossary"] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{
SlotName: "glossary",
MediaType: "text/plain; charset=utf-8",
Content: []byte(glossary),
Digest: "sha256:glossary",
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///tmp/glossary.txt"},
SizeBytes: int64(len(glossary)),
BindingSource: contracts.ReferenceBindingSourceConfig,
},
},
}
}
return contracts.ReferenceSet{Slots: slots}
}
func TestRunnerFailsWhenDNDSpellsExtractorReturnsMalformedOutput(t *testing.T) { func TestRunnerFailsWhenDNDSpellsExtractorReturnsMalformedOutput(t *testing.T) {
raw := readDNDSpellsFixture(t) raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t) resolved := resolveDNDSpellsPipeline(t)

View File

@@ -20,6 +20,7 @@ const (
reasonMissingRequiredField = "missing_required_field" reasonMissingRequiredField = "missing_required_field"
reasonMissingSourceRef = "missing_source_ref" reasonMissingSourceRef = "missing_source_ref"
reasonInvalidSourceRef = "invalid_source_ref" reasonInvalidSourceRef = "invalid_source_ref"
reasonSpellNotNearSource = "spell_not_near_source"
) )
var _ contracts.Validator = ShapeValidator{} var _ contracts.Validator = ShapeValidator{}
@@ -54,12 +55,15 @@ func (validator SourceRefValidator) Validate(ctx context.Context, req contracts.
} }
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates)) decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
var warnings []contracts.Warning
for _, candidate := range req.Candidates { for _, candidate := range req.Candidates {
decisions = append(decisions, validateSourceRefs(req.Source, candidate)) decisions = append(decisions, validateSourceRefs(req.Source, candidate))
warnings = append(warnings, sourceRelatednessWarnings(req.Source, candidate)...)
} }
return contracts.ValidationResult{ return contracts.ValidationResult{
ValidatorName: validator.Name(), ValidatorName: validator.Name(),
Decisions: decisions, Decisions: decisions,
Warnings: warnings,
}, nil }, nil
} }
@@ -88,6 +92,66 @@ func validateSourceRefs(doc *source.SourceDocument, candidate artifacts.Artifact
return validate.Approved(candidate.Index) return validate.Approved(candidate.Index)
} }
func sourceRelatednessWarnings(doc *source.SourceDocument, candidate artifacts.ArtifactCandidate) []contracts.Warning {
if doc == nil || len(candidate.SourceRefs) == 0 {
return nil
}
var payload SpellCast
if err := json.Unmarshal(candidate.Payload, &payload); err != nil {
return nil
}
spell := strings.TrimSpace(payload.Spell)
if spell == "" {
return nil
}
needle := strings.ToLower(spell)
for _, ref := range candidate.SourceRefs {
text, ok := sourceRefText(doc, ref)
if !ok {
continue
}
if strings.Contains(strings.ToLower(text), needle) {
return nil
}
}
return []contracts.Warning{
{
Scope: fmt.Sprintf("candidate.%d", candidate.Index),
ReasonCode: reasonSpellNotNearSource,
Message: fmt.Sprintf("spell %q was not found in the cited source text", spell),
},
}
}
func sourceRefText(doc *source.SourceDocument, ref source.SourceRef) (string, bool) {
if err := source.ValidateRef(doc, ref); err != nil {
return "", false
}
start := -1
end := -1
for i, unit := range doc.Units {
if unit.ID == ref.StartUnitID {
start = i
}
if unit.ID == ref.EndUnitID {
end = i
}
}
if start < 0 || end < start {
return "", false
}
var b strings.Builder
for i := start; i <= end; i++ {
if b.Len() > 0 {
b.WriteString("\n")
}
b.WriteString(doc.Units[i].Text)
}
return b.String(), true
}
func requiredSpellCastFields(payload SpellCast) []struct { func requiredSpellCastFields(payload SpellCast) []struct {
name string name string
value string value string

View File

@@ -51,6 +51,9 @@ func TestValidatorsApproveValidCandidate(t *testing.T) {
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err) t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
} }
assertSingleDecision(t, sourceRefResult, sourceRefValidatorName, 7, true, validate.ReasonApproved) assertSingleDecision(t, sourceRefResult, sourceRefValidatorName, 7, true, validate.ReasonApproved)
if len(sourceRefResult.Warnings) != 0 {
t.Fatalf("warnings = %#v, want none", sourceRefResult.Warnings)
}
} }
func TestShapeValidatorRejectsMalformedPayload(t *testing.T) { func TestShapeValidatorRejectsMalformedPayload(t *testing.T) {
@@ -167,6 +170,29 @@ func TestSourceRefValidatorRejectsInvalidRefs(t *testing.T) {
} }
} }
func TestSourceRefValidatorWarnsWhenSpellNameIsNotInCitedSource(t *testing.T) {
candidate := validSpellCandidate(31)
payload := validSpellPayload()
payload.Spell = "Shield"
candidate.Payload = mustSpellPayload(t, payload)
result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Source: promptSourceDocument(),
Candidates: []artifacts.ArtifactCandidate{candidate},
})
if err != nil {
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, result, sourceRefValidatorName, 31, true, validate.ReasonApproved)
if len(result.Warnings) != 1 {
t.Fatalf("warnings = %#v, want one relatedness warning", result.Warnings)
}
warning := result.Warnings[0]
if warning.ReasonCode != reasonSpellNotNearSource || !strings.Contains(warning.Message, "Shield") {
t.Fatalf("warning = %#v, want spell relatedness warning", warning)
}
}
func TestSourceRefValidatorRequiresSourceDocument(t *testing.T) { func TestSourceRefValidatorRequiresSourceDocument(t *testing.T) {
_, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{ _, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Candidates: []artifacts.ArtifactCandidate{validSpellCandidate(19)}, Candidates: []artifacts.ArtifactCandidate{validSpellCandidate(19)},

View File

@@ -209,6 +209,8 @@ type fakeChunker struct{}
func (fakeChunker) Key() string { return "fake/chunk" } func (fakeChunker) Key() string { return "fake/chunk" }
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) { func (fakeChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{}, nil return contracts.ChunkResult{}, nil
} }
@@ -221,6 +223,8 @@ func (fakeExtractor) ArtifactType() string { return "fake" }
func (fakeExtractor) SchemaVersion() string { return "v1" } func (fakeExtractor) SchemaVersion() string { return "v1" }
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeExtractor) Validators() []contracts.Validator { return nil } func (fakeExtractor) Validators() []contracts.Validator { return nil }
func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) { func (fakeExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {

View File

@@ -157,6 +157,10 @@ func (runnerSeriatimChunker) Key() string {
return "fake/chunk" return "fake/chunk"
} }
func (runnerSeriatimChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (runnerSeriatimChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) { func (runnerSeriatimChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{ return contracts.ChunkResult{
Chunks: []contracts.SourceChunk{ Chunks: []contracts.SourceChunk{
@@ -186,6 +190,10 @@ func (e *runnerSeriatimExtractor) SchemaVersion() string {
return "v1" return "v1"
} }
func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (e *runnerSeriatimExtractor) Validators() []contracts.Validator { func (e *runnerSeriatimExtractor) Validators() []contracts.Validator {
return nil return nil
} }

View File

@@ -25,6 +25,10 @@ func (n *Normalizer) Key() string {
return Key return Key
} }
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) { func (n *Normalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
if n == nil { if n == nil {
return contracts.NormalizeResult{}, normalizerErrorf("normalizer must not be nil") return contracts.NormalizeResult{}, normalizerErrorf("normalizer must not be nil")

View File

@@ -34,6 +34,13 @@ func TestModuleSpecAndRegister(t *testing.T) {
if !reflect.DeepEqual(spec, want) { if !reflect.DeepEqual(spec, want) {
t.Fatalf("registered spec = %#v, want %#v", spec, want) t.Fatalf("registered spec = %#v, want %#v", spec, want)
} }
normalizer, err := registry.Build(Key)
if err != nil {
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
}
if slots := normalizer.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
}
} }
func TestNormalizePassesThroughOrderAndValues(t *testing.T) { func TestNormalizePassesThroughOrderAndValues(t *testing.T) {

View File

@@ -142,6 +142,43 @@ func TestEncodePrettyPrintsJSON(t *testing.T) {
} }
} }
func TestEncodeIncludesManifestReferences(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{
RunID: "run-1",
References: []artifacts.ReferenceProvenance{
{
Stage: "extract",
LaneID: "events",
SlotName: "roster",
OriginType: "file",
OriginURI: "file:///tmp/roster.txt",
Digest: "sha256:reference",
MediaType: "text/plain; charset=utf-8",
SizeBytes: 12,
BindingSource: "config",
},
},
},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
manifest := decodeObject(t, fileBytes(t, result.Files, "manifest.json"))
references := manifest["references"].([]any)
if len(references) != 1 {
t.Fatalf("references = %#v, want one entry", references)
}
reference := references[0].(map[string]any)
if reference["lane_id"] != "events" || reference["slot_name"] != "roster" || reference["digest"] != "sha256:reference" {
t.Fatalf("reference manifest = %#v, want lane slot digest", reference)
}
if _, ok := reference["content"]; ok {
t.Fatalf("reference manifest = %#v, want no content field", reference)
}
}
func TestEncodeRejectsArtifactTypeWithoutSafeFileName(t *testing.T) { func TestEncodeRejectsArtifactTypeWithoutSafeFileName(t *testing.T) {
_, err := New().Encode(context.Background(), contracts.OutputRequest{ _, err := New().Encode(context.Background(), contracts.OutputRequest{
Approved: []artifacts.Artifact{artifact("///", "unsafe")}, Approved: []artifacts.Artifact{artifact("///", "unsafe")},