Compare commits
15 Commits
v0.1.0
...
f9999a73df
| Author | SHA1 | Date | |
|---|---|---|---|
| f9999a73df | |||
| 11d8187052 | |||
| 86bff552c1 | |||
| d3f790095e | |||
| 95218218e2 | |||
| e700df82d8 | |||
| e19cc02c4d | |||
| 8a5419448f | |||
| c8217549a8 | |||
| 2130414899 | |||
| 7f83a20fa6 | |||
| 317ab0472d | |||
| e5eb0ba5c8 | |||
| b95af4f87d | |||
| 11073b613c |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -1,3 +1,6 @@
|
|||||||
|
# build artifacts
|
||||||
|
./notarius
|
||||||
|
|
||||||
# ---> Go
|
# ---> Go
|
||||||
# If you prefer the allow list template instead of the deny list, see community template:
|
# If you prefer the allow list template instead of the deny list, see community template:
|
||||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||||
@@ -49,6 +52,7 @@ go.work.sum
|
|||||||
# Icon must end with two \r
|
# Icon must end with two \r
|
||||||
Icon
|
Icon
|
||||||
|
|
||||||
|
|
||||||
# Thumbnails
|
# Thumbnails
|
||||||
._*
|
._*
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,6 @@
|
|||||||
Please carefully review the documents in `docs/policy` before making any changes to this repository.
|
Please review `docs/internal/overview.md` for initial orientation in this repository.
|
||||||
- `architecture.md` provides the canonical high-level architecture policy for this repository.
|
|
||||||
- `documentation.md` provides the canonical documentation policy for this repository.
|
Additionally, please carefully review the relevant documents in `docs/policy` before making any changes to this repository.
|
||||||
|
- `development.md` defines the contributor workflow for this application.
|
||||||
|
- `architecture.md` provides the canonical high-level architecture policy for this repository, and should be reviewed before writing or changing any code.
|
||||||
|
- `documentation.md` provides the canonical documentation policy for this repository, and should be reviewed before writing or changing any documentation.
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ go run ./cmd/notarius pipelines list \
|
|||||||
The production CLI currently registers these module keys:
|
The production CLI currently registers these module keys:
|
||||||
|
|
||||||
- input: `seriatim`
|
- input: `seriatim`
|
||||||
- chunk: `generic`
|
- chunk: `generic`, `dnd/scenes`
|
||||||
- extract: `dnd/spells`
|
- extract: `dnd/spells`
|
||||||
- merge: `appendorder`
|
- merge: `appendorder`
|
||||||
- normalize: `noop`
|
- normalize: `noop`
|
||||||
|
|||||||
@@ -167,6 +167,7 @@ one configured profile.
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| input | `seriatim` | Reads Seriatim transcript JSON. |
|
| input | `seriatim` | Reads Seriatim transcript JSON. |
|
||||||
| chunk | `generic` | Splits source units into ordered chunks. |
|
| chunk | `generic` | Splits source units into ordered chunks. |
|
||||||
|
| chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. |
|
||||||
| extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. |
|
| extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. |
|
||||||
| merge | `appendorder` | Keeps candidates in append order. |
|
| merge | `appendorder` | Keeps candidates in append order. |
|
||||||
| normalize | `noop` | Passes merged artifacts through unchanged. |
|
| normalize | `noop` | Passes merged artifacts through unchanged. |
|
||||||
@@ -178,6 +179,9 @@ The `generic` chunker accepts:
|
|||||||
- `overlap_units`: non-negative integer, default `0`, and must be less than
|
- `overlap_units`: non-negative integer, default `0`, and must be less than
|
||||||
`max_units`.
|
`max_units`.
|
||||||
|
|
||||||
|
The `dnd/scenes` chunker requires transcript source capabilities, calls the
|
||||||
|
configured structured LLM provider, and does not accept module options.
|
||||||
|
|
||||||
## Diagnostics
|
## Diagnostics
|
||||||
|
|
||||||
`diagnostics` fields:
|
`diagnostics` fields:
|
||||||
|
|||||||
@@ -57,7 +57,19 @@ approved.
|
|||||||
"pipeline_id": "dnd-session",
|
"pipeline_id": "dnd-session",
|
||||||
"pipeline_digest": "sha256:...",
|
"pipeline_digest": "sha256:...",
|
||||||
"input_module": "seriatim",
|
"input_module": "seriatim",
|
||||||
"chunker": "generic",
|
"chunker": "dnd/scenes",
|
||||||
|
"module_metadata": {
|
||||||
|
"chunker": {
|
||||||
|
"prompt_id": "dnd.scenes",
|
||||||
|
"prompt_version": "v1",
|
||||||
|
"prompt_sha256": "sha256:...",
|
||||||
|
"response_schema_key": "dnd_scenes",
|
||||||
|
"response_schema_id": "notarius.dnd.scenes",
|
||||||
|
"response_schema_name": "notarius_dnd_scenes_v1",
|
||||||
|
"response_schema_version": "v1",
|
||||||
|
"response_schema_sha256": "sha256:..."
|
||||||
|
}
|
||||||
|
},
|
||||||
"source_digests": ["sha256:..."],
|
"source_digests": ["sha256:..."],
|
||||||
"extractors": ["dnd/spells"],
|
"extractors": ["dnd/spells"],
|
||||||
"merger": "appendorder",
|
"merger": "appendorder",
|
||||||
@@ -86,9 +98,15 @@ approved.
|
|||||||
|
|
||||||
Fields with empty values may be omitted by JSON encoding.
|
Fields with empty values may be omitted by JSON encoding.
|
||||||
|
|
||||||
|
`module_metadata` is omitted when no singleton module provides metadata.
|
||||||
|
|
||||||
`validation_status` is `approved` when no candidates were rejected and
|
`validation_status` is `approved` when no candidates were rejected and
|
||||||
`rejected` when one or more candidates were rejected.
|
`rejected` when one or more candidates were rejected.
|
||||||
|
|
||||||
|
Top-level `module_metadata` is reserved for singleton pipeline modules
|
||||||
|
(`input`, `chunker`, and `output`). Lane-owned module metadata remains under
|
||||||
|
`artifact_lanes[].metadata`.
|
||||||
|
|
||||||
## Artifact Files
|
## Artifact Files
|
||||||
|
|
||||||
Each artifact file has this shape:
|
Each artifact file has this shape:
|
||||||
|
|||||||
@@ -20,6 +20,11 @@ A production module package should provide:
|
|||||||
Module specs should describe capabilities accurately. Resolution uses specs to
|
Module specs should describe capabilities accurately. Resolution uses specs to
|
||||||
reject incompatible pipelines before execution.
|
reject incompatible pipelines before execution.
|
||||||
|
|
||||||
|
Chunk modules receive the structured LLM client through `contracts.ChunkRequest`
|
||||||
|
when they need model-backed chunking. The pipeline runner validates generic
|
||||||
|
chunk result invariants before extraction; module-owned policies may be stricter
|
||||||
|
but must stay within the module package.
|
||||||
|
|
||||||
## `seriatim` Input
|
## `seriatim` Input
|
||||||
|
|
||||||
Package: `internal/modules/input/seriatim`
|
Package: `internal/modules/input/seriatim`
|
||||||
@@ -44,6 +49,10 @@ The `generic` chunker splits source units into ordered chunks. It validates the
|
|||||||
source document, clones source units, assigns chunk IDs such as `chunk-000001`,
|
source document, clones source units, assigns chunk IDs such as `chunk-000001`,
|
||||||
and records chunk metadata for start unit, end unit, and unit count.
|
and records chunk metadata for start unit, end unit, and unit count.
|
||||||
|
|
||||||
|
The pipeline runner canonicalizes chunk units from the source document by ID
|
||||||
|
before extractors and mergers run. Chunker-owned context should stay in
|
||||||
|
`SourceChunk.Metadata`.
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
|
|
||||||
- `max_units`: positive integer, default `50`;
|
- `max_units`: positive integer, default `50`;
|
||||||
@@ -54,6 +63,39 @@ Provides:
|
|||||||
|
|
||||||
- `chunks`
|
- `chunks`
|
||||||
|
|
||||||
|
## `dnd/scenes` Chunker
|
||||||
|
|
||||||
|
Package: `internal/modules/chunk/dnd/scenes`
|
||||||
|
|
||||||
|
The `dnd/scenes` chunker uses the structured LLM client to divide transcript
|
||||||
|
source units into coherent D&D scenes. It renders embedded prompts, loads the
|
||||||
|
embedded structured response schema, validates model-authored source-unit
|
||||||
|
boundaries, and converts each scene into a deterministic source chunk.
|
||||||
|
|
||||||
|
Requires:
|
||||||
|
|
||||||
|
- `source.transcript`
|
||||||
|
|
||||||
|
Provides:
|
||||||
|
|
||||||
|
- `chunks`
|
||||||
|
- `chunks.scenes`
|
||||||
|
|
||||||
|
Options: none. Non-empty options are rejected.
|
||||||
|
|
||||||
|
The chunker enforces full source-unit coverage from the first source unit to the
|
||||||
|
last, exact source-unit IDs, sequential contiguous scenes, and no overlap. It
|
||||||
|
assigns chunk IDs such as `scene-000001` and stores scene metadata including
|
||||||
|
title, primary mode, participants, summary, boundary note, confidence, boundary
|
||||||
|
unit IDs, and unit count. Boundary caveats become warnings with reason code
|
||||||
|
`scene_boundary_caveat`. Whitespace-only caveats are treated as malformed
|
||||||
|
structured output rather than silently dropped.
|
||||||
|
|
||||||
|
Malformed model output fails explicitly rather than falling back to another
|
||||||
|
chunker. The chunker exposes prompt and response-schema provenance through
|
||||||
|
top-level `module_metadata.chunker` without raw prompts, raw schemas, source
|
||||||
|
text, or secrets.
|
||||||
|
|
||||||
## `dnd/spells` Extractor
|
## `dnd/spells` Extractor
|
||||||
|
|
||||||
Package: `internal/modules/extract/dnd/spells`
|
Package: `internal/modules/extract/dnd/spells`
|
||||||
@@ -78,7 +120,8 @@ Artifact type and schema version:
|
|||||||
- schema version: `v1`
|
- schema version: `v1`
|
||||||
|
|
||||||
The extractor adds prompt and response-schema provenance to lane manifest
|
The extractor adds prompt and response-schema provenance to lane manifest
|
||||||
metadata. Durable artifact payload details belong in the
|
metadata under `artifact_lanes[].metadata.extractor`. Durable artifact payload
|
||||||
|
details belong in the
|
||||||
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
|
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
|
||||||
|
|
||||||
## D&D Spell Validators
|
## D&D Spell Validators
|
||||||
|
|||||||
@@ -73,8 +73,39 @@ The runner:
|
|||||||
2. builds the input adapter and parses the raw input into a source document;
|
2. builds the input adapter and parses the raw input into a source document;
|
||||||
3. validates the source document;
|
3. validates the source document;
|
||||||
4. builds the chunker and produces source chunks;
|
4. builds the chunker and produces source chunks;
|
||||||
5. runs each selected artifact lane in sorted resolved order;
|
5. validates source chunks against framework invariants;
|
||||||
6. builds the output encoder and validates logical output file names.
|
6. runs each selected artifact lane in sorted resolved order;
|
||||||
|
7. builds the output encoder and validates logical output file names.
|
||||||
|
|
||||||
|
## Chunk Results
|
||||||
|
|
||||||
|
Chunkers implement `contracts.Chunker` and receive a `contracts.ChunkRequest`
|
||||||
|
with the validated source document, the structured LLM client, the configured
|
||||||
|
LLM profile, module options, and run metadata. Deterministic and LLM-backed
|
||||||
|
chunkers use the same contract; provider construction stays outside chunk
|
||||||
|
modules.
|
||||||
|
|
||||||
|
After `Chunk` returns, the runner appends chunker warnings before returning any
|
||||||
|
chunker error. When chunking succeeds, the runner validates generic chunk
|
||||||
|
invariants before running extractors:
|
||||||
|
|
||||||
|
- chunk IDs must be non-empty and unique in the chunk result;
|
||||||
|
- each chunk `SourceID` must match the source document ID;
|
||||||
|
- each chunk `Index` must match its zero-based returned order;
|
||||||
|
- each chunk must contain at least one source unit;
|
||||||
|
- a chunk must not repeat a source unit;
|
||||||
|
- every chunk source unit must exist in the source document;
|
||||||
|
- source units inside each chunk must appear in source-document order.
|
||||||
|
|
||||||
|
After validation, the runner rebuilds each chunk from source-document units by
|
||||||
|
ID, preserving the chunk boundary order and cloning chunk metadata. Extractors
|
||||||
|
and downstream stages therefore see canonical source units, while
|
||||||
|
`SourceChunk.Metadata` remains the supported place for chunker-owned context.
|
||||||
|
|
||||||
|
The framework does not require complete source-unit coverage and does not reject
|
||||||
|
overlap between different chunks. Stricter policies, such as full coverage or
|
||||||
|
non-overlap, belong to individual chunk modules when they are part of that
|
||||||
|
module's contract.
|
||||||
|
|
||||||
Within an artifact lane, the runner:
|
Within an artifact lane, the runner:
|
||||||
|
|
||||||
@@ -119,9 +150,15 @@ On successful execution, the manifest validation status is:
|
|||||||
|
|
||||||
## Manifest Population
|
## Manifest Population
|
||||||
|
|
||||||
The manifest records run ID, pipeline ID, pipeline digest, module keys, artifact
|
The manifest records run ID, pipeline ID, pipeline digest, module keys, top-level
|
||||||
lanes, LLM profile metadata, source digest, validation status, and timing.
|
module metadata, artifact lanes, LLM profile metadata, source digest,
|
||||||
|
validation status, and timing.
|
||||||
|
|
||||||
Modules can add non-secret manifest metadata by implementing
|
Singleton pipeline modules may add non-secret metadata by implementing
|
||||||
`contracts.ManifestMetadataProvider`. The D&D spell extractor uses this for
|
`contracts.ManifestMetadataProvider`. The runner records that metadata under
|
||||||
|
`module_metadata` with stable keys for `input`, `chunker`, and `output`.
|
||||||
|
|
||||||
|
Lane-owned modules may add non-secret metadata through
|
||||||
|
`artifact_lanes[].metadata`. The runner records extractor, merger, and
|
||||||
|
normalizer metadata there. The D&D spell extractor uses lane metadata for
|
||||||
prompt and response-schema provenance.
|
prompt and response-schema provenance.
|
||||||
|
|||||||
@@ -34,8 +34,8 @@ The `json` output module writes these files:
|
|||||||
|
|
||||||
- `index.json`: file index with paths to the manifest, artifact files,
|
- `index.json`: file index with paths to the manifest, artifact files,
|
||||||
rejected artifacts, and warnings.
|
rejected artifacts, and warnings.
|
||||||
- `manifest.json`: run manifest with resolved pipeline provenance, module keys,
|
- `manifest.json`: run manifest with resolved pipeline provenance, top-level
|
||||||
validation status, and timing.
|
module metadata, module keys, validation status, and timing.
|
||||||
- `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact
|
- `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact
|
||||||
type. For the current D&D spell extractor, this includes
|
type. For the current D&D spell extractor, this includes
|
||||||
`artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved.
|
`artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved.
|
||||||
@@ -63,7 +63,7 @@ Implemented diagnostics artifacts:
|
|||||||
- `effective-config.json`: resolved config with API keys redacted.
|
- `effective-config.json`: resolved config with API keys redacted.
|
||||||
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
|
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
|
||||||
- `run-manifest.json`: the same run manifest written to durable output when it
|
- `run-manifest.json`: the same run manifest written to durable output when it
|
||||||
is available.
|
is available, including top-level module metadata when present.
|
||||||
- `warnings.json`: warning list.
|
- `warnings.json`: warning list.
|
||||||
- `run-report.json`: counts, status, output path, diagnostics path, and run ID.
|
- `run-report.json`: counts, status, output path, diagnostics path, and run ID.
|
||||||
- `error.log`: failure message, written after diagnostics directory creation
|
- `error.log`: failure message, written after diagnostics directory creation
|
||||||
|
|||||||
305
docs/roadmap/implementation.md
Normal file
305
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,305 @@
|
|||||||
|
# Extraction References Implementation Plan
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Implement the extraction-reference feature described in
|
||||||
|
[references.md](references.md). This plan is decision-complete for an LLM coding
|
||||||
|
agent: implement each stage in order, keep the repository compiling after each
|
||||||
|
stage, and do not move planned behavior into non-roadmap docs until the relevant
|
||||||
|
behavior exists.
|
||||||
|
|
||||||
|
Core decisions to preserve:
|
||||||
|
|
||||||
|
- references are opaque framework inputs and domain semantics stay in extract
|
||||||
|
modules;
|
||||||
|
- references are not evidence and must not be addressable through `SourceRef`;
|
||||||
|
- reference binding is lane-scoped;
|
||||||
|
- extractors expose `ReferenceSlots()` directly on the first-class extractor
|
||||||
|
contract;
|
||||||
|
- token budgeting is deferred; enforce only UTF-8 text handling, empty-file
|
||||||
|
warnings, and declared `MaxBytes`;
|
||||||
|
- run manifests record references in a dedicated section, separate from
|
||||||
|
`source_digests`;
|
||||||
|
- CLI unbinding uses `--without-reference`.
|
||||||
|
|
||||||
|
## Stage 1: Contracts and Mechanical Adoption
|
||||||
|
|
||||||
|
Add the framework contracts needed to describe references without changing
|
||||||
|
runtime behavior. Slot declarations must be available without constructing
|
||||||
|
extractor modules, because pipeline/config validation should use registry
|
||||||
|
metadata rather than runtime module instances.
|
||||||
|
|
||||||
|
Implementation steps:
|
||||||
|
|
||||||
|
- In `internal/framework/contracts`, add reference model types:
|
||||||
|
`ReferenceSlot`, `ReferenceOrigin`, `ReferenceItem`,
|
||||||
|
`ResolvedReferenceSlot`, `ReferenceSet`, and a binding-source enum or string
|
||||||
|
constants for `config` and `cli`.
|
||||||
|
- Include `Name`, `Description`, `Required`, `AcceptedMediaTypes`, `Multiple`,
|
||||||
|
and `MaxBytes` on `ReferenceSlot`.
|
||||||
|
- Include slot name, media type, content bytes, digest, origin, size bytes, and
|
||||||
|
binding source on `ReferenceItem`.
|
||||||
|
- Add `References ReferenceSet` to `contracts.ExtractionRequest`.
|
||||||
|
- Add `ReferenceSlots() []ReferenceSlot` to `contracts.Extractor`.
|
||||||
|
- Extend extractor registration metadata so reference slots are also declared
|
||||||
|
through the extractor's registry spec. Prefer the smallest idiomatic change to
|
||||||
|
the existing registry model, such as adding `ReferenceSlots` to `ModuleSpec`
|
||||||
|
with validation that non-extractor modules leave it empty, unless the codebase
|
||||||
|
shape clearly supports a narrower extractor-specific spec.
|
||||||
|
- Update every concrete extractor and all extractor fakes/test doubles to
|
||||||
|
implement `ReferenceSlots()`. Existing extractors without references should
|
||||||
|
return `nil`.
|
||||||
|
- Add tests that compare a production extractor's runtime `ReferenceSlots()`
|
||||||
|
with its registered spec slots so the two declarations cannot drift.
|
||||||
|
- Add contract tests for empty reference sets, slot copying expectations if
|
||||||
|
helpers are introduced, and compile-time coverage through existing fakes.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
|
||||||
|
- `go test ./internal/framework/contracts`
|
||||||
|
- `go test ./internal/framework/pipeline`
|
||||||
|
- `go test ./...`
|
||||||
|
|
||||||
|
## Stage 2: Config Shape and Pipeline-Level Resolution
|
||||||
|
|
||||||
|
Add unresolved reference bindings to config and resolved lane bindings to the
|
||||||
|
pipeline model. Do not read reference files in this stage.
|
||||||
|
|
||||||
|
Implementation steps:
|
||||||
|
|
||||||
|
- Add `references` maps to file config parsing at both pipeline and artifact
|
||||||
|
lane level.
|
||||||
|
- Add corresponding fields to `pipeline.PipelineProfile` and
|
||||||
|
`pipeline.ArtifactLaneProfile`.
|
||||||
|
- Preserve deterministic map handling and duplicate-after-trim validation.
|
||||||
|
- Extend config cloning, effective config, redaction, validation, and tests for
|
||||||
|
the new fields.
|
||||||
|
- Add resolved reference binding structures to `internal/framework/pipeline`.
|
||||||
|
They should represent lane ID, slot name, source URI/path, and binding source,
|
||||||
|
but not file bytes.
|
||||||
|
- During `pipeline.ResolvePipeline`, collect selected lanes, read each lane
|
||||||
|
extractor's declared slots from registry metadata, and validate without
|
||||||
|
building extractor instances:
|
||||||
|
- every bound slot is declared by the lane extractor;
|
||||||
|
- required slots are bound after applying pipeline-level and lane-level config;
|
||||||
|
- required slots remain bound after any CLI unbinds supplied to resolution;
|
||||||
|
- selected lanes under `--only` are the only lanes considered.
|
||||||
|
- Apply pipeline-level bindings as defaults only to lanes whose extractor
|
||||||
|
declares the matching slot.
|
||||||
|
- Apply lane-level bindings as overrides or additions for that lane.
|
||||||
|
- Keep reference bindings out of source digests and artifact source references.
|
||||||
|
- Add tests proving reference-slot validation works through registry specs even
|
||||||
|
when extractor constructors would fail if called.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
|
||||||
|
- `go test ./internal/core/config`
|
||||||
|
- `go test ./internal/framework/pipeline`
|
||||||
|
- `go test ./...`
|
||||||
|
|
||||||
|
## Stage 3: CLI Reference Overrides and Unbinds
|
||||||
|
|
||||||
|
Add run-time CLI syntax for reference binding overrides and optional unbinding.
|
||||||
|
|
||||||
|
Implementation steps:
|
||||||
|
|
||||||
|
- Add repeatable `--reference` flags to `notarius run`.
|
||||||
|
Accepted forms:
|
||||||
|
- `slot=path` for unambiguous slot names across selected lanes;
|
||||||
|
- `lane.slot=path` for explicit lane-scoped binding.
|
||||||
|
- Add repeatable `--without-reference` flags to `notarius run`.
|
||||||
|
Accepted forms:
|
||||||
|
- `slot`;
|
||||||
|
- `lane.slot`.
|
||||||
|
- Reject empty paths for `--reference`; use `--without-reference` for unbinding.
|
||||||
|
- Reject malformed values with concise CLI errors before expensive work.
|
||||||
|
- Pass parsed override/unbind requests into config/pipeline resolution.
|
||||||
|
- Resolve flat CLI names only when exactly one selected lane declares the slot.
|
||||||
|
If multiple selected lanes declare the same slot, fail and instruct the user
|
||||||
|
to use `lane.slot`.
|
||||||
|
- Let CLI bindings override config bindings for the same lane and slot.
|
||||||
|
- Let CLI unbinds remove config-bound optional slots for the same lane and slot.
|
||||||
|
- Fail if unbinding leaves a required slot unbound.
|
||||||
|
- Add CLI tests for flat binding, lane-qualified binding, ambiguous flat
|
||||||
|
binding, malformed syntax, optional unbind, and required-slot unbind failure.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
|
||||||
|
- `go test ./internal/cli`
|
||||||
|
- `go test ./internal/core/config`
|
||||||
|
- `go test ./internal/framework/pipeline`
|
||||||
|
- `go test ./...`
|
||||||
|
|
||||||
|
## Stage 4: Run Preparation and Reference Materialization
|
||||||
|
|
||||||
|
Read, validate, digest, and materialize resolved file references before any LLM
|
||||||
|
call.
|
||||||
|
|
||||||
|
Implementation steps:
|
||||||
|
|
||||||
|
- Add a reference resolver/materializer near pipeline run preparation. Keep file
|
||||||
|
I/O out of pure config parsing.
|
||||||
|
- Ensure run preparation receives the loaded config path or config directory so
|
||||||
|
config-relative reference paths can be resolved after pure config parsing.
|
||||||
|
- Resolve config-relative paths relative to the config file path and
|
||||||
|
CLI-relative paths relative to the current working directory.
|
||||||
|
- For MVP, accept only UTF-8 text files. Reject non-UTF-8 content with an error
|
||||||
|
naming pipeline, lane, slot, and path.
|
||||||
|
- Compute `sha256:` content digests over the raw reference bytes.
|
||||||
|
- Populate `ReferenceItem` values with content bytes, media type, digest,
|
||||||
|
origin type `file`, normalized origin URI/path, size bytes, and binding source.
|
||||||
|
- Enforce declared `MaxBytes` when greater than zero. The error should name the
|
||||||
|
pipeline, lane, slot, actual size, limit, and path.
|
||||||
|
- Emit a warning for empty bound files, but do not fail.
|
||||||
|
- Add `ReferenceSet` values to the runner input or resolved pipeline path in a
|
||||||
|
way that keeps lane-scoped references available when calling each extractor.
|
||||||
|
- Pass the correct lane-specific `ReferenceSet` into
|
||||||
|
`contracts.ExtractionRequest`.
|
||||||
|
- Ensure no reference content is written to ordinary diagnostics, logs, errors,
|
||||||
|
or manifests.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
|
||||||
|
- Focused resolver/materializer tests for path resolution, digest stability,
|
||||||
|
UTF-8 rejection, empty-file warning, `MaxBytes`, and binding source.
|
||||||
|
- `go test ./internal/cli`
|
||||||
|
- `go test ./internal/framework/pipeline`
|
||||||
|
- `go test ./...`
|
||||||
|
|
||||||
|
## Stage 5: Prompt Template Reference Functions
|
||||||
|
|
||||||
|
Make references available to module-owned prompt templates.
|
||||||
|
|
||||||
|
Implementation steps:
|
||||||
|
|
||||||
|
- Extend `internal/framework/prompt` so prompt bundles can be compiled with
|
||||||
|
declared reference slots.
|
||||||
|
- Add `reference` and `hasreference` template functions.
|
||||||
|
- Validate at bundle build time, or the earliest feasible equivalent, that
|
||||||
|
templates reference only declared slots.
|
||||||
|
- Render a declared but unbound optional slot as an empty string.
|
||||||
|
- Ensure `hasreference` returns true only when the slot has at least one bound
|
||||||
|
item with content.
|
||||||
|
- Render multiple items deterministically if future `Multiple` support is
|
||||||
|
enabled; for MVP, reject multiple bindings unless the slot declares
|
||||||
|
`Multiple`.
|
||||||
|
- Keep prompt metadata hashes based on template source. Do not include rendered
|
||||||
|
reference content in prompt identity.
|
||||||
|
- Add deterministic rendering tests proving byte-identical output across runs
|
||||||
|
with the same reference bytes and config.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
|
||||||
|
- `go test ./internal/framework/prompt`
|
||||||
|
- `go test ./internal/modules/extract/dnd/spells`
|
||||||
|
- `go test ./...`
|
||||||
|
|
||||||
|
## Stage 6: Manifest and Diagnostics Provenance
|
||||||
|
|
||||||
|
Record reference provenance separately from source provenance.
|
||||||
|
|
||||||
|
Implementation steps:
|
||||||
|
|
||||||
|
- Add a dedicated references section to `artifacts.RunManifest`.
|
||||||
|
The shape should be lane-scoped and include lane ID, slot name, origin type,
|
||||||
|
origin URI/path, digest, media type, size bytes, and binding source.
|
||||||
|
- Do not add reference digests to `source_digests`.
|
||||||
|
- Include reference digests in any cache/idempotency key if such a key exists.
|
||||||
|
If no cache/idempotency key exists, add a test or comment documenting that no
|
||||||
|
additional key needs updating yet.
|
||||||
|
- Write a diagnostics artifact for resolved references that contains provenance
|
||||||
|
only, not full content, consistent with redacted effective config behavior.
|
||||||
|
- Ensure durable JSON output manifests include the new manifest section.
|
||||||
|
- Add manifest round-trip tests and a CLI/run test where two runs that differ
|
||||||
|
only in reference bytes produce distinguishable manifests.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
|
||||||
|
- `go test ./internal/core/artifacts`
|
||||||
|
- `go test ./internal/core/diagnostics`
|
||||||
|
- `go test ./internal/modules/output/json`
|
||||||
|
- `go test ./internal/cli`
|
||||||
|
- `go test ./...`
|
||||||
|
|
||||||
|
## Stage 7: D&D Spells Consumer
|
||||||
|
|
||||||
|
Use the new reference feature in the first production extractor.
|
||||||
|
|
||||||
|
Implementation steps:
|
||||||
|
|
||||||
|
- Declare optional `roster` and `glossary` slots on `dnd/spells`.
|
||||||
|
- Set accepted media type to text/UTF-8. Add conservative `MaxBytes` limits only
|
||||||
|
if a clear module-owned limit is chosen; otherwise leave `MaxBytes` unset.
|
||||||
|
- Update the D&D spells prompt bundle to include conditional reference sections
|
||||||
|
using `hasreference` and `reference`.
|
||||||
|
- Frame references as supporting material only. The prompt must instruct the
|
||||||
|
model to extract only spell-cast events present in the source transcript and
|
||||||
|
use references only for disambiguation.
|
||||||
|
- Update prompt metadata tests as needed while preserving template-hash
|
||||||
|
semantics.
|
||||||
|
- Add fixture coverage with no references, with roster/glossary references, and
|
||||||
|
with a roster that mentions a spell never cast in the transcript. The last
|
||||||
|
case must assert no spell-cast artifact is produced for the uncast spell.
|
||||||
|
- If existing deterministic source-reference validation can be extended
|
||||||
|
cleanly, add warning-level relatedness checks for spell names or close
|
||||||
|
variants near cited source text. If this becomes large, defer that validator
|
||||||
|
enhancement to a separate roadmap item and keep the prompt/regression fixture
|
||||||
|
guard in this stage.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
|
||||||
|
- `go test ./internal/modules/extract/dnd/spells`
|
||||||
|
- `go test ./internal/framework/pipeline`
|
||||||
|
- `go test ./internal/cli`
|
||||||
|
- `go test ./...`
|
||||||
|
|
||||||
|
## Stage 8: Canonical Documentation and Examples
|
||||||
|
|
||||||
|
Move implemented behavior out of roadmap-only status once code exists.
|
||||||
|
|
||||||
|
Implementation steps:
|
||||||
|
|
||||||
|
- Update `docs/cli.md` with `--reference` and `--without-reference` syntax,
|
||||||
|
precedence, ambiguity behavior, and examples.
|
||||||
|
- Update `docs/config.md` with pipeline-level and lane-level `references`
|
||||||
|
blocks.
|
||||||
|
- Update `docs/internal/modules.md` or the most appropriate internal docs with
|
||||||
|
module-author guidance for `ReferenceSlots()`, reference request delivery,
|
||||||
|
prompt functions, evidence exclusion, and provenance.
|
||||||
|
- Update `docs/internal/pipeline.md` with reference resolution lifecycle and
|
||||||
|
lane-scoped delivery.
|
||||||
|
- Update `docs/integrations/json-output.md` with the manifest reference
|
||||||
|
provenance shape.
|
||||||
|
- Update `docs/operations.md` or `docs/troubleshooting.md` for common reference
|
||||||
|
errors such as unknown slot, ambiguous flat override, missing required slot,
|
||||||
|
unreadable file, non-UTF-8 content, and `MaxBytes` failures.
|
||||||
|
- Add maintained example reference files and update `examples/dnd-spells.config.yml`
|
||||||
|
only after the CLI/config behavior is implemented and covered by tests.
|
||||||
|
- Keep future-only material in `docs/roadmap/references.md`; do not duplicate
|
||||||
|
canonical current behavior there after implementation.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
|
||||||
|
- `rg -n "references:|--reference|--without-reference|ReferenceSlots|reference \"|hasreference" docs examples`
|
||||||
|
- `go test ./...`
|
||||||
|
- `go vet ./...`
|
||||||
|
- `go build ./cmd/notarius`
|
||||||
|
|
||||||
|
## Final Acceptance Criteria
|
||||||
|
|
||||||
|
The feature is complete when:
|
||||||
|
|
||||||
|
- extractor modules can declare reference slots through the first-class
|
||||||
|
extractor contract;
|
||||||
|
- config and CLI can bind and unbind lane-scoped file references;
|
||||||
|
- selected-pipeline validation catches unknown, ambiguous, or missing required
|
||||||
|
references before any LLM call;
|
||||||
|
- run preparation materializes UTF-8 text references with digests, size checks,
|
||||||
|
and empty-file warnings;
|
||||||
|
- extractors receive lane-scoped resolved references;
|
||||||
|
- prompt templates can render `reference` and `hasreference` deterministically;
|
||||||
|
- D&D spell extraction uses optional roster and glossary references;
|
||||||
|
- run manifests and diagnostics record reference provenance without recording
|
||||||
|
full content or treating references as source evidence;
|
||||||
|
- canonical docs and maintained examples describe only implemented behavior;
|
||||||
|
- `go test ./...`, `go vet ./...`, and `go build ./cmd/notarius` pass.
|
||||||
238
docs/roadmap/references.md
Normal file
238
docs/roadmap/references.md
Normal file
@@ -0,0 +1,238 @@
|
|||||||
|
# Feature Roadmap: Extraction References
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
This document defines the target state and policy choices for the planned
|
||||||
|
extraction-reference feature in Notarius. It describes planned behavior, not
|
||||||
|
implemented behavior. The staged implementation plan lives in
|
||||||
|
[implementation.md](implementation.md).
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Extraction quality improves when an LLM-backed extractor receives stable
|
||||||
|
reference material alongside the source input. For the initial D&D spell
|
||||||
|
extractor, useful reference material includes a party roster, a player list, and
|
||||||
|
a campaign glossary.
|
||||||
|
|
||||||
|
Notarius should support passing this material to extractors as **named reference
|
||||||
|
items** without introducing domain-specific concepts into core or framework
|
||||||
|
packages. The framework should know only that:
|
||||||
|
|
||||||
|
- extractors declare named reference slots they accept;
|
||||||
|
- pipeline config and CLI flags bind content, initially files, to those slots;
|
||||||
|
- bound content is rendered into module-owned prompt templates;
|
||||||
|
- bound content is digested and recorded as run provenance.
|
||||||
|
|
||||||
|
Only extract modules should know what a "roster" or "glossary" means. Domain
|
||||||
|
semantics live in module-owned slot declarations and prompt templates.
|
||||||
|
|
||||||
|
## Definitions
|
||||||
|
|
||||||
|
- **Reference slot**: a named, typed-by-convention input declared by an
|
||||||
|
extractor, with a human-readable description, required/optional status, and
|
||||||
|
optional guardrails such as accepted media types and maximum bytes.
|
||||||
|
- **Reference item**: resolved content bound to a slot for a given run: slot
|
||||||
|
name, content bytes, media type, content digest, size, origin, and binding
|
||||||
|
source.
|
||||||
|
- **Reference binding**: the association of a slot name to a content source,
|
||||||
|
defined in pipeline config and overridable per run via CLI.
|
||||||
|
- **Reference set**: the lane-scoped collection of resolved reference items
|
||||||
|
delivered to an extractor.
|
||||||
|
|
||||||
|
## Architectural Principles
|
||||||
|
|
||||||
|
- References are opaque to the framework. Core and framework packages must not
|
||||||
|
interpret reference content or recognize domain slot names.
|
||||||
|
- References are inputs. Anything that can change extraction output must be
|
||||||
|
digested into the run manifest and participate in any cache or idempotency key.
|
||||||
|
- References are not evidence. `SourceRef` values must only ever reference
|
||||||
|
source units. Reference items must not receive unit IDs and must not be
|
||||||
|
addressable by source references.
|
||||||
|
- Slots are declared, not ad hoc. Binding an undeclared slot name, or omitting a
|
||||||
|
required slot, should fail before any LLM call.
|
||||||
|
- Reference delivery is lane-scoped. Pipeline-level bindings may apply to
|
||||||
|
multiple lanes, but each lane receives only the references declared by its
|
||||||
|
extractor after pipeline, lane, CLI override, and CLI unbind rules are
|
||||||
|
resolved.
|
||||||
|
- Optional slots degrade gracefully. Prompt templates should render cleanly
|
||||||
|
whether or not an optional slot is bound.
|
||||||
|
- Rendering must be deterministic. Identical source input, config, prompts, and
|
||||||
|
reference bytes should produce byte-identical rendered prompts. Reference
|
||||||
|
slots should render in declaration order, with stable binding order within a
|
||||||
|
slot.
|
||||||
|
|
||||||
|
## Target Contracts
|
||||||
|
|
||||||
|
Extractors should declare accepted reference slots directly on the extractor
|
||||||
|
contract. This is a first-class feature, so mechanical updates to existing
|
||||||
|
extractors and test fakes are acceptable.
|
||||||
|
|
||||||
|
The target slot declaration includes:
|
||||||
|
|
||||||
|
- `Name`;
|
||||||
|
- `Description`;
|
||||||
|
- `Required`;
|
||||||
|
- `AcceptedMediaTypes`;
|
||||||
|
- `Multiple`;
|
||||||
|
- `MaxBytes`.
|
||||||
|
|
||||||
|
Extractors with no reference needs return an empty slot list.
|
||||||
|
|
||||||
|
Resolved reference items should be content-bearing values, not unresolved file
|
||||||
|
paths. The MVP producer is "read this file," but the item shape should permit
|
||||||
|
future producers such as prior-run artifacts, derived summaries, or entity
|
||||||
|
registries without changing extractor-facing contracts.
|
||||||
|
|
||||||
|
The extraction request should carry the lane-scoped resolved reference set.
|
||||||
|
Framework and core code should treat the set as opaque bytes plus metadata.
|
||||||
|
|
||||||
|
## Binding Lifecycle
|
||||||
|
|
||||||
|
Reference handling should be split across existing lifecycle boundaries:
|
||||||
|
|
||||||
|
1. Config parsing records pipeline-level and lane-level reference bindings
|
||||||
|
without reading files.
|
||||||
|
2. Pipeline resolution validates selected lanes, declared extractor slots,
|
||||||
|
missing required slots, unknown bindings, and ambiguous flat CLI bindings.
|
||||||
|
3. Run preparation resolves paths, reads files, validates media type and size,
|
||||||
|
computes digests, and materializes reference items.
|
||||||
|
4. Extraction receives the lane-specific resolved reference set.
|
||||||
|
|
||||||
|
Config-relative paths resolve relative to the config file. CLI-relative paths
|
||||||
|
resolve relative to the current working directory.
|
||||||
|
|
||||||
|
## Configuration and CLI
|
||||||
|
|
||||||
|
Reference bindings should live in pipeline config because the initial use cases
|
||||||
|
are campaign-invariant more often than run-variant. Bindings should be supported
|
||||||
|
at two levels:
|
||||||
|
|
||||||
|
- pipeline level: defaults shared by artifact lanes whose extractors declare
|
||||||
|
matching slots;
|
||||||
|
- lane level: additions or overrides for a single artifact lane.
|
||||||
|
|
||||||
|
Illustrative config shape:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
pipelines:
|
||||||
|
dnd-session:
|
||||||
|
input: seriatim
|
||||||
|
references:
|
||||||
|
roster: ./campaign/party_roster.md
|
||||||
|
glossary: ./campaign/glossary.md
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract: dnd/spells
|
||||||
|
npcs:
|
||||||
|
extract: dnd/npcs
|
||||||
|
references:
|
||||||
|
npc_registry: ./campaign/npcs.md
|
||||||
|
```
|
||||||
|
|
||||||
|
Per-run CLI binding overrides should be repeatable:
|
||||||
|
|
||||||
|
```text
|
||||||
|
notarius run dnd-session --input session-014.json --reference roster=./alt_roster.md
|
||||||
|
```
|
||||||
|
|
||||||
|
Flat CLI slot names are allowed when unambiguous across selected lanes. Lane
|
||||||
|
qualified names, such as `spells.roster=./alt_roster.md`, disambiguate or target
|
||||||
|
a specific lane. CLI bindings override config bindings for the same lane and
|
||||||
|
slot.
|
||||||
|
|
||||||
|
Users should also be able to unbind a config-bound optional slot for a run with
|
||||||
|
an explicit repeatable flag:
|
||||||
|
|
||||||
|
```text
|
||||||
|
notarius run dnd-session --input session-014.json --without-reference roster
|
||||||
|
```
|
||||||
|
|
||||||
|
Unbinding a required slot should fail during pipeline/reference resolution.
|
||||||
|
|
||||||
|
## Prompt Template Integration
|
||||||
|
|
||||||
|
Prompt templates are module-owned. Template rendering should expose:
|
||||||
|
|
||||||
|
- `{{ reference "roster" }}`: renders the content of the bound item;
|
||||||
|
- `{{ hasreference "glossary" }}`: predicate for conditional sections, so
|
||||||
|
optional slots can be included only when bound.
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- Referencing an undeclared slot from a template is a module bug and should fail
|
||||||
|
at prompt registration/build time or the earliest feasible equivalent.
|
||||||
|
- Referencing a declared but unbound optional slot should render as empty;
|
||||||
|
templates should use `hasreference` to avoid dangling section headers.
|
||||||
|
- Rendering must be deterministic and independent of map iteration order.
|
||||||
|
- Prompt identity should be computed over the template, not the rendered prompt.
|
||||||
|
Reference digests are recorded separately in the manifest so a reference edit
|
||||||
|
is visible as a reference change, not a prompt change.
|
||||||
|
|
||||||
|
Reference content is repeated in every per-chunk prompt in the MVP. Per-slot or
|
||||||
|
per-chunk inclusion policies are deferred until cost data justifies them.
|
||||||
|
|
||||||
|
## Provenance
|
||||||
|
|
||||||
|
The run manifest must record resolved references separately from source
|
||||||
|
digests. For every bound lane and slot, it should record:
|
||||||
|
|
||||||
|
- lane ID;
|
||||||
|
- slot name;
|
||||||
|
- origin type and URI;
|
||||||
|
- content digest;
|
||||||
|
- media type;
|
||||||
|
- size in bytes;
|
||||||
|
- whether the binding came from config or CLI override.
|
||||||
|
|
||||||
|
Reference digests must participate in any idempotency/cache key alongside source
|
||||||
|
digests, prompt hashes, schema versions, model, and parameters. Two runs that
|
||||||
|
differ only in reference content must be distinguishable from the manifest
|
||||||
|
alone.
|
||||||
|
|
||||||
|
Diagnostics for a run should include the resolved binding set with digests, not
|
||||||
|
full reference content, consistent with the existing redacted-effective-config
|
||||||
|
pattern.
|
||||||
|
|
||||||
|
## Validation and Guardrails
|
||||||
|
|
||||||
|
### References Are Not Evidence
|
||||||
|
|
||||||
|
The primary new failure mode is the model extracting facts from references
|
||||||
|
rather than from the source input. For example, a roster may list a player
|
||||||
|
character's known spells, and the model might emit a spell-cast artifact for a
|
||||||
|
spell that was never cast in the session.
|
||||||
|
|
||||||
|
Defenses, in priority order:
|
||||||
|
|
||||||
|
1. **Structural.** `SourceRef` remains the only grounding mechanism and can only
|
||||||
|
reference source units. No contract change should make references
|
||||||
|
addressable as evidence.
|
||||||
|
2. **Prompt discipline.** Module templates should frame references explicitly as
|
||||||
|
reference material, such as "use the roster to resolve speakers to
|
||||||
|
characters; extract only events that occur in the transcript."
|
||||||
|
3. **Validator support.** The source-reference validator, or a sibling
|
||||||
|
deterministic validator, should warn when referenced source text does not
|
||||||
|
plausibly relate to the extracted fact. Severity should be `warn`, not
|
||||||
|
`fail`, because transcripts can use paraphrase, nicknames, and abbreviations.
|
||||||
|
4. **Regression fixtures.** Tests should include a fixture in which a bound
|
||||||
|
roster mentions a spell that is never cast in the transcript, asserting no
|
||||||
|
artifact record is produced for it.
|
||||||
|
|
||||||
|
### Size and Sanity Guardrails
|
||||||
|
|
||||||
|
- MVP accepts UTF-8 text content only. Other media types should be rejected with
|
||||||
|
a clear error.
|
||||||
|
- A slot-level `MaxBytes` value should be enforced when declared.
|
||||||
|
- Empty bound files should produce a warning because they are likely user error.
|
||||||
|
|
||||||
|
## Out of Scope
|
||||||
|
|
||||||
|
- Token budgeting and model context-window management for references.
|
||||||
|
- Non-file reference producers, including prior-run artifacts, derived
|
||||||
|
summaries, and entity registries.
|
||||||
|
- Per-chunk or per-slot inclusion policies.
|
||||||
|
- Structured or parsed references such as typed roster schemas. References are
|
||||||
|
opaque text handed to prompts.
|
||||||
|
- Reference caching, preprocessing, summarization, embedding, or retrieval.
|
||||||
|
- Making references addressable as evidence in any form.
|
||||||
|
|
||||||
@@ -177,6 +177,34 @@ Fix:
|
|||||||
|
|
||||||
Provider error messages are redacted for configured API key values.
|
Provider error messages are redacted for configured API key values.
|
||||||
|
|
||||||
|
## Scene Chunking Failure
|
||||||
|
|
||||||
|
Symptoms include:
|
||||||
|
|
||||||
|
- `dnd scenes chunker`
|
||||||
|
- `malformed structured output`
|
||||||
|
- `boundary_caveats`
|
||||||
|
- `start_unit_id`
|
||||||
|
- `end_unit_id`
|
||||||
|
- `gap`
|
||||||
|
- `overlap`
|
||||||
|
- `final scene`
|
||||||
|
- `complete structured output`
|
||||||
|
|
||||||
|
Fix:
|
||||||
|
|
||||||
|
- Validate the pipeline configuration and confirm the input module provides a
|
||||||
|
transcript source when using `chunk: dnd/scenes`.
|
||||||
|
- Confirm the LLM profile has a working OpenAI-compatible `base_url`, `model`,
|
||||||
|
and credentials.
|
||||||
|
- Inspect retained diagnostics for the run error and resolved pipeline.
|
||||||
|
- If the error names malformed structured output, retry with a model that
|
||||||
|
follows structured response schemas reliably.
|
||||||
|
- If the error names `boundary_caveats`, check for blank or whitespace-only
|
||||||
|
caveat text in the scene response.
|
||||||
|
- Scene boundaries must use exact source-unit IDs, cover the full source
|
||||||
|
document, be contiguous, and not overlap.
|
||||||
|
|
||||||
## Output Write Failure
|
## Output Write Failure
|
||||||
|
|
||||||
Symptoms include:
|
Symptoms include:
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
|
||||||
@@ -34,6 +35,9 @@ func productionRegistries() (pipeline.Registries, error) {
|
|||||||
if err := generic.Register(registries.Chunkers); err != nil {
|
if err := generic.Register(registries.Chunkers); err != nil {
|
||||||
return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err)
|
return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err)
|
||||||
}
|
}
|
||||||
|
if err := scenes.Register(registries.Chunkers); err != nil {
|
||||||
|
return pipeline.Registries{}, fmt.Errorf("register dnd scenes chunker: %w", err)
|
||||||
|
}
|
||||||
if err := spells.Register(registries.Extractors); err != nil {
|
if err := spells.Register(registries.Extractors); err != nil {
|
||||||
return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err)
|
return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
|
||||||
@@ -136,6 +137,11 @@ func TestProductionCatalogIncludesDefaultModules(t *testing.T) {
|
|||||||
got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(generic.Key) },
|
got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(generic.Key) },
|
||||||
want: generic.ModuleSpec(),
|
want: generic.ModuleSpec(),
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "dnd scenes chunker",
|
||||||
|
got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(scenes.Key) },
|
||||||
|
want: scenes.ModuleSpec(),
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "dnd spells extractor",
|
name: "dnd spells extractor",
|
||||||
got: func() (pipeline.ModuleSpec, bool) { return catalog.Extractors.Spec(spells.Key) },
|
got: func() (pipeline.ModuleSpec, bool) { return catalog.Extractors.Spec(spells.Key) },
|
||||||
@@ -186,6 +192,21 @@ func TestRunConfigValidateUsesProductionCatalogByDefault(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunConfigValidateAcceptsDNDScenesChunker(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells"))
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{})
|
||||||
|
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "is valid for pipeline") {
|
||||||
|
t.Fatalf("stdout = %q, want validation success", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) {
|
func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) {
|
||||||
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "missing/extract"))
|
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "missing/extract"))
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
@@ -1041,6 +1062,81 @@ func TestExampleFixtureRunWritesExpectedJSON(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExampleFixtureRunWithDNDScenesRecordsChunkerAndWarnings(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells"))
|
||||||
|
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
|
||||||
|
outputDir := t.TempDir()
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
client := newSceneRunLLMClient("Scene boundary was ambiguous.")
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
|
||||||
|
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if client.calls != 2 {
|
||||||
|
t.Fatalf("LLM calls = %d, want chunking and extraction calls", client.calls)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "1 warning") {
|
||||||
|
t.Fatalf("stderr = %q, want warning count", stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
runOutputDir := onlyChildDir(t, outputDir)
|
||||||
|
manifestBytes := readFile(t, filepath.Join(runOutputDir, "manifest.json"))
|
||||||
|
var manifest artifacts.RunManifest
|
||||||
|
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
|
||||||
|
t.Fatalf("unmarshal manifest: %v", err)
|
||||||
|
}
|
||||||
|
if manifest.Chunker != scenes.Key {
|
||||||
|
t.Fatalf("manifest chunker = %q, want %q", manifest.Chunker, scenes.Key)
|
||||||
|
}
|
||||||
|
chunkerMetadata := manifest.ModuleMetadata["chunker"]
|
||||||
|
if chunkerMetadata == nil {
|
||||||
|
t.Fatalf("module metadata chunker = %#v, want object", manifest.ModuleMetadata["chunker"])
|
||||||
|
}
|
||||||
|
wantMetadataKeys := []string{
|
||||||
|
"prompt_id",
|
||||||
|
"prompt_version",
|
||||||
|
"prompt_sha256",
|
||||||
|
"response_schema_key",
|
||||||
|
"response_schema_id",
|
||||||
|
"response_schema_name",
|
||||||
|
"response_schema_version",
|
||||||
|
"response_schema_sha256",
|
||||||
|
}
|
||||||
|
if len(chunkerMetadata) != len(wantMetadataKeys) {
|
||||||
|
t.Fatalf("chunker metadata keys = %#v, want %d keys", chunkerMetadata, len(wantMetadataKeys))
|
||||||
|
}
|
||||||
|
for _, key := range wantMetadataKeys {
|
||||||
|
value, ok := chunkerMetadata[key]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("chunker metadata missing key %q: %#v", key, chunkerMetadata)
|
||||||
|
}
|
||||||
|
if _, ok := value.(string); !ok {
|
||||||
|
t.Fatalf("chunker metadata[%q] = %#v, want string", key, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"prompt", "schema", "source", "text", "payload", "api_key", "secret", "token"} {
|
||||||
|
if _, ok := chunkerMetadata[forbidden]; ok {
|
||||||
|
t.Fatalf("chunker metadata leaked forbidden key %q: %#v", forbidden, chunkerMetadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"Source document ID:", "Aria casts Cure Wounds.", "spell_casts", "Scene boundary was ambiguous."} {
|
||||||
|
if strings.Contains(string(manifestBytes), forbidden) {
|
||||||
|
t.Fatalf("manifest leaked %q: %s", forbidden, manifestBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
warnings := string(readFile(t, filepath.Join(runOutputDir, "warnings.json")))
|
||||||
|
if !strings.Contains(warnings, "scene_boundary_caveat") || !strings.Contains(warnings, "Scene boundary was ambiguous.") {
|
||||||
|
t.Fatalf("warnings output = %s, want scene boundary caveat", warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExampleFixtureRunOnlySpells(t *testing.T) {
|
func TestExampleFixtureRunOnlySpells(t *testing.T) {
|
||||||
configPath := fixturePath(t, "examples/dnd-spells.config.yml")
|
configPath := fixturePath(t, "examples/dnd-spells.config.yml")
|
||||||
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
|
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
|
||||||
@@ -1220,6 +1316,18 @@ pipelines:
|
|||||||
`
|
`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mvpConfigYAMLWithChunk(pipelineID string, chunker string, extractor string) string {
|
||||||
|
return `version: 1
|
||||||
|
pipelines:
|
||||||
|
` + pipelineID + `:
|
||||||
|
input: seriatim
|
||||||
|
chunk: ` + chunker + `
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract: ` + extractor + `
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
|
func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
b.WriteString("version: 1\n")
|
b.WriteString("version: 1\n")
|
||||||
@@ -1287,6 +1395,7 @@ type fakeRunLLMClient struct {
|
|||||||
calls int
|
calls int
|
||||||
err error
|
err error
|
||||||
payload map[string]any
|
payload map[string]any
|
||||||
|
sceneCaveat string
|
||||||
}
|
}
|
||||||
|
|
||||||
func newFakeRunLLMClient(invalidSourceRef bool) *fakeRunLLMClient {
|
func newFakeRunLLMClient(invalidSourceRef bool) *fakeRunLLMClient {
|
||||||
@@ -1301,11 +1410,43 @@ func newMalformedRunLLMClient() *fakeRunLLMClient {
|
|||||||
return &fakeRunLLMClient{payload: map[string]any{}}
|
return &fakeRunLLMClient{payload: map[string]any{}}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newSceneRunLLMClient(caveat string) *fakeRunLLMClient {
|
||||||
|
return &fakeRunLLMClient{sceneCaveat: caveat}
|
||||||
|
}
|
||||||
|
|
||||||
func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
client.calls++
|
client.calls++
|
||||||
if client.err != nil {
|
if client.err != nil {
|
||||||
return contracts.StructuredCompletionResponse{}, client.err
|
return contracts.StructuredCompletionResponse{}, client.err
|
||||||
}
|
}
|
||||||
|
if req.StageName == scenes.Key && client.payload == nil {
|
||||||
|
payload := map[string]any{
|
||||||
|
"scenes": []map[string]any{
|
||||||
|
{
|
||||||
|
"start_unit_id": "seg-001",
|
||||||
|
"end_unit_id": "seg-002",
|
||||||
|
"short_title": "Opening spell",
|
||||||
|
"primary_mode": "Narrative",
|
||||||
|
"main_participants": []string{"Aria"},
|
||||||
|
"summary": "Aria casts a spell.",
|
||||||
|
"boundary_note": "The provided source units form one scene.",
|
||||||
|
"boundary_confidence": "High",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"boundary_caveats": []string{},
|
||||||
|
}
|
||||||
|
if client.sceneCaveat != "" {
|
||||||
|
payload["boundary_caveats"] = []string{client.sceneCaveat}
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(encoded, out); err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
|
}
|
||||||
|
return contracts.StructuredCompletionResponse{Content: encoded}, nil
|
||||||
|
}
|
||||||
startUnitID := "seg-001"
|
startUnitID := "seg-001"
|
||||||
if client.invalidSourceRef {
|
if client.invalidSourceRef {
|
||||||
startUnitID = "missing-segment"
|
startUnitID = "missing-segment"
|
||||||
|
|||||||
@@ -59,6 +59,7 @@ type RunManifest struct {
|
|||||||
Merger string `json:"merger,omitempty"`
|
Merger string `json:"merger,omitempty"`
|
||||||
Normalizer string `json:"normalizer,omitempty"`
|
Normalizer string `json:"normalizer,omitempty"`
|
||||||
OutputEncoder string `json:"output_encoder,omitempty"`
|
OutputEncoder string `json:"output_encoder,omitempty"`
|
||||||
|
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
|
||||||
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
||||||
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
||||||
SchemaVersion string `json:"schema_version,omitempty"`
|
SchemaVersion string `json:"schema_version,omitempty"`
|
||||||
|
|||||||
@@ -183,6 +183,42 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
|||||||
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
|
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunManifestIncludesTopLevelModuleMetadata(t *testing.T) {
|
||||||
|
manifest := RunManifest{
|
||||||
|
ModuleMetadata: map[string]map[string]any{
|
||||||
|
"chunker": {
|
||||||
|
"prompt_id": "dnd.scenes",
|
||||||
|
"prompt_version": "v1",
|
||||||
|
"prompt_sha256": "sha256:abc123",
|
||||||
|
"response_schema_key": "dnd_scenes",
|
||||||
|
"response_schema_name": "dnd_scenes",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
gotJSON, err := json.Marshal(manifest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var got map[string]any
|
||||||
|
if err := json.Unmarshal(gotJSON, &got); err != nil {
|
||||||
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
moduleMetadata, ok := got["module_metadata"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("module_metadata = %#v, want object", got["module_metadata"])
|
||||||
|
}
|
||||||
|
assertHasKeys(t, moduleMetadata, "chunker")
|
||||||
|
|
||||||
|
chunkerMetadata, ok := moduleMetadata["chunker"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("module_metadata.chunker = %#v, want object", moduleMetadata["chunker"])
|
||||||
|
}
|
||||||
|
assertHasKeys(t, chunkerMetadata, "prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_name")
|
||||||
|
}
|
||||||
|
|
||||||
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
|
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
@@ -98,6 +98,43 @@ func TestResolveSurfacesMissingCapabilityThroughCatalog(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) {
|
||||||
|
cfg := validConfig()
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
profile.Chunk = pipeline.Binding("dnd/scenes")
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"events": lane}
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
|
||||||
|
catalog := fakeCatalog(t,
|
||||||
|
pipeline.ModuleSpec{
|
||||||
|
Key: "fake/input",
|
||||||
|
Stage: pipeline.StageInput,
|
||||||
|
Provides: []string{"source.transcript"},
|
||||||
|
},
|
||||||
|
pipeline.ModuleSpec{
|
||||||
|
Key: "fake/extract",
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
Requires: []string{"chunks", "source.transcript"},
|
||||||
|
Provides: []string{"artifact"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
mustRegisterChunker(t, catalog.Chunkers, pipeline.ModuleSpec{
|
||||||
|
Key: "dnd/scenes",
|
||||||
|
Stage: pipeline.StageChunk,
|
||||||
|
Requires: []string{"source.transcript"},
|
||||||
|
Provides: []string{"chunks", "chunks.scenes"},
|
||||||
|
})
|
||||||
|
|
||||||
|
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if got := effective.ResolvedPipeline.Chunk.Module; got != "dnd/scenes" {
|
||||||
|
t.Fatalf("Chunk.Module = %q, want dnd/scenes", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
|
func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
|
||||||
cfg := validConfig()
|
cfg := validConfig()
|
||||||
first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ var _ contracts.Extractor = compositionExtractor{}
|
|||||||
var _ contracts.Merger = compositionMerger{}
|
var _ contracts.Merger = compositionMerger{}
|
||||||
var _ contracts.Normalizer = compositionNormalizer{}
|
var _ contracts.Normalizer = compositionNormalizer{}
|
||||||
var _ contracts.Validator = compositionValidator{}
|
var _ contracts.Validator = compositionValidator{}
|
||||||
|
var _ contracts.StructuredLLMClient = compositionLLMClient{}
|
||||||
var _ contracts.OutputEncoder = compositionOutputEncoder{}
|
var _ contracts.OutputEncoder = compositionOutputEncoder{}
|
||||||
|
|
||||||
func TestContractsComposeAcrossPackages(t *testing.T) {
|
func TestContractsComposeAcrossPackages(t *testing.T) {
|
||||||
@@ -39,6 +40,7 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
|
|||||||
|
|
||||||
chunking, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
chunking, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
||||||
Source: doc,
|
Source: doc,
|
||||||
|
LLMClient: compositionLLMClient{},
|
||||||
Metadata: map[string]any{"max_units": 2},
|
Metadata: map[string]any{"max_units": 2},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -164,6 +166,9 @@ func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.Chunk
|
|||||||
if req.Source == nil {
|
if req.Source == nil {
|
||||||
return contracts.ChunkResult{}, errors.New("source document is required")
|
return contracts.ChunkResult{}, errors.New("source document is required")
|
||||||
}
|
}
|
||||||
|
if req.LLMClient == nil {
|
||||||
|
return contracts.ChunkResult{}, errors.New("structured llm client is required")
|
||||||
|
}
|
||||||
|
|
||||||
return contracts.ChunkResult{
|
return contracts.ChunkResult{
|
||||||
Chunks: []contracts.SourceChunk{
|
Chunks: []contracts.SourceChunk{
|
||||||
@@ -178,6 +183,12 @@ func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.Chunk
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type compositionLLMClient struct{}
|
||||||
|
|
||||||
|
func (client compositionLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
|
return contracts.StructuredCompletionResponse{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
type compositionExtractor struct{}
|
type compositionExtractor struct{}
|
||||||
|
|
||||||
func (extractor compositionExtractor) Key() string {
|
func (extractor compositionExtractor) Key() string {
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ type SourceChunk struct {
|
|||||||
|
|
||||||
type ChunkRequest struct {
|
type ChunkRequest struct {
|
||||||
Source *source.SourceDocument `json:"-"`
|
Source *source.SourceDocument `json:"-"`
|
||||||
|
LLMClient StructuredLLMClient `json:"-"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
|
|||||||
@@ -117,6 +117,27 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFakeChunkerReceivesLLMClient(t *testing.T) {
|
||||||
|
doc := &source.SourceDocument{
|
||||||
|
ID: "source-1",
|
||||||
|
Kind: "document",
|
||||||
|
Format: "text/plain",
|
||||||
|
Digest: "sha256:abc123",
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
{ID: "u1", Kind: "section", Text: "Source text."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
client := fakeLLMClient{}
|
||||||
|
chunker := &recordingChunker{key: "llm-chunker"}
|
||||||
|
|
||||||
|
if _, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc, LLMClient: client}); err != nil {
|
||||||
|
t.Fatalf("Chunk() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if chunker.request.LLMClient == nil {
|
||||||
|
t.Fatal("ChunkRequest.LLMClient = nil, want structured LLM client")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
|
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
|
||||||
extractor := fakeExtractor{
|
extractor := fakeExtractor{
|
||||||
key: "generic-extractor",
|
key: "generic-extractor",
|
||||||
@@ -305,6 +326,20 @@ func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkRe
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type recordingChunker struct {
|
||||||
|
key string
|
||||||
|
request ChunkRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (chunker *recordingChunker) Key() string {
|
||||||
|
return chunker.key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
|
||||||
|
chunker.request = req
|
||||||
|
return fakeChunker{key: chunker.key}.Chunk(ctx, req)
|
||||||
|
}
|
||||||
|
|
||||||
type fakeExtractor struct {
|
type fakeExtractor struct {
|
||||||
key string
|
key string
|
||||||
artifactType string
|
artifactType string
|
||||||
|
|||||||
82
internal/framework/pipeline/chunk_validation.go
Normal file
82
internal/framework/pipeline/chunk_validation.go
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
|
||||||
|
sourceUnitIndexes := make(map[string]int, len(doc.Units))
|
||||||
|
sourceUnits := make(map[string]source.SourceUnit, len(doc.Units))
|
||||||
|
for index, unit := range doc.Units {
|
||||||
|
sourceUnitIndexes[unit.ID] = index
|
||||||
|
sourceUnits[unit.ID] = unit
|
||||||
|
}
|
||||||
|
|
||||||
|
canonicalChunks := make([]contracts.SourceChunk, 0, len(chunks))
|
||||||
|
seenChunkIDs := make(map[string]struct{}, len(chunks))
|
||||||
|
for chunkIndex, chunk := range chunks {
|
||||||
|
if strings.TrimSpace(chunk.ID) == "" {
|
||||||
|
return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
|
||||||
|
}
|
||||||
|
if _, ok := seenChunkIDs[chunk.ID]; ok {
|
||||||
|
return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
|
||||||
|
}
|
||||||
|
seenChunkIDs[chunk.ID] = struct{}{}
|
||||||
|
|
||||||
|
if chunk.SourceID != doc.ID {
|
||||||
|
return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
|
||||||
|
}
|
||||||
|
if chunk.Index != chunkIndex {
|
||||||
|
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
|
||||||
|
}
|
||||||
|
if len(chunk.Units) == 0 {
|
||||||
|
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
seenUnitIDs := make(map[string]struct{}, len(chunk.Units))
|
||||||
|
previousSourceIndex := -1
|
||||||
|
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
|
||||||
|
for unitIndex, unit := range chunk.Units {
|
||||||
|
if strings.TrimSpace(unit.ID) == "" {
|
||||||
|
return nil, fmt.Errorf("chunk %q unit[%d].id must not be empty", chunk.ID, unitIndex)
|
||||||
|
}
|
||||||
|
if _, ok := seenUnitIDs[unit.ID]; ok {
|
||||||
|
return nil, fmt.Errorf("chunk %q repeats source unit %q", chunk.ID, unit.ID)
|
||||||
|
}
|
||||||
|
seenUnitIDs[unit.ID] = struct{}{}
|
||||||
|
|
||||||
|
sourceIndex, ok := sourceUnitIndexes[unit.ID]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("chunk %q source unit %q was not found in source document %q", chunk.ID, unit.ID, doc.ID)
|
||||||
|
}
|
||||||
|
if sourceIndex <= previousSourceIndex {
|
||||||
|
return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
|
||||||
|
}
|
||||||
|
previousSourceIndex = sourceIndex
|
||||||
|
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
|
||||||
|
}
|
||||||
|
|
||||||
|
canonicalChunks = append(canonicalChunks, contracts.SourceChunk{
|
||||||
|
ID: chunk.ID,
|
||||||
|
SourceID: chunk.SourceID,
|
||||||
|
Index: chunk.Index,
|
||||||
|
Units: canonicalUnits,
|
||||||
|
Metadata: cloneMetadata(chunk.Metadata),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return canonicalChunks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
|
||||||
|
return source.SourceUnit{
|
||||||
|
ID: unit.ID,
|
||||||
|
Kind: unit.Kind,
|
||||||
|
Text: unit.Text,
|
||||||
|
Metadata: cloneMetadata(unit.Metadata),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -69,6 +69,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
|
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
|
||||||
}
|
}
|
||||||
|
attachModuleManifestMetadata(&output, "input", adapter)
|
||||||
doc, err := adapter.Parse(ctx, contracts.ParseRequest{
|
doc, err := adapter.Parse(ctx, contracts.ParseRequest{
|
||||||
SourceID: input.SourceID,
|
SourceID: input.SourceID,
|
||||||
Path: input.Path,
|
Path: input.Path,
|
||||||
@@ -89,8 +90,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
|
return failOutput(output), fmt.Errorf("build chunker %q: %w", input.Pipeline.Chunk.Module, err)
|
||||||
}
|
}
|
||||||
|
attachModuleManifestMetadata(&output, "chunker", chunker)
|
||||||
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
||||||
Source: doc,
|
Source: doc,
|
||||||
|
LLMClient: input.LLMClient,
|
||||||
LLMProfile: input.Pipeline.Chunk.LLMProfile,
|
LLMProfile: input.Pipeline.Chunk.LLMProfile,
|
||||||
Options: cloneOptions(input.Pipeline.Chunk.Options),
|
Options: cloneOptions(input.Pipeline.Chunk.Options),
|
||||||
Metadata: input.Metadata,
|
Metadata: input.Metadata,
|
||||||
@@ -102,10 +105,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
if len(chunkResult.Chunks) == 0 {
|
if len(chunkResult.Chunks) == 0 {
|
||||||
return failOutput(output), fmt.Errorf("chunker %q returned no chunks", chunker.Key())
|
return failOutput(output), fmt.Errorf("chunker %q returned no chunks", chunker.Key())
|
||||||
}
|
}
|
||||||
|
canonicalChunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
|
||||||
|
if err != nil {
|
||||||
|
return failOutput(output), fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
||||||
|
}
|
||||||
|
|
||||||
nextCandidateIndex := 0
|
nextCandidateIndex := 0
|
||||||
for _, lane := range input.Pipeline.ArtifactLanes {
|
for _, lane := range input.Pipeline.ArtifactLanes {
|
||||||
if err := r.runLane(ctx, input, doc, chunkResult.Chunks, lane, &output, &nextCandidateIndex); err != nil {
|
if err := r.runLane(ctx, input, doc, canonicalChunks, lane, &output, &nextCandidateIndex); err != nil {
|
||||||
return failOutput(output), err
|
return failOutput(output), err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -121,6 +128,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
|
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
|
||||||
}
|
}
|
||||||
|
attachModuleManifestMetadata(&output, "output", encoder)
|
||||||
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
|
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
|
||||||
Manifest: output.Manifest,
|
Manifest: output.Manifest,
|
||||||
Approved: output.Approved,
|
Approved: output.Approved,
|
||||||
@@ -382,14 +390,10 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
|
|||||||
|
|
||||||
metadata := make(map[string]any)
|
metadata := make(map[string]any)
|
||||||
for _, module := range modules {
|
for _, module := range modules {
|
||||||
provider, ok := module.(contracts.ManifestMetadataProvider)
|
moduleMetadata, ok := moduleManifestMetadata(module)
|
||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
|
|
||||||
if len(moduleMetadata) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key := manifestMetadataKey(module)
|
key := manifestMetadataKey(module)
|
||||||
if key == "" {
|
if key == "" {
|
||||||
continue
|
continue
|
||||||
@@ -403,6 +407,20 @@ func setLaneManifestMetadata(output *RunOutput, laneID string, modules ...any) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
|
||||||
|
if output == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
moduleMetadata, ok := moduleManifestMetadata(module)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if output.Manifest.ModuleMetadata == nil {
|
||||||
|
output.Manifest.ModuleMetadata = make(map[string]map[string]any)
|
||||||
|
}
|
||||||
|
output.Manifest.ModuleMetadata[moduleKey] = moduleMetadata
|
||||||
|
}
|
||||||
|
|
||||||
func manifestMetadataKey(module any) string {
|
func manifestMetadataKey(module any) string {
|
||||||
switch module.(type) {
|
switch module.(type) {
|
||||||
case contracts.Extractor:
|
case contracts.Extractor:
|
||||||
@@ -416,6 +434,19 @@ func manifestMetadataKey(module any) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func moduleManifestMetadata(module any) (map[string]any, bool) {
|
||||||
|
provider, ok := module.(contracts.ManifestMetadataProvider)
|
||||||
|
if !ok {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
moduleMetadata := cloneMetadata(provider.ManifestMetadata())
|
||||||
|
if len(moduleMetadata) == 0 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return moduleMetadata, true
|
||||||
|
}
|
||||||
|
|
||||||
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
|
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
|
||||||
out := make([]contracts.OutputFile, 0, len(result.Files))
|
out := make([]contracts.OutputFile, 0, len(result.Files))
|
||||||
for _, file := range result.Files {
|
for _, file := range result.Files {
|
||||||
|
|||||||
@@ -256,6 +256,191 @@ func TestRunRejectsChunkerBuildChunkAndEmptyChunkErrors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunRejectsInvalidChunks(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
chunks []contracts.SourceChunk
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "empty chunk id",
|
||||||
|
chunks: []contracts.SourceChunk{{ID: "", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}}},
|
||||||
|
want: "id must not be empty",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate chunk id",
|
||||||
|
chunks: []contracts.SourceChunk{
|
||||||
|
{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}},
|
||||||
|
{ID: "chunk-0", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u2")}},
|
||||||
|
},
|
||||||
|
want: "duplicated",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "wrong source id",
|
||||||
|
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "other-source", Index: 0, Units: []source.SourceUnit{unitWithID("u1")}}},
|
||||||
|
want: "source_id",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "wrong index",
|
||||||
|
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u1")}}},
|
||||||
|
want: "index",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty units",
|
||||||
|
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0}},
|
||||||
|
want: "units must not be empty",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "repeated unit inside chunk",
|
||||||
|
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1"), unitWithID("u1")}}},
|
||||||
|
want: "repeats source unit",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown unit",
|
||||||
|
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u9")}}},
|
||||||
|
want: "was not found",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "units out of source order",
|
||||||
|
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u2"), unitWithID("u1")}}},
|
||||||
|
want: "source document order",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
modules.chunker.chunks = test.chunks
|
||||||
|
|
||||||
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
|
||||||
|
assertRunError(t, err, test.want)
|
||||||
|
if output.Manifest.ValidationStatus != "failed" {
|
||||||
|
t.Fatalf("ValidationStatus = %q, want failed", output.Manifest.ValidationStatus)
|
||||||
|
}
|
||||||
|
if len(modules.extractors["extract-alpha"].requests) != 0 {
|
||||||
|
t.Fatalf("extractor calls = %d, want none after invalid chunks", len(modules.extractors["extract-alpha"].requests))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
modules.chunker.chunks = []contracts.SourceChunk{
|
||||||
|
{ID: "chunk-0", SourceID: "source-1", Index: 0, Units: []source.SourceUnit{unitWithID("u1"), unitWithID("u2")}},
|
||||||
|
{ID: "chunk-1", SourceID: "source-1", Index: 1, Units: []source.SourceUnit{unitWithID("u2")}},
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(output.Approved) != 2 {
|
||||||
|
t.Fatalf("len(Approved) = %d, want one candidate per accepted chunk", len(output.Approved))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
modules.input.doc = sourceDocumentWithUnitMetadata()
|
||||||
|
modules.chunker.chunks = []contracts.SourceChunk{
|
||||||
|
{
|
||||||
|
ID: "chunk-0",
|
||||||
|
SourceID: "source-1",
|
||||||
|
Index: 0,
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
{
|
||||||
|
ID: "u1",
|
||||||
|
Kind: "mutated-kind",
|
||||||
|
Text: "mutated text",
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"speaker": "chunker-speaker",
|
||||||
|
"note": "chunker note",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(output.Approved) != 1 {
|
||||||
|
t.Fatalf("len(Approved) = %d, want 1", len(output.Approved))
|
||||||
|
}
|
||||||
|
|
||||||
|
extractor := modules.extractors["extract-alpha"]
|
||||||
|
if len(extractor.requests) != 1 {
|
||||||
|
t.Fatalf("len(extractor requests) = %d, want 1", len(extractor.requests))
|
||||||
|
}
|
||||||
|
chunk := extractor.requests[0].Chunk
|
||||||
|
if chunk == nil {
|
||||||
|
t.Fatal("extractor chunk = nil, want canonical chunk")
|
||||||
|
}
|
||||||
|
if chunk.Units[0].ID != "u1" || chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" {
|
||||||
|
t.Fatalf("chunk unit = %#v, want source document unit values", chunk.Units[0])
|
||||||
|
}
|
||||||
|
if got := chunk.Units[0].Metadata["speaker"]; got != "source-speaker" {
|
||||||
|
t.Fatalf("chunk unit metadata = %#v, want source document metadata", chunk.Units[0].Metadata)
|
||||||
|
}
|
||||||
|
if got := chunk.Units[0].Metadata["topic"]; got != "source-topic" {
|
||||||
|
t.Fatalf("chunk unit metadata = %#v, want cloned source document metadata", chunk.Units[0].Metadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
modules.input.doc.Units[0].Kind = "changed-kind"
|
||||||
|
modules.input.doc.Units[0].Text = "changed text"
|
||||||
|
modules.input.doc.Units[0].Metadata["speaker"] = "changed-speaker"
|
||||||
|
if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Metadata["speaker"] != "source-speaker" {
|
||||||
|
t.Fatalf("chunk unit changed after source mutation: %#v", chunk.Units[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPreservesChunkMetadataDuringCanonicalization(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
modules.chunker.chunks = []contracts.SourceChunk{
|
||||||
|
{
|
||||||
|
ID: "chunk-0",
|
||||||
|
SourceID: "source-1",
|
||||||
|
Index: 0,
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
unitWithID("u1"),
|
||||||
|
},
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"scene_title": "Original scene",
|
||||||
|
"boundary_note": "Chunker note",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
extractor := modules.extractors["extract-alpha"]
|
||||||
|
if len(extractor.requests) != 1 || extractor.requests[0].Chunk == nil {
|
||||||
|
t.Fatalf("extractor requests = %#v, want one canonical chunk", extractor.requests)
|
||||||
|
}
|
||||||
|
if got := extractor.requests[0].Chunk.Metadata["scene_title"]; got != "Original scene" {
|
||||||
|
t.Fatalf("chunk metadata = %#v, want chunker metadata", extractor.requests[0].Chunk.Metadata)
|
||||||
|
}
|
||||||
|
if got := extractor.requests[0].Chunk.Metadata["boundary_note"]; got != "Chunker note" {
|
||||||
|
t.Fatalf("chunk metadata = %#v, want chunker metadata", extractor.requests[0].Chunk.Metadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
modules.chunker.chunks[0].Metadata["scene_title"] = "changed"
|
||||||
|
modules.chunker.chunks[0].Metadata["boundary_note"] = "changed"
|
||||||
|
if got := extractor.requests[0].Chunk.Metadata["scene_title"]; got != "Original scene" {
|
||||||
|
t.Fatalf("chunk metadata aliased to chunker map: %#v", extractor.requests[0].Chunk.Metadata)
|
||||||
|
}
|
||||||
|
if got := extractor.requests[0].Chunk.Metadata["boundary_note"]; got != "Chunker note" {
|
||||||
|
t.Fatalf("chunk metadata aliased to chunker map: %#v", extractor.requests[0].Chunk.Metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
|
func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
llmClient := fakeLLMClient{}
|
llmClient := fakeLLMClient{}
|
||||||
@@ -273,6 +458,9 @@ func TestRunExecutesChunksAndPassesChunkAndLLMClient(t *testing.T) {
|
|||||||
if !reflect.DeepEqual(extractor.seenChunkIDs, []string{"chunk-0", "chunk-1"}) {
|
if !reflect.DeepEqual(extractor.seenChunkIDs, []string{"chunk-0", "chunk-1"}) {
|
||||||
t.Fatalf("seen chunks = %#v, want both chunks", extractor.seenChunkIDs)
|
t.Fatalf("seen chunks = %#v, want both chunks", extractor.seenChunkIDs)
|
||||||
}
|
}
|
||||||
|
if len(modules.chunker.requests) != 1 || modules.chunker.requests[0].LLMClient == nil {
|
||||||
|
t.Fatalf("chunker LLM client = %#v, want client on chunk request", modules.chunker.requests)
|
||||||
|
}
|
||||||
if len(extractor.seenLLMClients) != 2 || extractor.seenLLMClients[0] == nil || extractor.seenLLMClients[1] == nil {
|
if len(extractor.seenLLMClients) != 2 || extractor.seenLLMClients[0] == nil || extractor.seenLLMClients[1] == nil {
|
||||||
t.Fatalf("seen LLM clients = %#v, want client for each chunk", extractor.seenLLMClients)
|
t.Fatalf("seen LLM clients = %#v, want client for each chunk", extractor.seenLLMClients)
|
||||||
}
|
}
|
||||||
@@ -371,6 +559,47 @@ func TestRunPassesModuleBindingConfigToStageRequests(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunRecordsTopLevelModuleMetadataForSingletonModules(t *testing.T) {
|
||||||
|
modules := defaultRunnerModules()
|
||||||
|
modules.input.manifestMetadata = map[string]any{
|
||||||
|
"input_profile": "input-metadata",
|
||||||
|
}
|
||||||
|
modules.chunker.manifestMetadata = map[string]any{
|
||||||
|
"prompt_id": "dnd.scenes",
|
||||||
|
"prompt_version": "v1",
|
||||||
|
"prompt_sha256": "sha256:chunker-prompt",
|
||||||
|
"response_schema_key": "dnd_scenes",
|
||||||
|
"response_schema_id": "schema-dnd-scenes",
|
||||||
|
"response_schema_name": "dnd_scenes",
|
||||||
|
}
|
||||||
|
modules.output.manifestMetadata = map[string]any{
|
||||||
|
"output_profile": "output-metadata",
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{Pipeline: resolvedPipeline()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if output.Manifest.ModuleMetadata == nil {
|
||||||
|
t.Fatal("ModuleMetadata = nil, want module metadata map")
|
||||||
|
}
|
||||||
|
if got := output.Manifest.ModuleMetadata["input"]; !reflect.DeepEqual(got, modules.input.manifestMetadata) {
|
||||||
|
t.Fatalf("input module metadata = %#v, want %#v", got, modules.input.manifestMetadata)
|
||||||
|
}
|
||||||
|
if got := output.Manifest.ModuleMetadata["chunker"]; !reflect.DeepEqual(got, modules.chunker.manifestMetadata) {
|
||||||
|
t.Fatalf("chunker module metadata = %#v, want %#v", got, modules.chunker.manifestMetadata)
|
||||||
|
}
|
||||||
|
if got := output.Manifest.ModuleMetadata["output"]; !reflect.DeepEqual(got, modules.output.manifestMetadata) {
|
||||||
|
t.Fatalf("output module metadata = %#v, want %#v", got, modules.output.manifestMetadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
modules.chunker.manifestMetadata["prompt_id"] = "changed"
|
||||||
|
if output.Manifest.ModuleMetadata["chunker"]["prompt_id"] != "dnd.scenes" {
|
||||||
|
t.Fatalf("chunker module metadata aliased to provider map: %#v", output.Manifest.ModuleMetadata["chunker"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
|
func TestRunPassesPerChunkCandidatesToMergeAndNormalize(t *testing.T) {
|
||||||
modules := defaultRunnerModules()
|
modules := defaultRunnerModules()
|
||||||
|
|
||||||
@@ -775,6 +1004,11 @@ func TestRunManifestIncludesExtractorMetadata(t *testing.T) {
|
|||||||
if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
|
if extractorMetadata["prompt_id"] != "test.prompt" || extractorMetadata["response_schema_name"] != "test_schema" {
|
||||||
t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
|
t.Fatalf("extractor metadata = %#v, want prompt and schema metadata", extractorMetadata)
|
||||||
}
|
}
|
||||||
|
if output.Manifest.ModuleMetadata != nil {
|
||||||
|
if _, ok := output.Manifest.ModuleMetadata["extractor"]; ok {
|
||||||
|
t.Fatalf("top-level module metadata includes lane metadata key: %#v", output.Manifest.ModuleMetadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
|
func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) {
|
||||||
@@ -966,6 +1200,7 @@ type runnerInputAdapter struct {
|
|||||||
key string
|
key string
|
||||||
doc *source.SourceDocument
|
doc *source.SourceDocument
|
||||||
err error
|
err error
|
||||||
|
manifestMetadata map[string]any
|
||||||
requests []contracts.ParseRequest
|
requests []contracts.ParseRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -978,11 +1213,16 @@ func (adapter *runnerInputAdapter) Parse(ctx context.Context, req contracts.Pars
|
|||||||
return adapter.doc, adapter.err
|
return adapter.doc, adapter.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (adapter *runnerInputAdapter) ManifestMetadata() map[string]any {
|
||||||
|
return adapter.manifestMetadata
|
||||||
|
}
|
||||||
|
|
||||||
type runnerChunker struct {
|
type runnerChunker struct {
|
||||||
key string
|
key string
|
||||||
chunks []contracts.SourceChunk
|
chunks []contracts.SourceChunk
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
err error
|
err error
|
||||||
|
manifestMetadata map[string]any
|
||||||
requests []contracts.ChunkRequest
|
requests []contracts.ChunkRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -998,6 +1238,10 @@ func (chunker *runnerChunker) Chunk(ctx context.Context, req contracts.ChunkRequ
|
|||||||
}, chunker.err
|
}, chunker.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (chunker *runnerChunker) ManifestMetadata() map[string]any {
|
||||||
|
return chunker.manifestMetadata
|
||||||
|
}
|
||||||
|
|
||||||
type runnerExtractor struct {
|
type runnerExtractor struct {
|
||||||
key string
|
key string
|
||||||
artifactType string
|
artifactType string
|
||||||
@@ -1142,6 +1386,7 @@ type runnerOutputEncoder struct {
|
|||||||
files []contracts.OutputFile
|
files []contracts.OutputFile
|
||||||
warnings []contracts.Warning
|
warnings []contracts.Warning
|
||||||
err error
|
err error
|
||||||
|
manifestMetadata map[string]any
|
||||||
requests []contracts.OutputRequest
|
requests []contracts.OutputRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1157,6 +1402,10 @@ func (encoder *runnerOutputEncoder) Encode(ctx context.Context, req contracts.Ou
|
|||||||
}, encoder.err
|
}, encoder.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (encoder *runnerOutputEncoder) ManifestMetadata() map[string]any {
|
||||||
|
return encoder.manifestMetadata
|
||||||
|
}
|
||||||
|
|
||||||
type fakeLLMClient struct{}
|
type fakeLLMClient struct{}
|
||||||
|
|
||||||
func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
@@ -1179,6 +1428,36 @@ func validSourceDocument() *source.SourceDocument {
|
|||||||
Digest: "sha256:source",
|
Digest: "sha256:source",
|
||||||
Units: []source.SourceUnit{
|
Units: []source.SourceUnit{
|
||||||
{ID: "u1", Kind: "unit", Text: "Source unit."},
|
{ID: "u1", Kind: "unit", Text: "Source unit."},
|
||||||
|
{ID: "u2", Kind: "unit", Text: "Second source unit."},
|
||||||
|
{ID: "u3", Kind: "unit", Text: "Third source unit."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sourceDocumentWithUnitMetadata() *source.SourceDocument {
|
||||||
|
return &source.SourceDocument{
|
||||||
|
ID: "source-1",
|
||||||
|
Kind: "document",
|
||||||
|
Format: "text/plain",
|
||||||
|
Digest: "sha256:source",
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
{
|
||||||
|
ID: "u1",
|
||||||
|
Kind: "source-kind",
|
||||||
|
Text: "source text",
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"speaker": "source-speaker",
|
||||||
|
"topic": "source-topic",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "u2",
|
||||||
|
Kind: "source-kind",
|
||||||
|
Text: "second source text",
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"speaker": "source-speaker-2",
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1194,6 +1473,19 @@ func sourceChunkWithID(id string, index int) contracts.SourceChunk {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func unitWithID(id string) source.SourceUnit {
|
||||||
|
switch id {
|
||||||
|
case "u1":
|
||||||
|
return source.SourceUnit{ID: "u1", Kind: "unit", Text: "Source unit."}
|
||||||
|
case "u2":
|
||||||
|
return source.SourceUnit{ID: "u2", Kind: "unit", Text: "Second source unit."}
|
||||||
|
case "u3":
|
||||||
|
return source.SourceUnit{ID: "u3", Kind: "unit", Text: "Third source unit."}
|
||||||
|
default:
|
||||||
|
return source.SourceUnit{ID: id, Kind: "unit", Text: "Unknown source unit."}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func warningReasons(warnings []contracts.Warning) []string {
|
func warningReasons(warnings []contracts.Warning) []string {
|
||||||
reasons := make([]string, 0, len(warnings))
|
reasons := make([]string, 0, len(warnings))
|
||||||
for _, warning := range warnings {
|
for _, warning := range warnings {
|
||||||
|
|||||||
6
internal/modules/chunk/dnd/scenes/assets.go
Normal file
6
internal/modules/chunk/dnd/scenes/assets.go
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
package scenes
|
||||||
|
|
||||||
|
import "embed"
|
||||||
|
|
||||||
|
//go:embed assets/prompts/*.md assets/schemas/*.json
|
||||||
|
var embeddedAssets embed.FS
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
You identify coherent scenes in Dungeons & Dragons session source units.
|
||||||
|
|
||||||
|
{{ hardening }}
|
||||||
|
|
||||||
|
Use only the provided source units. Source text may contain transcription
|
||||||
|
errors, repeated lines, incomplete sentences, and misheard proper nouns. Speaker
|
||||||
|
metadata, when present, may be treated as accurate.
|
||||||
|
|
||||||
|
Return only valid JSON matching the provided response schema.
|
||||||
71
internal/modules/chunk/dnd/scenes/assets/prompts/user.md
Normal file
71
internal/modules/chunk/dnd/scenes/assets/prompts/user.md
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
Source document ID: {{ .SourceID }}
|
||||||
|
|
||||||
|
Ordered source units:
|
||||||
|
{{ range .Units }}
|
||||||
|
- Unit ID: {{ .ID }}
|
||||||
|
Text: {{ .Text }}
|
||||||
|
{{ if .Metadata }}
|
||||||
|
Metadata:
|
||||||
|
{{ range .Metadata }}
|
||||||
|
- {{ .Key }}: {{ .Value }}
|
||||||
|
{{ end }}
|
||||||
|
{{ end }}
|
||||||
|
{{ end }}
|
||||||
|
|
||||||
|
Divide these source units into D&D scenes for the dnd/scenes chunk module.
|
||||||
|
|
||||||
|
A scene is a coherent unit of play. Start a new scene when there is a meaningful
|
||||||
|
change in location, objective, threat, activity, encounter, or mode of play.
|
||||||
|
|
||||||
|
Good reasons to start a new scene include:
|
||||||
|
- the party moves to a new location;
|
||||||
|
- a combat encounter begins or ends;
|
||||||
|
- combat changes into a substantially different phase;
|
||||||
|
- the party shifts between combat, exploration, social interaction, discussion,
|
||||||
|
planning, travel, rest, or downtime;
|
||||||
|
- a new NPC, faction, threat, or objective becomes central;
|
||||||
|
- the party completes one immediate goal and begins another;
|
||||||
|
- a major table-level rules discussion interrupts and materially changes play.
|
||||||
|
|
||||||
|
Do not start a new scene merely because:
|
||||||
|
- the speaker changes;
|
||||||
|
- a new combat round begins;
|
||||||
|
- a player asks a brief rules question;
|
||||||
|
- there is a joke, aside, or short table comment;
|
||||||
|
- a character takes a routine turn;
|
||||||
|
- the same encounter continues without a meaningful change in situation.
|
||||||
|
|
||||||
|
dnd/scenes boundary policy:
|
||||||
|
- cover the full provided source document from the first source unit to the last
|
||||||
|
source unit;
|
||||||
|
- return sequential scenes with no gaps;
|
||||||
|
- do not overlap scenes;
|
||||||
|
- preserve source-unit order;
|
||||||
|
- use exact source-unit IDs from the ordered source units;
|
||||||
|
- each scene must have start_unit_id and end_unit_id;
|
||||||
|
- do not include final chunk IDs or chunk indexes.
|
||||||
|
|
||||||
|
For each scene:
|
||||||
|
- short_title should be brief and factual;
|
||||||
|
- primary_mode must be Recap, Discussion, Combat, or Narrative;
|
||||||
|
- main_participants should include only principal characters, NPCs, factions, or
|
||||||
|
groups involved;
|
||||||
|
- summary should be factual and compact, usually one to three sentences;
|
||||||
|
- boundary_note should explain why the scene begins at start_unit_id and ends at
|
||||||
|
end_unit_id;
|
||||||
|
- boundary_confidence must be High, Medium, or Low.
|
||||||
|
|
||||||
|
Primary mode guidance:
|
||||||
|
- Use Recap for opening recap, initiative setup, session framing, or immediate
|
||||||
|
continuation from prior events.
|
||||||
|
- Use Discussion when the party is primarily discussing options or choosing a
|
||||||
|
course of action.
|
||||||
|
- Use Combat when active combat or combat-resolution mechanics dominate.
|
||||||
|
- Use Narrative for all other non-combat gameplay, including exploration, social
|
||||||
|
interactions, shopping, preparation, travel, rest, and downtime.
|
||||||
|
|
||||||
|
In boundary_caveats, list overall caveats about scene divisions. Include scenes
|
||||||
|
that could reasonably be split differently, combat phases that were kept
|
||||||
|
together, gradual transitions, or places where map context would have helped.
|
||||||
|
|
||||||
|
Return exactly one JSON object and no explanatory text.
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "notarius.dnd.scenes",
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"scenes",
|
||||||
|
"boundary_caveats"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"scenes": {
|
||||||
|
"type": "array",
|
||||||
|
"minItems": 1,
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"start_unit_id",
|
||||||
|
"end_unit_id",
|
||||||
|
"short_title",
|
||||||
|
"primary_mode",
|
||||||
|
"main_participants",
|
||||||
|
"summary",
|
||||||
|
"boundary_note",
|
||||||
|
"boundary_confidence"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"start_unit_id": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
|
},
|
||||||
|
"end_unit_id": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
|
},
|
||||||
|
"short_title": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
|
},
|
||||||
|
"primary_mode": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"Recap",
|
||||||
|
"Discussion",
|
||||||
|
"Combat",
|
||||||
|
"Narrative"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"main_participants": {
|
||||||
|
"type": "array",
|
||||||
|
"minItems": 1,
|
||||||
|
"items": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"summary": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
|
},
|
||||||
|
"boundary_note": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
|
},
|
||||||
|
"boundary_confidence": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"High",
|
||||||
|
"Medium",
|
||||||
|
"Low"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"boundary_caveats": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
311
internal/modules/chunk/dnd/scenes/chunker.go
Normal file
311
internal/modules/chunk/dnd/scenes/chunker.go
Normal file
@@ -0,0 +1,311 @@
|
|||||||
|
package scenes
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Key = "dnd/scenes"
|
||||||
|
|
||||||
|
var requiredCapabilities = []string{
|
||||||
|
"source.transcript",
|
||||||
|
}
|
||||||
|
|
||||||
|
var providedCapabilities = []string{
|
||||||
|
"chunks",
|
||||||
|
"chunks.scenes",
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ contracts.Chunker = (*Chunker)(nil)
|
||||||
|
var _ contracts.ManifestMetadataProvider = (*Chunker)(nil)
|
||||||
|
|
||||||
|
type Chunker struct{}
|
||||||
|
|
||||||
|
func New() *Chunker {
|
||||||
|
return &Chunker{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Chunker) Key() string {
|
||||||
|
return Key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Chunker) ManifestMetadata() map[string]any {
|
||||||
|
promptMetadata := scenesPromptBundle.Metadata()
|
||||||
|
metadata := map[string]any{
|
||||||
|
"prompt_id": PromptID,
|
||||||
|
"prompt_version": promptMetadata.PromptVersion,
|
||||||
|
"prompt_sha256": promptMetadata.SHA256,
|
||||||
|
"response_schema_key": string(ResponseSchemaKey),
|
||||||
|
"response_schema_id": ResponseSchemaID,
|
||||||
|
"response_schema_name": ResponseSchemaName,
|
||||||
|
}
|
||||||
|
if schema, err := loadResponseSchema(); err == nil {
|
||||||
|
metadata["response_schema_version"] = schema.Version
|
||||||
|
metadata["response_schema_sha256"] = schema.SHA256
|
||||||
|
}
|
||||||
|
return metadata
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||||
|
if c == nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")
|
||||||
|
}
|
||||||
|
if ctx == nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("context must not be nil")
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("context error before chunking: %w", err)
|
||||||
|
}
|
||||||
|
if req.Source == nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("source must not be nil")
|
||||||
|
}
|
||||||
|
if len(req.Source.Units) == 0 {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("source units must not be empty")
|
||||||
|
}
|
||||||
|
if err := source.ValidateDocument(req.Source); err != nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("validate source document: %w", err)
|
||||||
|
}
|
||||||
|
if req.LLMClient == nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("LLM client must not be nil")
|
||||||
|
}
|
||||||
|
if len(req.Options) > 0 {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("options are not supported")
|
||||||
|
}
|
||||||
|
|
||||||
|
system, user, _, err := renderPrompt(req)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("render prompt: %w", err)
|
||||||
|
}
|
||||||
|
schema, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("load response schema %q: %w", ResponseSchemaKey, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var response chunkResponse
|
||||||
|
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||||
|
StageName: Key,
|
||||||
|
Messages: []contracts.LLMMessage{
|
||||||
|
{Role: "system", Content: system},
|
||||||
|
{Role: "user", Content: user},
|
||||||
|
},
|
||||||
|
ResponseSchemaName: schema.Name,
|
||||||
|
ResponseSchema: schema.JSONSchema,
|
||||||
|
}, &response); err != nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("complete structured output: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
warnings, err := warningsFromCaveats(response.BoundaryCaveats)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
|
||||||
|
}
|
||||||
|
chunks, err := chunksFromResponse(req.Source, response)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ChunkResult{}, chunkerErrorf("malformed structured output: %w", err)
|
||||||
|
}
|
||||||
|
return contracts.ChunkResult{
|
||||||
|
Chunks: chunks,
|
||||||
|
Warnings: warnings,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ModuleSpec() pipeline.ModuleSpec {
|
||||||
|
return pipeline.ModuleSpec{
|
||||||
|
Key: Key,
|
||||||
|
Stage: pipeline.StageChunk,
|
||||||
|
Requires: append([]string(nil), requiredCapabilities...),
|
||||||
|
Provides: append([]string(nil), providedCapabilities...),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Register(registry *pipeline.ChunkerRegistry) error {
|
||||||
|
return registry.RegisterWithSpec(ModuleSpec(), func() (contracts.Chunker, error) {
|
||||||
|
return New(), nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]contracts.SourceChunk, error) {
|
||||||
|
if response.Scenes == nil {
|
||||||
|
return nil, fmt.Errorf("scenes must be present")
|
||||||
|
}
|
||||||
|
if len(response.Scenes) == 0 {
|
||||||
|
return nil, fmt.Errorf("scenes must not be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
unitIndexes := make(map[string]int, len(doc.Units))
|
||||||
|
for i, unit := range doc.Units {
|
||||||
|
unitIndexes[unit.ID] = i
|
||||||
|
}
|
||||||
|
|
||||||
|
chunks := make([]contracts.SourceChunk, 0, len(response.Scenes))
|
||||||
|
previousEnd := -1
|
||||||
|
for i, scene := range response.Scenes {
|
||||||
|
normalized, err := normalizeScene(i, scene)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
startIndex, ok := unitIndexes[normalized.StartUnitID]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("scene[%d] start_unit_id %q was not found", i, normalized.StartUnitID)
|
||||||
|
}
|
||||||
|
endIndex, ok := unitIndexes[normalized.EndUnitID]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("scene[%d] end_unit_id %q was not found", i, normalized.EndUnitID)
|
||||||
|
}
|
||||||
|
if startIndex > endIndex {
|
||||||
|
return nil, fmt.Errorf("scene[%d] start_unit_id %q appears after end_unit_id %q", i, normalized.StartUnitID, normalized.EndUnitID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if i == 0 && startIndex != 0 {
|
||||||
|
return nil, fmt.Errorf("first scene must start at first source unit %q", doc.Units[0].ID)
|
||||||
|
}
|
||||||
|
if i > 0 {
|
||||||
|
if startIndex <= previousEnd {
|
||||||
|
return nil, fmt.Errorf("scene[%d] overlaps previous scene", i)
|
||||||
|
}
|
||||||
|
if startIndex > previousEnd+1 {
|
||||||
|
return nil, fmt.Errorf("scene[%d] leaves a gap after previous scene", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
previousEnd = endIndex
|
||||||
|
|
||||||
|
units := cloneUnits(doc.Units[startIndex : endIndex+1])
|
||||||
|
chunks = append(chunks, contracts.SourceChunk{
|
||||||
|
ID: fmt.Sprintf("scene-%06d", i+1),
|
||||||
|
SourceID: doc.ID,
|
||||||
|
Index: i,
|
||||||
|
Units: units,
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"scene_title": normalized.ShortTitle,
|
||||||
|
"primary_mode": normalized.PrimaryMode,
|
||||||
|
"main_participants": append([]string(nil), normalized.MainParticipants...),
|
||||||
|
"summary": normalized.Summary,
|
||||||
|
"boundary_note": normalized.BoundaryNote,
|
||||||
|
"boundary_confidence": normalized.BoundaryConfidence,
|
||||||
|
"start_unit_id": normalized.StartUnitID,
|
||||||
|
"end_unit_id": normalized.EndUnitID,
|
||||||
|
"unit_count": len(units),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if previousEnd != len(doc.Units)-1 {
|
||||||
|
return nil, fmt.Errorf("final scene must end at final source unit %q", doc.Units[len(doc.Units)-1].ID)
|
||||||
|
}
|
||||||
|
return chunks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeScene(index int, scene sceneResponse) (sceneResponse, error) {
|
||||||
|
out := sceneResponse{
|
||||||
|
StartUnitID: strings.TrimSpace(scene.StartUnitID),
|
||||||
|
EndUnitID: strings.TrimSpace(scene.EndUnitID),
|
||||||
|
ShortTitle: strings.TrimSpace(scene.ShortTitle),
|
||||||
|
PrimaryMode: strings.TrimSpace(scene.PrimaryMode),
|
||||||
|
Summary: strings.TrimSpace(scene.Summary),
|
||||||
|
BoundaryNote: strings.TrimSpace(scene.BoundaryNote),
|
||||||
|
BoundaryConfidence: strings.TrimSpace(scene.BoundaryConfidence),
|
||||||
|
}
|
||||||
|
|
||||||
|
required := map[string]string{
|
||||||
|
"start_unit_id": out.StartUnitID,
|
||||||
|
"end_unit_id": out.EndUnitID,
|
||||||
|
"short_title": out.ShortTitle,
|
||||||
|
"primary_mode": out.PrimaryMode,
|
||||||
|
"summary": out.Summary,
|
||||||
|
"boundary_note": out.BoundaryNote,
|
||||||
|
"boundary_confidence": out.BoundaryConfidence,
|
||||||
|
}
|
||||||
|
for field, value := range required {
|
||||||
|
if value == "" {
|
||||||
|
return sceneResponse{}, fmt.Errorf("scene[%d] %s must not be empty", index, field)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !validPrimaryMode(out.PrimaryMode) {
|
||||||
|
return sceneResponse{}, fmt.Errorf("scene[%d] primary_mode %q is not supported", index, out.PrimaryMode)
|
||||||
|
}
|
||||||
|
if !validBoundaryConfidence(out.BoundaryConfidence) {
|
||||||
|
return sceneResponse{}, fmt.Errorf("scene[%d] boundary_confidence %q is not supported", index, out.BoundaryConfidence)
|
||||||
|
}
|
||||||
|
if len(scene.MainParticipants) == 0 {
|
||||||
|
return sceneResponse{}, fmt.Errorf("scene[%d] main_participants must not be empty", index)
|
||||||
|
}
|
||||||
|
out.MainParticipants = make([]string, 0, len(scene.MainParticipants))
|
||||||
|
for participantIndex, participant := range scene.MainParticipants {
|
||||||
|
trimmed := strings.TrimSpace(participant)
|
||||||
|
if trimmed == "" {
|
||||||
|
return sceneResponse{}, fmt.Errorf("scene[%d] main_participants[%d] must not be empty", index, participantIndex)
|
||||||
|
}
|
||||||
|
out.MainParticipants = append(out.MainParticipants, trimmed)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validPrimaryMode(value string) bool {
|
||||||
|
switch value {
|
||||||
|
case "Recap", "Discussion", "Combat", "Narrative":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validBoundaryConfidence(value string) bool {
|
||||||
|
switch value {
|
||||||
|
case "High", "Medium", "Low":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func warningsFromCaveats(caveats []string) ([]contracts.Warning, error) {
|
||||||
|
if len(caveats) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
warnings := make([]contracts.Warning, 0, len(caveats))
|
||||||
|
for i, caveat := range caveats {
|
||||||
|
trimmed := strings.TrimSpace(caveat)
|
||||||
|
if trimmed == "" {
|
||||||
|
return nil, fmt.Errorf("boundary_caveats[%d] must not be empty after trimming", i)
|
||||||
|
}
|
||||||
|
warnings = append(warnings, contracts.Warning{
|
||||||
|
Scope: Key,
|
||||||
|
ReasonCode: "scene_boundary_caveat",
|
||||||
|
Message: trimmed,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return warnings, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
|
||||||
|
out := make([]source.SourceUnit, 0, len(units))
|
||||||
|
for _, unit := range units {
|
||||||
|
out = append(out, source.SourceUnit{
|
||||||
|
ID: unit.ID,
|
||||||
|
Kind: unit.Kind,
|
||||||
|
Text: unit.Text,
|
||||||
|
Metadata: cloneMetadata(unit.Metadata),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneMetadata(metadata map[string]any) map[string]any {
|
||||||
|
if len(metadata) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]any, len(metadata))
|
||||||
|
for key, value := range metadata {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func chunkerErrorf(format string, args ...any) error {
|
||||||
|
return fmt.Errorf("dnd scenes chunker: "+format, args...)
|
||||||
|
}
|
||||||
484
internal/modules/chunk/dnd/scenes/chunker_test.go
Normal file
484
internal/modules/chunk/dnd/scenes/chunker_test.go
Normal file
@@ -0,0 +1,484 @@
|
|||||||
|
package scenes
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewModuleSpecAndRegister(t *testing.T) {
|
||||||
|
chunker := New()
|
||||||
|
if chunker == nil {
|
||||||
|
t.Fatal("New() = nil, want chunker")
|
||||||
|
}
|
||||||
|
if chunker.Key() != Key {
|
||||||
|
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := pipeline.ModuleSpec{
|
||||||
|
Key: Key,
|
||||||
|
Stage: pipeline.StageChunk,
|
||||||
|
Requires: []string{"source.transcript"},
|
||||||
|
Provides: []string{"chunks", "chunks.scenes"},
|
||||||
|
}
|
||||||
|
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||||
|
}
|
||||||
|
got := ModuleSpec()
|
||||||
|
got.Requires[0] = "changed"
|
||||||
|
got.Provides[0] = "changed"
|
||||||
|
if again := ModuleSpec(); !reflect.DeepEqual(again, want) {
|
||||||
|
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
registry := pipeline.NewChunkerRegistry()
|
||||||
|
if err := Register(registry); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
registered, ok := registry.Spec(Key)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("Spec(%q) ok = false, want true", Key)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(registered, want) {
|
||||||
|
t.Fatalf("registered spec = %#v, want %#v", registered, want)
|
||||||
|
}
|
||||||
|
built, err := registry.Build(Key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
||||||
|
}
|
||||||
|
if built.Key() != Key {
|
||||||
|
t.Fatalf("built Key() = %q, want %q", built.Key(), Key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegisterNilRegistryReturnsError(t *testing.T) {
|
||||||
|
err := Register(nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Register(nil) error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "chunker registry") {
|
||||||
|
t.Fatalf("Register(nil) error = %q, want registry context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
|
||||||
|
client := &fakeScenesLLMClient{
|
||||||
|
response: chunkResponse{
|
||||||
|
Scenes: []sceneResponse{
|
||||||
|
{
|
||||||
|
StartUnitID: "seg-001",
|
||||||
|
EndUnitID: "seg-002",
|
||||||
|
ShortTitle: " Goblin parley ",
|
||||||
|
PrimaryMode: "Discussion",
|
||||||
|
MainParticipants: []string{" Aria ", "Goblin scout"},
|
||||||
|
Summary: " The party negotiates with a scout. ",
|
||||||
|
BoundaryNote: " The scene covers the discussion before fighting starts. ",
|
||||||
|
BoundaryConfidence: "High",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
StartUnitID: "seg-003",
|
||||||
|
EndUnitID: "seg-004",
|
||||||
|
ShortTitle: "Ambush at the gate",
|
||||||
|
PrimaryMode: "Combat",
|
||||||
|
MainParticipants: []string{"Aria", "Goblin ambushers"},
|
||||||
|
Summary: "The goblins attack at the gate.",
|
||||||
|
BoundaryNote: "Combat begins and resolves the immediate threat.",
|
||||||
|
BoundaryConfidence: "Medium",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
BoundaryCaveats: []string{" The transition into combat is gradual. "},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Chunk() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(client.requests) != 1 {
|
||||||
|
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||||
|
}
|
||||||
|
req := client.requests[0]
|
||||||
|
if req.StageName != Key {
|
||||||
|
t.Fatalf("StageName = %q, want %q", req.StageName, Key)
|
||||||
|
}
|
||||||
|
schema, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if req.ResponseSchemaName != schema.Name {
|
||||||
|
t.Fatalf("ResponseSchemaName = %q, want %q", req.ResponseSchemaName, schema.Name)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(req.ResponseSchema, schema.JSONSchema) {
|
||||||
|
t.Fatal("ResponseSchema does not match D&D scenes schema")
|
||||||
|
}
|
||||||
|
if len(req.Messages) != 2 || req.Messages[0].Role != "system" || req.Messages[1].Role != "user" {
|
||||||
|
t.Fatalf("Messages = %#v, want system then user", req.Messages)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"session-alpha", "seg-001", "seg-004", "start_unit_id", "boundary_confidence"} {
|
||||||
|
if !strings.Contains(req.Messages[1].Content, want) {
|
||||||
|
t.Fatalf("user message = %q, want substring %q", req.Messages[1].Content, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"scene-000001", "scene-000002"}) {
|
||||||
|
t.Fatalf("chunk IDs = %#v, want deterministic scene IDs", got)
|
||||||
|
}
|
||||||
|
gotUnits := [][]string{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units)}
|
||||||
|
wantUnits := [][]string{{"seg-001", "seg-002"}, {"seg-003", "seg-004"}}
|
||||||
|
if !reflect.DeepEqual(gotUnits, wantUnits) {
|
||||||
|
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
|
||||||
|
}
|
||||||
|
first := result.Chunks[0]
|
||||||
|
if first.SourceID != "session-alpha" || first.Index != 0 {
|
||||||
|
t.Fatalf("first chunk = %#v, want source and index fields", first)
|
||||||
|
}
|
||||||
|
if first.Metadata["scene_title"] != "Goblin parley" ||
|
||||||
|
first.Metadata["primary_mode"] != "Discussion" ||
|
||||||
|
first.Metadata["summary"] != "The party negotiates with a scout." ||
|
||||||
|
first.Metadata["boundary_note"] != "The scene covers the discussion before fighting starts." ||
|
||||||
|
first.Metadata["boundary_confidence"] != "High" ||
|
||||||
|
first.Metadata["start_unit_id"] != "seg-001" ||
|
||||||
|
first.Metadata["end_unit_id"] != "seg-002" ||
|
||||||
|
first.Metadata["unit_count"] != 2 {
|
||||||
|
t.Fatalf("first metadata = %#v, want scene metadata", first.Metadata)
|
||||||
|
}
|
||||||
|
if got, ok := first.Metadata["main_participants"].([]string); !ok || !reflect.DeepEqual(got, []string{"Aria", "Goblin scout"}) {
|
||||||
|
t.Fatalf("main_participants = %#v, want trimmed participant slice", first.Metadata["main_participants"])
|
||||||
|
}
|
||||||
|
if got := result.Warnings; len(got) != 1 ||
|
||||||
|
got[0].Scope != Key ||
|
||||||
|
got[0].ReasonCode != "scene_boundary_caveat" ||
|
||||||
|
got[0].Message != "The transition into combat is gradual." {
|
||||||
|
t.Fatalf("Warnings = %#v, want boundary caveat warning", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
|
||||||
|
client := &fakeScenesLLMClient{
|
||||||
|
response: chunkResponse{
|
||||||
|
Scenes: validSceneResponse().Scenes,
|
||||||
|
BoundaryCaveats: []string{
|
||||||
|
" ",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Chunk() error = nil, want malformed structured output error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "dnd scenes chunker") || !strings.Contains(err.Error(), "malformed structured output") || !strings.Contains(err.Error(), "boundary_caveats[0]") {
|
||||||
|
t.Fatalf("Chunk() error = %q, want malformed boundary caveat context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) {
|
||||||
|
doc := sceneSourceDocument()
|
||||||
|
client := &fakeScenesLLMClient{response: validSceneResponse()}
|
||||||
|
|
||||||
|
result, err := New().Chunk(context.Background(), contracts.ChunkRequest{
|
||||||
|
Source: doc,
|
||||||
|
LLMClient: client,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Chunk() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
doc.Units[0].ID = "mutated"
|
||||||
|
doc.Units[0].Metadata["speaker"] = "mutated"
|
||||||
|
client.response.Scenes[0].MainParticipants[0] = "mutated"
|
||||||
|
|
||||||
|
if result.Chunks[0].Units[0].ID != "seg-001" {
|
||||||
|
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
|
||||||
|
}
|
||||||
|
if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" {
|
||||||
|
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
|
||||||
|
}
|
||||||
|
participants, ok := result.Chunks[0].Metadata["main_participants"].([]string)
|
||||||
|
if !ok || participants[0] != "Aria" {
|
||||||
|
t.Fatalf("participants = %#v, want defensive copy", result.Chunks[0].Metadata["main_participants"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkerManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T) {
|
||||||
|
metadata := New().ManifestMetadata()
|
||||||
|
|
||||||
|
tests := map[string]string{
|
||||||
|
"prompt_id": PromptID,
|
||||||
|
"prompt_version": ResponseSchemaVersion,
|
||||||
|
"response_schema_key": string(ResponseSchemaKey),
|
||||||
|
"response_schema_id": ResponseSchemaID,
|
||||||
|
"response_schema_name": ResponseSchemaName,
|
||||||
|
"response_schema_version": ResponseSchemaVersion,
|
||||||
|
}
|
||||||
|
for key, want := range tests {
|
||||||
|
if metadata[key] != want {
|
||||||
|
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
|
||||||
|
value, ok := metadata[key].(string)
|
||||||
|
if !ok || !strings.HasPrefix(value, "sha256:") {
|
||||||
|
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"prompt", "schema", "source", "text"} {
|
||||||
|
if _, ok := metadata[forbidden]; ok {
|
||||||
|
t.Fatalf("metadata includes raw %q field: %#v", forbidden, metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkRejectsInvalidRequests(t *testing.T) {
|
||||||
|
validClient := &fakeScenesLLMClient{response: validSceneResponse()}
|
||||||
|
validReq := chunkRequestWithClient(validClient)
|
||||||
|
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
invalidDoc := sceneSourceDocument()
|
||||||
|
invalidDoc.Units[0].ID = ""
|
||||||
|
emptyDoc := sceneSourceDocument()
|
||||||
|
emptyDoc.Units = nil
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
chunker *Chunker
|
||||||
|
ctx context.Context
|
||||||
|
req contracts.ChunkRequest
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "nil chunker", chunker: nil, ctx: context.Background(), req: validReq, want: "chunker"},
|
||||||
|
{name: "nil context", chunker: New(), ctx: nil, req: validReq, want: "context"},
|
||||||
|
{name: "canceled context", chunker: New(), ctx: canceledCtx, req: validReq, want: "context"},
|
||||||
|
{name: "nil source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{LLMClient: validClient}, want: "source"},
|
||||||
|
{name: "empty source units", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: emptyDoc, LLMClient: validClient}, want: "units"},
|
||||||
|
{name: "invalid source", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: invalidDoc, LLMClient: validClient}, want: "validate source document"},
|
||||||
|
{name: "nil LLM client", chunker: New(), ctx: context.Background(), req: contracts.ChunkRequest{Source: sceneSourceDocument()}, want: "LLM client"},
|
||||||
|
{name: "unsupported options", chunker: New(), ctx: context.Background(), req: requestWithOptions(validReq), want: "options"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := tt.chunker.Chunk(tt.ctx, tt.req)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Chunk() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
|
||||||
|
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkRejectsMalformedStructuredOutput(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
response chunkResponse
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{name: "missing scenes", response: chunkResponse{}, want: "scenes"},
|
||||||
|
{name: "empty scenes", response: chunkResponse{Scenes: []sceneResponse{}}, want: "scenes"},
|
||||||
|
{
|
||||||
|
name: "unknown boundary id",
|
||||||
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
||||||
|
scene("seg-001", "seg-999"),
|
||||||
|
}),
|
||||||
|
want: "was not found",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "out of order boundaries",
|
||||||
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
||||||
|
scene("seg-003", "seg-002"),
|
||||||
|
}),
|
||||||
|
want: "appears after",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "gap",
|
||||||
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
||||||
|
scene("seg-001", "seg-001"),
|
||||||
|
scene("seg-003", "seg-004"),
|
||||||
|
}),
|
||||||
|
want: "gap",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "overlap",
|
||||||
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
||||||
|
scene("seg-001", "seg-002"),
|
||||||
|
scene("seg-002", "seg-004"),
|
||||||
|
}),
|
||||||
|
want: "overlap",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "incomplete coverage",
|
||||||
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
||||||
|
scene("seg-001", "seg-003"),
|
||||||
|
}),
|
||||||
|
want: "final scene",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty metadata field",
|
||||||
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
||||||
|
{
|
||||||
|
StartUnitID: "seg-001",
|
||||||
|
EndUnitID: "seg-004",
|
||||||
|
ShortTitle: " ",
|
||||||
|
PrimaryMode: "Narrative",
|
||||||
|
MainParticipants: []string{"Aria"},
|
||||||
|
Summary: "Summary.",
|
||||||
|
BoundaryNote: "Note.",
|
||||||
|
BoundaryConfidence: "High",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
want: "short_title",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty participant",
|
||||||
|
response: replaceScenes(validSceneResponse(), []sceneResponse{
|
||||||
|
{
|
||||||
|
StartUnitID: "seg-001",
|
||||||
|
EndUnitID: "seg-004",
|
||||||
|
ShortTitle: "Title",
|
||||||
|
PrimaryMode: "Narrative",
|
||||||
|
MainParticipants: []string{"Aria", " "},
|
||||||
|
Summary: "Summary.",
|
||||||
|
BoundaryNote: "Note.",
|
||||||
|
BoundaryConfidence: "High",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
want: "main_participants",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
client := &fakeScenesLLMClient{response: tt.response}
|
||||||
|
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Chunk() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), tt.want) {
|
||||||
|
t.Fatalf("Chunk() error = %q, want module context and %q", err.Error(), tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunkWrapsLLMClientError(t *testing.T) {
|
||||||
|
client := &fakeScenesLLMClient{err: errors.New("provider unavailable")}
|
||||||
|
|
||||||
|
_, err := New().Chunk(context.Background(), chunkRequestWithClient(client))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Chunk() error = nil, want LLM error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "dnd scenes") || !strings.Contains(err.Error(), "provider unavailable") {
|
||||||
|
t.Fatalf("Chunk() error = %q, want wrapped LLM context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func chunkRequestWithClient(client contracts.StructuredLLMClient) contracts.ChunkRequest {
|
||||||
|
return contracts.ChunkRequest{
|
||||||
|
Source: sceneSourceDocument(),
|
||||||
|
LLMClient: client,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestWithOptions(req contracts.ChunkRequest) contracts.ChunkRequest {
|
||||||
|
req.Options = map[string]any{"max_units": 2}
|
||||||
|
return req
|
||||||
|
}
|
||||||
|
|
||||||
|
func sceneSourceDocument() *source.SourceDocument {
|
||||||
|
return &source.SourceDocument{
|
||||||
|
ID: "session-alpha",
|
||||||
|
Kind: "transcript",
|
||||||
|
Format: "application/vnd.seriatim.minimal+json",
|
||||||
|
Digest: "sha256:source",
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
{ID: "seg-001", Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}},
|
||||||
|
{ID: "seg-002", Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."},
|
||||||
|
{ID: "seg-003", Kind: "transcript_segment", Text: "The guards rush out with blades drawn."},
|
||||||
|
{ID: "seg-004", Kind: "transcript_segment", Text: "The party defeats the ambushers."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validSceneResponse() chunkResponse {
|
||||||
|
return chunkResponse{
|
||||||
|
Scenes: []sceneResponse{
|
||||||
|
scene("seg-001", "seg-004"),
|
||||||
|
},
|
||||||
|
BoundaryCaveats: []string{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func replaceScenes(response chunkResponse, scenes []sceneResponse) chunkResponse {
|
||||||
|
response.Scenes = scenes
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
|
||||||
|
func scene(startUnitID string, endUnitID string) sceneResponse {
|
||||||
|
return sceneResponse{
|
||||||
|
StartUnitID: startUnitID,
|
||||||
|
EndUnitID: endUnitID,
|
||||||
|
ShortTitle: "Scene title",
|
||||||
|
PrimaryMode: "Narrative",
|
||||||
|
MainParticipants: []string{"Aria"},
|
||||||
|
Summary: "A compact summary.",
|
||||||
|
BoundaryNote: "The source units form one coherent scene.",
|
||||||
|
BoundaryConfidence: "High",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func chunkIDs(chunks []contracts.SourceChunk) []string {
|
||||||
|
ids := make([]string, 0, len(chunks))
|
||||||
|
for _, chunk := range chunks {
|
||||||
|
ids = append(ids, chunk.ID)
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func unitIDs(units []source.SourceUnit) []string {
|
||||||
|
ids := make([]string, 0, len(units))
|
||||||
|
for _, unit := range units {
|
||||||
|
ids = append(ids, unit.ID)
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeScenesLLMClient struct {
|
||||||
|
response chunkResponse
|
||||||
|
err error
|
||||||
|
requests []contracts.StructuredCompletionRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (client *fakeScenesLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
|
client.requests = append(client.requests, contracts.StructuredCompletionRequest{
|
||||||
|
StageName: req.StageName,
|
||||||
|
Messages: append([]contracts.LLMMessage(nil), req.Messages...),
|
||||||
|
Model: req.Model,
|
||||||
|
ResponseSchemaName: req.ResponseSchemaName,
|
||||||
|
ResponseSchema: append(json.RawMessage(nil), req.ResponseSchema...),
|
||||||
|
})
|
||||||
|
if client.err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, client.err
|
||||||
|
}
|
||||||
|
|
||||||
|
target, ok := out.(*chunkResponse)
|
||||||
|
if !ok {
|
||||||
|
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||||
|
}
|
||||||
|
*target = client.response
|
||||||
|
content, err := json.Marshal(client.response)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
|
}
|
||||||
|
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||||
|
}
|
||||||
17
internal/modules/chunk/dnd/scenes/model.go
Normal file
17
internal/modules/chunk/dnd/scenes/model.go
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
package scenes
|
||||||
|
|
||||||
|
type chunkResponse struct {
|
||||||
|
Scenes []sceneResponse `json:"scenes"`
|
||||||
|
BoundaryCaveats []string `json:"boundary_caveats"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type sceneResponse struct {
|
||||||
|
StartUnitID string `json:"start_unit_id"`
|
||||||
|
EndUnitID string `json:"end_unit_id"`
|
||||||
|
ShortTitle string `json:"short_title"`
|
||||||
|
PrimaryMode string `json:"primary_mode"`
|
||||||
|
MainParticipants []string `json:"main_participants"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
BoundaryNote string `json:"boundary_note"`
|
||||||
|
BoundaryConfidence string `json:"boundary_confidence"`
|
||||||
|
}
|
||||||
97
internal/modules/chunk/dnd/scenes/prompt.go
Normal file
97
internal/modules/chunk/dnd/scenes/prompt.go
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
package scenes
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/prompt"
|
||||||
|
)
|
||||||
|
|
||||||
|
type promptData struct {
|
||||||
|
SourceID string
|
||||||
|
Units []promptUnit
|
||||||
|
}
|
||||||
|
|
||||||
|
type promptUnit struct {
|
||||||
|
ID string
|
||||||
|
Text string
|
||||||
|
Metadata []promptMetadata
|
||||||
|
}
|
||||||
|
|
||||||
|
type promptMetadata struct {
|
||||||
|
Key string
|
||||||
|
Value string
|
||||||
|
}
|
||||||
|
|
||||||
|
var scenesPromptBundle = mustLoadPromptBundle()
|
||||||
|
|
||||||
|
func mustLoadPromptBundle() *prompt.Bundle {
|
||||||
|
bundle, err := prompt.LoadBundle(embeddedAssets, prompt.Definition{
|
||||||
|
PromptID: PromptID,
|
||||||
|
Version: ResponseSchemaVersion,
|
||||||
|
EmbeddedPath: "assets/prompts",
|
||||||
|
SystemPath: "assets/prompts/system.md",
|
||||||
|
UserPath: "assets/prompts/user.md",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return bundle
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildPromptData(req contracts.ChunkRequest) (promptData, error) {
|
||||||
|
if req.Source == nil {
|
||||||
|
return promptData{}, fmt.Errorf("dnd scenes prompt: source must not be nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
data := promptData{
|
||||||
|
SourceID: req.Source.ID,
|
||||||
|
Units: make([]promptUnit, 0, len(req.Source.Units)),
|
||||||
|
}
|
||||||
|
for _, unit := range req.Source.Units {
|
||||||
|
data.Units = append(data.Units, promptUnit{
|
||||||
|
ID: unit.ID,
|
||||||
|
Text: unit.Text,
|
||||||
|
Metadata: selectedMetadata(unit),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderPrompt(req contracts.ChunkRequest) (system string, user string, metadata prompt.Metadata, err error) {
|
||||||
|
data, err := buildPromptData(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", prompt.Metadata{}, err
|
||||||
|
}
|
||||||
|
system, user, metadata, err = scenesPromptBundle.RenderUserSystem(data)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", prompt.Metadata{}, fmt.Errorf("dnd scenes prompt: %w", err)
|
||||||
|
}
|
||||||
|
return system, user, metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectedMetadata(unit source.SourceUnit) []promptMetadata {
|
||||||
|
if len(unit.Metadata) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
keys := []string{"speaker", "start", "end"}
|
||||||
|
metadata := make([]promptMetadata, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
value, ok := unit.Metadata[key]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rendered := strings.TrimSpace(fmt.Sprint(value))
|
||||||
|
if rendered == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
metadata = append(metadata, promptMetadata{
|
||||||
|
Key: key,
|
||||||
|
Value: rendered,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return metadata
|
||||||
|
}
|
||||||
155
internal/modules/chunk/dnd/scenes/prompt_test.go
Normal file
155
internal/modules/chunk/dnd/scenes/prompt_test.go
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
package scenes
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/prompt"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildPromptDataFromSourceDocument(t *testing.T) {
|
||||||
|
req := promptChunkRequest()
|
||||||
|
|
||||||
|
data, err := buildPromptData(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("buildPromptData() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.SourceID != "session-alpha" {
|
||||||
|
t.Fatalf("SourceID = %q, want session-alpha", data.SourceID)
|
||||||
|
}
|
||||||
|
if len(data.Units) != 2 {
|
||||||
|
t.Fatalf("len(Units) = %d, want 2", len(data.Units))
|
||||||
|
}
|
||||||
|
first := data.Units[0]
|
||||||
|
if first.ID != "seg-001" || first.Text != "Aria and Bram discuss whether to enter the ruins." {
|
||||||
|
t.Fatalf("first unit = %#v, want source unit data", first)
|
||||||
|
}
|
||||||
|
wantMetadata := []promptMetadata{
|
||||||
|
{Key: "speaker", Value: "Alice"},
|
||||||
|
{Key: "start", Value: "1.25"},
|
||||||
|
{Key: "end", Value: "3.5"},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(first.Metadata, wantMetadata) {
|
||||||
|
t.Fatalf("first.Metadata = %#v, want %#v", first.Metadata, wantMetadata)
|
||||||
|
}
|
||||||
|
if len(data.Units[1].Metadata) != 0 {
|
||||||
|
t.Fatalf("second.Metadata = %#v, want no selected metadata", data.Units[1].Metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildPromptDataDoesNotMutateRequest(t *testing.T) {
|
||||||
|
req := promptChunkRequest()
|
||||||
|
beforeSource := mustJSON(t, req.Source)
|
||||||
|
beforeRequest := mustJSON(t, req)
|
||||||
|
|
||||||
|
if _, err := buildPromptData(req); err != nil {
|
||||||
|
t.Fatalf("buildPromptData() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
afterSource := mustJSON(t, req.Source)
|
||||||
|
afterRequest := mustJSON(t, req)
|
||||||
|
if beforeSource != afterSource || beforeRequest != afterRequest {
|
||||||
|
t.Fatalf(
|
||||||
|
"request mutated:\nsource before: %s\nsource after: %s\nrequest before: %s\nrequest after: %s",
|
||||||
|
beforeSource,
|
||||||
|
afterSource,
|
||||||
|
beforeRequest,
|
||||||
|
afterRequest,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderPromptIncludesSourceUnitsAndMetadata(t *testing.T) {
|
||||||
|
system, user, metadata, err := renderPrompt(promptChunkRequest())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("renderPrompt() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(system, prompt.HardeningText()) {
|
||||||
|
t.Fatalf("system prompt = %q, want hardening text", system)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"session-alpha",
|
||||||
|
"seg-001",
|
||||||
|
"seg-002",
|
||||||
|
"Aria and Bram discuss whether to enter the ruins.",
|
||||||
|
"The goblins rush out and initiative begins.",
|
||||||
|
"speaker: Alice",
|
||||||
|
"start: 1.25",
|
||||||
|
"end: 3.5",
|
||||||
|
"start_unit_id",
|
||||||
|
"end_unit_id",
|
||||||
|
"primary_mode",
|
||||||
|
"boundary_confidence",
|
||||||
|
"Recap, Discussion, Combat, or Narrative",
|
||||||
|
"High, Medium, or Low",
|
||||||
|
"no gaps",
|
||||||
|
"do not overlap",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(user, want) {
|
||||||
|
t.Fatalf("user prompt = %q, want substring %q", user, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if metadata.PromptID != PromptID {
|
||||||
|
t.Fatalf("metadata.PromptID = %q, want %q", metadata.PromptID, PromptID)
|
||||||
|
}
|
||||||
|
if metadata.PromptVersion != ResponseSchemaVersion {
|
||||||
|
t.Fatalf("metadata.PromptVersion = %q, want %q", metadata.PromptVersion, ResponseSchemaVersion)
|
||||||
|
}
|
||||||
|
if metadata.EmbeddedPath != "assets/prompts" {
|
||||||
|
t.Fatalf("metadata.EmbeddedPath = %q, want assets/prompts", metadata.EmbeddedPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildPromptDataRejectsMissingSourceContext(t *testing.T) {
|
||||||
|
if _, err := buildPromptData(contracts.ChunkRequest{}); err == nil || !strings.Contains(err.Error(), "source") {
|
||||||
|
t.Fatalf("buildPromptData() error = %v, want source error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptChunkRequest() contracts.ChunkRequest {
|
||||||
|
return contracts.ChunkRequest{
|
||||||
|
Source: promptSourceDocument(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptSourceDocument() *source.SourceDocument {
|
||||||
|
return &source.SourceDocument{
|
||||||
|
ID: "session-alpha",
|
||||||
|
Kind: "transcript",
|
||||||
|
Format: "application/vnd.seriatim.minimal+json",
|
||||||
|
Digest: "sha256:test",
|
||||||
|
Units: []source.SourceUnit{
|
||||||
|
{
|
||||||
|
ID: "seg-001",
|
||||||
|
Kind: "transcript_segment",
|
||||||
|
Text: "Aria and Bram discuss whether to enter the ruins.",
|
||||||
|
Metadata: map[string]any{
|
||||||
|
"speaker": "Alice",
|
||||||
|
"start": json.Number("1.25"),
|
||||||
|
"end": json.Number("3.5"),
|
||||||
|
"ignored": "not rendered",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "seg-002",
|
||||||
|
Kind: "transcript_segment",
|
||||||
|
Text: "The goblins rush out and initiative begins.",
|
||||||
|
Metadata: map[string]any{"ignored": "not rendered"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustJSON(t *testing.T, value any) string {
|
||||||
|
t.Helper()
|
||||||
|
encoded, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
return string(encoded)
|
||||||
|
}
|
||||||
21
internal/modules/chunk/dnd/scenes/schema.go
Normal file
21
internal/modules/chunk/dnd/scenes/schema.go
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
package scenes
|
||||||
|
|
||||||
|
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
|
|
||||||
|
const (
|
||||||
|
PromptID = "dnd.scenes"
|
||||||
|
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scenes")
|
||||||
|
ResponseSchemaID = "notarius.dnd.scenes"
|
||||||
|
ResponseSchemaVersion = "v1"
|
||||||
|
ResponseSchemaName = "notarius_dnd_scenes_v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||||
|
return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{
|
||||||
|
Key: ResponseSchemaKey,
|
||||||
|
ID: ResponseSchemaID,
|
||||||
|
Version: ResponseSchemaVersion,
|
||||||
|
Name: ResponseSchemaName,
|
||||||
|
AssetPath: "assets/schemas/dnd_scenes.v1.json",
|
||||||
|
})
|
||||||
|
}
|
||||||
146
internal/modules/chunk/dnd/scenes/schema_test.go
Normal file
146
internal/modules/chunk/dnd/scenes/schema_test.go
Normal file
@@ -0,0 +1,146 @@
|
|||||||
|
package scenes
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadResponseSchemaForScenes(t *testing.T) {
|
||||||
|
schema, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if schema.Key != ResponseSchemaKey {
|
||||||
|
t.Fatalf("schema.Key = %q, want %q", schema.Key, ResponseSchemaKey)
|
||||||
|
}
|
||||||
|
if schema.ID != ResponseSchemaID {
|
||||||
|
t.Fatalf("schema.ID = %q, want %q", schema.ID, ResponseSchemaID)
|
||||||
|
}
|
||||||
|
if schema.Version != ResponseSchemaVersion {
|
||||||
|
t.Fatalf("schema.Version = %q, want %q", schema.Version, ResponseSchemaVersion)
|
||||||
|
}
|
||||||
|
if schema.Name != ResponseSchemaName {
|
||||||
|
t.Fatalf("schema.Name = %q, want %q", schema.Name, ResponseSchemaName)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(schema.SHA256, "sha256:") {
|
||||||
|
t.Fatalf("schema.SHA256 = %q, want sha256 prefix", schema.SHA256)
|
||||||
|
}
|
||||||
|
if !json.Valid(schema.JSONSchema) {
|
||||||
|
t.Fatalf("schema.JSONSchema is invalid JSON: %s", schema.JSONSchema)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResponseSchemaShapeUsesSourceUnitBoundaries(t *testing.T) {
|
||||||
|
schema, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded map[string]any
|
||||||
|
if err := json.Unmarshal(schema.JSONSchema, &decoded); err != nil {
|
||||||
|
t.Fatalf("Unmarshal() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if decoded["$id"] != ResponseSchemaID {
|
||||||
|
t.Fatalf("$id = %#v, want %q", decoded["$id"], ResponseSchemaID)
|
||||||
|
}
|
||||||
|
if decoded["additionalProperties"] != false {
|
||||||
|
t.Fatalf("additionalProperties = %#v, want false", decoded["additionalProperties"])
|
||||||
|
}
|
||||||
|
|
||||||
|
properties := decoded["properties"].(map[string]any)
|
||||||
|
if _, ok := properties["artifact_type"]; ok {
|
||||||
|
t.Fatal("schema includes artifact_type, want only scene response fields")
|
||||||
|
}
|
||||||
|
if _, ok := properties["session_scope"]; ok {
|
||||||
|
t.Fatal("schema includes session_scope, want no session wrapper")
|
||||||
|
}
|
||||||
|
|
||||||
|
sceneProperties := properties["scenes"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any)
|
||||||
|
for _, field := range []string{"scene_id", "start_segment_id", "end_segment_id"} {
|
||||||
|
if _, ok := sceneProperties[field]; ok {
|
||||||
|
t.Fatalf("scene schema includes old field %q", field)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, field := range []string{"start_unit_id", "end_unit_id"} {
|
||||||
|
property := sceneProperties[field].(map[string]any)
|
||||||
|
if property["type"] != "string" {
|
||||||
|
t.Fatalf("%s type = %#v, want string", field, property["type"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
modeEnum := sceneProperties["primary_mode"].(map[string]any)["enum"].([]any)
|
||||||
|
if !sameStrings(modeEnum, []string{"Recap", "Discussion", "Combat", "Narrative"}) {
|
||||||
|
t.Fatalf("primary_mode enum = %#v, want Recap/Discussion/Combat/Narrative", modeEnum)
|
||||||
|
}
|
||||||
|
confidenceEnum := sceneProperties["boundary_confidence"].(map[string]any)["enum"].([]any)
|
||||||
|
if !sameStrings(confidenceEnum, []string{"High", "Medium", "Low"}) {
|
||||||
|
t.Fatalf("boundary_confidence enum = %#v, want High/Medium/Low", confidenceEnum)
|
||||||
|
}
|
||||||
|
|
||||||
|
boundaryCaveatItems := decoded["properties"].(map[string]any)["boundary_caveats"].(map[string]any)["items"].(map[string]any)
|
||||||
|
if boundaryCaveatItems["type"] != "string" {
|
||||||
|
t.Fatalf("boundary_caveats.items.type = %#v, want string", boundaryCaveatItems["type"])
|
||||||
|
}
|
||||||
|
if boundaryCaveatItems["minLength"] != float64(1) {
|
||||||
|
t.Fatalf("boundary_caveats.items.minLength = %#v, want 1", boundaryCaveatItems["minLength"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResponseStructRejectsIntegerBoundaries(t *testing.T) {
|
||||||
|
raw := []byte(`{
|
||||||
|
"scenes": [
|
||||||
|
{
|
||||||
|
"start_unit_id": 1,
|
||||||
|
"end_unit_id": 3,
|
||||||
|
"short_title": "Ambush",
|
||||||
|
"primary_mode": "Combat",
|
||||||
|
"main_participants": ["Aria"],
|
||||||
|
"summary": "The party fights.",
|
||||||
|
"boundary_note": "Combat starts and resolves.",
|
||||||
|
"boundary_confidence": "High"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"boundary_caveats": []
|
||||||
|
}`)
|
||||||
|
|
||||||
|
var response chunkResponse
|
||||||
|
err := json.Unmarshal(raw, &response)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Unmarshal() error = nil, want integer boundary type error: %#v", response)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "string") {
|
||||||
|
t.Fatalf("Unmarshal() error = %v, want string type error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResponseSchemaJSONIsMutationSafe(t *testing.T) {
|
||||||
|
first, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
first.JSONSchema[0] = '['
|
||||||
|
|
||||||
|
second, err := loadResponseSchema()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if !json.Valid(second.JSONSchema) {
|
||||||
|
t.Fatalf("schema JSON was mutated: %s", second.JSONSchema)
|
||||||
|
}
|
||||||
|
if len(second.JSONSchema) > 0 && second.JSONSchema[0] == '[' {
|
||||||
|
t.Fatalf("schema JSON did not use defensive copy")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sameStrings(got []any, want []string) bool {
|
||||||
|
if len(got) != len(want) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user