Register D&D scene chunker

This commit is contained in:
2026-07-04 13:05:39 +00:00
parent 7f83a20fa6
commit 2130414899
7 changed files with 135 additions and 1 deletions

View File

@@ -117,7 +117,7 @@ go run ./cmd/notarius pipelines list \
The production CLI currently registers these module keys:
- input: `seriatim`
- chunk: `generic`
- chunk: `generic`, `dnd/scenes`
- extract: `dnd/spells`
- merge: `appendorder`
- normalize: `noop`

View File

@@ -167,6 +167,7 @@ one configured profile.
| --- | --- | --- |
| input | `seriatim` | Reads Seriatim transcript JSON. |
| chunk | `generic` | Splits source units into ordered chunks. |
| chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. |
| extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. |
| merge | `appendorder` | Keeps candidates in append order. |
| normalize | `noop` | Passes merged artifacts through unchanged. |
@@ -178,6 +179,9 @@ The `generic` chunker accepts:
- `overlap_units`: non-negative integer, default `0`, and must be less than
`max_units`.
The `dnd/scenes` chunker requires transcript source capabilities, calls the
configured structured LLM provider, and does not accept module options.
## Diagnostics
`diagnostics` fields:

View File

@@ -59,6 +59,37 @@ Provides:
- `chunks`
## `dnd/scenes` Chunker
Package: `internal/modules/chunk/dnd/scenes`
The `dnd/scenes` chunker uses the structured LLM client to divide transcript
source units into coherent D&D scenes. It renders embedded prompts, loads the
embedded structured response schema, validates model-authored source-unit
boundaries, and converts each scene into a deterministic source chunk.
Requires:
- `source.transcript`
Provides:
- `chunks`
- `chunks.scenes`
Options: none. Non-empty options are rejected.
The chunker enforces full source-unit coverage from the first source unit to the
last, exact source-unit IDs, sequential contiguous scenes, and no overlap. It
assigns chunk IDs such as `scene-000001` and stores scene metadata including
title, primary mode, participants, summary, boundary note, confidence, boundary
unit IDs, and unit count. Boundary caveats become warnings with reason code
`scene_boundary_caveat`.
Malformed model output fails explicitly rather than falling back to another
chunker. The chunker exposes prompt and response-schema provenance through its
metadata provider without raw prompts, raw schemas, source text, or secrets.
## `dnd/spells` Extractor
Package: `internal/modules/extract/dnd/spells`

View File

@@ -177,6 +177,31 @@ Fix:
Provider error messages are redacted for configured API key values.
## Scene Chunking Failure
Symptoms include:
- `dnd scenes chunker`
- `malformed structured output`
- `start_unit_id`
- `end_unit_id`
- `gap`
- `overlap`
- `final scene`
- `complete structured output`
Fix:
- Validate the pipeline configuration and confirm the input module provides a
transcript source when using `chunk: dnd/scenes`.
- Confirm the LLM profile has a working OpenAI-compatible `base_url`, `model`,
and credentials.
- Inspect retained diagnostics for the run error and resolved pipeline.
- If the error names malformed structured output, retry with a model that
follows structured response schemas reliably.
- Scene boundaries must use exact source-unit IDs, cover the full source
document, be contiguous, and not overlap.
## Output Write Failure
Symptoms include:

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
@@ -34,6 +35,9 @@ func productionRegistries() (pipeline.Registries, error) {
if err := generic.Register(registries.Chunkers); err != nil {
return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err)
}
if err := scenes.Register(registries.Chunkers); err != nil {
return pipeline.Registries{}, fmt.Errorf("register dnd scenes chunker: %w", err)
}
if err := spells.Register(registries.Extractors); err != nil {
return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err)
}

View File

@@ -16,6 +16,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
@@ -136,6 +137,11 @@ func TestProductionCatalogIncludesDefaultModules(t *testing.T) {
got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(generic.Key) },
want: generic.ModuleSpec(),
},
{
name: "dnd scenes chunker",
got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(scenes.Key) },
want: scenes.ModuleSpec(),
},
{
name: "dnd spells extractor",
got: func() (pipeline.ModuleSpec, bool) { return catalog.Extractors.Spec(spells.Key) },
@@ -186,6 +192,21 @@ func TestRunConfigValidateUsesProductionCatalogByDefault(t *testing.T) {
}
}
func TestRunConfigValidateAcceptsDNDScenesChunker(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells"))
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if !strings.Contains(stdout.String(), "is valid for pipeline") {
t.Fatalf("stdout = %q, want validation success", stdout.String())
}
}
func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "missing/extract"))
var stdout bytes.Buffer
@@ -1220,6 +1241,18 @@ pipelines:
`
}
func mvpConfigYAMLWithChunk(pipelineID string, chunker string, extractor string) string {
return `version: 1
pipelines:
` + pipelineID + `:
input: seriatim
chunk: ` + chunker + `
artifacts:
spells:
extract: ` + extractor + `
`
}
func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
var b strings.Builder
b.WriteString("version: 1\n")

View File

@@ -98,6 +98,43 @@ func TestResolveSurfacesMissingCapabilityThroughCatalog(t *testing.T) {
}
}
func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
profile.Chunk = pipeline.Binding("dnd/scenes")
lane := profile.Artifacts["events"]
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"events": lane}
cfg.Pipelines["example"] = profile
catalog := fakeCatalog(t,
pipeline.ModuleSpec{
Key: "fake/input",
Stage: pipeline.StageInput,
Provides: []string{"source.transcript"},
},
pipeline.ModuleSpec{
Key: "fake/extract",
Stage: pipeline.StageExtract,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"artifact"},
},
)
mustRegisterChunker(t, catalog.Chunkers, pipeline.ModuleSpec{
Key: "dnd/scenes",
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks", "chunks.scenes"},
})
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
if got := effective.ResolvedPipeline.Chunk.Module; got != "dnd/scenes" {
t.Fatalf("Chunk.Module = %q, want dnd/scenes", got)
}
}
func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
cfg := validConfig()
first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})