Register D&D scene chunker
This commit is contained in:
@@ -117,7 +117,7 @@ go run ./cmd/notarius pipelines list \
|
||||
The production CLI currently registers these module keys:
|
||||
|
||||
- input: `seriatim`
|
||||
- chunk: `generic`
|
||||
- chunk: `generic`, `dnd/scenes`
|
||||
- extract: `dnd/spells`
|
||||
- merge: `appendorder`
|
||||
- normalize: `noop`
|
||||
|
||||
@@ -167,6 +167,7 @@ one configured profile.
|
||||
| --- | --- | --- |
|
||||
| input | `seriatim` | Reads Seriatim transcript JSON. |
|
||||
| chunk | `generic` | Splits source units into ordered chunks. |
|
||||
| chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. |
|
||||
| extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. |
|
||||
| merge | `appendorder` | Keeps candidates in append order. |
|
||||
| normalize | `noop` | Passes merged artifacts through unchanged. |
|
||||
@@ -178,6 +179,9 @@ The `generic` chunker accepts:
|
||||
- `overlap_units`: non-negative integer, default `0`, and must be less than
|
||||
`max_units`.
|
||||
|
||||
The `dnd/scenes` chunker requires transcript source capabilities, calls the
|
||||
configured structured LLM provider, and does not accept module options.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
`diagnostics` fields:
|
||||
|
||||
@@ -59,6 +59,37 @@ Provides:
|
||||
|
||||
- `chunks`
|
||||
|
||||
## `dnd/scenes` Chunker
|
||||
|
||||
Package: `internal/modules/chunk/dnd/scenes`
|
||||
|
||||
The `dnd/scenes` chunker uses the structured LLM client to divide transcript
|
||||
source units into coherent D&D scenes. It renders embedded prompts, loads the
|
||||
embedded structured response schema, validates model-authored source-unit
|
||||
boundaries, and converts each scene into a deterministic source chunk.
|
||||
|
||||
Requires:
|
||||
|
||||
- `source.transcript`
|
||||
|
||||
Provides:
|
||||
|
||||
- `chunks`
|
||||
- `chunks.scenes`
|
||||
|
||||
Options: none. Non-empty options are rejected.
|
||||
|
||||
The chunker enforces full source-unit coverage from the first source unit to the
|
||||
last, exact source-unit IDs, sequential contiguous scenes, and no overlap. It
|
||||
assigns chunk IDs such as `scene-000001` and stores scene metadata including
|
||||
title, primary mode, participants, summary, boundary note, confidence, boundary
|
||||
unit IDs, and unit count. Boundary caveats become warnings with reason code
|
||||
`scene_boundary_caveat`.
|
||||
|
||||
Malformed model output fails explicitly rather than falling back to another
|
||||
chunker. The chunker exposes prompt and response-schema provenance through its
|
||||
metadata provider without raw prompts, raw schemas, source text, or secrets.
|
||||
|
||||
## `dnd/spells` Extractor
|
||||
|
||||
Package: `internal/modules/extract/dnd/spells`
|
||||
|
||||
@@ -177,6 +177,31 @@ Fix:
|
||||
|
||||
Provider error messages are redacted for configured API key values.
|
||||
|
||||
## Scene Chunking Failure
|
||||
|
||||
Symptoms include:
|
||||
|
||||
- `dnd scenes chunker`
|
||||
- `malformed structured output`
|
||||
- `start_unit_id`
|
||||
- `end_unit_id`
|
||||
- `gap`
|
||||
- `overlap`
|
||||
- `final scene`
|
||||
- `complete structured output`
|
||||
|
||||
Fix:
|
||||
|
||||
- Validate the pipeline configuration and confirm the input module provides a
|
||||
transcript source when using `chunk: dnd/scenes`.
|
||||
- Confirm the LLM profile has a working OpenAI-compatible `base_url`, `model`,
|
||||
and credentials.
|
||||
- Inspect retained diagnostics for the run error and resolved pipeline.
|
||||
- If the error names malformed structured output, retry with a model that
|
||||
follows structured response schemas reliably.
|
||||
- Scene boundaries must use exact source-unit IDs, cover the full source
|
||||
document, be contiguous, and not overlap.
|
||||
|
||||
## Output Write Failure
|
||||
|
||||
Symptoms include:
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
|
||||
@@ -34,6 +35,9 @@ func productionRegistries() (pipeline.Registries, error) {
|
||||
if err := generic.Register(registries.Chunkers); err != nil {
|
||||
return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err)
|
||||
}
|
||||
if err := scenes.Register(registries.Chunkers); err != nil {
|
||||
return pipeline.Registries{}, fmt.Errorf("register dnd scenes chunker: %w", err)
|
||||
}
|
||||
if err := spells.Register(registries.Extractors); err != nil {
|
||||
return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err)
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
|
||||
@@ -136,6 +137,11 @@ func TestProductionCatalogIncludesDefaultModules(t *testing.T) {
|
||||
got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(generic.Key) },
|
||||
want: generic.ModuleSpec(),
|
||||
},
|
||||
{
|
||||
name: "dnd scenes chunker",
|
||||
got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(scenes.Key) },
|
||||
want: scenes.ModuleSpec(),
|
||||
},
|
||||
{
|
||||
name: "dnd spells extractor",
|
||||
got: func() (pipeline.ModuleSpec, bool) { return catalog.Extractors.Spec(spells.Key) },
|
||||
@@ -186,6 +192,21 @@ func TestRunConfigValidateUsesProductionCatalogByDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigValidateAcceptsDNDScenesChunker(t *testing.T) {
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells"))
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{})
|
||||
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "is valid for pipeline") {
|
||||
t.Fatalf("stdout = %q, want validation success", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) {
|
||||
configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "missing/extract"))
|
||||
var stdout bytes.Buffer
|
||||
@@ -1220,6 +1241,18 @@ pipelines:
|
||||
`
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithChunk(pipelineID string, chunker string, extractor string) string {
|
||||
return `version: 1
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
chunk: ` + chunker + `
|
||||
artifacts:
|
||||
spells:
|
||||
extract: ` + extractor + `
|
||||
`
|
||||
}
|
||||
|
||||
func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("version: 1\n")
|
||||
|
||||
@@ -98,6 +98,43 @@ func TestResolveSurfacesMissingCapabilityThroughCatalog(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.Chunk = pipeline.Binding("dnd/scenes")
|
||||
lane := profile.Artifacts["events"]
|
||||
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"events": lane}
|
||||
cfg.Pipelines["example"] = profile
|
||||
|
||||
catalog := fakeCatalog(t,
|
||||
pipeline.ModuleSpec{
|
||||
Key: "fake/input",
|
||||
Stage: pipeline.StageInput,
|
||||
Provides: []string{"source.transcript"},
|
||||
},
|
||||
pipeline.ModuleSpec{
|
||||
Key: "fake/extract",
|
||||
Stage: pipeline.StageExtract,
|
||||
Requires: []string{"chunks", "source.transcript"},
|
||||
Provides: []string{"artifact"},
|
||||
},
|
||||
)
|
||||
mustRegisterChunker(t, catalog.Chunkers, pipeline.ModuleSpec{
|
||||
Key: "dnd/scenes",
|
||||
Stage: pipeline.StageChunk,
|
||||
Requires: []string{"source.transcript"},
|
||||
Provides: []string{"chunks", "chunks.scenes"},
|
||||
})
|
||||
|
||||
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
if got := effective.ResolvedPipeline.Chunk.Module; got != "dnd/scenes" {
|
||||
t.Fatalf("Chunk.Module = %q, want dnd/scenes", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
||||
|
||||
Reference in New Issue
Block a user