diff --git a/docs/cli.md b/docs/cli.md index 38b5643..fa92fe5 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -117,7 +117,7 @@ go run ./cmd/notarius pipelines list \ The production CLI currently registers these module keys: - input: `seriatim` -- chunk: `generic` +- chunk: `generic`, `dnd/scenes` - extract: `dnd/spells` - merge: `appendorder` - normalize: `noop` diff --git a/docs/config.md b/docs/config.md index 4ba29c9..0feb26a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -167,6 +167,7 @@ one configured profile. | --- | --- | --- | | input | `seriatim` | Reads Seriatim transcript JSON. | | chunk | `generic` | Splits source units into ordered chunks. | +| chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. | | extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. | | merge | `appendorder` | Keeps candidates in append order. | | normalize | `noop` | Passes merged artifacts through unchanged. | @@ -178,6 +179,9 @@ The `generic` chunker accepts: - `overlap_units`: non-negative integer, default `0`, and must be less than `max_units`. +The `dnd/scenes` chunker requires transcript source capabilities, calls the +configured structured LLM provider, and does not accept module options. + ## Diagnostics `diagnostics` fields: diff --git a/docs/internal/modules.md b/docs/internal/modules.md index 67f1269..c0692c6 100644 --- a/docs/internal/modules.md +++ b/docs/internal/modules.md @@ -59,6 +59,37 @@ Provides: - `chunks` +## `dnd/scenes` Chunker + +Package: `internal/modules/chunk/dnd/scenes` + +The `dnd/scenes` chunker uses the structured LLM client to divide transcript +source units into coherent D&D scenes. It renders embedded prompts, loads the +embedded structured response schema, validates model-authored source-unit +boundaries, and converts each scene into a deterministic source chunk. + +Requires: + +- `source.transcript` + +Provides: + +- `chunks` +- `chunks.scenes` + +Options: none. Non-empty options are rejected. + +The chunker enforces full source-unit coverage from the first source unit to the +last, exact source-unit IDs, sequential contiguous scenes, and no overlap. It +assigns chunk IDs such as `scene-000001` and stores scene metadata including +title, primary mode, participants, summary, boundary note, confidence, boundary +unit IDs, and unit count. Boundary caveats become warnings with reason code +`scene_boundary_caveat`. + +Malformed model output fails explicitly rather than falling back to another +chunker. The chunker exposes prompt and response-schema provenance through its +metadata provider without raw prompts, raw schemas, source text, or secrets. + ## `dnd/spells` Extractor Package: `internal/modules/extract/dnd/spells` diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 0d5e2c9..bea0560 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -177,6 +177,31 @@ Fix: Provider error messages are redacted for configured API key values. +## Scene Chunking Failure + +Symptoms include: + +- `dnd scenes chunker` +- `malformed structured output` +- `start_unit_id` +- `end_unit_id` +- `gap` +- `overlap` +- `final scene` +- `complete structured output` + +Fix: + +- Validate the pipeline configuration and confirm the input module provides a + transcript source when using `chunk: dnd/scenes`. +- Confirm the LLM profile has a working OpenAI-compatible `base_url`, `model`, + and credentials. +- Inspect retained diagnostics for the run error and resolved pipeline. +- If the error names malformed structured output, retry with a model that + follows structured response schemas reliably. +- Scene boundaries must use exact source-unit IDs, cover the full source + document, be contiguous, and not overlap. + ## Output Write Failure Symptoms include: diff --git a/internal/cli/catalog.go b/internal/cli/catalog.go index dd863ee..fd08a82 100644 --- a/internal/cli/catalog.go +++ b/internal/cli/catalog.go @@ -10,6 +10,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes" "gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic" "gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim" @@ -34,6 +35,9 @@ func productionRegistries() (pipeline.Registries, error) { if err := generic.Register(registries.Chunkers); err != nil { return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err) } + if err := scenes.Register(registries.Chunkers); err != nil { + return pipeline.Registries{}, fmt.Errorf("register dnd scenes chunker: %w", err) + } if err := spells.Register(registries.Extractors); err != nil { return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err) } diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index b05cf0f..89ea7b4 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -16,6 +16,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes" "gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic" "gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim" @@ -136,6 +137,11 @@ func TestProductionCatalogIncludesDefaultModules(t *testing.T) { got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(generic.Key) }, want: generic.ModuleSpec(), }, + { + name: "dnd scenes chunker", + got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(scenes.Key) }, + want: scenes.ModuleSpec(), + }, { name: "dnd spells extractor", got: func() (pipeline.ModuleSpec, bool) { return catalog.Extractors.Spec(spells.Key) }, @@ -186,6 +192,21 @@ func TestRunConfigValidateUsesProductionCatalogByDefault(t *testing.T) { } } +func TestRunConfigValidateAcceptsDNDScenesChunker(t *testing.T) { + configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells")) + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{}) + + if code != 0 { + t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "is valid for pipeline") { + t.Fatalf("stdout = %q, want validation success", stdout.String()) + } +} + func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) { configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "missing/extract")) var stdout bytes.Buffer @@ -1220,6 +1241,18 @@ pipelines: ` } +func mvpConfigYAMLWithChunk(pipelineID string, chunker string, extractor string) string { + return `version: 1 +pipelines: + ` + pipelineID + `: + input: seriatim + chunk: ` + chunker + ` + artifacts: + spells: + extract: ` + extractor + ` +` +} + func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string { var b strings.Builder b.WriteString("version: 1\n") diff --git a/internal/core/config/effective_config_test.go b/internal/core/config/effective_config_test.go index 7c120ef..2f136e4 100644 --- a/internal/core/config/effective_config_test.go +++ b/internal/core/config/effective_config_test.go @@ -98,6 +98,43 @@ func TestResolveSurfacesMissingCapabilityThroughCatalog(t *testing.T) { } } +func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) { + cfg := validConfig() + profile := cfg.Pipelines["example"] + profile.Chunk = pipeline.Binding("dnd/scenes") + lane := profile.Artifacts["events"] + profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"events": lane} + cfg.Pipelines["example"] = profile + + catalog := fakeCatalog(t, + pipeline.ModuleSpec{ + Key: "fake/input", + Stage: pipeline.StageInput, + Provides: []string{"source.transcript"}, + }, + pipeline.ModuleSpec{ + Key: "fake/extract", + Stage: pipeline.StageExtract, + Requires: []string{"chunks", "source.transcript"}, + Provides: []string{"artifact"}, + }, + ) + mustRegisterChunker(t, catalog.Chunkers, pipeline.ModuleSpec{ + Key: "dnd/scenes", + Stage: pipeline.StageChunk, + Requires: []string{"source.transcript"}, + Provides: []string{"chunks", "chunks.scenes"}, + }) + + effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog}) + if err != nil { + t.Fatalf("Resolve() error = %v, want nil", err) + } + if got := effective.ResolvedPipeline.Chunk.Module; got != "dnd/scenes" { + t.Fatalf("Chunk.Module = %q, want dnd/scenes", got) + } +} + func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) { cfg := validConfig() first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})