Bugfix in the seriatim input adapter
This commit is contained in:
@@ -3,9 +3,9 @@
|
|||||||
Notarius is a Go CLI for extracting structured artifacts from source material
|
Notarius is a Go CLI for extracting structured artifacts from source material
|
||||||
with explicit, configurable pipeline modules.
|
with explicit, configurable pipeline modules.
|
||||||
|
|
||||||
The current implementation reads Seriatim minimal transcript JSON, chunks the
|
The current implementation reads Seriatim transcript JSON, chunks the source
|
||||||
source units, extracts D&D spell-cast artifacts with an OpenAI-compatible LLM,
|
units, extracts D&D spell-cast artifacts with an OpenAI-compatible LLM, and
|
||||||
and writes JSON output plus diagnostics for each run.
|
writes JSON output plus diagnostics for each run.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
NOTARIUS_LLM_DEFAULT_BASE_URL=http://127.0.0.1:8080/v1 \
|
NOTARIUS_LLM_DEFAULT_BASE_URL=http://127.0.0.1:8080/v1 \
|
||||||
|
|||||||
@@ -165,7 +165,7 @@ one configured profile.
|
|||||||
|
|
||||||
| Slot | Key | Notes |
|
| Slot | Key | Notes |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| input | `seriatim` | Reads Seriatim minimal transcript JSON. |
|
| input | `seriatim` | Reads Seriatim transcript JSON. |
|
||||||
| chunk | `generic` | Splits source units into ordered chunks. |
|
| chunk | `generic` | Splits source units into ordered chunks. |
|
||||||
| extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. |
|
| extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. |
|
||||||
| merge | `appendorder` | Keeps candidates in append order. |
|
| merge | `appendorder` | Keeps candidates in append order. |
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# Seriatim Minimal Transcript JSON
|
# Seriatim Transcript JSON
|
||||||
|
|
||||||
This document is the external input contract for the implemented `seriatim`
|
This document is the external input contract for the implemented `seriatim`
|
||||||
input adapter.
|
input adapter.
|
||||||
@@ -8,7 +8,7 @@ input adapter.
|
|||||||
- Module key: `seriatim`
|
- Module key: `seriatim`
|
||||||
- Document kind: `transcript`
|
- Document kind: `transcript`
|
||||||
- Unit kind: `transcript_segment`
|
- Unit kind: `transcript_segment`
|
||||||
- Source format: `application/vnd.seriatim.minimal+json`
|
- Source format: `application/vnd.seriatim+json`
|
||||||
|
|
||||||
The adapter parses raw Seriatim JSON into a generic source document. It owns
|
The adapter parses raw Seriatim JSON into a generic source document. It owns
|
||||||
transcript-specific JSON parsing and metadata mapping; core source and pipeline
|
transcript-specific JSON parsing and metadata mapping; core source and pipeline
|
||||||
@@ -17,7 +17,8 @@ code stay source-format agnostic.
|
|||||||
## Accepted Shape
|
## Accepted Shape
|
||||||
|
|
||||||
The input must be one JSON object with top-level `metadata` and `segments`
|
The input must be one JSON object with top-level `metadata` and `segments`
|
||||||
fields:
|
fields. This covers the maintained minimal fixture and Seriatim intermediate
|
||||||
|
output that provides the same required segment fields.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
@@ -40,7 +41,8 @@ fields:
|
|||||||
The maintained example is
|
The maintained example is
|
||||||
[examples/seriatim-minimal-transcript.json](../../examples/seriatim-minimal-transcript.json).
|
[examples/seriatim-minimal-transcript.json](../../examples/seriatim-minimal-transcript.json).
|
||||||
|
|
||||||
Top-level metadata entries are preserved. Other segment fields are ignored.
|
Top-level metadata entries are preserved. Other segment fields, such as
|
||||||
|
`categories`, are ignored.
|
||||||
|
|
||||||
Multiple top-level JSON values are rejected.
|
Multiple top-level JSON values are rejected.
|
||||||
|
|
||||||
@@ -54,7 +56,8 @@ The adapter rejects:
|
|||||||
- missing, null, or non-object `metadata`;
|
- missing, null, or non-object `metadata`;
|
||||||
- missing, null, non-array, or empty `segments`;
|
- missing, null, non-array, or empty `segments`;
|
||||||
- segment values that are not objects;
|
- segment values that are not objects;
|
||||||
- non-string `id`, `speaker`, or `text`;
|
- segment `id` values that are neither strings nor numbers;
|
||||||
|
- non-string `speaker` or `text`;
|
||||||
- empty segment IDs;
|
- empty segment IDs;
|
||||||
- segment IDs with leading or trailing whitespace;
|
- segment IDs with leading or trailing whitespace;
|
||||||
- duplicate segment IDs;
|
- duplicate segment IDs;
|
||||||
@@ -72,7 +75,7 @@ The adapter maps input to `SourceDocument`:
|
|||||||
|
|
||||||
- `metadata` becomes `SourceDocument.Metadata`;
|
- `metadata` becomes `SourceDocument.Metadata`;
|
||||||
- `SourceDocument.Kind` is `transcript`;
|
- `SourceDocument.Kind` is `transcript`;
|
||||||
- `SourceDocument.Format` is `application/vnd.seriatim.minimal+json`;
|
- `SourceDocument.Format` is `application/vnd.seriatim+json`;
|
||||||
- `SourceDocument.Digest` is `sha256:<hex>` of the exact raw input bytes.
|
- `SourceDocument.Digest` is `sha256:<hex>` of the exact raw input bytes.
|
||||||
|
|
||||||
`SourceDocument.ID` is selected in this order:
|
`SourceDocument.ID` is selected in this order:
|
||||||
@@ -84,7 +87,8 @@ The adapter maps input to `SourceDocument`:
|
|||||||
|
|
||||||
Each segment becomes one `SourceUnit`:
|
Each segment becomes one `SourceUnit`:
|
||||||
|
|
||||||
- `segment.id` becomes `SourceUnit.ID`;
|
- `segment.id` becomes `SourceUnit.ID`; numeric IDs are converted to their JSON
|
||||||
|
number text, so `1` becomes `"1"`;
|
||||||
- `segment.text` becomes `SourceUnit.Text`;
|
- `segment.text` becomes `SourceUnit.Text`;
|
||||||
- `SourceUnit.Kind` is `transcript_segment`;
|
- `SourceUnit.Kind` is `transcript_segment`;
|
||||||
- `speaker`, `start`, and `end` are stored in source-unit metadata.
|
- `speaker`, `start`, and `end` are stored in source-unit metadata.
|
||||||
@@ -110,4 +114,7 @@ The module declares these provided capabilities:
|
|||||||
|
|
||||||
## Compatibility Limit
|
## Compatibility Limit
|
||||||
|
|
||||||
This contract covers only the minimal transcript JSON shape described here.
|
This contract covers only Seriatim transcript JSON with the top-level
|
||||||
|
`metadata` object and `segments` array described here. Broader Seriatim output
|
||||||
|
schemas are compatible only when they provide these required fields with the
|
||||||
|
accepted types.
|
||||||
|
|||||||
@@ -24,9 +24,9 @@ reject incompatible pipelines before execution.
|
|||||||
|
|
||||||
Package: `internal/modules/input/seriatim`
|
Package: `internal/modules/input/seriatim`
|
||||||
|
|
||||||
The `seriatim` adapter parses Seriatim minimal transcript JSON into a generic
|
The `seriatim` adapter parses Seriatim transcript JSON into a generic source
|
||||||
source document. It owns transcript JSON details, source ID selection, source
|
document. It owns transcript JSON details, source ID selection, source digest
|
||||||
digest creation, transcript segment validation, and segment metadata mapping.
|
creation, transcript segment validation, and segment metadata mapping.
|
||||||
|
|
||||||
Provides:
|
Provides:
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ const Key = "seriatim"
|
|||||||
const (
|
const (
|
||||||
DocumentKind = "transcript"
|
DocumentKind = "transcript"
|
||||||
UnitKind = "transcript_segment"
|
UnitKind = "transcript_segment"
|
||||||
Format = "application/vnd.seriatim.minimal+json"
|
Format = "application/vnd.seriatim+json"
|
||||||
)
|
)
|
||||||
|
|
||||||
var providedCapabilities = []string{
|
var providedCapabilities = []string{
|
||||||
|
|||||||
@@ -70,6 +70,35 @@ func TestParseValidMinimalTranscript(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseAcceptsNumericSegmentIDs(t *testing.T) {
|
||||||
|
raw := []byte(`{"metadata":{"application":"seriatim","version":"v1.5.0","output_schema":"seriatim-intermediate"},"segments":[{"id":1,"start":451.821,"end":469.685,"speaker":"Narrator","text":"The stone door opens.","categories":["scene"]},{"id":2,"start":470,"end":471,"speaker":"Player","text":"I step inside."}]}`)
|
||||||
|
|
||||||
|
doc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if got, want := doc.Metadata["output_schema"], "seriatim-intermediate"; got != want {
|
||||||
|
t.Fatalf("doc.Metadata[output_schema] = %#v, want %q", got, want)
|
||||||
|
}
|
||||||
|
if doc.Format != Format {
|
||||||
|
t.Fatalf("doc.Format = %q, want %q", doc.Format, Format)
|
||||||
|
}
|
||||||
|
if len(doc.Units) != 2 {
|
||||||
|
t.Fatalf("len(doc.Units) = %d, want 2", len(doc.Units))
|
||||||
|
}
|
||||||
|
if doc.Units[0].ID != "1" || doc.Units[1].ID != "2" {
|
||||||
|
t.Fatalf("unit IDs = %#v, want numeric IDs normalized to strings", []string{doc.Units[0].ID, doc.Units[1].ID})
|
||||||
|
}
|
||||||
|
ref := source.SourceRef{
|
||||||
|
SourceID: doc.ID,
|
||||||
|
StartUnitID: "1",
|
||||||
|
EndUnitID: "2",
|
||||||
|
}
|
||||||
|
if err := source.ValidateRef(doc, ref); err != nil {
|
||||||
|
t.Fatalf("ValidateRef() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseRequestSourceIDOverridesMetadataIDs(t *testing.T) {
|
func TestParseRequestSourceIDOverridesMetadataIDs(t *testing.T) {
|
||||||
raw := readFixture(t, "testdata/valid_minimal.json")
|
raw := readFixture(t, "testdata/valid_minimal.json")
|
||||||
|
|
||||||
@@ -176,6 +205,11 @@ func TestParseRejectsInvalidInput(t *testing.T) {
|
|||||||
raw: validJSONWithSegment(`"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
|
raw: validJSONWithSegment(`"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
|
||||||
wantErr: []string{"id", "empty"},
|
wantErr: []string{"id", "empty"},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "invalid segment id type",
|
||||||
|
raw: validJSONWithSegment(`"id":{},"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
|
||||||
|
wantErr: []string{"id", "string or number"},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "whitespace segment id",
|
name: "whitespace segment id",
|
||||||
raw: validJSONWithSegment(`"id":" s1 ","start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
|
raw: validJSONWithSegment(`"id":" s1 ","start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`),
|
||||||
|
|||||||
@@ -77,8 +77,8 @@ func decodeSegment(raw []byte, index int) (segment, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var decoded segment
|
var decoded segment
|
||||||
if err := decodeOptionalString(fields, "id", &decoded.ID); err != nil {
|
if err := decodeOptionalSegmentID(fields, "id", &decoded.ID); err != nil {
|
||||||
return segment{}, fmt.Errorf("segment[%d] id must be a string: %w", index, err)
|
return segment{}, fmt.Errorf("segment[%d] id must be a string or number: %w", index, err)
|
||||||
}
|
}
|
||||||
if err := decodeOptionalNumber(fields, "start", &decoded.Start); err != nil {
|
if err := decodeOptionalNumber(fields, "start", &decoded.Start); err != nil {
|
||||||
return segment{}, fmt.Errorf("segment[%d] start must be a number: %w", index, err)
|
return segment{}, fmt.Errorf("segment[%d] start must be a number: %w", index, err)
|
||||||
@@ -103,6 +103,27 @@ func decodeOptionalString(fields map[string]json.RawMessage, key string, out *st
|
|||||||
return decodeJSON(raw, out)
|
return decodeJSON(raw, out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func decodeOptionalSegmentID(fields map[string]json.RawMessage, key string, out *string) error {
|
||||||
|
raw, ok := fields[key]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var text string
|
||||||
|
if err := decodeJSON(raw, &text); err == nil {
|
||||||
|
*out = text
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var number json.Number
|
||||||
|
if err := decodeJSON(raw, &number); err == nil {
|
||||||
|
*out = number.String()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("must be a string or number")
|
||||||
|
}
|
||||||
|
|
||||||
func decodeOptionalNumber(fields map[string]json.RawMessage, key string, out *json.Number) error {
|
func decodeOptionalNumber(fields map[string]json.RawMessage, key string, out *json.Number) error {
|
||||||
raw, ok := fields[key]
|
raw, ok := fields[key]
|
||||||
if !ok {
|
if !ok {
|
||||||
|
|||||||
Reference in New Issue
Block a user