diff --git a/internal/modules/input/seriatim/adapter.go b/internal/modules/input/seriatim/adapter.go index f6f9aa6..7738541 100644 --- a/internal/modules/input/seriatim/adapter.go +++ b/internal/modules/input/seriatim/adapter.go @@ -2,7 +2,12 @@ package seriatim import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" + "math" + "strconv" + "strings" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" @@ -36,7 +41,46 @@ func (a *Adapter) Key() string { } func (a *Adapter) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) { - return nil, fmt.Errorf("seriatim input: source document parsing is not implemented") + if ctx == nil { + return nil, inputErrorf("context must not be nil") + } + if err := ctx.Err(); err != nil { + return nil, inputErrorf("context error before parsing: %w", err) + } + if len(req.Raw) == 0 { + return nil, inputErrorf("raw input must not be empty") + } + + parsed, err := decodeTranscript(req.Raw) + if err != nil { + return nil, inputErrorf("parse JSON: %w", err) + } + if len(parsed.Segments) == 0 { + return nil, inputErrorf("segments must not be empty") + } + + rawDigest := digest(req.Raw) + doc := &source.SourceDocument{ + ID: documentID(req.SourceID, parsed.Metadata, rawDigest), + Kind: DocumentKind, + Format: Format, + Digest: rawDigest, + Metadata: copyMetadata(parsed.Metadata), + } + + seenSegmentIDs := make(map[string]struct{}, len(parsed.Segments)) + for i, segment := range parsed.Segments { + unit, err := sourceUnit(segment, i, seenSegmentIDs) + if err != nil { + return nil, err + } + doc.Units = append(doc.Units, unit) + } + + if err := source.ValidateDocument(doc); err != nil { + return nil, inputErrorf("validate source document: %w", err) + } + return doc, nil } func ModuleSpec() pipeline.ModuleSpec { @@ -52,3 +96,109 @@ func Register(registry *pipeline.InputAdapterRegistry) error { return New(), nil }) } + +func sourceUnit(segment segment, index int, seen map[string]struct{}) (source.SourceUnit, error) { + segmentLabel := fmt.Sprintf("segment[%d]", index) + segmentID := strings.TrimSpace(segment.ID) + if segmentID == "" { + return source.SourceUnit{}, inputErrorf("%s id must not be empty", segmentLabel) + } + if segmentID != segment.ID { + return source.SourceUnit{}, inputErrorf("%s id %q must not contain leading or trailing whitespace", segmentLabel, segment.ID) + } + if _, ok := seen[segment.ID]; ok { + return source.SourceUnit{}, inputErrorf("segment id %q is duplicated", segment.ID) + } + seen[segment.ID] = struct{}{} + + speaker := strings.TrimSpace(segment.Speaker) + if speaker == "" { + return source.SourceUnit{}, inputErrorf("segment %q speaker must not be empty", segment.ID) + } + + start, err := validTimestamp(segment.Start, fmt.Sprintf("segment %q start", segment.ID)) + if err != nil { + return source.SourceUnit{}, err + } + end, err := validTimestamp(segment.End, fmt.Sprintf("segment %q end", segment.ID)) + if err != nil { + return source.SourceUnit{}, err + } + if end < start { + return source.SourceUnit{}, inputErrorf("segment %q end must be greater than or equal to start", segment.ID) + } + + if strings.TrimSpace(segment.Text) == "" { + return source.SourceUnit{}, inputErrorf("segment %q text must not be empty", segment.ID) + } + + return source.SourceUnit{ + ID: segment.ID, + Kind: UnitKind, + Text: segment.Text, + Metadata: map[string]any{ + MetadataSpeaker: segment.Speaker, + MetadataStart: segment.Start, + MetadataEnd: segment.End, + }, + }, nil +} + +func validTimestamp(value fmt.Stringer, label string) (float64, error) { + raw := strings.TrimSpace(value.String()) + if raw == "" { + return 0, inputErrorf("%s must not be empty", label) + } + parsed, err := strconv.ParseFloat(raw, 64) + if err != nil { + return 0, inputErrorf("%s must be a valid number: %w", label, err) + } + if math.IsInf(parsed, 0) || math.IsNaN(parsed) { + return 0, inputErrorf("%s must be finite", label) + } + if parsed < 0 { + return 0, inputErrorf("%s must not be negative", label) + } + return parsed, nil +} + +func documentID(requestedID string, metadata map[string]any, rawDigest string) string { + if id := strings.TrimSpace(requestedID); id != "" { + return id + } + if id := stringMetadata(metadata, "id"); id != "" { + return id + } + if id := stringMetadata(metadata, "source_id"); id != "" { + return id + } + return "seriatim:" + strings.TrimPrefix(rawDigest, "sha256:")[:16] +} + +func stringMetadata(metadata map[string]any, key string) string { + value, ok := metadata[key].(string) + if !ok { + return "" + } + return strings.TrimSpace(value) +} + +func copyMetadata(metadata map[string]any) map[string]any { + if len(metadata) == 0 { + return nil + } + copied := make(map[string]any, len(metadata)) + for key, value := range metadata { + copied[key] = value + } + return copied +} + +func digest(raw []byte) string { + sum := sha256.Sum256(raw) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func inputErrorf(format string, args ...any) error { + return fmt.Errorf("seriatim input: "+format, args...) +} diff --git a/internal/modules/input/seriatim/adapter_test.go b/internal/modules/input/seriatim/adapter_test.go new file mode 100644 index 0000000..ff58f2c --- /dev/null +++ b/internal/modules/input/seriatim/adapter_test.go @@ -0,0 +1,285 @@ +package seriatim + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +func TestParseValidMinimalTranscript(t *testing.T) { + raw := readFixture(t, "testdata/valid_minimal.json") + + doc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw}) + if err != nil { + t.Fatalf("Parse() error = %v, want nil", err) + } + + if doc.ID != "session-alpha" { + t.Fatalf("doc.ID = %q, want session-alpha", doc.ID) + } + if doc.Kind != DocumentKind { + t.Fatalf("doc.Kind = %q, want %q", doc.Kind, DocumentKind) + } + if doc.Format != Format { + t.Fatalf("doc.Format = %q, want %q", doc.Format, Format) + } + if doc.Digest != testDigest(raw) { + t.Fatalf("doc.Digest = %q, want %q", doc.Digest, testDigest(raw)) + } + if got := doc.Metadata["title"]; got != "Synthetic session transcript" { + t.Fatalf("doc.Metadata[title] = %#v, want Synthetic session transcript", got) + } + if len(doc.Units) != 2 { + t.Fatalf("len(doc.Units) = %d, want 2", len(doc.Units)) + } + + first := doc.Units[0] + if first.ID != "seg-001" { + t.Fatalf("first.ID = %q, want seg-001", first.ID) + } + if first.Kind != UnitKind { + t.Fatalf("first.Kind = %q, want %q", first.Kind, UnitKind) + } + if first.Text != "The stone door opens." { + t.Fatalf("first.Text = %q, want fixture text", first.Text) + } + if speaker, ok := Speaker(first); !ok || speaker != "Narrator" { + t.Fatalf("Speaker(first) = %q, %v; want Narrator, true", speaker, ok) + } + if start, ok := Start(first); !ok || start != json.Number("0") { + t.Fatalf("Start(first) = %q, %v; want 0, true", start, ok) + } + if end, ok := End(first); !ok || end != json.Number("4.5") { + t.Fatalf("End(first) = %q, %v; want 4.5, true", end, ok) + } + + ref := source.SourceRef{ + SourceID: doc.ID, + StartUnitID: doc.Units[0].ID, + EndUnitID: doc.Units[len(doc.Units)-1].ID, + } + if err := source.ValidateRef(doc, ref); err != nil { + t.Fatalf("ValidateRef() error = %v, want nil", err) + } +} + +func TestParseRequestSourceIDOverridesMetadataIDs(t *testing.T) { + raw := readFixture(t, "testdata/valid_minimal.json") + + doc, err := New().Parse(context.Background(), contracts.ParseRequest{ + SourceID: " requested-source ", + Raw: raw, + }) + if err != nil { + t.Fatalf("Parse() error = %v, want nil", err) + } + if doc.ID != "requested-source" { + t.Fatalf("doc.ID = %q, want requested-source", doc.ID) + } +} + +func TestParseFallbackDocumentIDIsDeterministic(t *testing.T) { + raw := []byte(`{"metadata":{},"segments":[{"id":"s1","start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."}]}`) + + first, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw}) + if err != nil { + t.Fatalf("first Parse() error = %v, want nil", err) + } + second, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw}) + if err != nil { + t.Fatalf("second Parse() error = %v, want nil", err) + } + + if first.ID != second.ID { + t.Fatalf("fallback IDs differ: %q vs %q", first.ID, second.ID) + } + if !strings.HasPrefix(first.ID, "seriatim:") { + t.Fatalf("fallback ID = %q, want seriatim prefix", first.ID) + } + if first.ID != "seriatim:"+strings.TrimPrefix(testDigest(raw), "sha256:")[:16] { + t.Fatalf("fallback ID = %q, want digest-derived ID", first.ID) + } +} + +func TestParseUsesMetadataSourceIDWhenMetadataIDIsAbsent(t *testing.T) { + raw := []byte(`{"metadata":{"source_id":" source-from-metadata "},"segments":[{"id":"s1","start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."}]}`) + + doc, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw}) + if err != nil { + t.Fatalf("Parse() error = %v, want nil", err) + } + if doc.ID != "source-from-metadata" { + t.Fatalf("doc.ID = %q, want source-from-metadata", doc.ID) + } +} + +func TestParseRejectsInvalidInput(t *testing.T) { + tests := []struct { + name string + raw []byte + wantErr []string + }{ + { + name: "malformed JSON", + raw: []byte(`{"metadata":`), + wantErr: []string{"seriatim input", "parse JSON"}, + }, + { + name: "trailing JSON", + raw: []byte(`{"metadata":{},"segments":[]} {}`), + wantErr: []string{"seriatim input", "trailing"}, + }, + { + name: "missing metadata", + raw: []byte(`{"segments":[]}`), + wantErr: []string{"metadata"}, + }, + { + name: "null metadata", + raw: []byte(`{"metadata":null,"segments":[]}`), + wantErr: []string{"metadata", "object"}, + }, + { + name: "metadata wrong type", + raw: []byte(`{"metadata":[],"segments":[]}`), + wantErr: []string{"metadata", "object"}, + }, + { + name: "missing segments", + raw: []byte(`{"metadata":{}}`), + wantErr: []string{"segments"}, + }, + { + name: "null segments", + raw: []byte(`{"metadata":{},"segments":null}`), + wantErr: []string{"segments", "array"}, + }, + { + name: "segments wrong type", + raw: []byte(`{"metadata":{},"segments":{}}`), + wantErr: []string{"segments", "array"}, + }, + { + name: "empty segments", + raw: []byte(`{"metadata":{},"segments":[]}`), + wantErr: []string{"segments", "empty"}, + }, + { + name: "missing segment id", + raw: validJSONWithSegment(`"start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`), + wantErr: []string{"id", "empty"}, + }, + { + name: "whitespace segment id", + raw: validJSONWithSegment(`"id":" s1 ","start":0,"end":1,"speaker":"Narrator","text":"Synthetic text."`), + wantErr: []string{"id", "whitespace"}, + }, + { + name: "empty text", + raw: validJSONWithSegment(`"id":"s1","start":0,"end":1,"speaker":"Narrator","text":" "`), + wantErr: []string{"text", "empty"}, + }, + { + name: "missing speaker", + raw: validJSONWithSegment(`"id":"s1","start":0,"end":1,"text":"Synthetic text."`), + wantErr: []string{"speaker", "empty"}, + }, + { + name: "missing start", + raw: validJSONWithSegment(`"id":"s1","end":1,"speaker":"Narrator","text":"Synthetic text."`), + wantErr: []string{"start", "empty"}, + }, + { + name: "missing end", + raw: validJSONWithSegment(`"id":"s1","start":0,"speaker":"Narrator","text":"Synthetic text."`), + wantErr: []string{"end", "empty"}, + }, + { + name: "negative start", + raw: validJSONWithSegment(`"id":"s1","start":-1,"end":1,"speaker":"Narrator","text":"Synthetic text."`), + wantErr: []string{"start", "negative"}, + }, + { + name: "non-numeric end", + raw: validJSONWithSegment(`"id":"s1","start":0,"end":"late","speaker":"Narrator","text":"Synthetic text."`), + wantErr: []string{"segments", "json"}, + }, + { + name: "non-finite timestamp", + raw: validJSONWithSegment(`"id":"s1","start":1e10000,"end":1e10000,"speaker":"Narrator","text":"Synthetic text."`), + wantErr: []string{"start", "valid number"}, + }, + { + name: "end before start", + raw: validJSONWithSegment(`"id":"s1","start":2,"end":1,"speaker":"Narrator","text":"Synthetic text."`), + wantErr: []string{"end", "start"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: tt.raw}) + if err == nil { + t.Fatal("Parse() error = nil, want error") + } + for _, want := range tt.wantErr { + if !strings.Contains(err.Error(), want) { + t.Fatalf("Parse() error = %q, want substring %q", err.Error(), want) + } + } + }) + } +} + +func TestParseRejectsDuplicateSegmentIDs(t *testing.T) { + raw := readFixture(t, "testdata/duplicate_segment_id.json") + + _, err := New().Parse(context.Background(), contracts.ParseRequest{Raw: raw}) + if err == nil { + t.Fatal("Parse() error = nil, want duplicate ID error") + } + if !strings.Contains(err.Error(), "duplicated") || !strings.Contains(err.Error(), "seg-001") { + t.Fatalf("Parse() error = %q, want duplicate segment context", err.Error()) + } +} + +func TestParseRejectsInvalidContextOrEmptyInput(t *testing.T) { + if _, err := New().Parse(nil, contracts.ParseRequest{Raw: []byte(`{}`)}); err == nil { + t.Fatal("Parse(nil context) error = nil, want error") + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := New().Parse(ctx, contracts.ParseRequest{Raw: []byte(`{}`)}); err == nil { + t.Fatal("Parse(canceled context) error = nil, want error") + } + + if _, err := New().Parse(context.Background(), contracts.ParseRequest{}); err == nil { + t.Fatal("Parse(empty input) error = nil, want error") + } +} + +func readFixture(t *testing.T, path string) []byte { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile(%q) error = %v, want nil", path, err) + } + return raw +} + +func validJSONWithSegment(segmentFields string) []byte { + return []byte(`{"metadata":{"id":"fixture"},"segments":[{` + segmentFields + `}]}`) +} + +func testDigest(raw []byte) string { + sum := sha256.Sum256(raw) + return "sha256:" + hex.EncodeToString(sum[:]) +} diff --git a/internal/modules/input/seriatim/model.go b/internal/modules/input/seriatim/model.go index 66202bb..ac1f8ac 100644 --- a/internal/modules/input/seriatim/model.go +++ b/internal/modules/input/seriatim/model.go @@ -3,6 +3,8 @@ package seriatim import ( "bytes" "encoding/json" + "fmt" + "io" ) type transcript struct { @@ -19,11 +21,55 @@ type segment struct { } func decodeTranscript(raw []byte) (transcript, error) { - var out transcript - decoder := json.NewDecoder(bytes.NewReader(raw)) - decoder.UseNumber() - if err := decoder.Decode(&out); err != nil { + var fields map[string]json.RawMessage + if err := decodeJSON(raw, &fields); err != nil { return transcript{}, err } - return out, nil + if fields == nil { + return transcript{}, fmt.Errorf("top-level value must be an object") + } + + metadataRaw, ok := fields["metadata"] + if !ok { + return transcript{}, fmt.Errorf("metadata is required") + } + var metadata map[string]any + if err := decodeJSON(metadataRaw, &metadata); err != nil { + return transcript{}, fmt.Errorf("metadata must be an object: %w", err) + } + if metadata == nil { + return transcript{}, fmt.Errorf("metadata must be an object") + } + + segmentsRaw, ok := fields["segments"] + if !ok { + return transcript{}, fmt.Errorf("segments are required") + } + var segments []segment + if err := decodeJSON(segmentsRaw, &segments); err != nil { + return transcript{}, fmt.Errorf("segments must be an array: %w", err) + } + if segments == nil { + return transcript{}, fmt.Errorf("segments must be an array") + } + + return transcript{ + Metadata: metadata, + Segments: segments, + }, nil +} + +func decodeJSON(raw []byte, out any) error { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + if err := decoder.Decode(out); err != nil { + return err + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + if err == nil { + return fmt.Errorf("unexpected trailing JSON value") + } + return err + } + return nil } diff --git a/internal/modules/input/seriatim/testdata/duplicate_segment_id.json b/internal/modules/input/seriatim/testdata/duplicate_segment_id.json new file mode 100644 index 0000000..e504e0e --- /dev/null +++ b/internal/modules/input/seriatim/testdata/duplicate_segment_id.json @@ -0,0 +1,21 @@ +{ + "metadata": { + "id": "duplicate-segment-fixture" + }, + "segments": [ + { + "id": "seg-001", + "start": 0, + "end": 1, + "speaker": "Narrator", + "text": "First segment." + }, + { + "id": "seg-001", + "start": 1, + "end": 2, + "speaker": "Player", + "text": "Duplicate segment." + } + ] +} diff --git a/internal/modules/input/seriatim/testdata/valid_minimal.json b/internal/modules/input/seriatim/testdata/valid_minimal.json new file mode 100644 index 0000000..339819e --- /dev/null +++ b/internal/modules/input/seriatim/testdata/valid_minimal.json @@ -0,0 +1,23 @@ +{ + "metadata": { + "id": "session-alpha", + "source_id": "fallback-session", + "title": "Synthetic session transcript" + }, + "segments": [ + { + "id": "seg-001", + "start": 0, + "end": 4.5, + "speaker": "Narrator", + "text": "The stone door opens." + }, + { + "id": "seg-002", + "start": 4.5, + "end": 8, + "speaker": "Player", + "text": "I cast light." + } + ] +}