Add canonical source unit provenance
This commit is contained in:
@@ -65,18 +65,21 @@ func (a *Adapter) Parse(ctx context.Context, req contracts.ParseRequest) (*sourc
|
||||
ID: documentID(req.SourceID, parsed.Metadata, rawDigest),
|
||||
Kind: DocumentKind,
|
||||
Format: Format,
|
||||
Digest: rawDigest,
|
||||
Metadata: copyMetadata(parsed.Metadata),
|
||||
}
|
||||
|
||||
seenSegmentIDs := make(map[int]struct{}, len(parsed.Segments))
|
||||
for i, segment := range parsed.Segments {
|
||||
unit, err := sourceUnit(segment, i, seenSegmentIDs)
|
||||
unit, err := sourceUnit(doc.ID, segment, i, seenSegmentIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc.Units = append(doc.Units, unit)
|
||||
}
|
||||
doc.Digest, err = source.DigestDocument(doc)
|
||||
if err != nil {
|
||||
return nil, inputErrorf("digest source document: %w", err)
|
||||
}
|
||||
|
||||
if err := source.ValidateDocument(doc); err != nil {
|
||||
return nil, inputErrorf("validate source document: %w", err)
|
||||
@@ -98,7 +101,7 @@ func Register(registry *pipeline.InputAdapterRegistry) error {
|
||||
})
|
||||
}
|
||||
|
||||
func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) {
|
||||
func sourceUnit(sourceID string, segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) {
|
||||
segmentLabel := fmt.Sprintf("segment[%d]", index)
|
||||
if segment.ID <= 0 {
|
||||
return source.SourceUnit{}, inputErrorf("%s id must be positive", segmentLabel)
|
||||
@@ -133,6 +136,11 @@ func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.Sourc
|
||||
ID: segment.ID,
|
||||
Kind: UnitKind,
|
||||
Text: segment.Text,
|
||||
Ref: source.SourceRef{
|
||||
SourceID: sourceID,
|
||||
StartUnitID: segment.ID,
|
||||
EndUnitID: segment.ID,
|
||||
},
|
||||
Metadata: map[string]any{
|
||||
MetadataSpeaker: segment.Speaker,
|
||||
MetadataStart: segment.Start,
|
||||
|
||||
Reference in New Issue
Block a user