Add canonical source unit provenance

This commit is contained in:
2026-07-17 05:19:57 +00:00
parent 15c369c509
commit 40709e4ad8
26 changed files with 254 additions and 34 deletions

View File

@@ -65,18 +65,21 @@ func (a *Adapter) Parse(ctx context.Context, req contracts.ParseRequest) (*sourc
ID: documentID(req.SourceID, parsed.Metadata, rawDigest),
Kind: DocumentKind,
Format: Format,
Digest: rawDigest,
Metadata: copyMetadata(parsed.Metadata),
}
seenSegmentIDs := make(map[int]struct{}, len(parsed.Segments))
for i, segment := range parsed.Segments {
unit, err := sourceUnit(segment, i, seenSegmentIDs)
unit, err := sourceUnit(doc.ID, segment, i, seenSegmentIDs)
if err != nil {
return nil, err
}
doc.Units = append(doc.Units, unit)
}
doc.Digest, err = source.DigestDocument(doc)
if err != nil {
return nil, inputErrorf("digest source document: %w", err)
}
if err := source.ValidateDocument(doc); err != nil {
return nil, inputErrorf("validate source document: %w", err)
@@ -98,7 +101,7 @@ func Register(registry *pipeline.InputAdapterRegistry) error {
})
}
func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) {
func sourceUnit(sourceID string, segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) {
segmentLabel := fmt.Sprintf("segment[%d]", index)
if segment.ID <= 0 {
return source.SourceUnit{}, inputErrorf("%s id must be positive", segmentLabel)
@@ -133,6 +136,11 @@ func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.Sourc
ID: segment.ID,
Kind: UnitKind,
Text: segment.Text,
Ref: source.SourceRef{
SourceID: sourceID,
StartUnitID: segment.ID,
EndUnitID: segment.ID,
},
Metadata: map[string]any{
MetadataSpeaker: segment.Speaker,
MetadataStart: segment.Start,