eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move chunks into the canonical source model

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 05:32:11 +00:00
parent 40709e4ad8
commit 075888c97f
33 changed files with 548 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
internal/framework/pipeline/debug.go
View File

@@ -104,14 +104,13 @@ type debugSourceDocument struct {
}
type debugSourceChunk struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
Index int `json:"index"`
StartUnitID int `json:"start_unit_id"`
EndUnitID int `json:"end_unit_id"`
Content debugBinaryEnvelope `json:"content"`
Units []source.SourceUnit `json:"units,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
ID string `json:"id"`
SourceID string `json:"source_id"`
Index int `json:"index"`
Ref source.SourceRef `json:"ref"`
Content debugBinaryEnvelope `json:"content"`
Units []source.SourceUnit `json:"units,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type debugExtractOutput struct {
@@ -448,20 +447,19 @@ func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocumen
}
}
func debugSourceChunkEnvelope(chunk contracts.SourceChunk) debugSourceChunk {
func debugSourceChunkEnvelope(chunk source.Chunk) debugSourceChunk {
return debugSourceChunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
StartUnitID: chunk.StartUnitID,
EndUnitID: chunk.EndUnitID,
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
Units: cloneSourceUnits(chunk.Units),
Metadata: redactSensitiveMap(chunk.Metadata),
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Ref: chunk.Ref,
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
Units: cloneSourceUnits(chunk.Units),
Metadata: redactSensitiveMap(chunk.Metadata),
}
}
func debugSourceChunkEnvelopes(chunks []contracts.SourceChunk) []debugSourceChunk {
func debugSourceChunkEnvelopes(chunks []source.Chunk) []debugSourceChunk {
if len(chunks) == 0 {
return nil
}