From 40709e4ad8ce50eb9aaa224d437477ea816fa051 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 17 Jul 2026 05:19:57 +0000 Subject: [PATCH] Add canonical source unit provenance --- docs/integrations/seriatim.md | 11 ++- docs/internal/modules.md | 7 +- docs/internal/pipeline.md | 5 ++ docs/policy/architecture.md | 3 +- internal/cli/compatibility_test.go | 7 +- internal/cli/run_test.go | 2 +- internal/core/source/digest.go | 35 ++++++++ internal/core/source/source.go | 1 + internal/core/source/source_test.go | 82 +++++++++++++++++++ internal/core/source/validation.go | 8 ++ internal/framework/checkpoint/recorder.go | 1 + .../framework/checkpoint/recorder_test.go | 7 +- .../framework/contracts/composition_test.go | 4 +- .../framework/pipeline/chunk_validation.go | 1 + internal/framework/pipeline/debug_test.go | 30 +++++++ .../framework/pipeline/input_registry_test.go | 2 +- .../pipeline/registry_integration_test.go | 2 +- internal/framework/pipeline/runner_test.go | 24 ++++-- .../pipeline/walking_skeleton_test.go | 5 ++ internal/modules/dnd/chunk/scenes/chunker.go | 1 + .../modules/dnd/chunk/scenes/chunker_test.go | 12 ++- .../dnd/extract/spells/test_helpers_test.go | 2 + .../modules/generic/chunk/units/chunker.go | 1 + .../generic/chunk/units/chunker_test.go | 5 ++ .../seriatim/input/transcript/adapter.go | 14 +++- .../seriatim/input/transcript/adapter_test.go | 16 +++- 26 files changed, 254 insertions(+), 34 deletions(-) create mode 100644 internal/core/source/digest.go create mode 100644 internal/framework/pipeline/debug_test.go diff --git a/docs/integrations/seriatim.md b/docs/integrations/seriatim.md index 287960c..0de334e 100644 --- a/docs/integrations/seriatim.md +++ b/docs/integrations/seriatim.md @@ -52,9 +52,14 @@ Notarius identifies the parsed source in this order: 2. `metadata.source_id`, when it is a non-empty string after trimming; 3. `seriatim:`. -The source digest recorded in output provenance is `sha256:` of the exact -raw input bytes. Segment IDs become the unit IDs used by artifact source -references. +The exact raw input SHA-256 remains the basis of the fallback source ID. The +source digest recorded in output provenance is instead the SHA-256 of the +canonical generic source document, excluding the digest field itself. It covers +the derived source identity, document kind and format, ordered units and their +self-references, and accepted metadata. Segment IDs become the unit IDs used by +artifact source references; each produced unit carries a self-reference whose +source ID is the derived document ID and whose start and end IDs both equal the +segment ID. ## Compatibility Limit diff --git a/docs/internal/modules.md b/docs/internal/modules.md index 79e4017..e85c916 100644 --- a/docs/internal/modules.md +++ b/docs/internal/modules.md @@ -40,9 +40,10 @@ evidence. The resolver and materializer behavior is described in ### `internal/modules/seriatim/input/transcript` The adapter decodes the supported transcript JSON, selects the source identity, -computes the raw-input digest, validates segments, and maps each segment into a -generic source unit with speaker and timestamp metadata. Its spec advertises the -transcript capabilities consumed by D&D modules. +computes canonical source provenance, validates segments, and maps each segment +into a generic source unit with a self-reference plus speaker and timestamp +metadata. Its spec advertises the transcript capabilities consumed by D&D +modules. Parsing is strict about required values and duplicate unit IDs but deliberately ignores unrelated Seriatim fields. The external format and derived-identity diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index db62b68..6623a65 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -74,6 +74,11 @@ selected input adapter. Later stage requests receive the generic source model; extract requests receive chunk-scoped input material, while chunk, merge, and normalize requests retain access to the original source material. +Source validation requires every unit to carry a canonical self-reference to +its containing document and its own unit ID. Explicit clone, checkpoint, and +debug boundaries retain that reference, and the canonical source digest covers +it deterministically. + `pipeline.RunOutput` carries the run manifest, accepted normalized results, rejected results, warnings, checkpoint events, and logical files returned by the output encoder. The CLI owns diagnostics and durable filesystem writes after the diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index 9d1b878..08da549 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -66,7 +66,8 @@ Framework stages operate on source documents, source units, and source references rather than format-specific structures. A source reference identifies an ordered range of generic source units. Framework code preserves those ranges and does not merge or rewrite them unless a stage module explicitly owns that -behavior. +behavior. Every source unit carries a validated self-reference to its containing +document and its own unit ID. Extract modules own artifact semantics, prompt use, response schemas, and domain interpretation. Domain-specific concepts remain in the relevant module, diff --git a/internal/cli/compatibility_test.go b/internal/cli/compatibility_test.go index 92511bc..2fa2de7 100644 --- a/internal/cli/compatibility_test.go +++ b/internal/cli/compatibility_test.go @@ -238,7 +238,7 @@ func TestMaintainedSeriatimToDNDCompatibilityBundle(t *testing.T) { if manifest.ValidationStatus != test.wantStatus || len(manifest.RejectedOutputs) != test.wantRejectedCount { t.Fatalf("manifest outcome = status %q rejected %#v, want %q/%d", manifest.ValidationStatus, manifest.RejectedOutputs, test.wantStatus, test.wantRejectedCount) } - if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:346f0b0b0cf2d8081ca222dbea47ff231ae9d81c8120ac51a4fc9fa6dfb6cc07"}) { + if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:1c98d94ae632fb10a2b56f684cd4fb1019cedb1a629e57dc0977cf4a54135be0"}) { t.Fatalf("source digests = %#v, want maintained fixture provenance", manifest.SourceDigests) } if got := manifestValidatorKeys(manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)); !reflect.DeepEqual(got, []string{ @@ -325,7 +325,10 @@ func TestProductionLLMCallersShareScheduledClient(t *testing.T) { client := frameworkllm.NewScheduledClient(underlying, scheduler) doc := &source.SourceDocument{ ID: "session-alpha", Kind: "transcript", Format: "application/json", Digest: "sha256:source", - Units: []source.SourceUnit{{ID: 1, Kind: "segment", Text: "Aria casts Cure Wounds."}, {ID: 2, Kind: "segment", Text: "The spell takes effect."}}, + Units: []source.SourceUnit{ + {ID: 1, Kind: "segment", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}}, + {ID: 2, Kind: "segment", Text: "The spell takes effect.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}, + }, } chunk := contracts.SourceChunk{ ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, StartUnitID: 1, EndUnitID: 2, diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index d240dc9..6c900cf 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -3594,7 +3594,7 @@ func (fakeRunInputAdapter) Parse(ctx context.Context, req contracts.ParseRequest Format: "test", Digest: "sha256:source", Units: []source.SourceUnit{ - {ID: 1, Kind: "text", Text: string(req.Raw)}, + {ID: 1, Kind: "text", Text: string(req.Raw), Ref: source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}, }, }, nil } diff --git a/internal/core/source/digest.go b/internal/core/source/digest.go new file mode 100644 index 0000000..cd7df81 --- /dev/null +++ b/internal/core/source/digest.go @@ -0,0 +1,35 @@ +package source + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" +) + +// DigestDocument returns a deterministic digest of the canonical source +// document content. The existing Digest field is excluded from its own digest. +func DigestDocument(doc *SourceDocument) (string, error) { + if doc == nil { + return "", fmt.Errorf("source document must not be nil") + } + payload := struct { + ID string `json:"id"` + Kind string `json:"kind"` + Format string `json:"format"` + Units []SourceUnit `json:"units"` + Metadata map[string]any `json:"metadata,omitempty"` + }{ + ID: doc.ID, + Kind: doc.Kind, + Format: doc.Format, + Units: doc.Units, + Metadata: doc.Metadata, + } + encoded, err := json.Marshal(payload) + if err != nil { + return "", fmt.Errorf("encode source document for digest: %w", err) + } + sum := sha256.Sum256(encoded) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} diff --git a/internal/core/source/source.go b/internal/core/source/source.go index eb16af6..9d87125 100644 --- a/internal/core/source/source.go +++ b/internal/core/source/source.go @@ -13,6 +13,7 @@ type SourceUnit struct { ID int `json:"id"` Kind string `json:"kind"` Text string `json:"text"` + Ref SourceRef `json:"ref"` Metadata map[string]any `json:"metadata,omitempty"` } diff --git a/internal/core/source/source_test.go b/internal/core/source/source_test.go index 4b938dc..0ece58e 100644 --- a/internal/core/source/source_test.go +++ b/internal/core/source/source_test.go @@ -142,6 +142,86 @@ func TestValidateDocumentDuplicateUnitIDs(t *testing.T) { } } +func TestValidateDocumentUnitReferences(t *testing.T) { + tests := []struct { + name string + mutate func(*SourceDocument) + wantErr string + }{ + { + name: "missing", + mutate: func(doc *SourceDocument) { doc.Units[0].Ref = SourceRef{} }, + wantErr: "source unit[0].ref: source ref source_id must not be empty", + }, + { + name: "foreign source", + mutate: func(doc *SourceDocument) { doc.Units[0].Ref.SourceID = "source-2" }, + wantErr: "source unit[0].ref: source ref source_id \"source-2\" does not match document id \"source-1\"", + }, + { + name: "non-self range", + mutate: func(doc *SourceDocument) { + doc.Units[0].Ref.StartUnitID = 2 + doc.Units[0].Ref.EndUnitID = 2 + }, + wantErr: "source unit[0].ref must identify source unit id 1", + }, + { + name: "reversed range", + mutate: func(doc *SourceDocument) { + doc.Units[0].Ref.StartUnitID = 2 + doc.Units[0].Ref.EndUnitID = 1 + }, + wantErr: "source unit[0].ref: source ref start_unit_id 2 appears after end_unit_id 1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + doc := validDocument() + tt.mutate(doc) + + err := ValidateDocument(doc) + if err == nil { + t.Fatal("ValidateDocument() error = nil, want unit reference error") + } + if err.Error() != tt.wantErr { + t.Fatalf("ValidateDocument() error = %q, want %q", err.Error(), tt.wantErr) + } + }) + } +} + +func TestDigestDocumentIsDeterministicAndIncludesUnitReference(t *testing.T) { + doc := validDocument() + doc.Metadata = map[string]any{"second": "value", "first": true} + first, err := DigestDocument(doc) + if err != nil { + t.Fatalf("DigestDocument() error = %v, want nil", err) + } + + reordered := validDocument() + reordered.Metadata = map[string]any{"first": true, "second": "value"} + second, err := DigestDocument(reordered) + if err != nil { + t.Fatalf("DigestDocument(reordered) error = %v, want nil", err) + } + if first != second { + t.Fatalf("digests = %q and %q, want deterministic map ordering", first, second) + } + + changed := validDocument() + changed.Metadata = map[string]any{"first": true, "second": "value"} + changed.Units[0].Ref.SourceID = "different-source" + changedDigest, err := DigestDocument(changed) + if err != nil { + t.Fatalf("DigestDocument(changed) error = %v, want nil", err) + } + if first == changedDigest { + t.Fatalf("digest = %q after reference change, want different digest", changedDigest) + } +} + func TestValidateRefValid(t *testing.T) { doc := validDocument() ref := SourceRef{ @@ -274,11 +354,13 @@ func validDocument() *SourceDocument { ID: 1, Kind: "paragraph", Text: "First unit.", + Ref: SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}, }, { ID: 2, Kind: "paragraph", Text: "Second unit.", + Ref: SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}, }, }, } diff --git a/internal/core/source/validation.go b/internal/core/source/validation.go index ffa85c6..9bea5ec 100644 --- a/internal/core/source/validation.go +++ b/internal/core/source/validation.go @@ -44,6 +44,14 @@ func ValidateDocument(doc *SourceDocument) error { } seenUnitIDs[unit.ID] = struct{}{} } + for i, unit := range doc.Units { + if err := ValidateRef(doc, unit.Ref); err != nil { + return fmt.Errorf("source unit[%d].ref: %w", i, err) + } + if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID { + return fmt.Errorf("source unit[%d].ref must identify source unit id %d", i, unit.ID) + } + } return nil } diff --git a/internal/framework/checkpoint/recorder.go b/internal/framework/checkpoint/recorder.go index 681d392..bebcdd9 100644 --- a/internal/framework/checkpoint/recorder.go +++ b/internal/framework/checkpoint/recorder.go @@ -414,6 +414,7 @@ func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit { ID: unit.ID, Kind: unit.Kind, Text: unit.Text, + Ref: unit.Ref, Metadata: cloneMetadata(unit.Metadata), }) } diff --git a/internal/framework/checkpoint/recorder_test.go b/internal/framework/checkpoint/recorder_test.go index 5f92e9c..bf65ed5 100644 --- a/internal/framework/checkpoint/recorder_test.go +++ b/internal/framework/checkpoint/recorder_test.go @@ -22,7 +22,7 @@ func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) { Kind: "document", Format: "text/plain", Digest: "sha256:source", - Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello"}}, + Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}}, } chunks := []contracts.SourceChunk{ { @@ -89,7 +89,7 @@ func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) { Kind: "document", Format: "text/plain", Digest: "sha256:source", - Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello"}}, + Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}}, } chunks := []contracts.SourceChunk{ { @@ -155,6 +155,9 @@ func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) { if !decision.Reused || sourceCheckpoint.Document.ID != "source-1" { t.Fatalf("source decision = %#v checkpoint=%#v, want reused", decision, sourceCheckpoint) } + if got, want := sourceCheckpoint.Document.Units[0].Ref, doc.Units[0].Ref; got != want { + t.Fatalf("checkpoint source unit ref = %#v, want %#v", got, want) + } chunkCheckpoint, decision := loader.Chunk("generic", doc.Digest) if !decision.Reused || len(chunkCheckpoint.Chunks) != 1 || string(chunkCheckpoint.Chunks[0].Content) != "chunk content" { t.Fatalf("chunk decision = %#v checkpoint=%#v, want reused", decision, chunkCheckpoint) diff --git a/internal/framework/contracts/composition_test.go b/internal/framework/contracts/composition_test.go index 542ba23..652c8a5 100644 --- a/internal/framework/contracts/composition_test.go +++ b/internal/framework/contracts/composition_test.go @@ -116,8 +116,8 @@ func (adapter compositionAdapter) Parse(ctx context.Context, req contracts.Parse Format: "text/plain", Digest: "sha256:abc123", Units: []source.SourceUnit{ - {ID: 1, Kind: "unit", Text: "First source unit."}, - {ID: 2, Kind: "unit", Text: "Second source unit."}, + {ID: 1, Kind: "unit", Text: "First source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 1, EndUnitID: 1}}, + {ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 2, EndUnitID: 2}}, }, }, nil } diff --git a/internal/framework/pipeline/chunk_validation.go b/internal/framework/pipeline/chunk_validation.go index 08b6021..8f9a1e5 100644 --- a/internal/framework/pipeline/chunk_validation.go +++ b/internal/framework/pipeline/chunk_validation.go @@ -102,6 +102,7 @@ func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit { ID: unit.ID, Kind: unit.Kind, Text: unit.Text, + Ref: unit.Ref, Metadata: cloneMetadata(unit.Metadata), } } diff --git a/internal/framework/pipeline/debug_test.go b/internal/framework/pipeline/debug_test.go new file mode 100644 index 0000000..d29c0a5 --- /dev/null +++ b/internal/framework/pipeline/debug_test.go @@ -0,0 +1,30 @@ +package pipeline + +import ( + "encoding/json" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" +) + +func TestDebugSourceDocumentPreservesUnitReferences(t *testing.T) { + doc := validSourceDocument() + envelope := debugSourceDocumentEnvelope(doc) + encoded, err := json.Marshal(envelope) + if err != nil { + t.Fatalf("Marshal(debug source document) error = %v, want nil", err) + } + + var decoded debugSourceDocument + if err := json.Unmarshal(encoded, &decoded); err != nil { + t.Fatalf("Unmarshal(debug source document) error = %v, want nil", err) + } + if got, want := decoded.Units[0].Ref, (source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}); got != want { + t.Fatalf("debug source unit ref = %#v, want %#v", got, want) + } + + envelope.Units[0].Ref.SourceID = "mutated" + if got := doc.Units[0].Ref.SourceID; got != "source-1" { + t.Fatalf("source document ref = %q after debug mutation, want source-1", got) + } +} diff --git a/internal/framework/pipeline/input_registry_test.go b/internal/framework/pipeline/input_registry_test.go index bf8d433..e995504 100644 --- a/internal/framework/pipeline/input_registry_test.go +++ b/internal/framework/pipeline/input_registry_test.go @@ -301,7 +301,7 @@ func (adapter fakeAdapter) Parse(ctx context.Context, req contracts.ParseRequest Format: "text/plain", Digest: "sha256:abc123", Units: []source.SourceUnit{ - {ID: 1, Kind: "unit", Text: "Source unit."}, + {ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: req.SourceID, StartUnitID: 1, EndUnitID: 1}}, }, }, nil } diff --git a/internal/framework/pipeline/registry_integration_test.go b/internal/framework/pipeline/registry_integration_test.go index 8443986..9b47acc 100644 --- a/internal/framework/pipeline/registry_integration_test.go +++ b/internal/framework/pipeline/registry_integration_test.go @@ -247,7 +247,7 @@ func integrationSourceDocument() *source.SourceDocument { Format: "text/plain", Digest: "sha256:abc123", Units: []source.SourceUnit{ - {ID: 1, Kind: "unit", Text: "Source unit."}, + {ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}, }, } } diff --git a/internal/framework/pipeline/runner_test.go b/internal/framework/pipeline/runner_test.go index c385774..b0e6877 100644 --- a/internal/framework/pipeline/runner_test.go +++ b/internal/framework/pipeline/runner_test.go @@ -402,6 +402,9 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) { if chunk.Units[0].ID != 1 || chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" { t.Fatalf("chunk unit = %#v, want source document unit values", chunk.Units[0]) } + if got := chunk.Units[0].Ref; got != modules.input.doc.Units[0].Ref { + t.Fatalf("chunk unit ref = %#v, want source document ref %#v", got, modules.input.doc.Units[0].Ref) + } if got := chunk.Units[0].Metadata["speaker"]; got != "source-speaker" { t.Fatalf("chunk unit metadata = %#v, want source document metadata", chunk.Units[0].Metadata) } @@ -411,8 +414,9 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) { modules.input.doc.Units[0].Kind = "changed-kind" modules.input.doc.Units[0].Text = "changed text" + modules.input.doc.Units[0].Ref.SourceID = "changed-source" modules.input.doc.Units[0].Metadata["speaker"] = "changed-speaker" - if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Metadata["speaker"] != "source-speaker" { + if chunk.Units[0].Kind != "source-kind" || chunk.Units[0].Text != "source text" || chunk.Units[0].Ref.SourceID != "source-1" || chunk.Units[0].Metadata["speaker"] != "source-speaker" { t.Fatalf("chunk unit changed after source mutation: %#v", chunk.Units[0]) } } @@ -2464,9 +2468,9 @@ func validSourceDocument() *source.SourceDocument { Format: "text/plain", Digest: "sha256:source", Units: []source.SourceUnit{ - {ID: 1, Kind: "unit", Text: "Source unit."}, - {ID: 2, Kind: "unit", Text: "Second source unit."}, - {ID: 3, Kind: "unit", Text: "Third source unit."}, + {ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}, + {ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}}, + {ID: 3, Kind: "unit", Text: "Third source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 3, EndUnitID: 3}}, }, } } @@ -2482,6 +2486,7 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument { ID: 1, Kind: "source-kind", Text: "source text", + Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}, Metadata: map[string]any{ "speaker": "source-speaker", "topic": "source-topic", @@ -2491,6 +2496,7 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument { ID: 2, Kind: "source-kind", Text: "second source text", + Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}, Metadata: map[string]any{ "speaker": "source-speaker-2", }, @@ -2523,15 +2529,15 @@ func sourceChunkWithContent(id string, index int, content []byte, mediaType stri func unitWithID(id string) source.SourceUnit { switch id { case "u1": - return source.SourceUnit{ID: 1, Kind: "unit", Text: "Source unit."} + return source.SourceUnit{ID: 1, Kind: "unit", Text: "Source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}} case "u2": - return source.SourceUnit{ID: 2, Kind: "unit", Text: "Second source unit."} + return source.SourceUnit{ID: 2, Kind: "unit", Text: "Second source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 2, EndUnitID: 2}} case "u3": - return source.SourceUnit{ID: 3, Kind: "unit", Text: "Third source unit."} + return source.SourceUnit{ID: 3, Kind: "unit", Text: "Third source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 3, EndUnitID: 3}} case "u9": - return source.SourceUnit{ID: 9, Kind: "unit", Text: "Unknown source unit."} + return source.SourceUnit{ID: 9, Kind: "unit", Text: "Unknown source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 9, EndUnitID: 9}} default: - return source.SourceUnit{ID: 99, Kind: "unit", Text: "Unknown source unit."} + return source.SourceUnit{ID: 99, Kind: "unit", Text: "Unknown source unit.", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 99, EndUnitID: 99}} } } diff --git a/internal/framework/pipeline/walking_skeleton_test.go b/internal/framework/pipeline/walking_skeleton_test.go index ca03106..ac019de 100644 --- a/internal/framework/pipeline/walking_skeleton_test.go +++ b/internal/framework/pipeline/walking_skeleton_test.go @@ -200,6 +200,11 @@ func (input walkingSkeletonInput) Parse(ctx context.Context, req contracts.Parse ID: unit.ID, Kind: "unit", Text: unit.Text, + Ref: source.SourceRef{ + SourceID: fixture.ID, + StartUnitID: unit.ID, + EndUnitID: unit.ID, + }, }) } return &source.SourceDocument{ diff --git a/internal/modules/dnd/chunk/scenes/chunker.go b/internal/modules/dnd/chunk/scenes/chunker.go index 3fe9f82..e4bbbcd 100644 --- a/internal/modules/dnd/chunk/scenes/chunker.go +++ b/internal/modules/dnd/chunk/scenes/chunker.go @@ -332,6 +332,7 @@ func cloneUnits(units []source.SourceUnit) []source.SourceUnit { ID: unit.ID, Kind: unit.Kind, Text: unit.Text, + Ref: unit.Ref, Metadata: cloneMetadata(unit.Metadata), }) } diff --git a/internal/modules/dnd/chunk/scenes/chunker_test.go b/internal/modules/dnd/chunk/scenes/chunker_test.go index 3681723..cbe2049 100644 --- a/internal/modules/dnd/chunk/scenes/chunker_test.go +++ b/internal/modules/dnd/chunk/scenes/chunker_test.go @@ -318,12 +318,16 @@ func TestChunkDefensivelyCopiesSourceUnitsAndMetadata(t *testing.T) { } doc.Units[0].ID = 99 + doc.Units[0].Ref.SourceID = "mutated" doc.Units[0].Metadata["speaker"] = "mutated" client.response.Scenes[0].MainParticipants[0] = "mutated" if result.Chunks[0].Units[0].ID != 1 { t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0]) } + if got := result.Chunks[0].Units[0].Ref.SourceID; got != "session-alpha" { + t.Fatalf("chunk unit ref changed after source mutation: %#v", result.Chunks[0].Units[0].Ref) + } if result.Chunks[0].Units[0].Metadata["speaker"] != "Alice" { t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata) } @@ -535,10 +539,10 @@ func sceneSourceDocument() *source.SourceDocument { Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source", Units: []source.SourceUnit{ - {ID: 1, Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Metadata: map[string]any{"speaker": "Alice"}}, - {ID: 2, Kind: "transcript_segment", Text: "The goblin scout describes the gate guards."}, - {ID: 3, Kind: "transcript_segment", Text: "The guards rush out with blades drawn."}, - {ID: 4, Kind: "transcript_segment", Text: "The party defeats the ambushers."}, + {ID: 1, Kind: "transcript_segment", Text: "Aria asks whether the goblin will parley.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, Metadata: map[string]any{"speaker": "Alice"}}, + {ID: 2, Kind: "transcript_segment", Text: "The goblin scout describes the gate guards.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}, + {ID: 3, Kind: "transcript_segment", Text: "The guards rush out with blades drawn.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 3, EndUnitID: 3}}, + {ID: 4, Kind: "transcript_segment", Text: "The party defeats the ambushers.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 4, EndUnitID: 4}}, }, } } diff --git a/internal/modules/dnd/extract/spells/test_helpers_test.go b/internal/modules/dnd/extract/spells/test_helpers_test.go index 7597c06..6c0f4cc 100644 --- a/internal/modules/dnd/extract/spells/test_helpers_test.go +++ b/internal/modules/dnd/extract/spells/test_helpers_test.go @@ -39,6 +39,7 @@ func promptSourceDocument() *source.SourceDocument { ID: 1, Kind: "transcript_segment", Text: "Aria raises her hand and casts Cure Wounds.", + Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, Metadata: map[string]any{ "speaker": "Alice", "start": json.Number("1.25"), @@ -50,6 +51,7 @@ func promptSourceDocument() *source.SourceDocument { ID: 2, Kind: "transcript_segment", Text: "The fighter's wounds begin to close.", + Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}, Metadata: map[string]any{"ignored": "not rendered"}, }, }, diff --git a/internal/modules/generic/chunk/units/chunker.go b/internal/modules/generic/chunk/units/chunker.go index 10afaee..9cebdc6 100644 --- a/internal/modules/generic/chunk/units/chunker.go +++ b/internal/modules/generic/chunk/units/chunker.go @@ -243,6 +243,7 @@ func cloneUnits(units []source.SourceUnit) []source.SourceUnit { ID: unit.ID, Kind: unit.Kind, Text: unit.Text, + Ref: unit.Ref, Metadata: cloneMetadata(unit.Metadata), }) } diff --git a/internal/modules/generic/chunk/units/chunker_test.go b/internal/modules/generic/chunk/units/chunker_test.go index 4832d35..959633b 100644 --- a/internal/modules/generic/chunk/units/chunker_test.go +++ b/internal/modules/generic/chunk/units/chunker_test.go @@ -169,11 +169,15 @@ func TestChunkDefensivelyCopiesUnits(t *testing.T) { } doc.Units[0].ID = 99 + doc.Units[0].Ref.SourceID = "changed" doc.Units[0].Metadata["speaker"] = "changed" if result.Chunks[0].Units[0].ID != 1 { t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0]) } + if got := result.Chunks[0].Units[0].Ref.SourceID; got != "source-1" { + t.Fatalf("chunk unit ref changed after source mutation: %#v", result.Chunks[0].Units[0].Ref) + } if result.Chunks[0].Units[0].Metadata["speaker"] != "speaker-001" { t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata) } @@ -186,6 +190,7 @@ func testSource(count int) *source.SourceDocument { ID: i, Kind: "unit", Text: "Text for " + zeroPad3(i), + Ref: source.SourceRef{SourceID: "source-1", StartUnitID: i, EndUnitID: i}, Metadata: map[string]any{ "speaker": "speaker-" + zeroPad3(i), }, diff --git a/internal/modules/seriatim/input/transcript/adapter.go b/internal/modules/seriatim/input/transcript/adapter.go index 68b4b93..11b9e5a 100644 --- a/internal/modules/seriatim/input/transcript/adapter.go +++ b/internal/modules/seriatim/input/transcript/adapter.go @@ -65,18 +65,21 @@ func (a *Adapter) Parse(ctx context.Context, req contracts.ParseRequest) (*sourc ID: documentID(req.SourceID, parsed.Metadata, rawDigest), Kind: DocumentKind, Format: Format, - Digest: rawDigest, Metadata: copyMetadata(parsed.Metadata), } seenSegmentIDs := make(map[int]struct{}, len(parsed.Segments)) for i, segment := range parsed.Segments { - unit, err := sourceUnit(segment, i, seenSegmentIDs) + unit, err := sourceUnit(doc.ID, segment, i, seenSegmentIDs) if err != nil { return nil, err } doc.Units = append(doc.Units, unit) } + doc.Digest, err = source.DigestDocument(doc) + if err != nil { + return nil, inputErrorf("digest source document: %w", err) + } if err := source.ValidateDocument(doc); err != nil { return nil, inputErrorf("validate source document: %w", err) @@ -98,7 +101,7 @@ func Register(registry *pipeline.InputAdapterRegistry) error { }) } -func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) { +func sourceUnit(sourceID string, segment segment, index int, seen map[int]struct{}) (source.SourceUnit, error) { segmentLabel := fmt.Sprintf("segment[%d]", index) if segment.ID <= 0 { return source.SourceUnit{}, inputErrorf("%s id must be positive", segmentLabel) @@ -133,6 +136,11 @@ func sourceUnit(segment segment, index int, seen map[int]struct{}) (source.Sourc ID: segment.ID, Kind: UnitKind, Text: segment.Text, + Ref: source.SourceRef{ + SourceID: sourceID, + StartUnitID: segment.ID, + EndUnitID: segment.ID, + }, Metadata: map[string]any{ MetadataSpeaker: segment.Speaker, MetadataStart: segment.Start, diff --git a/internal/modules/seriatim/input/transcript/adapter_test.go b/internal/modules/seriatim/input/transcript/adapter_test.go index 7cdb2f2..5550f95 100644 --- a/internal/modules/seriatim/input/transcript/adapter_test.go +++ b/internal/modules/seriatim/input/transcript/adapter_test.go @@ -30,8 +30,8 @@ func TestParseValidMinimalTranscript(t *testing.T) { if doc.Format != Format { t.Fatalf("doc.Format = %q, want %q", doc.Format, Format) } - if doc.Digest != testDigest(raw) { - t.Fatalf("doc.Digest = %q, want %q", doc.Digest, testDigest(raw)) + if doc.Digest == testDigest(raw) || !strings.HasPrefix(doc.Digest, "sha256:") { + t.Fatalf("doc.Digest = %q, want canonical source digest distinct from raw input digest", doc.Digest) } if got := doc.Metadata["title"]; got != "Synthetic session transcript" { t.Fatalf("doc.Metadata[title] = %#v, want Synthetic session transcript", got) @@ -39,6 +39,12 @@ func TestParseValidMinimalTranscript(t *testing.T) { if len(doc.Units) != 2 { t.Fatalf("len(doc.Units) = %d, want 2", len(doc.Units)) } + for i, unit := range doc.Units { + want := source.SourceRef{SourceID: doc.ID, StartUnitID: unit.ID, EndUnitID: unit.ID} + if unit.Ref != want { + t.Fatalf("doc.Units[%d].Ref = %#v, want %#v", i, unit.Ref, want) + } + } first := doc.Units[0] if first.ID != 1 { @@ -89,6 +95,12 @@ func TestParseAcceptsNumericSegmentIDs(t *testing.T) { if doc.Units[0].ID != 1 || doc.Units[1].ID != 2 { t.Fatalf("unit IDs = %#v, want numeric IDs", []int{doc.Units[0].ID, doc.Units[1].ID}) } + for i, unit := range doc.Units { + want := source.SourceRef{SourceID: doc.ID, StartUnitID: unit.ID, EndUnitID: unit.ID} + if unit.Ref != want { + t.Fatalf("doc.Units[%d].Ref = %#v, want %#v", i, unit.Ref, want) + } + } ref := source.SourceRef{ SourceID: doc.ID, StartUnitID: 1,