Move chunks into the canonical source model
This commit is contained in:
@@ -249,17 +249,17 @@ func TestRunRejectsChunkerBuildChunkAndEmptyChunkErrors(t *testing.T) {
|
||||
func TestRunRejectsInvalidChunks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
chunks []contracts.SourceChunk
|
||||
chunks []source.Chunk
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty chunk id",
|
||||
chunks: []contracts.SourceChunk{chunkWithUnits("", "source-1", 0, unitWithID("u1"))},
|
||||
chunks: []source.Chunk{chunkWithUnits("", "source-1", 0, unitWithID("u1"))},
|
||||
want: "id must not be empty",
|
||||
},
|
||||
{
|
||||
name: "duplicate chunk id",
|
||||
chunks: []contracts.SourceChunk{
|
||||
chunks: []source.Chunk{
|
||||
chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1")),
|
||||
chunkWithUnits("chunk-0", "source-1", 1, unitWithID("u2")),
|
||||
},
|
||||
@@ -267,58 +267,95 @@ func TestRunRejectsInvalidChunks(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "wrong source id",
|
||||
chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "other-source", 0, unitWithID("u1"))},
|
||||
chunks: []source.Chunk{chunkWithUnits("chunk-0", "other-source", 0, unitWithID("u1"))},
|
||||
want: "source_id",
|
||||
},
|
||||
{
|
||||
name: "wrong index",
|
||||
chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "source-1", 1, unitWithID("u1"))},
|
||||
chunks: []source.Chunk{chunkWithUnits("chunk-0", "source-1", 1, unitWithID("u1"))},
|
||||
want: "index",
|
||||
},
|
||||
{
|
||||
name: "missing ref",
|
||||
chunks: []source.Chunk{func() source.Chunk {
|
||||
chunk := chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"))
|
||||
chunk.Ref = source.SourceRef{}
|
||||
return chunk
|
||||
}()},
|
||||
want: "source_id",
|
||||
},
|
||||
{
|
||||
name: "foreign ref",
|
||||
chunks: []source.Chunk{func() source.Chunk {
|
||||
chunk := chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"))
|
||||
chunk.Ref.SourceID = "other-source"
|
||||
return chunk
|
||||
}()},
|
||||
want: "does not match document id",
|
||||
},
|
||||
{
|
||||
name: "unknown start id",
|
||||
chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 9, 1, unitWithID("u1"))},
|
||||
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 9, 1, unitWithID("u1"))},
|
||||
want: "start_unit_id",
|
||||
},
|
||||
{
|
||||
name: "unknown end id",
|
||||
chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 1, 9, unitWithID("u1"))},
|
||||
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 1, 9, unitWithID("u1"))},
|
||||
want: "end_unit_id",
|
||||
},
|
||||
{
|
||||
name: "reversed bounds",
|
||||
chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 2, 1, unitWithID("u1"), unitWithID("u2"))},
|
||||
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 2, 1, unitWithID("u1"), unitWithID("u2"))},
|
||||
want: "appears after",
|
||||
},
|
||||
{
|
||||
name: "empty units",
|
||||
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, StartUnitID: 1, EndUnitID: 1, Content: []byte(`{"units":[]}`), MediaType: "application/json"}},
|
||||
chunks: []source.Chunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}, Content: []byte(`{"units":[]}`), MediaType: "application/json"}},
|
||||
want: "units must not be empty",
|
||||
},
|
||||
{
|
||||
name: "empty content",
|
||||
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, StartUnitID: 1, EndUnitID: 1, MediaType: "application/json", Units: []source.SourceUnit{unitWithID("u1")}}},
|
||||
chunks: []source.Chunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}, MediaType: "application/json", Units: []source.SourceUnit{unitWithID("u1")}}},
|
||||
want: "content must not be empty",
|
||||
},
|
||||
{
|
||||
name: "empty media type",
|
||||
chunks: []contracts.SourceChunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, StartUnitID: 1, EndUnitID: 1, Content: []byte(`{"units":[1]}`), Units: []source.SourceUnit{unitWithID("u1")}}},
|
||||
chunks: []source.Chunk{{ID: "chunk-0", SourceID: "source-1", Index: 0, Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}, Content: []byte(`{"units":[1]}`), Units: []source.SourceUnit{unitWithID("u1")}}},
|
||||
want: "media_type must not be empty",
|
||||
},
|
||||
{
|
||||
name: "repeated unit inside chunk",
|
||||
chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"), unitWithID("u1"))},
|
||||
chunks: []source.Chunk{chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"), unitWithID("u1"))},
|
||||
want: "repeats source unit",
|
||||
},
|
||||
{
|
||||
name: "unknown unit",
|
||||
chunks: []contracts.SourceChunk{chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u9"))},
|
||||
chunks: []source.Chunk{chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u9"))},
|
||||
want: "was not found",
|
||||
},
|
||||
{
|
||||
name: "units out of source order",
|
||||
chunks: []contracts.SourceChunk{chunkWithBounds("chunk-0", "source-1", 0, 1, 2, unitWithID("u2"), unitWithID("u1"))},
|
||||
want: "source document order",
|
||||
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 1, 2, unitWithID("u2"), unitWithID("u1"))},
|
||||
want: "contiguous range",
|
||||
},
|
||||
{
|
||||
name: "noncontiguous units",
|
||||
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 1, 3, unitWithID("u1"), unitWithID("u3"))},
|
||||
want: "contiguous range",
|
||||
},
|
||||
{
|
||||
name: "ref does not match unit span",
|
||||
chunks: []source.Chunk{chunkWithRef("chunk-0", "source-1", 0, 1, 2, unitWithID("u1"))},
|
||||
want: "does not match unit span",
|
||||
},
|
||||
{
|
||||
name: "unit ref does not match source",
|
||||
chunks: []source.Chunk{func() source.Chunk {
|
||||
unit := unitWithID("u1")
|
||||
unit.Ref.SourceID = "other-source"
|
||||
return chunkWithRef("chunk-0", "source-1", 0, 1, 1, unit)
|
||||
}()},
|
||||
want: "ref does not match source document",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -342,7 +379,7 @@ func TestRunRejectsInvalidChunks(t *testing.T) {
|
||||
|
||||
func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
|
||||
modules := defaultRunnerModules()
|
||||
modules.chunker.chunks = []contracts.SourceChunk{
|
||||
modules.chunker.chunks = []source.Chunk{
|
||||
chunkWithUnits("chunk-0", "source-1", 0, unitWithID("u1"), unitWithID("u2")),
|
||||
chunkWithUnits("chunk-1", "source-1", 1, unitWithID("u2")),
|
||||
}
|
||||
@@ -359,20 +396,20 @@ func TestRunAllowsPartialCoverageAndOverlappingChunks(t *testing.T) {
|
||||
func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
|
||||
modules := defaultRunnerModules()
|
||||
modules.input.doc = sourceDocumentWithUnitMetadata()
|
||||
modules.chunker.chunks = []contracts.SourceChunk{
|
||||
modules.chunker.chunks = []source.Chunk{
|
||||
{
|
||||
ID: "chunk-0",
|
||||
SourceID: "source-1",
|
||||
Index: 0,
|
||||
StartUnitID: 1,
|
||||
EndUnitID: 1,
|
||||
Content: []byte(`{"units":[{"id":1}]}`),
|
||||
MediaType: "application/json",
|
||||
ID: "chunk-0",
|
||||
SourceID: "source-1",
|
||||
Index: 0,
|
||||
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
|
||||
Content: []byte(`{"units":[{"id":1}]}`),
|
||||
MediaType: "application/json",
|
||||
Units: []source.SourceUnit{
|
||||
{
|
||||
ID: 1,
|
||||
Kind: "mutated-kind",
|
||||
Text: "mutated text",
|
||||
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
|
||||
Metadata: map[string]any{
|
||||
"speaker": "chunker-speaker",
|
||||
"note": "chunker note",
|
||||
@@ -423,15 +460,14 @@ func TestRunCanonicalizesChunkUnitsBeforeExtraction(t *testing.T) {
|
||||
|
||||
func TestRunPreservesChunkMetadataDuringCanonicalization(t *testing.T) {
|
||||
modules := defaultRunnerModules()
|
||||
modules.chunker.chunks = []contracts.SourceChunk{
|
||||
modules.chunker.chunks = []source.Chunk{
|
||||
{
|
||||
ID: "chunk-0",
|
||||
SourceID: "source-1",
|
||||
Index: 0,
|
||||
StartUnitID: 1,
|
||||
EndUnitID: 1,
|
||||
Content: []byte(`{"units":[{"id":1}]}`),
|
||||
MediaType: "application/json",
|
||||
ID: "chunk-0",
|
||||
SourceID: "source-1",
|
||||
Index: 0,
|
||||
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
|
||||
Content: []byte(`{"units":[{"id":1}]}`),
|
||||
MediaType: "application/json",
|
||||
Units: []source.SourceUnit{
|
||||
unitWithID("u1"),
|
||||
},
|
||||
@@ -975,7 +1011,7 @@ func TestRunPassesPerChunkRawOutputsToMergeAndNormalize(t *testing.T) {
|
||||
|
||||
func TestRunPassesChunkContentAndMediaTypeToExtractors(t *testing.T) {
|
||||
modules := defaultRunnerModules()
|
||||
modules.chunker.chunks = []contracts.SourceChunk{
|
||||
modules.chunker.chunks = []source.Chunk{
|
||||
sourceChunkWithContent("chunk-0", 0, []byte(`{"chunk":0}`), "application/vnd.test+json"),
|
||||
}
|
||||
|
||||
@@ -1027,10 +1063,27 @@ func TestRunDoesNotPassCheckpointPathsToModules(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckpointChunkDigestIncludesCanonicalReference(t *testing.T) {
|
||||
chunk := sourceChunkWithID("chunk-0", 0)
|
||||
first, err := joinedChunkDigest([]source.Chunk{chunk})
|
||||
if err != nil {
|
||||
t.Fatalf("joinedChunkDigest() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
chunk.Ref.EndUnitID = 2
|
||||
second, err := joinedChunkDigest([]source.Chunk{chunk})
|
||||
if err != nil {
|
||||
t.Fatalf("joinedChunkDigest(changed ref) error = %v, want nil", err)
|
||||
}
|
||||
if first == second {
|
||||
t.Fatalf("checkpoint chunk digests = %q and %q, want provenance change to alter dependency identity", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReusesCheckpointedWorkflowOutputs(t *testing.T) {
|
||||
modules := defaultRunnerModules()
|
||||
doc := validSourceDocument()
|
||||
chunks := []contracts.SourceChunk{sourceChunkWithID("chunk-0", 0)}
|
||||
chunks := []source.Chunk{sourceChunkWithID("chunk-0", 0)}
|
||||
extractOutput := contracts.ExtractOutput{
|
||||
LaneID: "alpha",
|
||||
ExtractorKey: "extract-alpha",
|
||||
@@ -1468,7 +1521,7 @@ func TestRunCollectsStageWarnings(t *testing.T) {
|
||||
|
||||
func TestRunCollectsChunkValidatorWarnings(t *testing.T) {
|
||||
modules := defaultRunnerModules()
|
||||
modules.chunker.chunks = []contracts.SourceChunk{sourceChunkWithID("chunk-0", 0)}
|
||||
modules.chunker.chunks = []source.Chunk{sourceChunkWithID("chunk-0", 0)}
|
||||
validator := &runnerChainValidator{
|
||||
name: "chain-chunk",
|
||||
warnings: []contracts.Warning{{ReasonCode: "chunk-validator-warning", Message: "chunk validator warning"}},
|
||||
@@ -1904,7 +1957,7 @@ type runnerModules struct {
|
||||
func defaultRunnerModules() *runnerModules {
|
||||
return &runnerModules{
|
||||
input: &runnerInputAdapter{key: "input", doc: validSourceDocument()},
|
||||
chunker: &runnerChunker{key: "chunk", chunks: []contracts.SourceChunk{sourceChunkWithID("chunk-0", 0), sourceChunkWithID("chunk-1", 1)}},
|
||||
chunker: &runnerChunker{key: "chunk", chunks: []source.Chunk{sourceChunkWithID("chunk-0", 0), sourceChunkWithID("chunk-1", 1)}},
|
||||
extractors: map[string]*runnerExtractor{
|
||||
"extract-alpha": {key: "extract-alpha"},
|
||||
},
|
||||
@@ -2013,7 +2066,7 @@ func (adapter *runnerInputAdapter) ManifestMetadata() map[string]any {
|
||||
|
||||
type runnerChunker struct {
|
||||
key string
|
||||
chunks []contracts.SourceChunk
|
||||
chunks []source.Chunk
|
||||
warnings []contracts.Warning
|
||||
err error
|
||||
failureErr error
|
||||
@@ -2505,21 +2558,20 @@ func sourceDocumentWithUnitMetadata() *source.SourceDocument {
|
||||
}
|
||||
}
|
||||
|
||||
func sourceChunkWithID(id string, index int) contracts.SourceChunk {
|
||||
func sourceChunkWithID(id string, index int) source.Chunk {
|
||||
unit := unitWithID("u1")
|
||||
return contracts.SourceChunk{
|
||||
ID: id,
|
||||
SourceID: "source-1",
|
||||
Index: index,
|
||||
StartUnitID: unit.ID,
|
||||
EndUnitID: unit.ID,
|
||||
Content: []byte(`{"units":[{"id":1,"kind":"unit","text":"Source unit."}]}`),
|
||||
MediaType: "application/json",
|
||||
Units: []source.SourceUnit{unit},
|
||||
return source.Chunk{
|
||||
ID: id,
|
||||
SourceID: "source-1",
|
||||
Index: index,
|
||||
Ref: unit.Ref,
|
||||
Content: []byte(`{"units":[{"id":1,"kind":"unit","text":"Source unit."}]}`),
|
||||
MediaType: "application/json",
|
||||
Units: []source.SourceUnit{unit},
|
||||
}
|
||||
}
|
||||
|
||||
func sourceChunkWithContent(id string, index int, content []byte, mediaType string) contracts.SourceChunk {
|
||||
func sourceChunkWithContent(id string, index int, content []byte, mediaType string) source.Chunk {
|
||||
chunk := sourceChunkWithID(id, index)
|
||||
chunk.Content = append([]byte(nil), content...)
|
||||
chunk.MediaType = mediaType
|
||||
@@ -2541,25 +2593,24 @@ func unitWithID(id string) source.SourceUnit {
|
||||
}
|
||||
}
|
||||
|
||||
func chunkWithUnits(id string, sourceID string, index int, units ...source.SourceUnit) contracts.SourceChunk {
|
||||
func chunkWithUnits(id string, sourceID string, index int, units ...source.SourceUnit) source.Chunk {
|
||||
startUnitID, endUnitID := 1, 1
|
||||
if len(units) > 0 {
|
||||
startUnitID = units[0].ID
|
||||
endUnitID = units[len(units)-1].ID
|
||||
}
|
||||
return chunkWithBounds(id, sourceID, index, startUnitID, endUnitID, units...)
|
||||
return chunkWithRef(id, sourceID, index, startUnitID, endUnitID, units...)
|
||||
}
|
||||
|
||||
func chunkWithBounds(id string, sourceID string, index int, startUnitID int, endUnitID int, units ...source.SourceUnit) contracts.SourceChunk {
|
||||
return contracts.SourceChunk{
|
||||
ID: id,
|
||||
SourceID: sourceID,
|
||||
Index: index,
|
||||
StartUnitID: startUnitID,
|
||||
EndUnitID: endUnitID,
|
||||
Content: []byte(`{"units":[1]}`),
|
||||
MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), units...),
|
||||
func chunkWithRef(id string, sourceID string, index int, startUnitID int, endUnitID int, units ...source.SourceUnit) source.Chunk {
|
||||
return source.Chunk{
|
||||
ID: id,
|
||||
SourceID: sourceID,
|
||||
Index: index,
|
||||
Ref: source.SourceRef{SourceID: sourceID, StartUnitID: startUnitID, EndUnitID: endUnitID},
|
||||
Content: []byte(`{"units":[1]}`),
|
||||
MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), units...),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user