231 lines
9.2 KiB
Go
231 lines
9.2 KiB
Go
package source
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestCanonicalizeChunkAnnotations(t *testing.T) {
|
|
original := ChunkAnnotations{
|
|
"domain/items": json.RawMessage(` { "z": [3, 2, 1], "a": 1.0 } `),
|
|
}
|
|
canonical, err := CanonicalizeChunkAnnotations(original)
|
|
if err != nil {
|
|
t.Fatalf("CanonicalizeChunkAnnotations() error = %v, want nil", err)
|
|
}
|
|
if got, want := string(canonical["domain/items"]), `{"a":1.0,"z":[3,2,1]}`; got != want {
|
|
t.Fatalf("canonical annotation = %q, want %q", got, want)
|
|
}
|
|
original["domain/items"][0] = '['
|
|
if got := string(canonical["domain/items"]); got != `{"a":1.0,"z":[3,2,1]}` {
|
|
t.Fatalf("canonical annotation changed after input mutation: %q", got)
|
|
}
|
|
canonical["domain/items"][0] = '['
|
|
if original["domain/items"][0] == '[' && bytes.Equal(original["domain/items"], canonical["domain/items"]) {
|
|
t.Fatal("input and canonical annotation share value storage")
|
|
}
|
|
}
|
|
|
|
func TestCanonicalizeChunkAnnotationsRejectsInvalidValues(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
annotations ChunkAnnotations
|
|
want string
|
|
}{
|
|
{name: "blank namespace", annotations: ChunkAnnotations{" \t": json.RawMessage(`true`)}, want: "namespace must not be empty"},
|
|
{name: "untrimmed namespace", annotations: ChunkAnnotations{" items ": json.RawMessage(`true`)}, want: "leading or trailing whitespace"},
|
|
{name: "invalid JSON", annotations: ChunkAnnotations{"items": json.RawMessage(`{"x":`)}, want: "valid JSON"},
|
|
{name: "trailing JSON", annotations: ChunkAnnotations{"items": json.RawMessage(`true false`)}, want: "exactly one JSON value"},
|
|
{name: "non-finite number", annotations: ChunkAnnotations{"items": json.RawMessage(`NaN`)}, want: "valid JSON"},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
_, err := CanonicalizeChunkAnnotations(tt.annotations)
|
|
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
|
t.Fatalf("CanonicalizeChunkAnnotations() error = %v, want containing %q", err, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestValidateChunkAnnotationsRequiresCanonicalJSON(t *testing.T) {
|
|
if err := ValidateChunkAnnotations(ChunkAnnotations{"items": json.RawMessage(` {"b":2,"a":1}`)}); err == nil || !strings.Contains(err.Error(), "canonical JSON") {
|
|
t.Fatalf("ValidateChunkAnnotations() error = %v, want canonical JSON error", err)
|
|
}
|
|
if err := ValidateChunkAnnotations(ChunkAnnotations{"items": json.RawMessage(`{"a":1,"b":2}`)}); err != nil {
|
|
t.Fatalf("ValidateChunkAnnotations(canonical) error = %v, want nil", err)
|
|
}
|
|
}
|
|
|
|
func TestCloneChunkPlanDoesNotShareAnnotationBytes(t *testing.T) {
|
|
plan := validChunkPlan(planDocument())
|
|
cloned := CloneChunkPlan(plan)
|
|
cloned.Annotations["plan"][0] = '['
|
|
cloned.Ranges[0].Annotations["range"][0] = '['
|
|
if string(plan.Annotations["plan"]) != `{"value":1}` || string(plan.Ranges[0].Annotations["range"]) != `{"value":2}` {
|
|
t.Fatal("CloneChunkPlan() shares annotation value storage")
|
|
}
|
|
}
|
|
|
|
func TestValidateChunkPlanRanges(t *testing.T) {
|
|
doc := planDocument()
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*ChunkPlan)
|
|
want string
|
|
}{
|
|
{name: "source mismatch", mutate: func(plan *ChunkPlan) { plan.SourceDigest = "sha256:other" }, want: "does not match"},
|
|
{name: "missing ranges", mutate: func(plan *ChunkPlan) { plan.Ranges = nil }, want: "ranges must not be empty"},
|
|
{name: "missing start", mutate: func(plan *ChunkPlan) { plan.Ranges[0].StartUnitID = 99 }, want: "start_unit_id 99 was not found"},
|
|
{name: "missing end", mutate: func(plan *ChunkPlan) { plan.Ranges[0].EndUnitID = 99 }, want: "end_unit_id 99 was not found"},
|
|
{name: "backward range", mutate: func(plan *ChunkPlan) { plan.Ranges[0] = ChunkRange{StartUnitID: 30, EndUnitID: 10} }, want: "appears after end_unit_id"},
|
|
{name: "duplicate start", mutate: func(plan *ChunkPlan) { plan.Ranges[1].StartUnitID = plan.Ranges[0].StartUnitID }, want: "does not appear after"},
|
|
{name: "backward starts", mutate: func(plan *ChunkPlan) {
|
|
plan.Ranges = []ChunkRange{{StartUnitID: 30, EndUnitID: 50}, {StartUnitID: 20, EndUnitID: 40}}
|
|
}, want: "does not appear after"},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
plan := validChunkPlan(doc)
|
|
tt.mutate(&plan)
|
|
err := ValidateChunkPlan(doc, plan)
|
|
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
|
t.Fatalf("ValidateChunkPlan() error = %v, want containing %q", err, tt.want)
|
|
}
|
|
})
|
|
}
|
|
|
|
for name, ranges := range map[string][]ChunkRange{
|
|
"gap": {{StartUnitID: 10, EndUnitID: 20}, {StartUnitID: 40, EndUnitID: 50}},
|
|
"overlap": {{StartUnitID: 10, EndUnitID: 30}, {StartUnitID: 20, EndUnitID: 50}},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
plan := validChunkPlan(doc)
|
|
plan.Ranges = ranges
|
|
if err := ValidateChunkPlan(doc, plan); err != nil {
|
|
t.Fatalf("ValidateChunkPlan() error = %v, want nil", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDigestChunkPlanIsStableAndCoversLogicalPlan(t *testing.T) {
|
|
doc := planDocument()
|
|
plan := validChunkPlan(doc)
|
|
first, err := DigestChunkPlan(plan)
|
|
if err != nil {
|
|
t.Fatalf("DigestChunkPlan() error = %v, want nil", err)
|
|
}
|
|
reformatted := CloneChunkPlan(plan)
|
|
reformatted.Annotations["plan"] = json.RawMessage(` { "value" : 1 } `)
|
|
second, err := DigestChunkPlan(reformatted)
|
|
if err != nil {
|
|
t.Fatalf("DigestChunkPlan(reformatted) error = %v, want nil", err)
|
|
}
|
|
if first != second {
|
|
t.Fatalf("digests = %q and %q, want stable canonical annotation digest", first, second)
|
|
}
|
|
|
|
changes := []func(*ChunkPlan){
|
|
func(value *ChunkPlan) { value.Ranges[0].EndUnitID = 30 },
|
|
func(value *ChunkPlan) { value.Annotations["plan"] = json.RawMessage(`{"value":2}`) },
|
|
func(value *ChunkPlan) { value.Ranges[0].Annotations["range"] = json.RawMessage(`{"value":3}`) },
|
|
}
|
|
for i, change := range changes {
|
|
changed := CloneChunkPlan(plan)
|
|
change(&changed)
|
|
digest, err := DigestChunkPlan(changed)
|
|
if err != nil {
|
|
t.Fatalf("DigestChunkPlan(change %d) error = %v", i, err)
|
|
}
|
|
if digest == first {
|
|
t.Fatalf("DigestChunkPlan(change %d) = %q, want changed digest", i, digest)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMaterializeChunkPlanExactOutputAndMutationSafety(t *testing.T) {
|
|
doc := planDocument()
|
|
plan := validChunkPlan(doc)
|
|
plan.Ranges = []ChunkRange{
|
|
{StartUnitID: 10, EndUnitID: 30, Annotations: ChunkAnnotations{"range": json.RawMessage(`{"value":2}`)}},
|
|
{StartUnitID: 20, EndUnitID: 50, Annotations: ChunkAnnotations{"range": json.RawMessage(`{"value":3}`)}},
|
|
}
|
|
chunks, err := MaterializeChunkPlan(doc, plan)
|
|
if err != nil {
|
|
t.Fatalf("MaterializeChunkPlan() error = %v, want nil", err)
|
|
}
|
|
if len(chunks) != 2 {
|
|
t.Fatalf("chunks = %d, want 2", len(chunks))
|
|
}
|
|
first := chunks[0]
|
|
if first.ID != "chunk-000001" || first.SourceID != doc.ID || first.Index != 0 || first.MediaType != "application/json" {
|
|
t.Fatalf("first chunk identity = %#v", first)
|
|
}
|
|
if want := (SourceRef{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 30}); first.Ref != want {
|
|
t.Fatalf("first ref = %#v, want %#v", first.Ref, want)
|
|
}
|
|
if got, want := unitIDs(first.Units), []int{10, 20, 30}; !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("first unit ids = %#v, want %#v", got, want)
|
|
}
|
|
wantContent, _ := json.Marshal(struct {
|
|
Units []SourceUnit `json:"units"`
|
|
}{Units: doc.Units[:3]})
|
|
if !bytes.Equal(first.Content, wantContent) {
|
|
t.Fatalf("first content = %s, want %s", first.Content, wantContent)
|
|
}
|
|
if !reflect.DeepEqual(first.Metadata, map[string]any{"start_unit_id": 10, "end_unit_id": 30, "unit_count": 3}) {
|
|
t.Fatalf("first metadata = %#v", first.Metadata)
|
|
}
|
|
if string(first.Annotations["range"]) != `{"value":2}` || string(first.PlanAnnotations["plan"]) != `{"value":1}` {
|
|
t.Fatalf("first annotations = %#v / %#v", first.Annotations, first.PlanAnnotations)
|
|
}
|
|
if got, want := unitIDs(chunks[1].Units), []int{20, 30, 40, 50}; !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("overlapping unit ids = %#v, want %#v", got, want)
|
|
}
|
|
|
|
again, err := MaterializeChunkPlan(doc, plan)
|
|
if err != nil {
|
|
t.Fatalf("MaterializeChunkPlan(repeated) error = %v", err)
|
|
}
|
|
if !reflect.DeepEqual(chunks, again) {
|
|
t.Fatalf("repeated materialization differs:\nfirst: %#v\nagain: %#v", chunks, again)
|
|
}
|
|
chunks[0].Units[0].Text = "mutated"
|
|
chunks[0].Annotations["range"][0] = '['
|
|
chunks[0].PlanAnnotations["plan"][0] = '['
|
|
if doc.Units[0].Text == "mutated" || string(plan.Ranges[0].Annotations["range"]) != `{"value":2}` || string(plan.Annotations["plan"]) != `{"value":1}` {
|
|
t.Fatal("materialized chunk shares owned plan or source storage")
|
|
}
|
|
}
|
|
|
|
func planDocument() *SourceDocument {
|
|
doc := &SourceDocument{ID: "source-plan", Kind: "test", Format: "application/test", Digest: "sha256:source-plan"}
|
|
for _, id := range []int{10, 20, 30, 40, 50} {
|
|
doc.Units = append(doc.Units, SourceUnit{ID: id, Kind: "line", Text: "unit", Ref: SourceRef{SourceID: doc.ID, StartUnitID: id, EndUnitID: id}})
|
|
}
|
|
return doc
|
|
}
|
|
|
|
func validChunkPlan(doc *SourceDocument) ChunkPlan {
|
|
return ChunkPlan{
|
|
SourceDigest: doc.Digest,
|
|
Ranges: []ChunkRange{
|
|
{StartUnitID: 10, EndUnitID: 20, Annotations: ChunkAnnotations{"range": json.RawMessage(`{"value":2}`)}},
|
|
{StartUnitID: 30, EndUnitID: 50},
|
|
},
|
|
Annotations: ChunkAnnotations{"plan": json.RawMessage(`{"value":1}`)},
|
|
}
|
|
}
|
|
|
|
func unitIDs(units []SourceUnit) []int {
|
|
ids := make([]int, len(units))
|
|
for i, unit := range units {
|
|
ids[i] = unit.ID
|
|
}
|
|
return ids
|
|
}
|