package chunkmap import ( "bytes" "encoding/json" "os" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" ) func TestBuildAndSerializeAcceptedChunkMap(t *testing.T) { request := acceptedBuildRequest(t) value, err := Build(request) if err != nil { t.Fatalf("Build() error = %v", err) } if value.SourceID != request.Source.ID || len(value.Chunks) != 2 || value.Chunks[0].UnitCount != 2 || value.Chunks[1].SourceRef.StartUnitID != 20 { t.Fatalf("Build() = %#v, want exact accepted chunk structure", value) } if value.PlanAnnotations == nil || value.Chunks[1].Annotations == nil { t.Fatalf("Build() annotations = %#v, want explicit maps", value) } artifact, err := Serialize(request) if err != nil { t.Fatalf("Serialize() error = %v", err) } if artifact.Kind != ArtifactKind || artifact.Schema.ID != SchemaID || artifact.Schema.Name != SchemaName || artifact.Schema.Version != SchemaVersion || artifact.MediaType != MediaType || artifact.Metadata != nil { t.Fatalf("Serialize() = %#v, want fixed artifact envelope without metadata", artifact) } decoded, err := New().Decode(artifact.Content) if err != nil { t.Fatalf("Decode(Serialize()) error = %v", err) } if decoded.PlanDigest != value.PlanDigest || decoded.Chunks[0].ID != "chunk-000001" || decoded.Chunks[1].UnitCount != 1 { t.Fatalf("Decode(Serialize()) = %#v, want durable chunk map", decoded) } } func TestCodecRoundTripsValidFixture(t *testing.T) { fixture, err := os.ReadFile("testdata/source_chunk_map.v1.json") if err != nil { t.Fatal(err) } codec := New() value, err := codec.Decode(fixture) if err != nil { t.Fatalf("Decode(fixture) error = %v", err) } encoded, err := codec.Encode(value) if err != nil { t.Fatalf("Encode(decoded fixture) error = %v", err) } if !bytes.Equal(encoded, bytes.TrimSpace(fixture)) { t.Fatalf("fixture does not use canonical encoding\nwant: %s\n got: %s", fixture, encoded) } value.PlanAnnotations["test/chunker"][0] = '[' value.Chunks[0].Annotations["test/chunker"][0] = '[' decoded, err := codec.Decode(encoded) if err != nil { t.Fatalf("Decode(encoded) after mutation error = %v", err) } if string(decoded.PlanAnnotations["test/chunker"]) != `{"label":"fixture"}` || string(decoded.Chunks[0].Annotations["test/chunker"]) != `{"category":"sample"}` { t.Fatalf("Decode() reused mutable chunk-map storage: %#v", decoded) } } func TestBuildCanonicalizesAnnotationFormatting(t *testing.T) { first := acceptedBuildRequest(t) second := acceptedBuildRequest(t) second.Plan.Annotations["test/chunker"] = json.RawMessage(" { \n \t\"label\" : \"fixture\" \n } ") canonical, err := source.CanonicalizeChunkPlan(second.Plan) if err != nil { t.Fatalf("CanonicalizeChunkPlan() error = %v", err) } second.Chunks, err = source.MaterializeChunkPlan(second.Source, canonical) if err != nil { t.Fatalf("MaterializeChunkPlan() error = %v", err) } firstArtifact, err := Serialize(first) if err != nil { t.Fatalf("Serialize(first) error = %v", err) } secondArtifact, err := Serialize(second) if err != nil { t.Fatalf("Serialize(second) error = %v", err) } if !bytes.Equal(firstArtifact.Content, secondArtifact.Content) { t.Fatalf("serialized content differs only because annotation whitespace changed\nfirst: %s\nsecond: %s", firstArtifact.Content, secondArtifact.Content) } } func TestBuildRejectsChunksOutsideAcceptedPlan(t *testing.T) { request := acceptedBuildRequest(t) request.Chunks[0].Units[0].ID = 999 if _, err := Build(request); err == nil { t.Fatal("Build() error = nil, want rejection for chunk units outside accepted source range") } } func TestCodecRejectsInvalidDurableBoundaries(t *testing.T) { value, err := Build(acceptedBuildRequest(t)) if err != nil { t.Fatal(err) } for _, test := range []struct { name string mutate func(*ChunkMap) }{ {name: "blank identity", mutate: func(value *ChunkMap) { value.RequestedChunker = " " }}, {name: "malformed digest", mutate: func(value *ChunkMap) { value.SourceDigest = "sha256:ABC" }}, {name: "index mismatch", mutate: func(value *ChunkMap) { value.Chunks[1].Index = 4 }}, {name: "duplicate chunk id", mutate: func(value *ChunkMap) { value.Chunks[1].ID = value.Chunks[0].ID }}, {name: "source mismatch", mutate: func(value *ChunkMap) { value.Chunks[0].SourceRef.SourceID = "other" }}, {name: "invalid range", mutate: func(value *ChunkMap) { value.Chunks[0].SourceRef.StartUnitID = 0 }}, {name: "invalid count", mutate: func(value *ChunkMap) { value.Chunks[0].UnitCount = 0 }}, {name: "invalid namespace", mutate: func(value *ChunkMap) { value.PlanAnnotations[" "] = json.RawMessage(`null`) }}, {name: "invalid annotation", mutate: func(value *ChunkMap) { value.Chunks[0].Annotations["test/chunker"] = json.RawMessage(`{`) }}, {name: "plan digest mismatch", mutate: func(value *ChunkMap) { value.PlanDigest = "sha256:" + strings.Repeat("a", 64) }}, } { t.Run(test.name, func(t *testing.T) { candidate := clone(value) test.mutate(&candidate) if _, err := New().Encode(candidate); err == nil { t.Fatal("Encode() error = nil, want invalid durable value rejection") } }) } content, err := New().Encode(value) if err != nil { t.Fatal(err) } for _, raw := range [][]byte{ append(append([]byte(nil), content[:len(content)-1]...), []byte(`,"unknown":true}`)...), append(append([]byte(nil), content...), []byte(` {}`)...), } { if _, err := New().Decode(raw); err == nil { t.Fatalf("Decode(%s) error = nil, want strict JSON rejection", raw) } } formatted := bytes.Replace(content, []byte(`{"label":"fixture"}`), []byte("{\n \"label\": \"fixture\"\n}"), 1) decoded, err := New().Decode(formatted) if err != nil { t.Fatalf("Decode(formatted annotations) error = %v", err) } if string(decoded.PlanAnnotations["test/chunker"]) != `{"label":"fixture"}` { t.Fatalf("decoded annotation = %s, want canonical JSON", decoded.PlanAnnotations["test/chunker"]) } } func TestDecodeEnforcesRequiredSchemaFieldsAndTypes(t *testing.T) { request := acceptedBuildRequest(t) request.Plan.Annotations = nil var err error request.Chunks, err = source.MaterializeChunkPlan(request.Source, request.Plan) if err != nil { t.Fatal(err) } artifact, err := Serialize(request) if err != nil { t.Fatal(err) } for _, test := range []struct { name string mutate func(map[string]any) }{ {name: "missing plan annotations", mutate: func(value map[string]any) { delete(value, "plan_annotations") }}, {name: "null plan annotations", mutate: func(value map[string]any) { value["plan_annotations"] = nil }}, {name: "missing first index", mutate: func(value map[string]any) { delete(chunkDocument(value, 0), "index") }}, {name: "null first index", mutate: func(value map[string]any) { chunkDocument(value, 0)["index"] = nil }}, {name: "missing empty chunk annotations", mutate: func(value map[string]any) { delete(chunkDocument(value, 1), "annotations") }}, {name: "null empty chunk annotations", mutate: func(value map[string]any) { chunkDocument(value, 1)["annotations"] = nil }}, {name: "explicit empty llm profile", mutate: func(value map[string]any) { value["producer"].(map[string]any)["llm_profile"] = "" }}, } { t.Run(test.name, func(t *testing.T) { value := decodeJSONDocument(t, artifact.Content) test.mutate(value) content, err := json.Marshal(value) if err != nil { t.Fatal(err) } if _, err := New().Decode(content); err == nil { t.Fatalf("Decode(%s) error = nil, want schema rejection", content) } }) } } func TestEncodeDoesNotMutateValue(t *testing.T) { value, err := Build(acceptedBuildRequest(t)) if err != nil { t.Fatal(err) } value.Chunks[0].Annotations["test/chunker"] = json.RawMessage(" { \n \"category\" : \"sample\" \n } ") before := clone(value) if _, err := New().Encode(value); err != nil { t.Fatalf("Encode() error = %v", err) } if !reflect.DeepEqual(value, before) { t.Fatalf("Encode() mutated value:\nbefore: %#v\nafter: %#v", before, value) } } func TestChunkMapOwnershipIsIndependent(t *testing.T) { request := acceptedBuildRequest(t) first, err := Build(request) if err != nil { t.Fatal(err) } request.Plan.Annotations["test/chunker"][0] = '[' first.PlanAnnotations["test/chunker"][0] = '[' second, err := Build(acceptedBuildRequest(t)) if err != nil { t.Fatal(err) } if string(second.PlanAnnotations["test/chunker"]) != `{"label":"fixture"}` { t.Fatalf("Build() shared mutable annotations: %s", second.PlanAnnotations["test/chunker"]) } } func decodeJSONDocument(t *testing.T, content []byte) map[string]any { t.Helper() decoder := json.NewDecoder(bytes.NewReader(content)) decoder.UseNumber() var value map[string]any if err := decoder.Decode(&value); err != nil { t.Fatal(err) } return value } func chunkDocument(value map[string]any, index int) map[string]any { return value["chunks"].([]any)[index].(map[string]any) } func acceptedBuildRequest(t *testing.T) BuildRequest { t.Helper() document := &source.SourceDocument{ ID: "source-test", Kind: "transcript", Format: "application/json", Units: []source.SourceUnit{ {ID: 10, Kind: "segment", Text: "First unit.", Ref: source.SourceRef{SourceID: "source-test", StartUnitID: 10, EndUnitID: 10}}, {ID: 3, Kind: "segment", Text: "Second unit.", Ref: source.SourceRef{SourceID: "source-test", StartUnitID: 3, EndUnitID: 3}}, {ID: 20, Kind: "segment", Text: "Third unit.", Ref: source.SourceRef{SourceID: "source-test", StartUnitID: 20, EndUnitID: 20}}, }, } digest, err := source.DigestDocument(document) if err != nil { t.Fatal(err) } document.Digest = digest plan := source.ChunkPlan{ SourceDigest: digest, Annotations: source.ChunkAnnotations{"test/chunker": json.RawMessage(`{"label":"fixture"}`)}, Ranges: []source.ChunkRange{ {StartUnitID: 10, EndUnitID: 3, Annotations: source.ChunkAnnotations{"test/chunker": json.RawMessage(`{"category":"sample"}`)}}, {StartUnitID: 20, EndUnitID: 20}, }, } chunks, err := source.MaterializeChunkPlan(document, plan) if err != nil { t.Fatal(err) } return BuildRequest{ Source: document, Plan: plan, Chunks: chunks, RequestedChunker: "chunk/requested", Producer: Producer{InputModule: "input/producer", ChunkModule: "chunk/producer", LLMProfile: "profile/test"}, } }