eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add opt-in JSON chunk map export

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-23 15:02:01 +00:00
parent 97c9a8e5ce
commit 16a998055c
2 changed files with 261 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
internal/modules/generic/output/json/encoder_test.go
View File

@@ -8,6 +8,8 @@ import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
@@ -37,6 +39,40 @@ func TestModuleSpecAndRegister(t *testing.T) {
if err := registry.ValidateOptions(Key, map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
t.Fatalf("ValidateOptions() error = %v, want unknown option error", err)
}
if err := registry.ValidateOptions(Key, map[string]any{"include_chunk_map": true}); err != nil {
t.Fatalf("ValidateOptions() error = %v, want nil", err)
}
}
func TestDecodeOptions(t *testing.T) {
for _, test := range []struct {
name string
options map[string]any
want Options
wantErr string
}{
{name: "omitted", want: Options{}},
{name: "disabled", options: map[string]any{"include_chunk_map": false}, want: Options{}},
{name: "enabled", options: map[string]any{"include_chunk_map": true}, want: Options{IncludeChunkMap: true}},
{name: "wrong type", options: map[string]any{"include_chunk_map": "true"}, wantErr: "must be a boolean"},
{name: "unknown", options: map[string]any{"unknown": true}, wantErr: "unknown option"},
} {
t.Run(test.name, func(t *testing.T) {
got, err := DecodeOptions(test.options)
if test.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
t.Fatalf("DecodeOptions() error = %v, want %q", err, test.wantErr)
}
return
}
if err != nil {
t.Fatalf("DecodeOptions() error = %v, want nil", err)
}
if got != test.want {
t.Fatalf("DecodeOptions() = %#v, want %#v", got, test.want)
}
})
}
}
func TestEncodeReturnsLogicalFilesForNormalizedOutputs(t *testing.T) {
@@ -129,6 +165,133 @@ func TestEncodeIncludesRejectedAndWarningsWhenEmpty(t *testing.T) {
}
}
func TestEncodeChunkMapExportIsOptIn(t *testing.T) {
artifact := acceptedChunkMapArtifact(t)
request := contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
ChunkMap: &artifact,
}
for _, test := range []struct {
name string
encoder *Encoder
}{
{name: "default", encoder: New()},
{name: "explicitly disabled", encoder: NewWithOptions(Options{IncludeChunkMap: false})},
} {
t.Run(test.name, func(t *testing.T) {
result, err := test.encoder.Encode(context.Background(), request)
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if got, want := outputFileNames(result.Files), []string{"index.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) {
t.Fatalf("file names = %#v, want %#v", got, want)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
if _, ok := index["chunk_map"]; ok {
t.Fatalf("index = %#v, want no chunk map descriptor", index)
}
})
}
}
func TestEncodeIncludesValidatedChunkMap(t *testing.T) {
artifact := acceptedChunkMapArtifact(t)
result, err := NewWithOptions(Options{IncludeChunkMap: true}).Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []contracts.SerializedOutput{
normalizeOutput("spells", `{"spell_casts":[]}`),
},
ChunkMap: &artifact,
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if got, want := outputFileNames(result.Files), []string{"chunk-map.json", "index.json", "lanes/spells.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) {
t.Fatalf("file names = %#v, want %#v", got, want)
}
chunkMapFile := fileBytes(t, result.Files, chunkMapFileName)
if !stdjson.Valid(chunkMapFile) || !strings.HasSuffix(string(chunkMapFile), "\n") || !strings.Contains(string(chunkMapFile), "\n \"source_id\"") {
t.Fatalf("chunk map file = %q, want pretty valid newline-terminated JSON", chunkMapFile)
}
if _, err := chunkmap.New().Decode(chunkMapFile); err != nil {
t.Fatalf("Decode(chunk map file) error = %v, want nil", err)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
if got, want := index["chunk_map"], map[string]any{
"artifact_kind": "source/chunk-map",
"file": chunkMapFileName,
"media_type": chunkmap.MediaType,
"schema_id": chunkmap.SchemaID,
"schema_name": chunkmap.SchemaName,
"schema_version": chunkmap.SchemaVersion,
}; !reflect.DeepEqual(got, want) {
t.Fatalf("chunk map descriptor = %#v, want %#v", got, want)
}
for _, entry := range index["output_files"].([]any) {
if entry.(map[string]any)["file"] == chunkMapFileName {
t.Fatalf("output_files = %#v, want no chunk map", index["output_files"])
}
}
}
func TestEncodeOmitsChunkMapWithoutAcceptedArtifact(t *testing.T) {
result, err := NewWithOptions(Options{IncludeChunkMap: true}).Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if got := outputFileNames(result.Files); containsString(got, chunkMapFileName) {
t.Fatalf("file names = %#v, want no chunk map", got)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
if _, ok := index["chunk_map"]; ok {
t.Fatalf("index = %#v, want no chunk map descriptor", index)
}
}
func TestEncodeRejectsInvalidChunkMapArtifact(t *testing.T) {
artifact := acceptedChunkMapArtifact(t)
for _, test := range []struct {
name string
mutate func(*contracts.SerializedArtifact)
}{
{name: "kind", mutate: func(artifact *contracts.SerializedArtifact) { artifact.Kind = "other/chunk-map" }},
{name: "schema", mutate: func(artifact *contracts.SerializedArtifact) { artifact.Schema.Version = "v2" }},
{name: "media type", mutate: func(artifact *contracts.SerializedArtifact) { artifact.MediaType = "text/plain" }},
{name: "content", mutate: func(artifact *contracts.SerializedArtifact) { artifact.Content = []byte(`{}`) }},
} {
t.Run(test.name, func(t *testing.T) {
candidate := contracts.CloneSerializedArtifact(artifact)
test.mutate(&candidate)
_, err := NewWithOptions(Options{IncludeChunkMap: true}).Encode(context.Background(), contracts.OutputRequest{ChunkMap: &candidate})
if err == nil {
t.Fatal("Encode() error = nil, want invalid chunk map error")
}
})
}
}
func TestEncodeChunkMapDoesNotMutateRequest(t *testing.T) {
artifact := acceptedChunkMapArtifact(t)
artifact.Metadata = map[string]any{"owner": "caller"}
request := contracts.OutputRequest{ChunkMap: &artifact}
before := contracts.CloneSerializedArtifact(artifact)
result, err := NewWithOptions(Options{IncludeChunkMap: true}).Encode(context.Background(), request)
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if !reflect.DeepEqual(*request.ChunkMap, before) {
t.Fatalf("chunk map artifact mutated:\nbefore: %#v\nafter: %#v", before, *request.ChunkMap)
}
request.ChunkMap.Content[0] = '['
request.ChunkMap.Schema.JSONSchema[0] = '['
request.ChunkMap.Metadata["owner"] = "changed"
if _, err := chunkmap.New().Decode(fileBytes(t, result.Files, chunkMapFileName)); err != nil {
t.Fatalf("chunk map output changed after request mutation: %v", err)
}
}
func TestEncodePrettyPrintsJSON(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
@@ -380,6 +543,39 @@ func normalizeOutput(laneID string, content string) contracts.SerializedOutput {
}
}
func acceptedChunkMapArtifact(t *testing.T) contracts.SerializedArtifact {
t.Helper()
document := &source.SourceDocument{
ID: "source-1",
Kind: "text",
Format: "text/plain",
Units: []source.SourceUnit{{
ID: 1,
Kind: "text",
Text: "Accepted source content.",
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
}},
}
digest, err := source.DigestDocument(document)
if err != nil {
t.Fatal(err)
}
document.Digest = digest
plan := source.ChunkPlan{SourceDigest: digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}
chunks, err := source.MaterializeChunkPlan(document, plan)
if err != nil {
t.Fatal(err)
}
artifact, err := chunkmap.Serialize(chunkmap.BuildRequest{
Source: document, Plan: plan, Chunks: chunks, RequestedChunker: "generic/units",
Producer: chunkmap.Producer{InputModule: "test/input", ChunkModule: "generic/units"},
})
if err != nil {
t.Fatal(err)
}
return artifact
}
func outputFileNames(files []contracts.OutputFile) []string {
names := make([]string, 0, len(files))
for _, file := range files {