Organize generic and Seriatim modules by domain
This commit is contained in:
272
internal/modules/generic/output/json/encoder.go
Normal file
272
internal/modules/generic/output/json/encoder.go
Normal file
@@ -0,0 +1,272 @@
|
||||
package json
|
||||
|
||||
import (
|
||||
"context"
|
||||
stdjson "encoding/json"
|
||||
"fmt"
|
||||
"mime"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const Key = "json"
|
||||
|
||||
const contentTypeJSON = "application/json"
|
||||
|
||||
var safeOutputFileChar = regexp.MustCompile(`[^A-Za-z0-9._-]`)
|
||||
|
||||
var _ contracts.OutputEncoder = (*Encoder)(nil)
|
||||
|
||||
type Encoder struct{}
|
||||
|
||||
func New() *Encoder {
|
||||
return &Encoder{}
|
||||
}
|
||||
|
||||
func (e *Encoder) Key() string {
|
||||
return Key
|
||||
}
|
||||
|
||||
func (e *Encoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
if e == nil {
|
||||
return contracts.OutputResult{}, encoderErrorf("encoder must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.OutputResult{}, encoderErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.OutputResult{}, encoderErrorf("context error before encoding: %w", err)
|
||||
}
|
||||
|
||||
files, err := logicalFiles(req)
|
||||
if err != nil {
|
||||
return contracts.OutputResult{}, err
|
||||
}
|
||||
return contracts.OutputResult{Files: files}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageOutput,
|
||||
Requires: []string{"normalized"},
|
||||
Provides: []string{"encoded"},
|
||||
}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.OutputEncoderRegistry) error {
|
||||
return registry.RegisterWithSpec(ModuleSpec(), func() (contracts.OutputEncoder, error) {
|
||||
return New(), nil
|
||||
})
|
||||
}
|
||||
|
||||
type indexFile struct {
|
||||
ManifestFile string `json:"manifest_file"`
|
||||
OutputFiles []outputFileIndex `json:"output_files"`
|
||||
RejectedFile string `json:"rejected_file"`
|
||||
WarningsFile string `json:"warnings_file"`
|
||||
}
|
||||
|
||||
type outputFileIndex struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
File string `json:"file"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
SchemaID string `json:"schema_id,omitempty"`
|
||||
SchemaName string `json:"schema_name,omitempty"`
|
||||
SchemaVer string `json:"schema_version,omitempty"`
|
||||
}
|
||||
|
||||
type rejectedFile struct {
|
||||
Rejected []contracts.RejectedOutput `json:"rejected"`
|
||||
}
|
||||
|
||||
type warningsFile struct {
|
||||
Warnings []contracts.Warning `json:"warnings"`
|
||||
}
|
||||
|
||||
func logicalFiles(req contracts.OutputRequest) ([]contracts.OutputFile, error) {
|
||||
outputs := cloneNormalizeOutputs(req.NormalizeOutputs)
|
||||
sort.SliceStable(outputs, func(i, j int) bool {
|
||||
return outputs[i].LaneID < outputs[j].LaneID
|
||||
})
|
||||
|
||||
outputIndexes := make([]outputFileIndex, 0, len(outputs))
|
||||
files := make([]contracts.OutputFile, 0, len(outputs)+4)
|
||||
manifestFile, err := jsonFile("manifest.json", req.Manifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, manifestFile)
|
||||
|
||||
usedOutputFiles := make(map[string]string, len(outputs))
|
||||
for _, output := range outputs {
|
||||
name, err := outputFileName(output.LaneID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if existingLane, ok := usedOutputFiles[name]; ok {
|
||||
return nil, encoderErrorf("lanes %q and %q produce duplicate output file %q", existingLane, output.LaneID, name)
|
||||
}
|
||||
usedOutputFiles[name] = output.LaneID
|
||||
outputIndexes = append(outputIndexes, outputFileIndex{
|
||||
LaneID: output.LaneID,
|
||||
MediaType: output.Payload.MediaType,
|
||||
File: name,
|
||||
ModuleKey: output.NormalizerKey,
|
||||
SchemaID: output.Schema.ID,
|
||||
SchemaName: output.Schema.Name,
|
||||
SchemaVer: output.Schema.Version,
|
||||
})
|
||||
file, err := rawOutputFile(name, output.Payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, file)
|
||||
}
|
||||
|
||||
index := indexFile{
|
||||
ManifestFile: "manifest.json",
|
||||
OutputFiles: outputIndexes,
|
||||
RejectedFile: "rejected.json",
|
||||
WarningsFile: "warnings.json",
|
||||
}
|
||||
indexOutput, err := jsonFile("index.json", index)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rejectedOutput, err := jsonFile("rejected.json", rejectedFile{Rejected: cloneRejected(req.Rejected)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
warningsOutput, err := jsonFile("warnings.json", warningsFile{Warnings: cloneWarnings(req.Warnings)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, indexOutput, rejectedOutput, warningsOutput)
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].Name < files[j].Name
|
||||
})
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func rawOutputFile(name string, payload contracts.RawPayload) (contracts.OutputFile, error) {
|
||||
content := append([]byte(nil), payload.Content...)
|
||||
if len(content) == 0 {
|
||||
content = []byte("null")
|
||||
}
|
||||
mediaType := strings.TrimSpace(payload.MediaType)
|
||||
if mediaType == "" {
|
||||
mediaType = "application/octet-stream"
|
||||
}
|
||||
if !isJSONMediaType(mediaType) {
|
||||
return contracts.OutputFile{}, encoderErrorf("normalized output %q has unsupported media type %q", name, mediaType)
|
||||
}
|
||||
var decoded any
|
||||
if err := stdjson.Unmarshal(content, &decoded); err != nil {
|
||||
return contracts.OutputFile{}, encoderErrorf("normalized output %q contains invalid JSON: %w", name, err)
|
||||
}
|
||||
pretty, err := marshalPretty(decoded)
|
||||
if err != nil {
|
||||
return contracts.OutputFile{}, err
|
||||
}
|
||||
return contracts.OutputFile{
|
||||
Name: name,
|
||||
ContentType: mediaType,
|
||||
Bytes: pretty,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func isJSONMediaType(mediaType string) bool {
|
||||
base, _, err := mime.ParseMediaType(strings.TrimSpace(mediaType))
|
||||
if err != nil {
|
||||
base = strings.TrimSpace(mediaType)
|
||||
}
|
||||
return strings.EqualFold(base, contentTypeJSON)
|
||||
}
|
||||
|
||||
func jsonFile(name string, value any) (contracts.OutputFile, error) {
|
||||
data, err := marshalPretty(value)
|
||||
if err != nil {
|
||||
return contracts.OutputFile{}, encoderErrorf("encode %s: %w", name, err)
|
||||
}
|
||||
return contracts.OutputFile{
|
||||
Name: name,
|
||||
ContentType: contentTypeJSON,
|
||||
Bytes: data,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func marshalPretty(value any) ([]byte, error) {
|
||||
data, err := stdjson.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(data, '\n'), nil
|
||||
}
|
||||
|
||||
func outputFileName(laneID string) (string, error) {
|
||||
sanitized := safeOutputFileChar.ReplaceAllString(strings.TrimSpace(laneID), "_")
|
||||
for strings.Contains(sanitized, "..") {
|
||||
sanitized = strings.ReplaceAll(sanitized, "..", "__")
|
||||
}
|
||||
sanitized = strings.Trim(sanitized, "._")
|
||||
if sanitized == "" {
|
||||
return "", encoderErrorf("lane id %q cannot produce a safe file name", laneID)
|
||||
}
|
||||
return "lanes/" + sanitized + ".json", nil
|
||||
}
|
||||
|
||||
func cloneNormalizeOutputs(outputs []contracts.NormalizeOutput) []contracts.NormalizeOutput {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]contracts.NormalizeOutput, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
output.Payload = cloneRawPayload(output.Payload)
|
||||
out = append(out, output)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneRawPayload(payload contracts.RawPayload) contracts.RawPayload {
|
||||
return contracts.RawPayload{
|
||||
Content: append([]byte(nil), payload.Content...),
|
||||
MediaType: payload.MediaType,
|
||||
Metadata: cloneMetadata(payload.Metadata),
|
||||
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
|
||||
if len(rejected) == 0 {
|
||||
return []contracts.RejectedOutput{}
|
||||
}
|
||||
return append([]contracts.RejectedOutput(nil), rejected...)
|
||||
}
|
||||
|
||||
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
|
||||
if len(warnings) == 0 {
|
||||
return []contracts.Warning{}
|
||||
}
|
||||
return append([]contracts.Warning(nil), warnings...)
|
||||
}
|
||||
|
||||
func cloneMetadata(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(metadata))
|
||||
for key, value := range metadata {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func encoderErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("json output encoder: "+format, args...)
|
||||
}
|
||||
425
internal/modules/generic/output/json/encoder_test.go
Normal file
425
internal/modules/generic/output/json/encoder_test.go
Normal file
@@ -0,0 +1,425 @@
|
||||
package json
|
||||
|
||||
import (
|
||||
"context"
|
||||
stdjson "encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestModuleSpecAndRegister(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageOutput,
|
||||
Requires: []string{"normalized"},
|
||||
Provides: []string{"encoded"},
|
||||
}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
registry := pipeline.NewOutputEncoderRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
spec, ok := registry.Spec(Key)
|
||||
if !ok {
|
||||
t.Fatalf("Spec(%q) ok = false, want true", Key)
|
||||
}
|
||||
if !reflect.DeepEqual(spec, want) {
|
||||
t.Fatalf("registered spec = %#v, want %#v", spec, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeReturnsLogicalFilesForNormalizedOutputs(t *testing.T) {
|
||||
req := contracts.OutputRequest{
|
||||
Manifest: artifacts.RunManifest{RunID: "run-1", PipelineID: "pipeline-1"},
|
||||
NormalizeOutputs: []contracts.NormalizeOutput{
|
||||
normalizeOutput("spells", `{"spell_casts":[{"spell":"Cure Wounds"}]}`),
|
||||
normalizeOutput("notes/items", `{"items":[{"name":"Torch"}]}`),
|
||||
},
|
||||
Rejected: []contracts.RejectedOutput{
|
||||
{
|
||||
Stage: "extract",
|
||||
LaneID: "spells",
|
||||
ModuleKey: "dnd/spells",
|
||||
ChunkID: "chunk-1",
|
||||
ChunkIndex: 1,
|
||||
ReasonCode: "invalid",
|
||||
Message: "not accepted",
|
||||
AttemptCount: 1,
|
||||
ValidatorName: "validator",
|
||||
},
|
||||
},
|
||||
Warnings: []contracts.Warning{{ReasonCode: "warning", Message: "check source"}},
|
||||
}
|
||||
|
||||
result, err := New().Encode(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
wantNames := []string{
|
||||
"index.json",
|
||||
"lanes/notes_items.json",
|
||||
"lanes/spells.json",
|
||||
"manifest.json",
|
||||
"rejected.json",
|
||||
"warnings.json",
|
||||
}
|
||||
if got := outputFileNames(result.Files); !reflect.DeepEqual(got, wantNames) {
|
||||
t.Fatalf("file names = %#v, want %#v", got, wantNames)
|
||||
}
|
||||
|
||||
for _, file := range result.Files {
|
||||
if !strings.HasSuffix(string(file.Bytes), "\n") {
|
||||
t.Fatalf("%s does not end with newline: %q", file.Name, string(file.Bytes))
|
||||
}
|
||||
if file.ContentType == contentTypeJSON && !stdjson.Valid(file.Bytes) {
|
||||
t.Fatalf("%s has invalid JSON: %s", file.Name, file.Bytes)
|
||||
}
|
||||
}
|
||||
|
||||
spells := decodeObject(t, fileBytes(t, result.Files, "lanes/spells.json"))
|
||||
spellCasts := spells["spell_casts"].([]any)
|
||||
if spellCasts[0].(map[string]any)["spell"] != "Cure Wounds" {
|
||||
t.Fatalf("spells output = %#v, want raw normalized content", spells)
|
||||
}
|
||||
|
||||
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
|
||||
outputFiles := index["output_files"].([]any)
|
||||
if len(outputFiles) != 2 {
|
||||
t.Fatalf("len(index output_files) = %d, want 2", len(outputFiles))
|
||||
}
|
||||
firstIndex := outputFiles[0].(map[string]any)
|
||||
secondIndex := outputFiles[1].(map[string]any)
|
||||
if firstIndex["lane_id"] != "notes/items" || secondIndex["lane_id"] != "spells" {
|
||||
t.Fatalf("output_files = %#v, want sorted by lane id", outputFiles)
|
||||
}
|
||||
|
||||
rejected := decodeObject(t, fileBytes(t, result.Files, "rejected.json"))
|
||||
if got := rejected["rejected"].([]any); len(got) != 1 {
|
||||
t.Fatalf("rejected = %#v, want one rejected output", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeIncludesRejectedAndWarningsWhenEmpty(t *testing.T) {
|
||||
result, err := New().Encode(context.Background(), contracts.OutputRequest{
|
||||
Manifest: artifacts.RunManifest{RunID: "run-1"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
rejected := decodeObject(t, fileBytes(t, result.Files, "rejected.json"))
|
||||
if got := rejected["rejected"].([]any); len(got) != 0 {
|
||||
t.Fatalf("rejected = %#v, want empty array", got)
|
||||
}
|
||||
warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json"))
|
||||
if got := warnings["warnings"].([]any); len(got) != 0 {
|
||||
t.Fatalf("warnings = %#v, want empty array", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodePrettyPrintsJSON(t *testing.T) {
|
||||
result, err := New().Encode(context.Background(), contracts.OutputRequest{
|
||||
Manifest: artifacts.RunManifest{RunID: "run-1"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
manifest := string(fileBytes(t, result.Files, "manifest.json"))
|
||||
if !strings.Contains(manifest, "\n \"run_id\": \"run-1\"\n") {
|
||||
t.Fatalf("manifest JSON = %q, want two-space indentation", manifest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeIncludesManifestReferences(t *testing.T) {
|
||||
result, err := New().Encode(context.Background(), contracts.OutputRequest{
|
||||
Manifest: artifacts.RunManifest{
|
||||
RunID: "run-1",
|
||||
References: []artifacts.ReferenceProvenance{
|
||||
{
|
||||
Stage: "extract",
|
||||
LaneID: "events",
|
||||
SlotName: "roster",
|
||||
OriginType: "file",
|
||||
OriginURI: "file:///tmp/roster.txt",
|
||||
Digest: "sha256:reference",
|
||||
MediaType: "text/plain; charset=utf-8",
|
||||
SizeBytes: 12,
|
||||
BindingSource: "config",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
manifest := decodeObject(t, fileBytes(t, result.Files, "manifest.json"))
|
||||
references := manifest["references"].([]any)
|
||||
if len(references) != 1 {
|
||||
t.Fatalf("references = %#v, want one entry", references)
|
||||
}
|
||||
reference := references[0].(map[string]any)
|
||||
if reference["lane_id"] != "events" || reference["slot_name"] != "roster" || reference["digest"] != "sha256:reference" {
|
||||
t.Fatalf("reference manifest = %#v, want lane slot digest", reference)
|
||||
}
|
||||
if _, ok := reference["content"]; ok {
|
||||
t.Fatalf("reference manifest = %#v, want no content field", reference)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeIncludesManifestRawOutputProvenance(t *testing.T) {
|
||||
result, err := New().Encode(context.Background(), contracts.OutputRequest{
|
||||
Manifest: artifacts.RunManifest{
|
||||
RunID: "run-1",
|
||||
NormalizedOutputs: []artifacts.NormalizedOutputManifest{
|
||||
{
|
||||
LaneID: "spells",
|
||||
ModuleKey: "noop",
|
||||
SourceID: "source-1",
|
||||
MediaType: contentTypeJSON,
|
||||
Schema: artifacts.OutputSchemaProvenance{
|
||||
ID: "schema-id",
|
||||
Name: "schema-name",
|
||||
Version: "v1",
|
||||
},
|
||||
},
|
||||
},
|
||||
RejectedOutputs: []artifacts.RejectedOutputManifest{
|
||||
{
|
||||
Stage: "extract",
|
||||
LaneID: "spells",
|
||||
ModuleKey: "dnd/spells",
|
||||
ChunkID: "chunk-0",
|
||||
ReasonCode: "raw_output_rejected",
|
||||
AttemptCount: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
manifest := decodeObject(t, fileBytes(t, result.Files, "manifest.json"))
|
||||
normalized := manifest["normalized_outputs"].([]any)
|
||||
if len(normalized) != 1 {
|
||||
t.Fatalf("normalized_outputs = %#v, want one entry", normalized)
|
||||
}
|
||||
normalizedEntry := normalized[0].(map[string]any)
|
||||
if normalizedEntry["lane_id"] != "spells" || normalizedEntry["media_type"] != contentTypeJSON {
|
||||
t.Fatalf("normalized output manifest = %#v, want lane and media type", normalizedEntry)
|
||||
}
|
||||
rejected := manifest["rejected_outputs"].([]any)
|
||||
if len(rejected) != 1 {
|
||||
t.Fatalf("rejected_outputs = %#v, want one entry", rejected)
|
||||
}
|
||||
rejectedEntry := rejected[0].(map[string]any)
|
||||
if rejectedEntry["attempt_count"] != float64(2) || rejectedEntry["chunk_id"] != "chunk-0" {
|
||||
t.Fatalf("rejected output manifest = %#v, want attempt count and chunk", rejectedEntry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeRejectsLaneIDWithoutSafeFileName(t *testing.T) {
|
||||
_, err := New().Encode(context.Background(), contracts.OutputRequest{
|
||||
NormalizeOutputs: []contracts.NormalizeOutput{normalizeOutput("///", `{"value":true}`)},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Encode() error = nil, want unsafe lane id error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "json output encoder") || !strings.Contains(err.Error(), "safe file name") {
|
||||
t.Fatalf("Encode() error = %q, want safe file name context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeSanitizesParentPathSequences(t *testing.T) {
|
||||
result, err := New().Encode(context.Background(), contracts.OutputRequest{
|
||||
NormalizeOutputs: []contracts.NormalizeOutput{normalizeOutput("dnd..spell.", `{"value":true}`)},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if got := outputFileNames(result.Files); !containsString(got, "lanes/dnd__spell.json") {
|
||||
t.Fatalf("file names = %#v, want sanitized output filename", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeRejectsInvalidJSONAndUnsupportedMediaTypes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
output contracts.NormalizeOutput
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "invalid JSON",
|
||||
output: normalizeOutput("spells", `{"spell_casts":[`),
|
||||
want: "invalid JSON",
|
||||
},
|
||||
{
|
||||
name: "unsupported media type",
|
||||
output: func() contracts.NormalizeOutput {
|
||||
output := normalizeOutput("spells", `{"spell_casts":[]}`)
|
||||
output.Payload.MediaType = "text/plain"
|
||||
return output
|
||||
}(),
|
||||
want: "unsupported media type",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := New().Encode(context.Background(), contracts.OutputRequest{
|
||||
NormalizeOutputs: []contracts.NormalizeOutput{test.output},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Encode() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Encode() error = %q, want %q", err.Error(), test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeRejectsSanitizedFilenameCollisions(t *testing.T) {
|
||||
_, err := New().Encode(context.Background(), contracts.OutputRequest{
|
||||
NormalizeOutputs: []contracts.NormalizeOutput{
|
||||
normalizeOutput("a/b", `{"value":"slash"}`),
|
||||
normalizeOutput("a?b", `{"value":"question"}`),
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Encode() error = nil, want duplicate file error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "duplicate output file") {
|
||||
t.Fatalf("Encode() error = %q, want duplicate file context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDoesNotMutateInputs(t *testing.T) {
|
||||
req := contracts.OutputRequest{
|
||||
Manifest: artifacts.RunManifest{RunID: "run-1"},
|
||||
NormalizeOutputs: []contracts.NormalizeOutput{
|
||||
normalizeOutput("spells", `{"name":"original"}`),
|
||||
},
|
||||
Rejected: []contracts.RejectedOutput{
|
||||
{Stage: "extract", LaneID: "spells", Message: "not accepted"},
|
||||
},
|
||||
Warnings: []contracts.Warning{{ReasonCode: "warning", Message: "message"}},
|
||||
}
|
||||
before := mustMarshal(t, req)
|
||||
|
||||
result, err := New().Encode(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v, want nil", err)
|
||||
}
|
||||
after := mustMarshal(t, req)
|
||||
if before != after {
|
||||
t.Fatalf("request mutated:\nbefore: %s\nafter: %s", before, after)
|
||||
}
|
||||
|
||||
req.NormalizeOutputs[0].Payload.Content[0] = '['
|
||||
req.NormalizeOutputs[0].Payload.Metadata["name"] = "changed"
|
||||
req.Rejected[0].Message = "changed"
|
||||
req.Warnings[0].Message = "changed"
|
||||
|
||||
if !stdjson.Valid(fileBytes(t, result.Files, "lanes/spells.json")) {
|
||||
t.Fatal("output changed after request mutation")
|
||||
}
|
||||
warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json"))
|
||||
gotWarnings := warnings["warnings"].([]any)
|
||||
if gotWarnings[0].(map[string]any)["message"] != "message" {
|
||||
t.Fatalf("warnings output changed after request mutation: %#v", gotWarnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputFilesDoNotContainWarnings(t *testing.T) {
|
||||
result, err := New().Encode(context.Background(), contracts.OutputRequest{
|
||||
NormalizeOutputs: []contracts.NormalizeOutput{normalizeOutput("spells", `{"spell":"Shield"}`)},
|
||||
Warnings: []contracts.Warning{
|
||||
{ReasonCode: "pipeline-warning", Message: "warning"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
outputFile := decodeObject(t, fileBytes(t, result.Files, "lanes/spells.json"))
|
||||
if _, ok := outputFile["warnings"]; ok {
|
||||
t.Fatalf("output file contains warnings: %#v", outputFile)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeOutput(laneID string, content string) contracts.NormalizeOutput {
|
||||
return contracts.NormalizeOutput{
|
||||
LaneID: laneID,
|
||||
NormalizerKey: "noop",
|
||||
SourceID: "source-1",
|
||||
Schema: contracts.ResponseSchema{
|
||||
ID: "schema-id",
|
||||
Name: "schema-name",
|
||||
Version: "v1",
|
||||
},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(content),
|
||||
MediaType: contentTypeJSON,
|
||||
Metadata: map[string]any{"name": laneID},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func outputFileNames(files []contracts.OutputFile) []string {
|
||||
names := make([]string, 0, len(files))
|
||||
for _, file := range files {
|
||||
names = append(names, file.Name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func containsString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fileBytes(t *testing.T, files []contracts.OutputFile, name string) []byte {
|
||||
t.Helper()
|
||||
for _, file := range files {
|
||||
if file.Name == name {
|
||||
return file.Bytes
|
||||
}
|
||||
}
|
||||
t.Fatalf("file %q not found in %#v", name, outputFileNames(files))
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodeObject(t *testing.T, data []byte) map[string]any {
|
||||
t.Helper()
|
||||
var got map[string]any
|
||||
if err := stdjson.Unmarshal(data, &got); err != nil {
|
||||
t.Fatalf("Unmarshal() error = %v, want nil\n%s", err, data)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func mustMarshal(t *testing.T, value any) string {
|
||||
t.Helper()
|
||||
data, err := stdjson.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v, want nil", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
Reference in New Issue
Block a user