365 lines
11 KiB
Go
365 lines
11 KiB
Go
package json
|
|
|
|
import (
|
|
"context"
|
|
stdjson "encoding/json"
|
|
"fmt"
|
|
"mime"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
)
|
|
|
|
const Key = "json"
|
|
|
|
const contentTypeJSON = "application/json"
|
|
|
|
const chunkMapFileName = "chunk-map.json"
|
|
|
|
var safeOutputFileChar = regexp.MustCompile(`[^A-Za-z0-9._-]`)
|
|
|
|
var _ contracts.OutputEncoder = (*Encoder)(nil)
|
|
|
|
type Options struct {
|
|
IncludeChunkMap bool
|
|
}
|
|
|
|
type Encoder struct {
|
|
options Options
|
|
}
|
|
|
|
func New() *Encoder {
|
|
return NewWithOptions(Options{})
|
|
}
|
|
|
|
func NewWithOptions(options Options) *Encoder {
|
|
return &Encoder{options: options}
|
|
}
|
|
|
|
func (e *Encoder) Key() string {
|
|
return Key
|
|
}
|
|
|
|
func (e *Encoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
|
if e == nil {
|
|
return contracts.OutputResult{}, encoderErrorf("encoder must not be nil")
|
|
}
|
|
if ctx == nil {
|
|
return contracts.OutputResult{}, encoderErrorf("context must not be nil")
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return contracts.OutputResult{}, encoderErrorf("context error before encoding: %w", err)
|
|
}
|
|
|
|
files, err := logicalFiles(req, e.options)
|
|
if err != nil {
|
|
return contracts.OutputResult{}, err
|
|
}
|
|
return contracts.OutputResult{Files: files}, nil
|
|
}
|
|
|
|
func ModuleSpec() pipeline.ModuleSpec {
|
|
return pipeline.ModuleSpec{
|
|
Key: Key,
|
|
Stage: pipeline.StageOutput,
|
|
Requires: []string{"normalized"},
|
|
Provides: []string{"encoded"},
|
|
}
|
|
}
|
|
|
|
func Register(registry *pipeline.OutputEncoderRegistry) error {
|
|
return registry.RegisterBuilderWithSpec(ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.OutputEncoder, error) {
|
|
options, err := DecodeOptions(request.Options)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Encoder{options: options}, nil
|
|
})
|
|
}
|
|
|
|
func validateOptions(options map[string]any) error {
|
|
_, err := DecodeOptions(options)
|
|
return err
|
|
}
|
|
|
|
func DecodeOptions(options map[string]any) (Options, error) {
|
|
if err := pipeline.RejectUnknownOptions(options, "include_chunk_map"); err != nil {
|
|
return Options{}, encoderErrorf("%w", err)
|
|
}
|
|
if value, ok := options["include_chunk_map"]; ok {
|
|
enabled, ok := value.(bool)
|
|
if !ok {
|
|
return Options{}, encoderErrorf("option %q must be a boolean", "include_chunk_map")
|
|
}
|
|
return Options{IncludeChunkMap: enabled}, nil
|
|
}
|
|
return Options{}, nil
|
|
}
|
|
|
|
type indexFile struct {
|
|
ManifestFile string `json:"manifest_file"`
|
|
OutputFiles []outputFileIndex `json:"output_files"`
|
|
RejectedFile string `json:"rejected_file"`
|
|
WarningsFile string `json:"warnings_file"`
|
|
ChunkMap *chunkMapIndex `json:"chunk_map,omitempty"`
|
|
}
|
|
|
|
type chunkMapIndex struct {
|
|
ArtifactKind contracts.ArtifactKind `json:"artifact_kind"`
|
|
File string `json:"file"`
|
|
MediaType string `json:"media_type"`
|
|
SchemaID string `json:"schema_id"`
|
|
SchemaName string `json:"schema_name"`
|
|
SchemaVersion string `json:"schema_version"`
|
|
}
|
|
|
|
type outputFileIndex struct {
|
|
LaneID string `json:"lane_id"`
|
|
MediaType string `json:"media_type,omitempty"`
|
|
File string `json:"file"`
|
|
ModuleKey string `json:"module_key,omitempty"`
|
|
SchemaID string `json:"schema_id,omitempty"`
|
|
SchemaName string `json:"schema_name,omitempty"`
|
|
SchemaVer string `json:"schema_version,omitempty"`
|
|
}
|
|
|
|
type rejectedFile struct {
|
|
Rejected []contracts.RejectedOutput `json:"rejected"`
|
|
}
|
|
|
|
type warningsFile struct {
|
|
Warnings []contracts.Warning `json:"warnings"`
|
|
}
|
|
|
|
func logicalFiles(req contracts.OutputRequest, options Options) ([]contracts.OutputFile, error) {
|
|
outputs := cloneNormalizeOutputs(req.NormalizeOutputs)
|
|
sort.SliceStable(outputs, func(i, j int) bool {
|
|
return outputs[i].LaneID < outputs[j].LaneID
|
|
})
|
|
|
|
outputIndexes := make([]outputFileIndex, 0, len(outputs))
|
|
files := make([]contracts.OutputFile, 0, len(outputs)+4)
|
|
manifestFile, err := jsonFile("manifest.json", req.Manifest)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
files = append(files, manifestFile)
|
|
|
|
usedOutputFiles := make(map[string]string, len(outputs))
|
|
for _, output := range outputs {
|
|
name, err := outputFileName(output.LaneID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if existingLane, ok := usedOutputFiles[name]; ok {
|
|
return nil, encoderErrorf("lanes %q and %q produce duplicate output file %q", existingLane, output.LaneID, name)
|
|
}
|
|
usedOutputFiles[name] = output.LaneID
|
|
outputIndexes = append(outputIndexes, outputFileIndex{
|
|
LaneID: output.LaneID,
|
|
MediaType: output.Artifact.MediaType,
|
|
File: name,
|
|
ModuleKey: output.NormalizerKey,
|
|
SchemaID: output.Artifact.Schema.ID,
|
|
SchemaName: output.Artifact.Schema.Name,
|
|
SchemaVer: output.Artifact.Schema.Version,
|
|
})
|
|
file, err := serializedOutputFile(name, output.Artifact)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
files = append(files, file)
|
|
}
|
|
|
|
index := indexFile{
|
|
ManifestFile: "manifest.json",
|
|
OutputFiles: outputIndexes,
|
|
RejectedFile: "rejected.json",
|
|
WarningsFile: "warnings.json",
|
|
}
|
|
if options.IncludeChunkMap && req.ChunkMap != nil {
|
|
chunkMapOutput, chunkMapDescriptor, err := serializedChunkMapFile(*req.ChunkMap)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
files = append(files, chunkMapOutput)
|
|
index.ChunkMap = &chunkMapDescriptor
|
|
}
|
|
indexOutput, err := jsonFile("index.json", index)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rejectedOutput, err := jsonFile("rejected.json", rejectedFile{Rejected: cloneRejected(req.Rejected)})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
warningsOutput, err := jsonFile("warnings.json", warningsFile{Warnings: cloneWarnings(req.Warnings)})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
files = append(files, indexOutput, rejectedOutput, warningsOutput)
|
|
sort.Slice(files, func(i, j int) bool {
|
|
return files[i].Name < files[j].Name
|
|
})
|
|
return files, nil
|
|
}
|
|
|
|
func serializedChunkMapFile(artifact contracts.SerializedArtifact) (contracts.OutputFile, chunkMapIndex, error) {
|
|
if artifact.Kind != chunkmap.ArtifactKind {
|
|
return contracts.OutputFile{}, chunkMapIndex{}, encoderErrorf("chunk map has unexpected artifact kind %q", artifact.Kind)
|
|
}
|
|
if artifact.Schema.ID != chunkmap.SchemaID || artifact.Schema.Name != chunkmap.SchemaName || artifact.Schema.Version != chunkmap.SchemaVersion {
|
|
return contracts.OutputFile{}, chunkMapIndex{}, encoderErrorf("chunk map has unexpected schema identity")
|
|
}
|
|
if strings.TrimSpace(artifact.MediaType) != chunkmap.MediaType {
|
|
return contracts.OutputFile{}, chunkMapIndex{}, encoderErrorf("chunk map has unsupported media type %q", artifact.MediaType)
|
|
}
|
|
if _, err := chunkmap.New().Decode(artifact.Content); err != nil {
|
|
return contracts.OutputFile{}, chunkMapIndex{}, encoderErrorf("decode chunk map: %w", err)
|
|
}
|
|
file, err := serializedOutputFile(chunkMapFileName, artifact)
|
|
if err != nil {
|
|
return contracts.OutputFile{}, chunkMapIndex{}, err
|
|
}
|
|
return file, chunkMapIndex{
|
|
ArtifactKind: artifact.Kind,
|
|
File: chunkMapFileName,
|
|
MediaType: chunkmap.MediaType,
|
|
SchemaID: artifact.Schema.ID,
|
|
SchemaName: artifact.Schema.Name,
|
|
SchemaVersion: artifact.Schema.Version,
|
|
}, nil
|
|
}
|
|
|
|
func serializedOutputFile(name string, artifact contracts.SerializedArtifact) (contracts.OutputFile, error) {
|
|
content := append([]byte(nil), artifact.Content...)
|
|
if len(content) == 0 {
|
|
content = []byte("null")
|
|
}
|
|
mediaType := strings.TrimSpace(artifact.MediaType)
|
|
if mediaType == "" {
|
|
mediaType = "application/octet-stream"
|
|
}
|
|
if !isJSONMediaType(mediaType) {
|
|
return contracts.OutputFile{}, encoderErrorf("normalized output %q has unsupported media type %q", name, mediaType)
|
|
}
|
|
var decoded any
|
|
if err := stdjson.Unmarshal(content, &decoded); err != nil {
|
|
return contracts.OutputFile{}, encoderErrorf("normalized output %q contains invalid JSON: %w", name, err)
|
|
}
|
|
pretty, err := marshalPretty(decoded)
|
|
if err != nil {
|
|
return contracts.OutputFile{}, err
|
|
}
|
|
return contracts.OutputFile{
|
|
Name: name,
|
|
ContentType: mediaType,
|
|
Bytes: pretty,
|
|
}, nil
|
|
}
|
|
|
|
func isJSONMediaType(mediaType string) bool {
|
|
base, _, err := mime.ParseMediaType(strings.TrimSpace(mediaType))
|
|
if err != nil {
|
|
base = strings.TrimSpace(mediaType)
|
|
}
|
|
return strings.EqualFold(base, contentTypeJSON)
|
|
}
|
|
|
|
func jsonFile(name string, value any) (contracts.OutputFile, error) {
|
|
data, err := marshalPretty(value)
|
|
if err != nil {
|
|
return contracts.OutputFile{}, encoderErrorf("encode %s: %w", name, err)
|
|
}
|
|
return contracts.OutputFile{
|
|
Name: name,
|
|
ContentType: contentTypeJSON,
|
|
Bytes: data,
|
|
}, nil
|
|
}
|
|
|
|
func marshalPretty(value any) ([]byte, error) {
|
|
data, err := stdjson.MarshalIndent(value, "", " ")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return append(data, '\n'), nil
|
|
}
|
|
|
|
func outputFileName(laneID string) (string, error) {
|
|
sanitized := safeOutputFileChar.ReplaceAllString(strings.TrimSpace(laneID), "_")
|
|
for strings.Contains(sanitized, "..") {
|
|
sanitized = strings.ReplaceAll(sanitized, "..", "__")
|
|
}
|
|
sanitized = strings.Trim(sanitized, "._")
|
|
if sanitized == "" {
|
|
return "", encoderErrorf("lane id %q cannot produce a safe file name", laneID)
|
|
}
|
|
return "lanes/" + sanitized + ".json", nil
|
|
}
|
|
|
|
func cloneNormalizeOutputs(outputs []contracts.SerializedOutput) []contracts.SerializedOutput {
|
|
if len(outputs) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]contracts.SerializedOutput, 0, len(outputs))
|
|
for _, output := range outputs {
|
|
out = append(out, contracts.CloneSerializedOutput(output))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
|
|
if len(rejected) == 0 {
|
|
return []contracts.RejectedOutput{}
|
|
}
|
|
return append([]contracts.RejectedOutput(nil), rejected...)
|
|
}
|
|
|
|
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
|
|
if len(warnings) == 0 {
|
|
return []contracts.Warning{}
|
|
}
|
|
return append([]contracts.Warning(nil), warnings...)
|
|
}
|
|
|
|
func cloneMetadata(metadata map[string]any) map[string]any {
|
|
if len(metadata) == 0 {
|
|
return nil
|
|
}
|
|
out := make(map[string]any, len(metadata))
|
|
for key, value := range metadata {
|
|
out[key] = cloneJSONMetadataValue(value)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneJSONMetadataValue(value any) any {
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
return cloneMetadata(typed)
|
|
case []any:
|
|
out := make([]any, len(typed))
|
|
for i := range typed {
|
|
out[i] = cloneJSONMetadataValue(typed[i])
|
|
}
|
|
return out
|
|
case stdjson.RawMessage:
|
|
return append(stdjson.RawMessage(nil), typed...)
|
|
case []byte:
|
|
return append([]byte(nil), typed...)
|
|
case []string:
|
|
return append([]string(nil), typed...)
|
|
default:
|
|
return value
|
|
}
|
|
}
|
|
|
|
func encoderErrorf(format string, args ...any) error {
|
|
return fmt.Errorf("json output encoder: "+format, args...)
|
|
}
|