package json import ( "bytes" "context" stdjson "encoding/json" "fmt" "io" "mime" "regexp" "sort" "strings" "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) const Key = "json" const contentTypeJSON = "application/json" const chunkMapFileName = "chunk-map.json" var safeOutputFileChar = regexp.MustCompile(`[^A-Za-z0-9._-]`) var _ contracts.OutputEncoder = (*Encoder)(nil) type Options struct { IncludeChunkMap bool } type Encoder struct { options Options } func New() *Encoder { return NewWithOptions(Options{}) } func NewWithOptions(options Options) *Encoder { return &Encoder{options: options} } func (e *Encoder) Key() string { return Key } func (e *Encoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) { if e == nil { return contracts.OutputResult{}, encoderErrorf("encoder must not be nil") } if ctx == nil { return contracts.OutputResult{}, encoderErrorf("context must not be nil") } if err := ctx.Err(); err != nil { return contracts.OutputResult{}, encoderErrorf("context error before encoding: %w", err) } files, err := logicalFiles(req, e.options) if err != nil { return contracts.OutputResult{}, err } return contracts.OutputResult{Files: files}, nil } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ Key: Key, Stage: pipeline.StageOutput, Requires: []string{"normalized"}, Provides: []string{"encoded"}, } } func Register(registry *pipeline.OutputEncoderRegistry) error { return registry.RegisterBuilderWithSpec(ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.OutputEncoder, error) { options, err := DecodeOptions(request.Options) if err != nil { return nil, err } return &Encoder{options: options}, nil }) } func validateOptions(options map[string]any) error { _, err := DecodeOptions(options) return err } func DecodeOptions(options map[string]any) (Options, error) { if err := pipeline.RejectUnknownOptions(options, "include_chunk_map"); err != nil { return Options{}, encoderErrorf("%w", err) } if value, ok := options["include_chunk_map"]; ok { enabled, ok := value.(bool) if !ok { return Options{}, encoderErrorf("option %q must be a boolean", "include_chunk_map") } return Options{IncludeChunkMap: enabled}, nil } return Options{}, nil } type indexFile struct { ManifestFile string `json:"manifest_file"` OutputFiles []outputFileIndex `json:"output_files"` RejectedFile string `json:"rejected_file"` WarningsFile string `json:"warnings_file"` ChunkMap *chunkMapIndex `json:"chunk_map,omitempty"` } type chunkMapIndex struct { ArtifactKind contracts.ArtifactKind `json:"artifact_kind"` File string `json:"file"` MediaType string `json:"media_type"` SchemaID string `json:"schema_id"` SchemaName string `json:"schema_name"` SchemaVersion string `json:"schema_version"` } type outputFileIndex struct { LaneID string `json:"lane_id"` MediaType string `json:"media_type,omitempty"` File string `json:"file"` ModuleKey string `json:"module_key,omitempty"` SchemaID string `json:"schema_id,omitempty"` SchemaName string `json:"schema_name,omitempty"` SchemaVer string `json:"schema_version,omitempty"` } type rejectedFile struct { Rejected []contracts.RejectedOutput `json:"rejected"` } type warningsFile struct { Warnings []contracts.Warning `json:"warnings"` } func logicalFiles(req contracts.OutputRequest, options Options) ([]contracts.OutputFile, error) { outputs := cloneNormalizeOutputs(req.NormalizeOutputs) sort.SliceStable(outputs, func(i, j int) bool { return outputs[i].LaneID < outputs[j].LaneID }) outputIndexes := make([]outputFileIndex, 0, len(outputs)) files := make([]contracts.OutputFile, 0, len(outputs)+4) manifestFile, err := jsonFile("manifest.json", req.Manifest) if err != nil { return nil, err } files = append(files, manifestFile) usedOutputFiles := make(map[string]string, len(outputs)) for _, output := range outputs { name, err := outputFileName(output.LaneID) if err != nil { return nil, err } if existingLane, ok := usedOutputFiles[name]; ok { return nil, encoderErrorf("lanes %q and %q produce duplicate output file %q", existingLane, output.LaneID, name) } usedOutputFiles[name] = output.LaneID outputIndexes = append(outputIndexes, outputFileIndex{ LaneID: output.LaneID, MediaType: output.Artifact.MediaType, File: name, ModuleKey: output.NormalizerKey, SchemaID: output.Artifact.Schema.ID, SchemaName: output.Artifact.Schema.Name, SchemaVer: output.Artifact.Schema.Version, }) file, err := serializedOutputFile(name, output.Artifact) if err != nil { return nil, err } files = append(files, file) } index := indexFile{ ManifestFile: "manifest.json", OutputFiles: outputIndexes, RejectedFile: "rejected.json", WarningsFile: "warnings.json", } if options.IncludeChunkMap && req.ChunkMap != nil { chunkMapOutput, chunkMapDescriptor, err := serializedChunkMapFile(*req.ChunkMap) if err != nil { return nil, err } files = append(files, chunkMapOutput) index.ChunkMap = &chunkMapDescriptor } indexOutput, err := jsonFile("index.json", index) if err != nil { return nil, err } rejectedOutput, err := jsonFile("rejected.json", rejectedFile{Rejected: cloneRejected(req.Rejected)}) if err != nil { return nil, err } warningsOutput, err := jsonFile("warnings.json", warningsFile{Warnings: cloneWarnings(req.Warnings)}) if err != nil { return nil, err } files = append(files, indexOutput, rejectedOutput, warningsOutput) sort.Slice(files, func(i, j int) bool { return files[i].Name < files[j].Name }) return files, nil } func serializedChunkMapFile(artifact contracts.SerializedArtifact) (contracts.OutputFile, chunkMapIndex, error) { if artifact.Kind != chunkmap.ArtifactKind { return contracts.OutputFile{}, chunkMapIndex{}, encoderErrorf("chunk map has unexpected artifact kind %q", artifact.Kind) } if artifact.Schema.ID != chunkmap.SchemaID || artifact.Schema.Name != chunkmap.SchemaName || artifact.Schema.Version != chunkmap.SchemaVersion { return contracts.OutputFile{}, chunkMapIndex{}, encoderErrorf("chunk map has unexpected schema identity") } if strings.TrimSpace(artifact.MediaType) != chunkmap.MediaType { return contracts.OutputFile{}, chunkMapIndex{}, encoderErrorf("chunk map has unsupported media type %q", artifact.MediaType) } if _, err := chunkmap.New().Decode(artifact.Content); err != nil { return contracts.OutputFile{}, chunkMapIndex{}, encoderErrorf("decode chunk map: %w", err) } file, err := serializedOutputFile(chunkMapFileName, artifact) if err != nil { return contracts.OutputFile{}, chunkMapIndex{}, err } return file, chunkMapIndex{ ArtifactKind: artifact.Kind, File: chunkMapFileName, MediaType: chunkmap.MediaType, SchemaID: artifact.Schema.ID, SchemaName: artifact.Schema.Name, SchemaVersion: artifact.Schema.Version, }, nil } func serializedOutputFile(name string, artifact contracts.SerializedArtifact) (contracts.OutputFile, error) { content := append([]byte(nil), artifact.Content...) if len(content) == 0 { content = []byte("null") } mediaType := strings.TrimSpace(artifact.MediaType) if mediaType == "" { mediaType = "application/octet-stream" } if !isJSONMediaType(mediaType) { return contracts.OutputFile{}, encoderErrorf("normalized output %q has unsupported media type %q", name, mediaType) } decoder := stdjson.NewDecoder(bytes.NewReader(content)) decoder.UseNumber() var decoded any if err := decoder.Decode(&decoded); err != nil { return contracts.OutputFile{}, encoderErrorf("normalized output %q contains invalid JSON: %w", name, err) } var trailing any if err := decoder.Decode(&trailing); err != io.EOF { return contracts.OutputFile{}, encoderErrorf("normalized output %q contains multiple JSON values", name) } pretty, err := marshalPretty(decoded) if err != nil { return contracts.OutputFile{}, err } return contracts.OutputFile{ Name: name, ContentType: mediaType, Bytes: pretty, }, nil } func isJSONMediaType(mediaType string) bool { base, _, err := mime.ParseMediaType(strings.TrimSpace(mediaType)) if err != nil { base = strings.TrimSpace(mediaType) } return strings.EqualFold(base, contentTypeJSON) } func jsonFile(name string, value any) (contracts.OutputFile, error) { data, err := marshalPretty(value) if err != nil { return contracts.OutputFile{}, encoderErrorf("encode %s: %w", name, err) } return contracts.OutputFile{ Name: name, ContentType: contentTypeJSON, Bytes: data, }, nil } func marshalPretty(value any) ([]byte, error) { data, err := stdjson.MarshalIndent(value, "", " ") if err != nil { return nil, err } return append(data, '\n'), nil } func outputFileName(laneID string) (string, error) { sanitized := safeOutputFileChar.ReplaceAllString(strings.TrimSpace(laneID), "_") for strings.Contains(sanitized, "..") { sanitized = strings.ReplaceAll(sanitized, "..", "__") } sanitized = strings.Trim(sanitized, "._") if sanitized == "" { return "", encoderErrorf("lane id %q cannot produce a safe file name", laneID) } return "lanes/" + sanitized + ".json", nil } func cloneNormalizeOutputs(outputs []contracts.SerializedOutput) []contracts.SerializedOutput { if len(outputs) == 0 { return nil } out := make([]contracts.SerializedOutput, 0, len(outputs)) for _, output := range outputs { out = append(out, contracts.CloneSerializedOutput(output)) } return out } func cloneRejected(rejected []contracts.RejectedOutput) []contracts.RejectedOutput { if len(rejected) == 0 { return []contracts.RejectedOutput{} } return append([]contracts.RejectedOutput(nil), rejected...) } func cloneWarnings(warnings []contracts.Warning) []contracts.Warning { if len(warnings) == 0 { return []contracts.Warning{} } return append([]contracts.Warning(nil), warnings...) } func cloneMetadata(metadata map[string]any) map[string]any { if len(metadata) == 0 { return nil } out := make(map[string]any, len(metadata)) for key, value := range metadata { out[key] = cloneJSONMetadataValue(value) } return out } func cloneJSONMetadataValue(value any) any { switch typed := value.(type) { case map[string]any: return cloneMetadata(typed) case []any: out := make([]any, len(typed)) for i := range typed { out[i] = cloneJSONMetadataValue(typed[i]) } return out case stdjson.RawMessage: return append(stdjson.RawMessage(nil), typed...) case []byte: return append([]byte(nil), typed...) case []string: return append([]string(nil), typed...) default: return value } } func encoderErrorf(format string, args ...any) error { return fmt.Errorf("json output encoder: "+format, args...) }