Add production default pipeline modules
This commit is contained in:
253
internal/modules/output/json/encoder.go
Normal file
253
internal/modules/output/json/encoder.go
Normal file
@@ -0,0 +1,253 @@
|
||||
package json
|
||||
|
||||
import (
|
||||
"context"
|
||||
stdjson "encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const Key = "json"
|
||||
|
||||
const contentTypeJSON = "application/json"
|
||||
|
||||
var safeArtifactFileChar = regexp.MustCompile(`[^A-Za-z0-9._-]`)
|
||||
|
||||
var _ contracts.OutputEncoder = (*Encoder)(nil)
|
||||
|
||||
type Encoder struct{}
|
||||
|
||||
func New() *Encoder {
|
||||
return &Encoder{}
|
||||
}
|
||||
|
||||
func (e *Encoder) Key() string {
|
||||
return Key
|
||||
}
|
||||
|
||||
func (e *Encoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
if e == nil {
|
||||
return contracts.OutputResult{}, encoderErrorf("encoder must not be nil")
|
||||
}
|
||||
if ctx == nil {
|
||||
return contracts.OutputResult{}, encoderErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.OutputResult{}, encoderErrorf("context error before encoding: %w", err)
|
||||
}
|
||||
|
||||
files, err := logicalFiles(req)
|
||||
if err != nil {
|
||||
return contracts.OutputResult{}, err
|
||||
}
|
||||
return contracts.OutputResult{Files: files}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageOutput,
|
||||
Requires: []string{"normalized"},
|
||||
Provides: []string{"encoded"},
|
||||
}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.OutputEncoderRegistry) error {
|
||||
return registry.RegisterWithSpec(ModuleSpec(), func() (contracts.OutputEncoder, error) {
|
||||
return New(), nil
|
||||
})
|
||||
}
|
||||
|
||||
type indexFile struct {
|
||||
ManifestFile string `json:"manifest_file"`
|
||||
ArtifactFiles []artifactFileIndex `json:"artifact_files"`
|
||||
RejectedFile string `json:"rejected_file"`
|
||||
WarningsFile string `json:"warnings_file"`
|
||||
}
|
||||
|
||||
type artifactFileIndex struct {
|
||||
ArtifactType string `json:"artifact_type"`
|
||||
File string `json:"file"`
|
||||
}
|
||||
|
||||
type artifactFile struct {
|
||||
ArtifactType string `json:"artifact_type"`
|
||||
Artifacts []artifacts.Artifact `json:"artifacts"`
|
||||
}
|
||||
|
||||
type rejectedFile struct {
|
||||
Rejected []artifacts.RejectedArtifact `json:"rejected"`
|
||||
}
|
||||
|
||||
type warningsFile struct {
|
||||
Warnings []contracts.Warning `json:"warnings"`
|
||||
}
|
||||
|
||||
func logicalFiles(req contracts.OutputRequest) ([]contracts.OutputFile, error) {
|
||||
artifactsByType := make(map[string][]artifacts.Artifact)
|
||||
for _, artifact := range req.Approved {
|
||||
artifactsByType[artifact.ArtifactType] = append(artifactsByType[artifact.ArtifactType], cloneArtifact(artifact))
|
||||
}
|
||||
|
||||
artifactTypes := make([]string, 0, len(artifactsByType))
|
||||
for artifactType := range artifactsByType {
|
||||
artifactTypes = append(artifactTypes, artifactType)
|
||||
}
|
||||
sort.Strings(artifactTypes)
|
||||
|
||||
artifactIndexes := make([]artifactFileIndex, 0, len(artifactTypes))
|
||||
files := make([]contracts.OutputFile, 0, len(artifactTypes)+4)
|
||||
manifestFile, err := jsonFile("manifest.json", req.Manifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, manifestFile)
|
||||
|
||||
usedArtifactFiles := make(map[string]string, len(artifactTypes))
|
||||
for _, artifactType := range artifactTypes {
|
||||
name, err := artifactFileName(artifactType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if existingType, ok := usedArtifactFiles[name]; ok {
|
||||
return nil, encoderErrorf("artifact types %q and %q produce duplicate output file %q", existingType, artifactType, name)
|
||||
}
|
||||
usedArtifactFiles[name] = artifactType
|
||||
artifactIndexes = append(artifactIndexes, artifactFileIndex{
|
||||
ArtifactType: artifactType,
|
||||
File: name,
|
||||
})
|
||||
file, err := jsonFile(name, artifactFile{
|
||||
ArtifactType: artifactType,
|
||||
Artifacts: artifactsByType[artifactType],
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, file)
|
||||
}
|
||||
|
||||
index := indexFile{
|
||||
ManifestFile: "manifest.json",
|
||||
ArtifactFiles: artifactIndexes,
|
||||
RejectedFile: "rejected.json",
|
||||
WarningsFile: "warnings.json",
|
||||
}
|
||||
indexOutput, err := jsonFile("index.json", index)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rejectedOutput, err := jsonFile("rejected.json", rejectedFile{Rejected: cloneRejected(req.Rejected)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
warningsOutput, err := jsonFile("warnings.json", warningsFile{Warnings: cloneWarnings(req.Warnings)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, indexOutput, rejectedOutput, warningsOutput)
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].Name < files[j].Name
|
||||
})
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func jsonFile(name string, value any) (contracts.OutputFile, error) {
|
||||
data, err := marshalPretty(value)
|
||||
if err != nil {
|
||||
return contracts.OutputFile{}, encoderErrorf("encode %s: %w", name, err)
|
||||
}
|
||||
return contracts.OutputFile{
|
||||
Name: name,
|
||||
ContentType: contentTypeJSON,
|
||||
Bytes: data,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func marshalPretty(value any) ([]byte, error) {
|
||||
data, err := stdjson.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(data, '\n'), nil
|
||||
}
|
||||
|
||||
func artifactFileName(artifactType string) (string, error) {
|
||||
sanitized := safeArtifactFileChar.ReplaceAllString(strings.TrimSpace(artifactType), "_")
|
||||
for strings.Contains(sanitized, "..") {
|
||||
sanitized = strings.ReplaceAll(sanitized, "..", "__")
|
||||
}
|
||||
sanitized = strings.Trim(sanitized, "._")
|
||||
if sanitized == "" {
|
||||
return "", encoderErrorf("artifact type %q cannot produce a safe file name", artifactType)
|
||||
}
|
||||
return "artifacts/" + sanitized + ".json", nil
|
||||
}
|
||||
|
||||
func cloneArtifact(artifact artifacts.Artifact) artifacts.Artifact {
|
||||
return artifacts.Artifact{
|
||||
ExtractorKey: artifact.ExtractorKey,
|
||||
ArtifactType: artifact.ArtifactType,
|
||||
SchemaVersion: artifact.SchemaVersion,
|
||||
Payload: append(stdjson.RawMessage(nil), artifact.Payload...),
|
||||
SourceRefs: append([]source.SourceRef(nil), artifact.SourceRefs...),
|
||||
Metadata: cloneMetadata(artifact.Metadata),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneRejected(rejected []artifacts.RejectedArtifact) []artifacts.RejectedArtifact {
|
||||
if len(rejected) == 0 {
|
||||
return []artifacts.RejectedArtifact{}
|
||||
}
|
||||
out := make([]artifacts.RejectedArtifact, 0, len(rejected))
|
||||
for _, item := range rejected {
|
||||
out = append(out, artifacts.RejectedArtifact{
|
||||
Candidate: cloneCandidate(item.Candidate),
|
||||
ValidatorName: item.ValidatorName,
|
||||
ReasonCode: item.ReasonCode,
|
||||
Message: item.Message,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneCandidate(candidate artifacts.ArtifactCandidate) artifacts.ArtifactCandidate {
|
||||
return artifacts.ArtifactCandidate{
|
||||
Index: candidate.Index,
|
||||
ExtractorKey: candidate.ExtractorKey,
|
||||
ArtifactType: candidate.ArtifactType,
|
||||
SchemaVersion: candidate.SchemaVersion,
|
||||
Payload: append(stdjson.RawMessage(nil), candidate.Payload...),
|
||||
SourceRefs: append([]source.SourceRef(nil), candidate.SourceRefs...),
|
||||
Metadata: cloneMetadata(candidate.Metadata),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
|
||||
if len(warnings) == 0 {
|
||||
return []contracts.Warning{}
|
||||
}
|
||||
return append([]contracts.Warning(nil), warnings...)
|
||||
}
|
||||
|
||||
func cloneMetadata(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(metadata))
|
||||
for key, value := range metadata {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func encoderErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("json output encoder: "+format, args...)
|
||||
}
|
||||
Reference in New Issue
Block a user