Add production default pipeline modules

2026-07-04 00:55:18 +00:00
parent bc4203a264
commit 0ad96618fc
15 changed files with 1651 additions and 299 deletions
--- a/internal/modules/output/json/encoder.go
+++ b/internal/modules/output/json/encoder.go
@@ -0,0 +1,253 @@
+package json
+
+import (
+	"context"
+	stdjson "encoding/json"
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
+	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+const Key = "json"
+
+const contentTypeJSON = "application/json"
+
+var safeArtifactFileChar = regexp.MustCompile(`[^A-Za-z0-9._-]`)
+
+var _ contracts.OutputEncoder = (*Encoder)(nil)
+
+type Encoder struct{}
+
+func New() *Encoder {
+	return &Encoder{}
+}
+
+func (e *Encoder) Key() string {
+	return Key
+}
+
+func (e *Encoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
+	if e == nil {
+		return contracts.OutputResult{}, encoderErrorf("encoder must not be nil")
+	}
+	if ctx == nil {
+		return contracts.OutputResult{}, encoderErrorf("context must not be nil")
+	}
+	if err := ctx.Err(); err != nil {
+		return contracts.OutputResult{}, encoderErrorf("context error before encoding: %w", err)
+	}
+
+	files, err := logicalFiles(req)
+	if err != nil {
+		return contracts.OutputResult{}, err
+	}
+	return contracts.OutputResult{Files: files}, nil
+}
+
+func ModuleSpec() pipeline.ModuleSpec {
+	return pipeline.ModuleSpec{
+		Key:      Key,
+		Stage:    pipeline.StageOutput,
+		Requires: []string{"normalized"},
+		Provides: []string{"encoded"},
+	}
+}
+
+func Register(registry *pipeline.OutputEncoderRegistry) error {
+	return registry.RegisterWithSpec(ModuleSpec(), func() (contracts.OutputEncoder, error) {
+		return New(), nil
+	})
+}
+
+type indexFile struct {
+	ManifestFile  string              `json:"manifest_file"`
+	ArtifactFiles []artifactFileIndex `json:"artifact_files"`
+	RejectedFile  string              `json:"rejected_file"`
+	WarningsFile  string              `json:"warnings_file"`
+}
+
+type artifactFileIndex struct {
+	ArtifactType string `json:"artifact_type"`
+	File         string `json:"file"`
+}
+
+type artifactFile struct {
+	ArtifactType string               `json:"artifact_type"`
+	Artifacts    []artifacts.Artifact `json:"artifacts"`
+}
+
+type rejectedFile struct {
+	Rejected []artifacts.RejectedArtifact `json:"rejected"`
+}
+
+type warningsFile struct {
+	Warnings []contracts.Warning `json:"warnings"`
+}
+
+func logicalFiles(req contracts.OutputRequest) ([]contracts.OutputFile, error) {
+	artifactsByType := make(map[string][]artifacts.Artifact)
+	for _, artifact := range req.Approved {
+		artifactsByType[artifact.ArtifactType] = append(artifactsByType[artifact.ArtifactType], cloneArtifact(artifact))
+	}
+
+	artifactTypes := make([]string, 0, len(artifactsByType))
+	for artifactType := range artifactsByType {
+		artifactTypes = append(artifactTypes, artifactType)
+	}
+	sort.Strings(artifactTypes)
+
+	artifactIndexes := make([]artifactFileIndex, 0, len(artifactTypes))
+	files := make([]contracts.OutputFile, 0, len(artifactTypes)+4)
+	manifestFile, err := jsonFile("manifest.json", req.Manifest)
+	if err != nil {
+		return nil, err
+	}
+	files = append(files, manifestFile)
+
+	usedArtifactFiles := make(map[string]string, len(artifactTypes))
+	for _, artifactType := range artifactTypes {
+		name, err := artifactFileName(artifactType)
+		if err != nil {
+			return nil, err
+		}
+		if existingType, ok := usedArtifactFiles[name]; ok {
+			return nil, encoderErrorf("artifact types %q and %q produce duplicate output file %q", existingType, artifactType, name)
+		}
+		usedArtifactFiles[name] = artifactType
+		artifactIndexes = append(artifactIndexes, artifactFileIndex{
+			ArtifactType: artifactType,
+			File:         name,
+		})
+		file, err := jsonFile(name, artifactFile{
+			ArtifactType: artifactType,
+			Artifacts:    artifactsByType[artifactType],
+		})
+		if err != nil {
+			return nil, err
+		}
+		files = append(files, file)
+	}
+
+	index := indexFile{
+		ManifestFile:  "manifest.json",
+		ArtifactFiles: artifactIndexes,
+		RejectedFile:  "rejected.json",
+		WarningsFile:  "warnings.json",
+	}
+	indexOutput, err := jsonFile("index.json", index)
+	if err != nil {
+		return nil, err
+	}
+	rejectedOutput, err := jsonFile("rejected.json", rejectedFile{Rejected: cloneRejected(req.Rejected)})
+	if err != nil {
+		return nil, err
+	}
+	warningsOutput, err := jsonFile("warnings.json", warningsFile{Warnings: cloneWarnings(req.Warnings)})
+	if err != nil {
+		return nil, err
+	}
+	files = append(files, indexOutput, rejectedOutput, warningsOutput)
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].Name < files[j].Name
+	})
+	return files, nil
+}
+
+func jsonFile(name string, value any) (contracts.OutputFile, error) {
+	data, err := marshalPretty(value)
+	if err != nil {
+		return contracts.OutputFile{}, encoderErrorf("encode %s: %w", name, err)
+	}
+	return contracts.OutputFile{
+		Name:        name,
+		ContentType: contentTypeJSON,
+		Bytes:       data,
+	}, nil
+}
+
+func marshalPretty(value any) ([]byte, error) {
+	data, err := stdjson.MarshalIndent(value, "", "  ")
+	if err != nil {
+		return nil, err
+	}
+	return append(data, '\n'), nil
+}
+
+func artifactFileName(artifactType string) (string, error) {
+	sanitized := safeArtifactFileChar.ReplaceAllString(strings.TrimSpace(artifactType), "_")
+	for strings.Contains(sanitized, "..") {
+		sanitized = strings.ReplaceAll(sanitized, "..", "__")
+	}
+	sanitized = strings.Trim(sanitized, "._")
+	if sanitized == "" {
+		return "", encoderErrorf("artifact type %q cannot produce a safe file name", artifactType)
+	}
+	return "artifacts/" + sanitized + ".json", nil
+}
+
+func cloneArtifact(artifact artifacts.Artifact) artifacts.Artifact {
+	return artifacts.Artifact{
+		ExtractorKey:  artifact.ExtractorKey,
+		ArtifactType:  artifact.ArtifactType,
+		SchemaVersion: artifact.SchemaVersion,
+		Payload:       append(stdjson.RawMessage(nil), artifact.Payload...),
+		SourceRefs:    append([]source.SourceRef(nil), artifact.SourceRefs...),
+		Metadata:      cloneMetadata(artifact.Metadata),
+	}
+}
+
+func cloneRejected(rejected []artifacts.RejectedArtifact) []artifacts.RejectedArtifact {
+	if len(rejected) == 0 {
+		return []artifacts.RejectedArtifact{}
+	}
+	out := make([]artifacts.RejectedArtifact, 0, len(rejected))
+	for _, item := range rejected {
+		out = append(out, artifacts.RejectedArtifact{
+			Candidate:     cloneCandidate(item.Candidate),
+			ValidatorName: item.ValidatorName,
+			ReasonCode:    item.ReasonCode,
+			Message:       item.Message,
+		})
+	}
+	return out
+}
+
+func cloneCandidate(candidate artifacts.ArtifactCandidate) artifacts.ArtifactCandidate {
+	return artifacts.ArtifactCandidate{
+		Index:         candidate.Index,
+		ExtractorKey:  candidate.ExtractorKey,
+		ArtifactType:  candidate.ArtifactType,
+		SchemaVersion: candidate.SchemaVersion,
+		Payload:       append(stdjson.RawMessage(nil), candidate.Payload...),
+		SourceRefs:    append([]source.SourceRef(nil), candidate.SourceRefs...),
+		Metadata:      cloneMetadata(candidate.Metadata),
+	}
+}
+
+func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
+	if len(warnings) == 0 {
+		return []contracts.Warning{}
+	}
+	return append([]contracts.Warning(nil), warnings...)
+}
+
+func cloneMetadata(metadata map[string]any) map[string]any {
+	if len(metadata) == 0 {
+		return nil
+	}
+	out := make(map[string]any, len(metadata))
+	for key, value := range metadata {
+		out[key] = value
+	}
+	return out
+}
+
+func encoderErrorf(format string, args ...any) error {
+	return fmt.Errorf("json output encoder: "+format, args...)
+}
--- a/internal/modules/output/json/encoder_test.go
+++ b/internal/modules/output/json/encoder_test.go
@@ -0,0 +1,316 @@
+package json
+
+import (
+	"context"
+	stdjson "encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
+	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+func TestModuleSpecAndRegister(t *testing.T) {
+	want := pipeline.ModuleSpec{
+		Key:      Key,
+		Stage:    pipeline.StageOutput,
+		Requires: []string{"normalized"},
+		Provides: []string{"encoded"},
+	}
+	if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
+		t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
+	}
+
+	registry := pipeline.NewOutputEncoderRegistry()
+	if err := Register(registry); err != nil {
+		t.Fatalf("Register() error = %v, want nil", err)
+	}
+	spec, ok := registry.Spec(Key)
+	if !ok {
+		t.Fatalf("Spec(%q) ok = false, want true", Key)
+	}
+	if !reflect.DeepEqual(spec, want) {
+		t.Fatalf("registered spec = %#v, want %#v", spec, want)
+	}
+}
+
+func TestEncodeReturnsLogicalFilesGroupedByArtifactType(t *testing.T) {
+	req := contracts.OutputRequest{
+		Manifest: artifacts.RunManifest{RunID: "run-1", PipelineID: "pipeline-1"},
+		Approved: []artifacts.Artifact{
+			artifact("dnd.spell-cast", "first"),
+			artifact("notes/item", "item"),
+			artifact("dnd.spell-cast", "second"),
+		},
+		Rejected: []artifacts.RejectedArtifact{
+			{
+				Candidate:     candidate("bad type", "bad"),
+				ValidatorName: "validator",
+				ReasonCode:    "invalid",
+				Message:       "not accepted",
+			},
+		},
+		Warnings: []contracts.Warning{{ReasonCode: "warning", Message: "check source"}},
+	}
+
+	result, err := New().Encode(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Encode() error = %v, want nil", err)
+	}
+
+	wantNames := []string{
+		"artifacts/dnd.spell-cast.json",
+		"artifacts/notes_item.json",
+		"index.json",
+		"manifest.json",
+		"rejected.json",
+		"warnings.json",
+	}
+	if got := outputFileNames(result.Files); !reflect.DeepEqual(got, wantNames) {
+		t.Fatalf("file names = %#v, want %#v", got, wantNames)
+	}
+	for _, file := range result.Files {
+		if file.ContentType != contentTypeJSON {
+			t.Fatalf("%s ContentType = %q, want %q", file.Name, file.ContentType, contentTypeJSON)
+		}
+		if !strings.HasSuffix(string(file.Bytes), "\n") {
+			t.Fatalf("%s does not end with newline: %q", file.Name, string(file.Bytes))
+		}
+		if !stdjson.Valid(file.Bytes) {
+			t.Fatalf("%s has invalid JSON: %s", file.Name, file.Bytes)
+		}
+	}
+
+	spellFile := decodeObject(t, fileBytes(t, result.Files, "artifacts/dnd.spell-cast.json"))
+	if spellFile["artifact_type"] != "dnd.spell-cast" {
+		t.Fatalf("artifact_type = %#v, want dnd.spell-cast", spellFile["artifact_type"])
+	}
+	spells := spellFile["artifacts"].([]any)
+	if len(spells) != 2 {
+		t.Fatalf("len(spells) = %d, want 2", len(spells))
+	}
+	firstPayload := spells[0].(map[string]any)["payload"].(map[string]any)
+	secondPayload := spells[1].(map[string]any)["payload"].(map[string]any)
+	if firstPayload["name"] != "first" || secondPayload["name"] != "second" {
+		t.Fatalf("spell order payloads = %#v then %#v, want runner order", firstPayload, secondPayload)
+	}
+
+	index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
+	artifactFiles := index["artifact_files"].([]any)
+	if len(artifactFiles) != 2 {
+		t.Fatalf("len(index artifact_files) = %d, want 2", len(artifactFiles))
+	}
+	firstIndex := artifactFiles[0].(map[string]any)
+	secondIndex := artifactFiles[1].(map[string]any)
+	if firstIndex["artifact_type"] != "dnd.spell-cast" || secondIndex["artifact_type"] != "notes/item" {
+		t.Fatalf("artifact_files = %#v, want sorted by artifact type", artifactFiles)
+	}
+}
+
+func TestEncodeIncludesRejectedAndWarningsWhenEmpty(t *testing.T) {
+	result, err := New().Encode(context.Background(), contracts.OutputRequest{
+		Manifest: artifacts.RunManifest{RunID: "run-1"},
+	})
+	if err != nil {
+		t.Fatalf("Encode() error = %v, want nil", err)
+	}
+
+	rejected := decodeObject(t, fileBytes(t, result.Files, "rejected.json"))
+	if got := rejected["rejected"].([]any); len(got) != 0 {
+		t.Fatalf("rejected = %#v, want empty array", got)
+	}
+	warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json"))
+	if got := warnings["warnings"].([]any); len(got) != 0 {
+		t.Fatalf("warnings = %#v, want empty array", got)
+	}
+}
+
+func TestEncodePrettyPrintsJSON(t *testing.T) {
+	result, err := New().Encode(context.Background(), contracts.OutputRequest{
+		Manifest: artifacts.RunManifest{RunID: "run-1"},
+	})
+	if err != nil {
+		t.Fatalf("Encode() error = %v, want nil", err)
+	}
+
+	manifest := string(fileBytes(t, result.Files, "manifest.json"))
+	if !strings.Contains(manifest, "\n  \"run_id\": \"run-1\"\n") {
+		t.Fatalf("manifest JSON = %q, want two-space indentation", manifest)
+	}
+}
+
+func TestEncodeRejectsArtifactTypeWithoutSafeFileName(t *testing.T) {
+	_, err := New().Encode(context.Background(), contracts.OutputRequest{
+		Approved: []artifacts.Artifact{artifact("///", "unsafe")},
+	})
+	if err == nil {
+		t.Fatal("Encode() error = nil, want unsafe artifact type error")
+	}
+	if !strings.Contains(err.Error(), "json output encoder") || !strings.Contains(err.Error(), "safe file name") {
+		t.Fatalf("Encode() error = %q, want safe file name context", err.Error())
+	}
+}
+
+func TestEncodeSanitizesParentPathSequences(t *testing.T) {
+	result, err := New().Encode(context.Background(), contracts.OutputRequest{
+		Approved: []artifacts.Artifact{artifact("dnd..spell.", "spell")},
+	})
+	if err != nil {
+		t.Fatalf("Encode() error = %v, want nil", err)
+	}
+
+	if got := outputFileNames(result.Files); !containsString(got, "artifacts/dnd__spell.json") {
+		t.Fatalf("file names = %#v, want sanitized artifact filename", got)
+	}
+}
+
+func TestEncodeRejectsSanitizedFilenameCollisions(t *testing.T) {
+	_, err := New().Encode(context.Background(), contracts.OutputRequest{
+		Approved: []artifacts.Artifact{
+			artifact("a/b", "slash"),
+			artifact("a?b", "question"),
+		},
+	})
+	if err == nil {
+		t.Fatal("Encode() error = nil, want duplicate file error")
+	}
+	if !strings.Contains(err.Error(), "duplicate output file") {
+		t.Fatalf("Encode() error = %q, want duplicate file context", err.Error())
+	}
+}
+
+func TestEncodeDoesNotMutateInputs(t *testing.T) {
+	req := contracts.OutputRequest{
+		Manifest: artifacts.RunManifest{RunID: "run-1"},
+		Approved: []artifacts.Artifact{
+			artifact("dnd.spell", "original"),
+		},
+		Rejected: []artifacts.RejectedArtifact{
+			{
+				Candidate:     candidate("bad", "rejected"),
+				ValidatorName: "validator",
+				ReasonCode:    "invalid",
+				Message:       "not accepted",
+			},
+		},
+		Warnings: []contracts.Warning{{ReasonCode: "warning", Message: "message"}},
+	}
+	before := mustMarshal(t, req)
+
+	result, err := New().Encode(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Encode() error = %v, want nil", err)
+	}
+	after := mustMarshal(t, req)
+	if before != after {
+		t.Fatalf("request mutated:\nbefore: %s\nafter:  %s", before, after)
+	}
+
+	req.Approved[0].Payload[0] = '['
+	req.Approved[0].SourceRefs[0].StartUnitID = "changed"
+	req.Approved[0].Metadata["name"] = "changed"
+	req.Rejected[0].Candidate.Payload[0] = '['
+	req.Warnings[0].Message = "changed"
+
+	if !stdjson.Valid(fileBytes(t, result.Files, "artifacts/dnd.spell.json")) {
+		t.Fatal("artifact output changed after request mutation")
+	}
+	warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json"))
+	gotWarnings := warnings["warnings"].([]any)
+	if gotWarnings[0].(map[string]any)["message"] != "message" {
+		t.Fatalf("warnings output changed after request mutation: %#v", gotWarnings)
+	}
+}
+
+func TestArtifactFilesDoNotContainWarnings(t *testing.T) {
+	result, err := New().Encode(context.Background(), contracts.OutputRequest{
+		Approved: []artifacts.Artifact{artifact("dnd.spell", "spell")},
+		Warnings: []contracts.Warning{
+			{ReasonCode: "pipeline-warning", Message: "warning"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Encode() error = %v, want nil", err)
+	}
+
+	artifactFile := decodeObject(t, fileBytes(t, result.Files, "artifacts/dnd.spell.json"))
+	if _, ok := artifactFile["warnings"]; ok {
+		t.Fatalf("artifact file contains warnings: %#v", artifactFile)
+	}
+}
+
+func artifact(artifactType, name string) artifacts.Artifact {
+	return artifacts.Artifact{
+		ExtractorKey:  "extractor",
+		ArtifactType:  artifactType,
+		SchemaVersion: "v1",
+		Payload:       stdjson.RawMessage(`{"name":"` + name + `"}`),
+		SourceRefs: []source.SourceRef{
+			{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u1"},
+		},
+		Metadata: map[string]any{"name": name},
+	}
+}
+
+func candidate(artifactType, name string) artifacts.ArtifactCandidate {
+	return artifacts.ArtifactCandidate{
+		Index:         1,
+		ExtractorKey:  "extractor",
+		ArtifactType:  artifactType,
+		SchemaVersion: "v1",
+		Payload:       stdjson.RawMessage(`{"name":"` + name + `"}`),
+		SourceRefs: []source.SourceRef{
+			{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u1"},
+		},
+		Metadata: map[string]any{"name": name},
+	}
+}
+
+func outputFileNames(files []contracts.OutputFile) []string {
+	names := make([]string, 0, len(files))
+	for _, file := range files {
+		names = append(names, file.Name)
+	}
+	return names
+}
+
+func containsString(values []string, want string) bool {
+	for _, value := range values {
+		if value == want {
+			return true
+		}
+	}
+	return false
+}
+
+func fileBytes(t *testing.T, files []contracts.OutputFile, name string) []byte {
+	t.Helper()
+	for _, file := range files {
+		if file.Name == name {
+			return file.Bytes
+		}
+	}
+	t.Fatalf("file %q not found in %#v", name, outputFileNames(files))
+	return nil
+}
+
+func decodeObject(t *testing.T, data []byte) map[string]any {
+	t.Helper()
+	var got map[string]any
+	if err := stdjson.Unmarshal(data, &got); err != nil {
+		t.Fatalf("Unmarshal() error = %v, want nil\n%s", err, data)
+	}
+	return got
+}
+
+func mustMarshal(t *testing.T, value any) string {
+	t.Helper()
+	data, err := stdjson.Marshal(value)
+	if err != nil {
+		t.Fatalf("Marshal() error = %v, want nil", err)
+	}
+	return string(data)
+}