eric/distributor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Add Markdown HTML publication

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-31 02:28:48 +00:00
parent e296361042
commit 5408f14195
21 changed files with 628 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
internal/publish/execute.go
View File

@@ -36,20 +36,23 @@ func Execute(ctx context.Context, req Request, plan Plan) error {
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, managedOutputPaths(writtenOutputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
}
for _, output := range plan.Outputs {
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
if err != nil {
cleanup()
return err
}
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
if err != nil {
cleanup()
return err
}
data, err := req.SourceBackend.ReadFile(ctx, sourcePath)
if err != nil {
cleanup()
return err
data := output.Data
if output.Kind == state.OutputKindSource {
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
if err != nil {
cleanup()
return err
}
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
if err != nil {
cleanup()
return err
}
}
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: false, PreferAtomic: true}); err != nil {
cleanup()

70
internal/publish/output.go
View File

@@ -1,29 +1,62 @@
package publish
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/transform"
)
func PlanSourceOutputs(req Request) ([]Output, error) {
if !req.Publish.Source {
return nil, nil
}
outputs := make([]Output, 0, len(req.SourceBundle.Manifest.Files))
seen := make(map[string]struct{}, len(req.SourceBundle.Manifest.Files))
for _, file := range req.SourceBundle.Manifest.Files {
if _, exists := seen[file.Path]; exists {
return nil, fmt.Errorf("destination output path collision: %s", file.Path)
func PlanOutputs(ctx context.Context, req Request) ([]Output, error) {
var outputs []Output
if req.Publish.Source {
sourceOutputs, err := PlanSourceOutputs(req)
if err != nil {
return nil, err
}
seen[file.Path] = struct{}{}
outputs = append(outputs, sourceOutputs...)
}
if req.Publish.HTML {
generatedOutputs, err := markdownTransformer().Generate(ctx, transform.Request{
SourceBundle: req.SourceBundle,
SourceBackend: req.SourceBackend,
})
if err != nil {
return nil, err
}
if len(generatedOutputs) == 0 {
return nil, fmt.Errorf("publish html requested but no markdown source files were found")
}
for _, generated := range generatedOutputs {
outputs = append(outputs, Output{
SourcePath: generated.SourcePath,
DestinationPath: generated.Path,
Kind: state.OutputKindGenerated,
Transform: generated.Transform,
Data: generated.Data,
SHA256: generated.SHA256,
Size: generated.Size,
})
}
}
if err := rejectOutputCollisions(outputs); err != nil {
return nil, err
}
return outputs, nil
}
func PlanSourceOutputs(req Request) ([]Output, error) {
outputs := make([]Output, 0, len(req.SourceBundle.Manifest.Files))
for _, file := range req.SourceBundle.Manifest.Files {
if err := storage.ValidatePath(file.Path); err != nil {
return nil, fmt.Errorf("destination output path %q: %w", file.Path, err)
}
outputs = append(outputs, Output{
SourcePath: file.Path,
DestinationPath: file.Path,
Kind: state.OutputKindSource,
SHA256: file.SHA256,
Size: file.Size,
})
@@ -31,13 +64,28 @@ func PlanSourceOutputs(req Request) ([]Output, error) {
return outputs, nil
}
func rejectOutputCollisions(outputs []Output) error {
seen := make(map[string]struct{}, len(outputs))
for _, output := range outputs {
if err := storage.ValidatePath(output.DestinationPath); err != nil {
return fmt.Errorf("destination output path %q: %w", output.DestinationPath, err)
}
if _, exists := seen[output.DestinationPath]; exists {
return fmt.Errorf("destination output path collision: %s", output.DestinationPath)
}
seen[output.DestinationPath] = struct{}{}
}
return nil
}
func stateOutputs(outputs []Output) []state.OutputFile {
files := make([]state.OutputFile, 0, len(outputs))
for _, output := range outputs {
files = append(files, state.OutputFile{
Path: output.DestinationPath,
Kind: state.OutputKindSource,
Kind: output.Kind,
SourcePath: output.SourcePath,
Transform: output.Transform,
SHA256: output.SHA256,
Size: output.Size,
})

86
internal/publish/output_test.go
View File

@@ -1,29 +1,101 @@
package publish
import (
"context"
"testing"
"time"
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
)
func TestPlanSourceOutputsRejectsCollision(t *testing.T) {
_, err := PlanSourceOutputs(Request{
func TestPlanOutputsRejectsCollision(t *testing.T) {
sourceBackend := fake.New()
if _, err := sourceBackend.WriteFile(context.Background(), "report.md", []byte("# Report\n"), storage.WriteOptions{}); err != nil {
t.Fatalf("WriteFile report.md error = %v", err)
}
if _, err := sourceBackend.WriteFile(context.Background(), "report.html", []byte("<p>source html</p>\n"), storage.WriteOptions{}); err != nil {
t.Fatalf("WriteFile report.html error = %v", err)
}
reportDigest := bundle.FileDigest([]byte("# Report\n"))
htmlDigest := bundle.FileDigest([]byte("<p>source html</p>\n"))
files := []bundle.ManifestFile{
{Path: "report.md", SHA256: reportDigest, Size: 9},
{Path: "report.html", SHA256: htmlDigest, Size: 19},
}
_, err := PlanOutputs(context.Background(), Request{
SourceBackend: sourceBackend,
SourceBundle: bundle.Bundle{
Manifest: bundle.Manifest{
SchemaVersion: 1,
ID: "bundle",
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
Files: []bundle.ManifestFile{
{Path: "report.md", SHA256: "sha256:1111111111111111111111111111111111111111111111111111111111111111", Size: 1},
{Path: "report.md", SHA256: "sha256:2222222222222222222222222222222222222222222222222222222222222222", Size: 1},
},
Digest: bundle.BundleDigest(files),
Files: files,
},
},
Publish: config.PublishPolicy{Source: true},
Publish: config.PublishPolicy{Source: true, HTML: true},
Transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
})
if err == nil {
t.Fatal("PlanSourceOutputs() error = nil, want collision")
}
}
func TestPlanOutputsRejectsHTMLWithoutMarkdown(t *testing.T) {
sourceBackend := fake.New()
if _, err := sourceBackend.WriteFile(context.Background(), "summary.txt", []byte("Summary\n"), storage.WriteOptions{}); err != nil {
t.Fatalf("WriteFile summary.txt error = %v", err)
}
files := []bundle.ManifestFile{{Path: "summary.txt", SHA256: bundle.FileDigest([]byte("Summary\n")), Size: 8}}
_, err := PlanOutputs(context.Background(), Request{
SourceBackend: sourceBackend,
SourceBundle: bundle.Bundle{Manifest: bundle.Manifest{
SchemaVersion: 1,
ID: "bundle",
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
Digest: bundle.BundleDigest(files),
Files: files,
}},
Publish: config.PublishPolicy{HTML: true},
Transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
})
if err == nil {
t.Fatal("PlanOutputs() error = nil, want no markdown failure")
}
}
func TestBuildRejectsHTMLWithoutTransform(t *testing.T) {
sourceBackend := fake.New()
destinationBackend := fake.New()
if _, err := sourceBackend.WriteFile(context.Background(), "report.md", []byte("# Report\n"), storage.WriteOptions{}); err != nil {
t.Fatalf("WriteFile report.md error = %v", err)
}
files := []bundle.ManifestFile{{Path: "report.md", SHA256: bundle.FileDigest([]byte("# Report\n")), Size: 9}}
_, err := Build(context.Background(), Request{
PipelineID: "reports",
DestinationID: "archive",
SourceBackend: sourceBackend,
DestinationBackend: destinationBackend,
DestinationBundlePath: "",
SourceBundle: bundle.Bundle{Manifest: bundle.Manifest{
SchemaVersion: 1,
ID: "bundle",
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
Digest: bundle.BundleDigest(files),
Files: files,
}},
Publish: config.PublishPolicy{HTML: true},
Transfer: config.TransferPolicy{
OnDestinationSame: config.TransferActionSkip,
OnDestinationOlder: config.TransferActionReplace,
OnDestinationNewer: config.TransferActionSkip,
OnConflict: config.TransferActionFail,
},
})
if err == nil {
t.Fatal("Build() error = nil, want missing transform error")
}
}

22
internal/publish/plan.go
View File

@@ -8,6 +8,8 @@ import (
"gitea.maximumdirect.net/eric/distributor/internal/config"
"gitea.maximumdirect.net/eric/distributor/internal/state"
"gitea.maximumdirect.net/eric/distributor/internal/storage"
"gitea.maximumdirect.net/eric/distributor/internal/transform"
markdowntransform "gitea.maximumdirect.net/eric/distributor/internal/transform/markdown"
)
type Action string
@@ -29,6 +31,7 @@ type Request struct {
DestinationBackend storage.Backend
DestinationBundlePath string
Publish config.PublishPolicy
Transform config.Transform
Transfer config.TransferPolicy
DistributorVersion string
}
@@ -48,6 +51,9 @@ type Plan struct {
type Output struct {
SourcePath string
DestinationPath string
Kind string
Transform string
Data []byte
SHA256 string
Size int64
}
@@ -56,7 +62,7 @@ func Build(ctx context.Context, req Request) (Plan, error) {
if err := validateRequest(req); err != nil {
return Plan{}, err
}
outputs, err := PlanSourceOutputs(req)
outputs, err := PlanOutputs(ctx, req)
if err != nil {
return Plan{}, err
}
@@ -96,15 +102,21 @@ func validateRequest(req Request) error {
if req.DestinationBackend == nil {
return fmt.Errorf("destination backend is required")
}
if req.Publish.HTML {
return fmt.Errorf("publish html is not implemented")
if !req.Publish.Source && !req.Publish.HTML {
return fmt.Errorf("publish source or html must be enabled")
}
if !req.Publish.Source {
return fmt.Errorf("publish source must be enabled")
if req.Publish.HTML {
if req.Transform.MarkdownToHTML == nil || !req.Transform.MarkdownToHTML.Enabled || req.Transform.MarkdownToHTML.Mode != config.TransformModeSidecar {
return fmt.Errorf("publish html requires markdown_to_html transform enabled with sidecar mode")
}
}
return nil
}
func markdownTransformer() transform.Transformer {
return markdowntransform.New()
}
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy) (Action, string) {
switch comparison.Outcome {
case state.OutcomeDestinationAbsent: