diff --git a/docs/cli.md b/docs/cli.md index ea13070..e96702d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -20,7 +20,7 @@ distributor inspect `version` prints the application name and version. The default development version is `dev`; release builds may replace it at build time. -`run --config ` executes configured local-to-local pipelines that publish source files only. +`run --config ` executes configured local-to-local pipelines that publish source files, generated HTML files, or both. `run --config --dry-run` discovers source bundles, inspects destination state, and prints planned actions without writing files. @@ -28,7 +28,7 @@ distributor inspect `inspect ` validates discovered local source bundles and prints a concise normalized summary. -Remote backends and HTML publication are not implemented yet. +Remote backends are not implemented yet. ## Flag reference @@ -70,3 +70,9 @@ Publish the local example: ```sh go run ./cmd/distributor run --config examples/local-publish.yml ``` + +Publish the local HTML example: + +```sh +go run ./cmd/distributor run --config examples/local-html.yml +``` diff --git a/docs/config.md b/docs/config.md index 0716b2b..a48ec36 100644 --- a/docs/config.md +++ b/docs/config.md @@ -10,7 +10,7 @@ If `--config` is omitted during run, the built-in default path is: /usr/local/etc/distributor/config.yml ``` -The current implementation supports local-to-local source-file publication. Remote backends and HTML publication are not implemented yet. +The current implementation supports local-to-local publication of source files, generated HTML files, or both. Remote backends are not implemented yet. ## Minimal config @@ -74,7 +74,7 @@ Backends: Destination policy: - `publish.source`: publish source artifacts. -- `publish.html`: publish generated HTML artifacts. +- `publish.html`: publish generated HTML artifacts from Markdown source files. - `transfer.on_destination_same`: `skip` or `fail`, defaults to `skip`. - `transfer.on_destination_older`: `replace` or `fail`, defaults to `replace`. - `transfer.on_destination_newer`: `skip` or `fail`, defaults to `skip`. @@ -82,6 +82,8 @@ Destination policy: When `publish.html` is true, `transform.markdown_to_html.enabled: true` and `transform.markdown_to_html.mode: sidecar` are required. +Markdown-to-HTML sidecar generation writes `report.html` for `report.md` and does not mutate the source bundle. + ## Secrets Do not put literal secrets in config files. S3 credentials may refer to environment variable names with: diff --git a/docs/internal/publish.md b/docs/internal/publish.md index baf7742..dfc1d2a 100644 --- a/docs/internal/publish.md +++ b/docs/internal/publish.md @@ -8,7 +8,7 @@ Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, publish policy, transfer policy, destination bundle path, and existing destination state. -Output is a plan with an action, reason, and selected source outputs. Execution writes selected source files and `.distributor.json` for publish or replacement actions. +Output is a plan with an action, reason, and selected source or generated outputs. Execution writes selected source files, generated files, and `.distributor.json` for publish or replacement actions. ## Actions @@ -16,7 +16,7 @@ Supported actions are publish new, replace older destination, skip same source, ## Boundaries -The current implementation publishes source files only. HTML generation and remote backend execution are not implemented. +The current implementation publishes source files and Markdown-to-HTML sidecar outputs. Remote backend execution is not implemented. The package uses `internal/state` for destination comparison and `internal/storage` for IO. It does not parse CLI flags or load config files. diff --git a/docs/internal/transform.md b/docs/internal/transform.md new file mode 100644 index 0000000..277ab02 --- /dev/null +++ b/docs/internal/transform.md @@ -0,0 +1,23 @@ +# Transform + +## Purpose + +`internal/transform` defines generated publication artifacts. `internal/transform/markdown` implements Markdown-to-HTML sidecar generation. + +## Inputs and outputs + +Inputs are a validated source bundle and source backend. Outputs include destination path, source path, transform id, generated bytes, SHA-256, and size. + +## Markdown behavior + +Markdown files ending in `.md` generate `.html` files in the same logical directory. Non-Markdown files do not generate outputs. Raw HTML embedded in Markdown is not passed through by the renderer. + +Generated HTML is deterministic for the same source content and transform configuration. + +## Boundaries + +Transforms do not publish files, mutate source bundles, or write destination state. Publish planning selects and writes transform outputs. + +## Tests + +Before changing transform behavior, inspect tests under `internal/transform`. diff --git a/docs/operations.md b/docs/operations.md index d9ef6f7..7654967 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -14,6 +14,12 @@ Run the local publication: go run ./cmd/distributor run --config examples/local-publish.yml ``` +Run the local HTML publication: + +```sh +go run ./cmd/distributor run --config examples/local-html.yml +``` + ## Filesystem layout Source bundles are discovered beneath the configured local source root. Destination bundle paths preserve the source bundle path relative to that source root. @@ -22,7 +28,7 @@ The maintained example writes under `workspace/`, which is ignored by Git. ## Destination state -Each published destination bundle contains `.distributor.json`. This state file records the source manifest and copied source outputs. It is the authoritative marker that a destination path is managed by `distributor`. +Each published destination bundle contains `.distributor.json`. This state file records the source manifest, copied source outputs, and generated outputs. It is the authoritative marker that a destination path is managed by `distributor`. `manifest.json` from the source bundle is not copied as destination state. @@ -36,4 +42,4 @@ If a write fails during local publication, `distributor` removes outputs written ## Caveats -Only local-to-local source-file publication is implemented. SSH, S3, HTML generation, notification, and force overwrite behavior are not implemented. +Only local-to-local publication is implemented. SSH, S3, notification, and force overwrite behavior are not implemented. diff --git a/examples/local-html.yml b/examples/local-html.yml new file mode 100644 index 0000000..90c8183 --- /dev/null +++ b/examples/local-html.yml @@ -0,0 +1,16 @@ +pipelines: + - id: example-html-bundle + source: + backend: local + path: examples/source-bundle + destinations: + - id: local-html + backend: local + path: workspace/published/html-bundle + publish: + source: false + html: true + transform: + markdown_to_html: + enabled: true + mode: sidecar diff --git a/go.mod b/go.mod index ea77c63..b84ddb7 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module gitea.maximumdirect.net/eric/distributor go 1.26 -require gopkg.in/yaml.v3 v3.0.1 +require ( + github.com/yuin/goldmark v1.8.2 + gopkg.in/yaml.v3 v3.0.1 +) diff --git a/go.sum b/go.sum index a62c313..f3e50ca 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= +github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/app/run.go b/internal/app/run.go index cedb9fb..27f20ba 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "strings" "gitea.maximumdirect.net/eric/distributor/internal/adapters/local" "gitea.maximumdirect.net/eric/distributor/internal/bundle" @@ -73,6 +74,7 @@ func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error DestinationBackend: destinationBackend, DestinationBundlePath: sourceBundle.RootRelativePath, Publish: *destination.Publish, + Transform: destination.Transform, Transfer: destination.Transfer, DistributorVersion: Version, } @@ -102,5 +104,16 @@ func writePlanLine(w io.Writer, plan publish.Plan, planErr error) { fmt.Fprintf(w, " - bundle=%s destination=%s action=error reason=%q\n", displayBundlePath(plan.BundlePath), plan.DestinationID, planErr.Error()) return } - fmt.Fprintf(w, " - bundle=%s destination=%s action=%s outputs=%d reason=%q\n", displayBundlePath(plan.BundlePath), plan.DestinationID, plan.Action, len(plan.Outputs), plan.Reason) + fmt.Fprintf(w, " - bundle=%s destination=%s action=%s outputs=%s reason=%q\n", displayBundlePath(plan.BundlePath), plan.DestinationID, plan.Action, outputSummary(plan.Outputs), plan.Reason) +} + +func outputSummary(outputs []publish.Output) string { + if len(outputs) == 0 { + return "none" + } + paths := make([]string, 0, len(outputs)) + for _, output := range outputs { + paths = append(paths, output.DestinationPath) + } + return strings.Join(paths, ",") } diff --git a/internal/app/run_test.go b/internal/app/run_test.go index 17d970e..4be6910 100644 --- a/internal/app/run_test.go +++ b/internal/app/run_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -34,7 +35,7 @@ func TestRunDryRunPrintsConfigSummary(t *testing.T) { for _, want := range []string{ "Configured pipelines: 1", "- reports: source=local bundles=1 destinations=1", - "bundle=. destination=archive action=publish_new", + "bundle=. destination=archive action=publish_new outputs=report.md,summary.txt", } { if !strings.Contains(output, want) { t.Fatalf("Run() output = %q, want substring %q", output, want) @@ -72,6 +73,103 @@ func TestRunPublishesNewLocalBundle(t *testing.T) { } } +func TestRunPublishesHTMLOnly(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) + + err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, false, true)}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + assertFileContains(t, filepath.Join(destinationRoot, "report.html"), "

Report

") + if _, err := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(err) { + t.Fatalf("report.md stat error = %v, want not exist", err) + } + destinationState := readStateFile(t, filepath.Join(destinationRoot, ".distributor.json")) + if got, want := len(destinationState.Outputs), 1; got != want { + t.Fatalf("state output count = %d, want %d", got, want) + } + output := destinationState.Outputs[0] + if output.Kind != state.OutputKindGenerated || output.Transform != "markdown_to_html" || output.Path != "report.html" || output.SourcePath != "report.md" { + t.Fatalf("generated output metadata = %#v", output) + } +} + +func TestRunPublishesSourceAndHTML(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) + + err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, true, true)}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n") + assertFileContains(t, filepath.Join(destinationRoot, "report.html"), "

Sunny.

") + assertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n") + destinationState := readStateFile(t, filepath.Join(destinationRoot, ".distributor.json")) + if got, want := len(destinationState.Outputs), 3; got != want { + t.Fatalf("state output count = %d, want %d", got, want) + } +} + +func TestRunDoesNotMutateSourceBundle(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) + sourcePath := filepath.Join(sourceRoot, "report.md") + before, err := os.ReadFile(sourcePath) + if err != nil { + t.Fatalf("read source before: %v", err) + } + + err = Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, true, true)}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + after, err := os.ReadFile(sourcePath) + if err != nil { + t.Fatalf("read source after: %v", err) + } + if string(after) != string(before) { + t.Fatalf("source changed from %q to %q", before, after) + } +} + +func TestRunFailsOnOutputPathCollision(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{ExtraFiles: []testFile{{Path: "report.html", Data: "

source html

\n"}}}) + + err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, true, true)}) + if err == nil || !strings.Contains(err.Error(), "destination output path collision") { + t.Fatalf("Run() error = %v, want collision", err) + } + if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 { + t.Fatalf("destination entries = %v err=%v, want empty", entries, err) + } +} + +func TestRunDryRunReportsGeneratedOutputs(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) + + var stdout bytes.Buffer + err := Run(context.Background(), RunOptions{ + ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, false, true), + DryRun: true, + Stdout: &stdout, + }) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if !strings.Contains(stdout.String(), "outputs=report.html") { + t.Fatalf("stdout = %q, want generated output path", stdout.String()) + } +} + func TestRunSkipsWhenDestinationStateMatches(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() @@ -194,8 +292,14 @@ func TestRunDryRunDoesNotWrite(t *testing.T) { } type testBundleOptions struct { - ID string - Created time.Time + ID string + Created time.Time + ExtraFiles []testFile +} + +type testFile struct { + Path string + Data string } func writeSourceBundle(t *testing.T, root, relative string, opts testBundleOptions) bundle.Manifest { @@ -217,6 +321,12 @@ func writeSourceBundle(t *testing.T, root, relative string, opts testBundleOptio {path: "report.md", data: "# Report\nSunny.\n"}, {path: "summary.txt", data: "Summary\n"}, } + for _, extra := range opts.ExtraFiles { + files = append(files, struct { + path string + data string + }{path: extra.Path, data: extra.Data}) + } manifestFiles := make([]bundle.ManifestFile, 0, len(files)) for _, file := range files { if err := os.WriteFile(filepath.Join(bundleRoot, filepath.FromSlash(file.path)), []byte(file.data), 0o600); err != nil { @@ -248,6 +358,19 @@ func writeSourceBundle(t *testing.T, root, relative string, opts testBundleOptio func writeLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string { t.Helper() + return writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, true, false) +} + +func writeLocalConfigWithPolicy(t *testing.T, sourceRoot, destinationRoot string, publishSource, publishHTML bool) string { + t.Helper() + transformConfig := "" + if publishHTML { + transformConfig = ` + transform: + markdown_to_html: + enabled: true + mode: sidecar` + } return writeConfigFile(t, ` pipelines: - id: reports @@ -259,8 +382,8 @@ pipelines: backend: local path: `+destinationRoot+` publish: - source: true - html: false + source: `+fmt.Sprintf("%t", publishSource)+` + html: `+fmt.Sprintf("%t", publishHTML)+transformConfig+` `) } @@ -341,3 +464,14 @@ func assertFile(t *testing.T, path, want string) { t.Fatalf("%s = %q, want %q", path, got, want) } } + +func assertFileContains(t *testing.T, path, want string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read file %s: %v", path, err) + } + if !strings.Contains(string(data), want) { + t.Fatalf("%s = %q, want substring %q", path, data, want) + } +} diff --git a/internal/config/load_test.go b/internal/config/load_test.go index 940d3a5..470a8c6 100644 --- a/internal/config/load_test.go +++ b/internal/config/load_test.go @@ -265,6 +265,7 @@ func TestExampleConfigsLoad(t *testing.T) { for _, path := range []string{ "../../examples/local-to-local.yml", "../../examples/local-publish.yml", + "../../examples/local-html.yml", "../../examples/fan-out.yml", } { t.Run(path, func(t *testing.T) { diff --git a/internal/publish/execute.go b/internal/publish/execute.go index 2077c7d..eaf6179 100644 --- a/internal/publish/execute.go +++ b/internal/publish/execute.go @@ -36,20 +36,23 @@ func Execute(ctx context.Context, req Request, plan Plan) error { _ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, managedOutputPaths(writtenOutputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}) } for _, output := range plan.Outputs { - sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath) - if err != nil { - cleanup() - return err - } destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath) if err != nil { cleanup() return err } - data, err := req.SourceBackend.ReadFile(ctx, sourcePath) - if err != nil { - cleanup() - return err + data := output.Data + if output.Kind == state.OutputKindSource { + sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath) + if err != nil { + cleanup() + return err + } + data, err = req.SourceBackend.ReadFile(ctx, sourcePath) + if err != nil { + cleanup() + return err + } } if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: false, PreferAtomic: true}); err != nil { cleanup() diff --git a/internal/publish/output.go b/internal/publish/output.go index 83abcd6..0bf1264 100644 --- a/internal/publish/output.go +++ b/internal/publish/output.go @@ -1,29 +1,62 @@ package publish import ( + "context" "fmt" "gitea.maximumdirect.net/eric/distributor/internal/state" "gitea.maximumdirect.net/eric/distributor/internal/storage" + "gitea.maximumdirect.net/eric/distributor/internal/transform" ) -func PlanSourceOutputs(req Request) ([]Output, error) { - if !req.Publish.Source { - return nil, nil - } - outputs := make([]Output, 0, len(req.SourceBundle.Manifest.Files)) - seen := make(map[string]struct{}, len(req.SourceBundle.Manifest.Files)) - for _, file := range req.SourceBundle.Manifest.Files { - if _, exists := seen[file.Path]; exists { - return nil, fmt.Errorf("destination output path collision: %s", file.Path) +func PlanOutputs(ctx context.Context, req Request) ([]Output, error) { + var outputs []Output + if req.Publish.Source { + sourceOutputs, err := PlanSourceOutputs(req) + if err != nil { + return nil, err } - seen[file.Path] = struct{}{} + outputs = append(outputs, sourceOutputs...) + } + if req.Publish.HTML { + generatedOutputs, err := markdownTransformer().Generate(ctx, transform.Request{ + SourceBundle: req.SourceBundle, + SourceBackend: req.SourceBackend, + }) + if err != nil { + return nil, err + } + if len(generatedOutputs) == 0 { + return nil, fmt.Errorf("publish html requested but no markdown source files were found") + } + for _, generated := range generatedOutputs { + outputs = append(outputs, Output{ + SourcePath: generated.SourcePath, + DestinationPath: generated.Path, + Kind: state.OutputKindGenerated, + Transform: generated.Transform, + Data: generated.Data, + SHA256: generated.SHA256, + Size: generated.Size, + }) + } + } + if err := rejectOutputCollisions(outputs); err != nil { + return nil, err + } + return outputs, nil +} + +func PlanSourceOutputs(req Request) ([]Output, error) { + outputs := make([]Output, 0, len(req.SourceBundle.Manifest.Files)) + for _, file := range req.SourceBundle.Manifest.Files { if err := storage.ValidatePath(file.Path); err != nil { return nil, fmt.Errorf("destination output path %q: %w", file.Path, err) } outputs = append(outputs, Output{ SourcePath: file.Path, DestinationPath: file.Path, + Kind: state.OutputKindSource, SHA256: file.SHA256, Size: file.Size, }) @@ -31,13 +64,28 @@ func PlanSourceOutputs(req Request) ([]Output, error) { return outputs, nil } +func rejectOutputCollisions(outputs []Output) error { + seen := make(map[string]struct{}, len(outputs)) + for _, output := range outputs { + if err := storage.ValidatePath(output.DestinationPath); err != nil { + return fmt.Errorf("destination output path %q: %w", output.DestinationPath, err) + } + if _, exists := seen[output.DestinationPath]; exists { + return fmt.Errorf("destination output path collision: %s", output.DestinationPath) + } + seen[output.DestinationPath] = struct{}{} + } + return nil +} + func stateOutputs(outputs []Output) []state.OutputFile { files := make([]state.OutputFile, 0, len(outputs)) for _, output := range outputs { files = append(files, state.OutputFile{ Path: output.DestinationPath, - Kind: state.OutputKindSource, + Kind: output.Kind, SourcePath: output.SourcePath, + Transform: output.Transform, SHA256: output.SHA256, Size: output.Size, }) diff --git a/internal/publish/output_test.go b/internal/publish/output_test.go index fd4899c..f5f583a 100644 --- a/internal/publish/output_test.go +++ b/internal/publish/output_test.go @@ -1,29 +1,101 @@ package publish import ( + "context" "testing" "time" "gitea.maximumdirect.net/eric/distributor/internal/bundle" "gitea.maximumdirect.net/eric/distributor/internal/config" + "gitea.maximumdirect.net/eric/distributor/internal/storage" + "gitea.maximumdirect.net/eric/distributor/internal/storage/fake" ) -func TestPlanSourceOutputsRejectsCollision(t *testing.T) { - _, err := PlanSourceOutputs(Request{ +func TestPlanOutputsRejectsCollision(t *testing.T) { + sourceBackend := fake.New() + if _, err := sourceBackend.WriteFile(context.Background(), "report.md", []byte("# Report\n"), storage.WriteOptions{}); err != nil { + t.Fatalf("WriteFile report.md error = %v", err) + } + if _, err := sourceBackend.WriteFile(context.Background(), "report.html", []byte("

source html

\n"), storage.WriteOptions{}); err != nil { + t.Fatalf("WriteFile report.html error = %v", err) + } + reportDigest := bundle.FileDigest([]byte("# Report\n")) + htmlDigest := bundle.FileDigest([]byte("

source html

\n")) + files := []bundle.ManifestFile{ + {Path: "report.md", SHA256: reportDigest, Size: 9}, + {Path: "report.html", SHA256: htmlDigest, Size: 19}, + } + _, err := PlanOutputs(context.Background(), Request{ + SourceBackend: sourceBackend, SourceBundle: bundle.Bundle{ Manifest: bundle.Manifest{ SchemaVersion: 1, ID: "bundle", Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC), - Files: []bundle.ManifestFile{ - {Path: "report.md", SHA256: "sha256:1111111111111111111111111111111111111111111111111111111111111111", Size: 1}, - {Path: "report.md", SHA256: "sha256:2222222222222222222222222222222222222222222222222222222222222222", Size: 1}, - }, + Digest: bundle.BundleDigest(files), + Files: files, }, }, - Publish: config.PublishPolicy{Source: true}, + Publish: config.PublishPolicy{Source: true, HTML: true}, + Transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}}, }) if err == nil { t.Fatal("PlanSourceOutputs() error = nil, want collision") } } + +func TestPlanOutputsRejectsHTMLWithoutMarkdown(t *testing.T) { + sourceBackend := fake.New() + if _, err := sourceBackend.WriteFile(context.Background(), "summary.txt", []byte("Summary\n"), storage.WriteOptions{}); err != nil { + t.Fatalf("WriteFile summary.txt error = %v", err) + } + files := []bundle.ManifestFile{{Path: "summary.txt", SHA256: bundle.FileDigest([]byte("Summary\n")), Size: 8}} + _, err := PlanOutputs(context.Background(), Request{ + SourceBackend: sourceBackend, + SourceBundle: bundle.Bundle{Manifest: bundle.Manifest{ + SchemaVersion: 1, + ID: "bundle", + Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC), + Digest: bundle.BundleDigest(files), + Files: files, + }}, + Publish: config.PublishPolicy{HTML: true}, + Transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}}, + }) + if err == nil { + t.Fatal("PlanOutputs() error = nil, want no markdown failure") + } +} + +func TestBuildRejectsHTMLWithoutTransform(t *testing.T) { + sourceBackend := fake.New() + destinationBackend := fake.New() + if _, err := sourceBackend.WriteFile(context.Background(), "report.md", []byte("# Report\n"), storage.WriteOptions{}); err != nil { + t.Fatalf("WriteFile report.md error = %v", err) + } + files := []bundle.ManifestFile{{Path: "report.md", SHA256: bundle.FileDigest([]byte("# Report\n")), Size: 9}} + _, err := Build(context.Background(), Request{ + PipelineID: "reports", + DestinationID: "archive", + SourceBackend: sourceBackend, + DestinationBackend: destinationBackend, + DestinationBundlePath: "", + SourceBundle: bundle.Bundle{Manifest: bundle.Manifest{ + SchemaVersion: 1, + ID: "bundle", + Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC), + Digest: bundle.BundleDigest(files), + Files: files, + }}, + Publish: config.PublishPolicy{HTML: true}, + Transfer: config.TransferPolicy{ + OnDestinationSame: config.TransferActionSkip, + OnDestinationOlder: config.TransferActionReplace, + OnDestinationNewer: config.TransferActionSkip, + OnConflict: config.TransferActionFail, + }, + }) + if err == nil { + t.Fatal("Build() error = nil, want missing transform error") + } +} diff --git a/internal/publish/plan.go b/internal/publish/plan.go index 874aed8..b80904e 100644 --- a/internal/publish/plan.go +++ b/internal/publish/plan.go @@ -8,6 +8,8 @@ import ( "gitea.maximumdirect.net/eric/distributor/internal/config" "gitea.maximumdirect.net/eric/distributor/internal/state" "gitea.maximumdirect.net/eric/distributor/internal/storage" + "gitea.maximumdirect.net/eric/distributor/internal/transform" + markdowntransform "gitea.maximumdirect.net/eric/distributor/internal/transform/markdown" ) type Action string @@ -29,6 +31,7 @@ type Request struct { DestinationBackend storage.Backend DestinationBundlePath string Publish config.PublishPolicy + Transform config.Transform Transfer config.TransferPolicy DistributorVersion string } @@ -48,6 +51,9 @@ type Plan struct { type Output struct { SourcePath string DestinationPath string + Kind string + Transform string + Data []byte SHA256 string Size int64 } @@ -56,7 +62,7 @@ func Build(ctx context.Context, req Request) (Plan, error) { if err := validateRequest(req); err != nil { return Plan{}, err } - outputs, err := PlanSourceOutputs(req) + outputs, err := PlanOutputs(ctx, req) if err != nil { return Plan{}, err } @@ -96,15 +102,21 @@ func validateRequest(req Request) error { if req.DestinationBackend == nil { return fmt.Errorf("destination backend is required") } - if req.Publish.HTML { - return fmt.Errorf("publish html is not implemented") + if !req.Publish.Source && !req.Publish.HTML { + return fmt.Errorf("publish source or html must be enabled") } - if !req.Publish.Source { - return fmt.Errorf("publish source must be enabled") + if req.Publish.HTML { + if req.Transform.MarkdownToHTML == nil || !req.Transform.MarkdownToHTML.Enabled || req.Transform.MarkdownToHTML.Mode != config.TransformModeSidecar { + return fmt.Errorf("publish html requires markdown_to_html transform enabled with sidecar mode") + } } return nil } +func markdownTransformer() transform.Transformer { + return markdowntransform.New() +} + func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy) (Action, string) { switch comparison.Outcome { case state.OutcomeDestinationAbsent: diff --git a/internal/transform/markdown/markdown.go b/internal/transform/markdown/markdown.go new file mode 100644 index 0000000..89bd13d --- /dev/null +++ b/internal/transform/markdown/markdown.go @@ -0,0 +1,57 @@ +package markdown + +import ( + "bytes" + "context" + "fmt" + "strings" + + "github.com/yuin/goldmark" + + "gitea.maximumdirect.net/eric/distributor/internal/bundle" + "gitea.maximumdirect.net/eric/distributor/internal/storage" + "gitea.maximumdirect.net/eric/distributor/internal/transform" +) + +type Transformer struct { + renderer goldmark.Markdown +} + +func New() *Transformer { + return &Transformer{renderer: goldmark.New()} +} + +func (t *Transformer) Generate(ctx context.Context, req transform.Request) ([]transform.Output, error) { + if t.renderer == nil { + t.renderer = goldmark.New() + } + var outputs []transform.Output + for _, file := range req.SourceBundle.Manifest.Files { + if !strings.HasSuffix(file.Path, ".md") { + continue + } + sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, file.Path) + if err != nil { + return nil, err + } + data, err := req.SourceBackend.ReadFile(ctx, sourcePath) + if err != nil { + return nil, fmt.Errorf("read markdown source %q: %w", file.Path, err) + } + var rendered bytes.Buffer + if err := t.renderer.Convert(data, &rendered); err != nil { + return nil, fmt.Errorf("render markdown source %q: %w", file.Path, err) + } + html := wrapHTML(rendered.Bytes()) + outputPath := strings.TrimSuffix(file.Path, ".md") + ".html" + outputs = append(outputs, transform.Output{ + Path: outputPath, + SourcePath: file.Path, + Transform: transform.MarkdownToHTML, + Data: html, + SHA256: bundle.FileDigest(html), + Size: int64(len(html)), + }) + } + return outputs, nil +} diff --git a/internal/transform/markdown/markdown_test.go b/internal/transform/markdown/markdown_test.go new file mode 100644 index 0000000..9dce626 --- /dev/null +++ b/internal/transform/markdown/markdown_test.go @@ -0,0 +1,113 @@ +package markdown + +import ( + "context" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/distributor/internal/bundle" + "gitea.maximumdirect.net/eric/distributor/internal/storage" + "gitea.maximumdirect.net/eric/distributor/internal/storage/fake" + "gitea.maximumdirect.net/eric/distributor/internal/transform" +) + +func TestGenerateMarkdownSidecar(t *testing.T) { + backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n") + outputs, err := New().Generate(context.Background(), transform.Request{SourceBackend: backend, SourceBundle: sourceBundle}) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + if got, want := len(outputs), 1; got != want { + t.Fatalf("output count = %d, want %d", got, want) + } + output := outputs[0] + if output.Path != "report.html" { + t.Fatalf("path = %q, want report.html", output.Path) + } + if output.SourcePath != "report.md" || output.Transform != transform.MarkdownToHTML { + t.Fatalf("metadata = %#v", output) + } + html := string(output.Data) + for _, want := range []string{"", "

Title

", "

Hello.

"} { + if !strings.Contains(html, want) { + t.Fatalf("html = %q, want substring %q", html, want) + } + } + if output.SHA256 != bundle.FileDigest(output.Data) || output.Size != int64(len(output.Data)) { + t.Fatalf("digest/size metadata = %s/%d", output.SHA256, output.Size) + } +} + +func TestGenerateIgnoresNonMarkdown(t *testing.T) { + backend := fake.New() + if _, err := backend.WriteFile(context.Background(), "summary.txt", []byte("Summary\n"), storage.WriteOptions{}); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + sourceBundle := bundle.Bundle{Manifest: bundle.Manifest{ + SchemaVersion: 1, + ID: "bundle", + Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC), + Files: []bundle.ManifestFile{{Path: "summary.txt", SHA256: bundle.FileDigest([]byte("Summary\n")), Size: 8}}, + }} + outputs, err := New().Generate(context.Background(), transform.Request{SourceBackend: backend, SourceBundle: sourceBundle}) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + if len(outputs) != 0 { + t.Fatalf("outputs = %#v, want none", outputs) + } +} + +func TestGenerateDoesNotPassRawHTML(t *testing.T) { + backend, sourceBundle := markdownFixture(t, "# Title\n\n\n") + outputs, err := New().Generate(context.Background(), transform.Request{SourceBackend: backend, SourceBundle: sourceBundle}) + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + html := string(outputs[0].Data) + if strings.Contains(html, "