diff --git a/docs/config.md b/docs/config.md index 76df4a1..5b669ca 100644 --- a/docs/config.md +++ b/docs/config.md @@ -54,7 +54,7 @@ pipelines: ## HTML Publication -To publish generated HTML from Markdown files: +To publish generated sidecar HTML from Markdown files: ```yaml publish: @@ -68,6 +68,21 @@ transform: Sidecar generation writes `report.html` for `report.md`. It does not mutate the source bundle. +To publish a single Markdown file as `index.html`: + +```yaml +publish: + source: false + html: true +transform: + markdown_to_html: + enabled: true + mode: index + input: report.md +``` + +When `mode: index` omits `input`, the source manifest must list exactly one Markdown file. + ## Reference Top level: @@ -164,7 +179,15 @@ Publish policy: - `publish.source`: publish source artifacts. - `publish.html`: publish generated HTML artifacts from Markdown source files. -At least one output type must be enabled. When `publish.html` is true, `transform.markdown_to_html.enabled` must be `true` and `transform.markdown_to_html.mode` must be `sidecar`. +At least one output type must be enabled. When `publish.html` is true, `transform.markdown_to_html.enabled` must be `true`. + +Markdown-to-HTML transform: + +- `transform.markdown_to_html.enabled`: enables Markdown-to-HTML generation for destinations with `publish.html: true`. +- `transform.markdown_to_html.mode`: optional; defaults to `sidecar`. Accepted values are `sidecar` and `index`. +- `transform.markdown_to_html.input`: optional source manifest path for `index` mode. It must identify a listed Markdown file. + +`sidecar` mode renders each manifest-listed `.md` file to a same-directory `.html` output. `index` mode renders one selected Markdown file to `index.html` at the destination bundle path. Enabled Markdown-to-HTML config is rejected when `publish.html` is false, and `input` is valid only with `mode: index`. Transfer policy: @@ -184,6 +207,7 @@ Defaults are applied after YAML decoding and before validation: - SSH `host_key_policy: accept-new` - S3 `region: us-east-1` - S3 `force_path_style: true` +- `transform.markdown_to_html.mode: sidecar` when a Markdown-to-HTML transform block is present and mode is omitted - `publish.source: true` - `publish.html: false` - `transfer.on_destination_same: skip` @@ -216,6 +240,7 @@ Maintained examples live under [examples](../examples/): - `local-to-local.yml`: minimal local config. - `local-publish.yml`: runnable local source publication. - `local-html.yml`: runnable local HTML publication. +- `local-index.yml`: runnable local `index.html` publication. - `fan-out.yml`: runnable local fan-out publication to source and HTML destinations. - `ssh-destination.yml`: environment-gated local-to-SSH publication example. - `s3-destination.yml`: environment-gated local-to-S3 publication example. diff --git a/docs/integrations/markdown.md b/docs/integrations/markdown.md index a15d883..ffce1c6 100644 --- a/docs/integrations/markdown.md +++ b/docs/integrations/markdown.md @@ -12,7 +12,12 @@ Rendering uses `github.com/yuin/goldmark`. The exact dependency version is pinne `internal/transform/markdown.New` constructs the renderer with `goldmark.New()` and no project-specific extensions or renderer options. -For each source bundle file ending in `.md`, the transform reads the Markdown source and generates an HTML sidecar in the same logical directory. The output path replaces the `.md` suffix with `.html`, so `report.md` produces `report.html`. Non-Markdown source files produce no Markdown outputs. +The transform supports two output modes: + +- `sidecar`: reads each source bundle file ending in `.md` and generates an HTML sidecar in the same logical directory. The output path replaces the `.md` suffix with `.html`, so `report.md` produces `report.html`. Non-Markdown source files produce no Markdown outputs. +- `index`: renders one selected Markdown source to `index.html` at the destination bundle path. + +In `index` mode, `transform.markdown_to_html.input` can name the source manifest path to render. If `input` is omitted, the manifest must list exactly one Markdown file. The selected input must be a safe relative source path, must be listed in the source manifest, and must end in `.md`. Raw HTML embedded in Markdown is not passed through by the current renderer behavior. Tests allow Goldmark's disabled-or-escaped raw HTML output forms and reject literal script tags in generated HTML. @@ -42,7 +47,7 @@ Generated outputs record: Markdown rendering does not mutate source bundles, publish files, write `.distributor.json`, select outputs, or choose transfer actions. Publish planning decides whether generated HTML is selected for a destination. -Only sidecar output mode is supported for current behavior. +Publish planning chooses the configured mode and input for each destination. Markdown rendering does not inspect destinations, publish files, write `.distributor.json`, or choose transfer actions. ## Tests @@ -52,4 +57,4 @@ Before changing Markdown renderer behavior, inspect and run: go test ./internal/transform/markdown ``` -The tests cover sidecar naming, ignored non-Markdown files, raw HTML handling, deterministic output, digest metadata, and size metadata. +The tests cover sidecar naming, index input selection, ignored non-Markdown files, raw HTML handling, deterministic output, digest metadata, and size metadata. diff --git a/docs/internal/config.md b/docs/internal/config.md index 2a4a0da..0ce3c72 100644 --- a/docs/internal/config.md +++ b/docs/internal/config.md @@ -24,6 +24,7 @@ Defaults are applied in `ApplyDefaults`: - SSH backend `port` defaults to `22`; - SSH backend `host_key_policy` defaults to `accept-new`; - destination publish policy defaults to source output only; +- Markdown-to-HTML mode defaults to `sidecar` when a transform block is present and mode is omitted; - `transfer.on_destination_same` defaults to `skip`; - `transfer.on_destination_older` defaults to `replace`; - `transfer.on_destination_newer` defaults to `skip`; @@ -35,7 +36,7 @@ Validation requires at least one pipeline, slug-like unique pipeline ids, one so Transfer validation accepts `replace` for `on_destination_newer` and `on_conflict`, but publish planning honors those destructive actions only when the current run explicitly requests force. -`ValidatePublishTransformPolicy` is shared with publish planning so destination policy combinations are checked consistently. Publishing HTML requires an enabled Markdown-to-HTML transform in `sidecar` mode. A publish policy must select source output, HTML output, or both. +`ValidatePublishTransformPolicy` is shared with publish planning so destination policy combinations are checked consistently. Publishing HTML requires an enabled Markdown-to-HTML transform in `sidecar` or `index` mode. Enabled Markdown-to-HTML config is rejected when `publish.html` is false. `input` is accepted only for enabled `index` mode. A publish policy must select source output, HTML output, or both. ## Executable support boundary diff --git a/docs/internal/publish.md b/docs/internal/publish.md index d92f0d8..7a9522c 100644 --- a/docs/internal/publish.md +++ b/docs/internal/publish.md @@ -22,7 +22,7 @@ Execution fails if a write, delete, state serialization, or context check fails. ## Boundaries -The current implementation publishes source files and Markdown-to-HTML sidecar outputs. Backend behavior is supplied through `internal/storage`; app runtime currently supplies local, SSH, and S3 backends. +The current implementation publishes source files and Markdown-to-HTML outputs. Markdown sidecar mode writes same-directory `.html` outputs, and Markdown index mode writes `index.html`. Backend behavior is supplied through `internal/storage`; app runtime currently supplies local, SSH, and S3 backends. The package uses `internal/state` for destination comparison, `internal/storage` for IO, and the shared `internal/config` publish/transform policy helper for request validation. It resolves transforms through a narrow resolver supplied by the caller; concrete transform registration is owned by the app layer. It does not parse CLI flags or load config files. diff --git a/docs/internal/transform.md b/docs/internal/transform.md index 0de180b..58c9a21 100644 --- a/docs/internal/transform.md +++ b/docs/internal/transform.md @@ -2,11 +2,11 @@ ## Purpose -`internal/transform` defines generated publication artifacts. `internal/transform/markdown` implements Markdown-to-HTML sidecar generation. +`internal/transform` defines generated publication artifacts. `internal/transform/markdown` implements Markdown-to-HTML generation. ## Inputs and outputs -Inputs are a validated source bundle and source backend. Outputs include destination path, source path, transform id, generated bytes, SHA-256, and size. +Inputs are a validated source bundle, source backend, and transform options supplied by publish planning. Outputs include destination path, source path, transform id, generated bytes, SHA-256, and size. ## Registry @@ -14,7 +14,7 @@ Inputs are a validated source bundle and source backend. Outputs include destina ## Markdown behavior -Markdown files ending in `.md` generate `.html` files in the same logical directory. Non-Markdown files do not generate outputs. Raw HTML embedded in Markdown is not passed through by the renderer. +Markdown sidecar mode renders files ending in `.md` to `.html` files in the same logical directory. Markdown index mode renders one selected manifest-listed Markdown file to `index.html`. Non-Markdown files do not generate sidecar outputs. Raw HTML embedded in Markdown is not passed through by the renderer. Generated HTML is deterministic for the same source content and transform configuration. @@ -22,7 +22,7 @@ See `docs/integrations/markdown.md` for the Goldmark integration contract. ## Failure behavior -Transform resolution fails when a requested transform id is not registered. Markdown rendering fails when the source file cannot be read or rendered. Publish planning fails when HTML output is requested and the selected transform produces no outputs for a bundle. +Transform resolution fails when a requested transform id is not registered. Markdown rendering fails when the source file cannot be read or rendered. Index input selection fails when the configured input is unsafe, not listed, not Markdown, or when no configured input can be inferred from exactly one manifest-listed Markdown file. Publish planning fails when HTML output is requested and the selected transform produces no outputs for a bundle. ## Boundaries @@ -42,5 +42,6 @@ Before changing transform behavior, inspect tests under: - Source bundle files are never mutated by transforms. - Generated outputs record destination path, source path, transform id, SHA-256, and size. - Markdown sidecar naming changes only the `.md` extension to `.html`. +- Markdown index mode always writes `index.html`. - Non-Markdown source files do not generate Markdown outputs. - Transform registration stays outside publish planning. diff --git a/docs/operations.md b/docs/operations.md index 6aeba0b..bbfd259 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -26,6 +26,12 @@ Run the local HTML publication: go run ./cmd/distributor run --config examples/local-html.yml ``` +Run the local `index.html` publication: + +```sh +go run ./cmd/distributor run --config examples/local-index.yml +``` + Preview local fan-out publication: ```sh @@ -115,6 +121,37 @@ go run ./cmd/distributor validate Use repeated `--file` flags to preserve a specific file order. If no `--file` flags are provided, the command scans the bundle directory recursively using the same filtering rules as `pkg/bundle.BuildManifest`. +## Static HTML Publication + +Markdown-to-HTML publication can write sidecar files or a fixed `index.html`. + +Use sidecar mode when each Markdown source should keep a matching HTML filename: + +```yaml +publish: + source: false + html: true +transform: + markdown_to_html: + enabled: true + mode: sidecar +``` + +Use index mode for static-site destinations that should serve a bundle through `index.html`: + +```yaml +publish: + source: false + html: true +transform: + markdown_to_html: + enabled: true + mode: index + input: report.md +``` + +If `input` is omitted in index mode, the source manifest must list exactly one Markdown file. Generated HTML is recorded in `.distributor.json` with `kind: generated`, `source_path`, `transform: markdown_to_html`, digest, and size metadata. + ## Source Validation and Inspection `validate` and `inspect` can operate on a local path or on one configured pipeline source. Configured source mode requires both `--config` and `--pipeline`; it loads the normal config, resolves `secrets.directory`, opens only the selected source backend, and does not open any destinations. diff --git a/docs/policy/development.md b/docs/policy/development.md index e4de34a..a120dee 100644 --- a/docs/policy/development.md +++ b/docs/policy/development.md @@ -19,7 +19,7 @@ Use it with `docs/policy/architecture.md` and `docs/policy/documentation.md`. - `internal/storage/fake`: in-memory backend for tests. - `internal/publish`: destination inspection, output planning, reconciliation, execution, managed cleanup, and explicit forced replacement. - `internal/transform`: transform interface and registry. -- `internal/transform/markdown`: Markdown-to-HTML sidecar transform. +- `internal/transform/markdown`: Markdown-to-HTML transform. - `internal/notify`: notification interface and current no-op notifier. - `internal/testutil`: shared test fixtures. Production code must not import this package. - `docs`: current user, operator, policy, internal, and roadmap documentation. @@ -128,8 +128,9 @@ When adding or changing commands or flags: 3. Add or update CLI tests in `internal/cli`. 4. Update `docs/cli.md` if syntax, flags, output expectations, or workflows change. -`validate` and `inspect` are local path commands. `run` loads configured -pipelines and executes local, SSH, and S3 backends. +`validate` and `inspect` support a local path shortcut and configured +source-only diagnostics. `run` loads configured pipelines and executes local, +SSH, and S3 backends. ## Storage Backends diff --git a/examples/local-index.yml b/examples/local-index.yml new file mode 100644 index 0000000..e7266ce --- /dev/null +++ b/examples/local-index.yml @@ -0,0 +1,16 @@ +pipelines: + - id: example-index-bundle + source: + backend: local + path: examples/source-bundle + destinations: + - id: local-index + backend: local + path: workspace/published/index-bundle + publish: + source: false + html: true + transform: + markdown_to_html: + enabled: true + mode: index diff --git a/internal/app/run_test.go b/internal/app/run_test.go index d77feb5..5f0a765 100644 --- a/internal/app/run_test.go +++ b/internal/app/run_test.go @@ -372,6 +372,59 @@ func TestRunPublishesHTMLOnly(t *testing.T) { } } +func TestRunPublishesHTMLIndexWithExplicitInput(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{ + ExtraFiles: []testFile{{Path: "notes.md", Data: "# Notes\nHidden.\n"}}, + }) + + err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, "report.md")}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + assertFileContains(t, filepath.Join(destinationRoot, "index.html"), "

Report

") + if _, err := os.Stat(filepath.Join(destinationRoot, "report.html")); !os.IsNotExist(err) { + t.Fatalf("report.html stat error = %v, want not exist", err) + } + destinationState := readStateFile(t, filepath.Join(destinationRoot, storage.StateFileName)) + if got, want := len(destinationState.Outputs), 1; got != want { + t.Fatalf("state output count = %d, want %d", got, want) + } + output := destinationState.Outputs[0] + if output.Kind != state.OutputKindGenerated || output.Transform != "markdown_to_html" || output.Path != "index.html" || output.SourcePath != "report.md" { + t.Fatalf("generated output metadata = %#v", output) + } +} + +func TestRunPublishesHTMLIndexWithSingleMarkdownFallback(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) + + err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, "")}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + assertFileContains(t, filepath.Join(destinationRoot, "index.html"), "

Report

") +} + +func TestRunFailsIndexModeWithAmbiguousMarkdownInput(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{ + ExtraFiles: []testFile{{Path: "notes.md", Data: "# Notes\n"}}, + }) + + err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, "")}) + if err == nil || !strings.Contains(err.Error(), "multiple markdown source files") { + t.Fatalf("Run() error = %v, want ambiguous input error", err) + } + if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 { + t.Fatalf("destination entries = %v err=%v, want empty", entries, err) + } +} + func TestRunPublishesSourceAndHTML(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() @@ -427,6 +480,22 @@ func TestRunFailsOnOutputPathCollision(t *testing.T) { } } +func TestRunFailsOnIndexOutputPathCollision(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{ + ExtraFiles: []testFile{{Path: "index.html", Data: "

source index

\n"}}, + }) + + err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, true, true, config.TransformModeIndex, "report.md")}) + if err == nil || !strings.Contains(err.Error(), "destination output path collision") { + t.Fatalf("Run() error = %v, want collision", err) + } + if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 { + t.Fatalf("destination entries = %v err=%v, want empty", entries, err) + } +} + func TestRunDryRunReportsGeneratedOutputs(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() @@ -446,6 +515,73 @@ func TestRunDryRunReportsGeneratedOutputs(t *testing.T) { } } +func TestRunDryRunReportsIndexOutputWithoutWriting(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) + + var stdout bytes.Buffer + err := Run(context.Background(), RunOptions{ + ConfigPath: writeLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, ""), + DryRun: true, + Stdout: &stdout, + }) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if !strings.Contains(stdout.String(), "outputs=index.html") { + t.Fatalf("stdout = %q, want index output path", stdout.String()) + } + if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 { + t.Fatalf("destination entries = %v err=%v, want empty", entries, err) + } +} + +func TestRunSourceOnlyDoesNotWriteIndexOutput(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{}) + + err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, true, false, config.TransformModeIndex, "")}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n") + if _, err := os.Stat(filepath.Join(destinationRoot, "index.html")); !os.IsNotExist(err) { + t.Fatalf("index.html stat error = %v, want not exist", err) + } +} + +func TestRunReplacesHTMLIndexOutput(t *testing.T) { + sourceRoot := t.TempDir() + destinationRoot := t.TempDir() + writeSourceBundle(t, sourceRoot, "", testBundleOptions{ + Created: testutil.DefaultCreated, + Files: []testFile{ + {Path: "report.md", Data: "# Report\nOld.\n"}, + {Path: "summary.txt", Data: "Summary\n"}, + }, + }) + configPath := writeLocalConfigWithMarkdownTransform(t, sourceRoot, destinationRoot, false, true, config.TransformModeIndex, "") + if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil { + t.Fatalf("first Run() error = %v", err) + } + assertFileContains(t, filepath.Join(destinationRoot, "index.html"), "

Old.

") + + writeSourceBundle(t, sourceRoot, "", testBundleOptions{ + Created: testutil.DefaultCreated.Add(time.Hour), + Files: []testFile{ + {Path: "report.md", Data: "# Report\nNew.\n"}, + {Path: "summary.txt", Data: "Summary\n"}, + }, + }) + + if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil { + t.Fatalf("second Run() error = %v", err) + } + assertFileContains(t, filepath.Join(destinationRoot, "index.html"), "

New.

") +} + func TestRunSkipsWhenDestinationStateMatches(t *testing.T) { sourceRoot := t.TempDir() destinationRoot := t.TempDir() @@ -727,6 +863,7 @@ func TestRunDryRunDoesNotWrite(t *testing.T) { type testBundleOptions struct { ID string Created time.Time + Files []testFile ExtraFiles []testFile } @@ -737,6 +874,13 @@ type testFile struct { func writeSourceBundle(t *testing.T, root, relative string, opts testBundleOptions) bundle.Manifest { t.Helper() + var files []testutil.SourceFile + if opts.Files != nil { + files = make([]testutil.SourceFile, 0, len(opts.Files)) + for _, file := range opts.Files { + files = append(files, testutil.SourceFile{Path: file.Path, Data: file.Data}) + } + } extraFiles := make([]testutil.SourceFile, 0, len(opts.ExtraFiles)) for _, file := range opts.ExtraFiles { extraFiles = append(extraFiles, testutil.SourceFile{Path: file.Path, Data: file.Data}) @@ -744,6 +888,7 @@ func writeSourceBundle(t *testing.T, root, relative string, opts testBundleOptio return testutil.WriteSourceBundle(t, root, relative, testutil.BundleOptions{ ID: opts.ID, Created: opts.Created, + Files: files, ExtraFiles: extraFiles, }) } @@ -779,6 +924,34 @@ pipelines: `) } +func writeLocalConfigWithMarkdownTransform(t *testing.T, sourceRoot, destinationRoot string, publishSource, publishHTML bool, mode, input string) string { + t.Helper() + enabled := publishHTML + inputConfig := "" + if input != "" { + inputConfig = ` + input: ` + input + } + return writeConfigFile(t, ` +pipelines: + - id: reports + source: + backend: local + path: `+sourceRoot+` + destinations: + - id: archive + backend: local + path: `+destinationRoot+` + publish: + source: `+fmt.Sprintf("%t", publishSource)+` + html: `+fmt.Sprintf("%t", publishHTML)+` + transform: + markdown_to_html: + enabled: `+fmt.Sprintf("%t", enabled)+` + mode: `+mode+inputConfig+` +`) +} + func writeFanoutConfig(t *testing.T, sourceRoot, firstDestination, secondDestination string) string { t.Helper() return testutil.WriteFanoutLocalConfig(t, sourceRoot, firstDestination, secondDestination) diff --git a/internal/config/config.go b/internal/config/config.go index 4e9fa7c..d67819a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -77,6 +77,7 @@ type Transform struct { type MarkdownToHTML struct { Enabled bool `yaml:"enabled"` Mode string `yaml:"mode"` + Input string `yaml:"input"` } type TransferPolicy struct { diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 39f9c52..d2944c6 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -1,5 +1,7 @@ package config +import "gitea.maximumdirect.net/eric/distributor/internal/transform" + const DefaultConfigPath = "/usr/local/etc/distributor/config.yml" const ( @@ -19,7 +21,8 @@ const ( ) const ( - TransformModeSidecar = "sidecar" + TransformModeSidecar = transform.MarkdownModeSidecar + TransformModeIndex = transform.MarkdownModeIndex ) const DefaultS3Region = "us-east-1" @@ -37,6 +40,9 @@ func ApplyDefaults(cfg *Config) { if destination.Publish == nil { destination.Publish = &PublishPolicy{Source: true} } + if destination.Transform.MarkdownToHTML != nil && destination.Transform.MarkdownToHTML.Mode == "" { + destination.Transform.MarkdownToHTML.Mode = TransformModeSidecar + } if destination.Transfer.OnDestinationSame == "" { destination.Transfer.OnDestinationSame = TransferActionSkip } diff --git a/internal/config/load_test.go b/internal/config/load_test.go index 4091fa6..88a0cfb 100644 --- a/internal/config/load_test.go +++ b/internal/config/load_test.go @@ -94,6 +94,58 @@ pipelines: } } +func TestLoadFileValidMarkdownIndexConfig(t *testing.T) { + cfg := loadConfig(t, ` +pipelines: + - id: static-site + source: + backend: local + path: /var/spool/reports + destinations: + - id: web + backend: local + path: /srv/www/reports + publish: + source: false + html: true + transform: + markdown_to_html: + enabled: true + mode: index + input: report.md +`) + + markdown := cfg.Pipelines[0].Destinations[0].Transform.MarkdownToHTML + if markdown == nil || markdown.Mode != TransformModeIndex || markdown.Input != "report.md" { + t.Fatalf("markdown config = %#v, want index input", markdown) + } +} + +func TestLoadFileDefaultsMarkdownModeToSidecar(t *testing.T) { + cfg := loadConfig(t, ` +pipelines: + - id: static-site + source: + backend: local + path: /var/spool/reports + destinations: + - id: web + backend: local + path: /srv/www/reports + publish: + source: false + html: true + transform: + markdown_to_html: + enabled: true +`) + + markdown := cfg.Pipelines[0].Destinations[0].Transform.MarkdownToHTML + if markdown == nil || markdown.Mode != TransformModeSidecar { + t.Fatalf("markdown mode = %#v, want sidecar default", markdown) + } +} + func TestLoadFileValidBackendConfigs(t *testing.T) { tests := map[string]string{ "local": ` @@ -464,6 +516,7 @@ func TestExampleConfigsLoad(t *testing.T) { "../../examples/local-to-local.yml", "../../examples/local-publish.yml", "../../examples/local-html.yml", + "../../examples/local-index.yml", "../../examples/fan-out.yml", "../../examples/ssh-destination.yml", "../../examples/s3-destination.yml", diff --git a/internal/config/validate.go b/internal/config/validate.go index 4abf7d5..6bb9d0e 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -149,18 +149,25 @@ func ValidatePublishTransformPolicy(publish PublishPolicy, transform Transform) if transform.MarkdownToHTML == nil { return nil } + mode := transform.MarkdownToHTML.Mode + if mode == "" { + mode = TransformModeSidecar + } + if mode != TransformModeSidecar && mode != TransformModeIndex { + return fmt.Errorf("transform.markdown_to_html.mode must be %s or %s", TransformModeSidecar, TransformModeIndex) + } + if transform.MarkdownToHTML.Input != "" && !transform.MarkdownToHTML.Enabled { + return fmt.Errorf("transform.markdown_to_html.input requires transform.markdown_to_html.enabled to be true") + } + if transform.MarkdownToHTML.Input != "" && mode != TransformModeIndex { + return fmt.Errorf("transform.markdown_to_html.input is only valid when mode is %s", TransformModeIndex) + } + if transform.MarkdownToHTML.Enabled && !publish.HTML { + return fmt.Errorf("transform.markdown_to_html.enabled requires publish.html to be true") + } if publish.HTML && !transform.MarkdownToHTML.Enabled { return fmt.Errorf("transform.markdown_to_html.enabled must be true when publish.html is true") } - if publish.HTML && transform.MarkdownToHTML.Mode != TransformModeSidecar { - return fmt.Errorf("transform.markdown_to_html.mode must be %s", TransformModeSidecar) - } - if transform.MarkdownToHTML.Enabled && transform.MarkdownToHTML.Mode != TransformModeSidecar { - return fmt.Errorf("transform.markdown_to_html.mode must be %s", TransformModeSidecar) - } - if !transform.MarkdownToHTML.Enabled && transform.MarkdownToHTML.Mode != "" && transform.MarkdownToHTML.Mode != TransformModeSidecar { - return fmt.Errorf("transform.markdown_to_html.mode must be %s", TransformModeSidecar) - } return nil } diff --git a/internal/config/validate_test.go b/internal/config/validate_test.go index c3c1c72..dc16d33 100644 --- a/internal/config/validate_test.go +++ b/internal/config/validate_test.go @@ -91,6 +91,30 @@ func publishTransformPolicyCases() []publishTransformPolicyCase { Mode: TransformModeSidecar, }}, }, + { + name: "html only default mode allowed", + publish: PublishPolicy{HTML: true}, + transform: Transform{MarkdownToHTML: &MarkdownToHTML{ + Enabled: true, + }}, + }, + { + name: "html only index allowed", + publish: PublishPolicy{HTML: true}, + transform: Transform{MarkdownToHTML: &MarkdownToHTML{ + Enabled: true, + Mode: TransformModeIndex, + }}, + }, + { + name: "html only index input allowed", + publish: PublishPolicy{HTML: true}, + transform: Transform{MarkdownToHTML: &MarkdownToHTML{ + Enabled: true, + Mode: TransformModeIndex, + Input: "report.md", + }}, + }, { name: "source and html sidecar allowed", publish: PublishPolicy{Source: true, HTML: true}, @@ -127,6 +151,15 @@ func publishTransformPolicyCases() []publishTransformPolicyCase { }}, wantErr: true, }, + { + name: "source only enabled transform rejected", + publish: PublishPolicy{Source: true}, + transform: Transform{MarkdownToHTML: &MarkdownToHTML{ + Enabled: true, + Mode: TransformModeSidecar, + }}, + wantErr: true, + }, { name: "enabled markdown wrong mode rejected", publish: PublishPolicy{Source: true}, @@ -136,6 +169,16 @@ func publishTransformPolicyCases() []publishTransformPolicyCase { }}, wantErr: true, }, + { + name: "sidecar input rejected", + publish: PublishPolicy{HTML: true}, + transform: Transform{MarkdownToHTML: &MarkdownToHTML{ + Enabled: true, + Mode: TransformModeSidecar, + Input: "report.md", + }}, + wantErr: true, + }, { name: "disabled markdown empty mode allowed", publish: PublishPolicy{Source: true}, @@ -151,6 +194,24 @@ func publishTransformPolicyCases() []publishTransformPolicyCase { Mode: TransformModeSidecar, }}, }, + { + name: "disabled markdown index mode allowed", + publish: PublishPolicy{Source: true}, + transform: Transform{MarkdownToHTML: &MarkdownToHTML{ + Enabled: false, + Mode: TransformModeIndex, + }}, + }, + { + name: "disabled markdown input rejected", + publish: PublishPolicy{Source: true}, + transform: Transform{MarkdownToHTML: &MarkdownToHTML{ + Enabled: false, + Mode: TransformModeIndex, + Input: "report.md", + }}, + wantErr: true, + }, { name: "disabled markdown wrong mode rejected", publish: PublishPolicy{Source: true}, diff --git a/internal/publish/output.go b/internal/publish/output.go index ac9a414..6a96bdf 100644 --- a/internal/publish/output.go +++ b/internal/publish/output.go @@ -26,6 +26,10 @@ func PlanOutputs(ctx context.Context, req Request) ([]Output, error) { generatedOutputs, err := transformer.Generate(ctx, transform.Request{ SourceBundle: req.SourceBundle, SourceBackend: req.SourceBackend, + Markdown: transform.MarkdownOptions{ + Mode: req.Transform.MarkdownToHTML.Mode, + Input: req.Transform.MarkdownToHTML.Input, + }, }) if err != nil { return nil, err diff --git a/internal/publish/output_test.go b/internal/publish/output_test.go index 6075906..b0172ff 100644 --- a/internal/publish/output_test.go +++ b/internal/publish/output_test.go @@ -103,6 +103,35 @@ func TestPlanOutputsUsesRegisteredTransformer(t *testing.T) { } } +func TestPlanOutputsPassesMarkdownOptions(t *testing.T) { + data := []byte("

Generated

\n") + transformer := &recordingTransformer{outputs: []transform.Output{{ + Path: "index.html", + SourcePath: "report.md", + Transform: transform.MarkdownToHTML, + Data: data, + SHA256: bundle.FileDigest(data), + Size: int64(len(data)), + }}} + + _, err := PlanOutputs(context.Background(), Request{ + Publish: config.PublishPolicy{HTML: true}, + Transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{ + Enabled: true, + Mode: config.TransformModeIndex, + Input: "report.md", + }}, + Transformers: testResolver{transform.MarkdownToHTML: transformer}, + }) + + if err != nil { + t.Fatalf("PlanOutputs() error = %v", err) + } + if transformer.request.Markdown.Mode != config.TransformModeIndex || transformer.request.Markdown.Input != "report.md" { + t.Fatalf("markdown options = %#v, want index/report.md", transformer.request.Markdown) + } +} + func TestBuildRejectsHTMLWithoutTransform(t *testing.T) { sourceBackend := fake.New() destinationBackend := fake.New() @@ -148,6 +177,14 @@ func TestValidateRequestChecksPublishTransformPolicy(t *testing.T) { Mode: config.TransformModeSidecar, }}, }, + { + name: "html only index allowed", + publish: config.PublishPolicy{HTML: true}, + transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{ + Enabled: true, + Mode: config.TransformModeIndex, + }}, + }, { name: "source and html sidecar allowed", publish: config.PublishPolicy{Source: true, HTML: true}, @@ -184,6 +221,15 @@ func TestValidateRequestChecksPublishTransformPolicy(t *testing.T) { }}, wantErr: true, }, + { + name: "source only enabled transform rejected", + publish: config.PublishPolicy{Source: true}, + transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{ + Enabled: true, + Mode: config.TransformModeSidecar, + }}, + wantErr: true, + }, { name: "enabled markdown wrong mode rejected", publish: config.PublishPolicy{Source: true}, @@ -253,3 +299,14 @@ type testTransformer struct { func (t testTransformer) Generate(context.Context, transform.Request) ([]transform.Output, error) { return t.outputs, t.err } + +type recordingTransformer struct { + outputs []transform.Output + request transform.Request + err error +} + +func (t *recordingTransformer) Generate(_ context.Context, req transform.Request) ([]transform.Output, error) { + t.request = req + return t.outputs, t.err +} diff --git a/internal/transform/markdown/markdown.go b/internal/transform/markdown/markdown.go index 89bd13d..8bfde9b 100644 --- a/internal/transform/markdown/markdown.go +++ b/internal/transform/markdown/markdown.go @@ -25,24 +25,26 @@ func (t *Transformer) Generate(ctx context.Context, req transform.Request) ([]tr if t.renderer == nil { t.renderer = goldmark.New() } + switch markdownMode(req.Markdown.Mode) { + case transform.MarkdownModeSidecar: + return t.generateSidecars(ctx, req) + case transform.MarkdownModeIndex: + return t.generateIndex(ctx, req) + default: + return nil, fmt.Errorf("markdown mode %q is not supported", req.Markdown.Mode) + } +} + +func (t *Transformer) generateSidecars(ctx context.Context, req transform.Request) ([]transform.Output, error) { var outputs []transform.Output for _, file := range req.SourceBundle.Manifest.Files { if !strings.HasSuffix(file.Path, ".md") { continue } - sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, file.Path) + html, err := t.render(ctx, req, file.Path) if err != nil { return nil, err } - data, err := req.SourceBackend.ReadFile(ctx, sourcePath) - if err != nil { - return nil, fmt.Errorf("read markdown source %q: %w", file.Path, err) - } - var rendered bytes.Buffer - if err := t.renderer.Convert(data, &rendered); err != nil { - return nil, fmt.Errorf("render markdown source %q: %w", file.Path, err) - } - html := wrapHTML(rendered.Bytes()) outputPath := strings.TrimSuffix(file.Path, ".md") + ".html" outputs = append(outputs, transform.Output{ Path: outputPath, @@ -55,3 +57,78 @@ func (t *Transformer) Generate(ctx context.Context, req transform.Request) ([]tr } return outputs, nil } + +func (t *Transformer) generateIndex(ctx context.Context, req transform.Request) ([]transform.Output, error) { + input, err := selectIndexInput(req) + if err != nil { + return nil, err + } + html, err := t.render(ctx, req, input) + if err != nil { + return nil, err + } + return []transform.Output{{ + Path: "index.html", + SourcePath: input, + Transform: transform.MarkdownToHTML, + Data: html, + SHA256: bundle.FileDigest(html), + Size: int64(len(html)), + }}, nil +} + +func selectIndexInput(req transform.Request) (string, error) { + if req.Markdown.Input != "" { + if err := storage.ValidatePath(req.Markdown.Input); err != nil { + return "", fmt.Errorf("markdown input %q: %w", req.Markdown.Input, err) + } + for _, file := range req.SourceBundle.Manifest.Files { + if file.Path != req.Markdown.Input { + continue + } + if !strings.HasSuffix(file.Path, ".md") { + return "", fmt.Errorf("markdown input %q must end in .md", req.Markdown.Input) + } + return file.Path, nil + } + return "", fmt.Errorf("markdown input %q is not listed in the source manifest", req.Markdown.Input) + } + + var markdownFiles []string + for _, file := range req.SourceBundle.Manifest.Files { + if strings.HasSuffix(file.Path, ".md") { + markdownFiles = append(markdownFiles, file.Path) + } + } + switch len(markdownFiles) { + case 0: + return "", fmt.Errorf("markdown index mode requires one markdown source file or transform.markdown_to_html.input") + case 1: + return markdownFiles[0], nil + default: + return "", fmt.Errorf("markdown index mode found multiple markdown source files; set transform.markdown_to_html.input") + } +} + +func (t *Transformer) render(ctx context.Context, req transform.Request, sourceFile string) ([]byte, error) { + sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, sourceFile) + if err != nil { + return nil, err + } + data, err := req.SourceBackend.ReadFile(ctx, sourcePath) + if err != nil { + return nil, fmt.Errorf("read markdown source %q: %w", sourceFile, err) + } + var rendered bytes.Buffer + if err := t.renderer.Convert(data, &rendered); err != nil { + return nil, fmt.Errorf("render markdown source %q: %w", sourceFile, err) + } + return wrapHTML(rendered.Bytes()), nil +} + +func markdownMode(mode string) string { + if mode == "" { + return transform.MarkdownModeSidecar + } + return mode +} diff --git a/internal/transform/markdown/markdown_test.go b/internal/transform/markdown/markdown_test.go index 55eed36..ff89b91 100644 --- a/internal/transform/markdown/markdown_test.go +++ b/internal/transform/markdown/markdown_test.go @@ -40,6 +40,124 @@ func TestGenerateMarkdownSidecar(t *testing.T) { } } +func TestGenerateMarkdownIndexExplicitInput(t *testing.T) { + backend := fake.New() + sourceBundle := testutil.WriteFakeSourceBundle(t, backend, "", testutil.BundleOptions{ + ID: "bundle", + Files: []testutil.SourceFile{ + {Path: "report.md", Data: "# Report\n"}, + {Path: "notes.md", Data: "# Notes\n"}, + }, + }) + + outputs, err := New().Generate(context.Background(), transform.Request{ + SourceBackend: backend, + SourceBundle: sourceBundle, + Markdown: transform.MarkdownOptions{Mode: transform.MarkdownModeIndex, Input: "notes.md"}, + }) + + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + if got, want := len(outputs), 1; got != want { + t.Fatalf("output count = %d, want %d", got, want) + } + output := outputs[0] + if output.Path != "index.html" || output.SourcePath != "notes.md" || output.Transform != transform.MarkdownToHTML { + t.Fatalf("output metadata = %#v, want index from notes.md", output) + } + if !strings.Contains(string(output.Data), "

Notes

") { + t.Fatalf("html = %q, want notes content", output.Data) + } + if output.SHA256 != bundle.FileDigest(output.Data) || output.Size != int64(len(output.Data)) { + t.Fatalf("digest/size metadata = %s/%d", output.SHA256, output.Size) + } +} + +func TestGenerateMarkdownIndexSelectsOnlyMarkdownFile(t *testing.T) { + backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n") + + outputs, err := New().Generate(context.Background(), transform.Request{ + SourceBackend: backend, + SourceBundle: sourceBundle, + Markdown: transform.MarkdownOptions{Mode: transform.MarkdownModeIndex}, + }) + + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + if got, want := outputs[0].Path, "index.html"; got != want { + t.Fatalf("path = %q, want %q", got, want) + } + if got, want := outputs[0].SourcePath, "report.md"; got != want { + t.Fatalf("source path = %q, want %q", got, want) + } +} + +func TestGenerateMarkdownIndexRejectsAmbiguousInput(t *testing.T) { + backend := fake.New() + sourceBundle := testutil.WriteFakeSourceBundle(t, backend, "", testutil.BundleOptions{ + ID: "bundle", + Files: []testutil.SourceFile{ + {Path: "report.md", Data: "# Report\n"}, + {Path: "notes.md", Data: "# Notes\n"}, + }, + }) + + _, err := New().Generate(context.Background(), transform.Request{ + SourceBackend: backend, + SourceBundle: sourceBundle, + Markdown: transform.MarkdownOptions{Mode: transform.MarkdownModeIndex}, + }) + + if err == nil || !strings.Contains(err.Error(), "multiple markdown source files") { + t.Fatalf("Generate() error = %v, want ambiguous input error", err) + } +} + +func TestGenerateMarkdownIndexRejectsMissingMarkdown(t *testing.T) { + backend := fake.New() + sourceBundle := testutil.WriteFakeSourceBundle(t, backend, "", testutil.BundleOptions{ + ID: "bundle", + Files: []testutil.SourceFile{{Path: "summary.txt", Data: "Summary\n"}}, + }) + + _, err := New().Generate(context.Background(), transform.Request{ + SourceBackend: backend, + SourceBundle: sourceBundle, + Markdown: transform.MarkdownOptions{Mode: transform.MarkdownModeIndex}, + }) + + if err == nil || !strings.Contains(err.Error(), "requires one markdown source file") { + t.Fatalf("Generate() error = %v, want missing markdown error", err) + } +} + +func TestGenerateMarkdownIndexRejectsInvalidExplicitInput(t *testing.T) { + backend, sourceBundle := markdownFixture(t, "# Title\n") + tests := []struct { + name string + input string + wantError string + }{ + {name: "unsafe", input: "../report.md", wantError: "markdown input"}, + {name: "not listed", input: "missing.md", wantError: "not listed"}, + {name: "not markdown", input: "summary.txt", wantError: "must end in .md"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := New().Generate(context.Background(), transform.Request{ + SourceBackend: backend, + SourceBundle: sourceBundle, + Markdown: transform.MarkdownOptions{Mode: transform.MarkdownModeIndex, Input: tt.input}, + }) + if err == nil || !strings.Contains(err.Error(), tt.wantError) { + t.Fatalf("Generate() error = %v, want substring %q", err, tt.wantError) + } + }) + } +} + func TestGenerateIgnoresNonMarkdown(t *testing.T) { backend := fake.New() if _, err := backend.WriteFile(context.Background(), "summary.txt", []byte("Summary\n"), storage.WriteOptions{}); err != nil { diff --git a/internal/transform/plan.go b/internal/transform/plan.go index 3b98ea1..a9fd758 100644 --- a/internal/transform/plan.go +++ b/internal/transform/plan.go @@ -1,3 +1,8 @@ package transform const MarkdownToHTML = "markdown_to_html" + +const ( + MarkdownModeSidecar = "sidecar" + MarkdownModeIndex = "index" +) diff --git a/internal/transform/transform.go b/internal/transform/transform.go index 8c1ffa2..b92a6c4 100644 --- a/internal/transform/transform.go +++ b/internal/transform/transform.go @@ -19,6 +19,12 @@ type Output struct { type Request struct { SourceBundle bundle.Bundle SourceBackend storage.Backend + Markdown MarkdownOptions +} + +type MarkdownOptions struct { + Mode string + Input string } type Transformer interface {