Compare commits
8 Commits
0818733b19
...
78f92154ab
| Author | SHA1 | Date | |
|---|---|---|---|
| 78f92154ab | |||
| 5408f14195 | |||
| e296361042 | |||
| aeee91940d | |||
| 518944e601 | |||
| 3c2f36a6e5 | |||
| 29dbad2967 | |||
| 22d0424232 |
10
README.md
10
README.md
@@ -1,5 +1,11 @@
|
|||||||
# distributor
|
# distributor
|
||||||
|
|
||||||
`distributor` is a planned Go application for validating and publishing manifested Markdown bundles.
|
`distributor` validates and publishes manifested report bundles.
|
||||||
|
|
||||||
Implementation has not started yet. Current design and implementation planning lives under `docs/roadmap/`.
|
Run the local example pipeline:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/local-publish.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
See [docs/cli.md](docs/cli.md), [docs/config.md](docs/config.md), and [docs/operations.md](docs/operations.md) for the implemented CLI, configuration, and operating notes. Current design and implementation planning lives under `docs/roadmap/`.
|
||||||
|
|||||||
12
cmd/distributor/main.go
Normal file
12
cmd/distributor/main.go
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/cli"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
os.Exit(cli.Execute(context.Background(), os.Args[1:], os.Stdout, os.Stderr))
|
||||||
|
}
|
||||||
78
docs/cli.md
Normal file
78
docs/cli.md
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
# Distributor CLI
|
||||||
|
|
||||||
|
## Shortest useful command
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/local-publish.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
This validates and publishes the example source bundle to `workspace/published/source-bundle`.
|
||||||
|
|
||||||
|
## Command overview
|
||||||
|
|
||||||
|
```sh
|
||||||
|
distributor --help
|
||||||
|
distributor version
|
||||||
|
distributor run
|
||||||
|
distributor validate
|
||||||
|
distributor inspect
|
||||||
|
```
|
||||||
|
|
||||||
|
`version` prints the application name and version. The default development version is `dev`; release builds may replace it at build time.
|
||||||
|
|
||||||
|
`run --config <path>` executes configured local-to-local pipelines that publish source files, generated HTML files, or both.
|
||||||
|
|
||||||
|
`run --config <path> --dry-run` discovers source bundles, inspects destination state, and prints planned actions plus a final status summary without writing files.
|
||||||
|
|
||||||
|
`validate <path>` validates a local source bundle directory or a local tree containing source bundles.
|
||||||
|
|
||||||
|
`inspect <path>` validates discovered local source bundles and prints a concise normalized summary.
|
||||||
|
|
||||||
|
Remote backends are not implemented yet.
|
||||||
|
|
||||||
|
## Flag reference
|
||||||
|
|
||||||
|
The root command supports:
|
||||||
|
|
||||||
|
- `--help`, `-h`: print root help.
|
||||||
|
|
||||||
|
Each subcommand supports:
|
||||||
|
|
||||||
|
- `--help`, `-h`: print command-specific help.
|
||||||
|
|
||||||
|
`run` supports:
|
||||||
|
|
||||||
|
- `--config <path>`: config file to load.
|
||||||
|
- `--dry-run`: validate config, print planned actions and final status, and do not publish.
|
||||||
|
|
||||||
|
## Common workflows
|
||||||
|
|
||||||
|
Validate a source bundle:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor validate examples/source-bundle
|
||||||
|
```
|
||||||
|
|
||||||
|
Inspect a source bundle:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor inspect examples/source-bundle
|
||||||
|
```
|
||||||
|
|
||||||
|
Preview local publication without writing:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/local-publish.yml --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Publish the local example:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/local-publish.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Publish the local HTML example:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/local-html.yml
|
||||||
|
```
|
||||||
96
docs/config.md
Normal file
96
docs/config.md
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
# Distributor Configuration
|
||||||
|
|
||||||
|
## Config file location
|
||||||
|
|
||||||
|
`distributor run --config <path>` loads the YAML config at the path provided by `--config`.
|
||||||
|
|
||||||
|
If `--config` is omitted during run, the built-in default path is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/usr/local/etc/distributor/config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
The current implementation supports local-to-local publication of source files, generated HTML files, or both. Remote backends are not implemented yet.
|
||||||
|
|
||||||
|
## Minimal config
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /var/spool/distributor/reports
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /srv/reports/archive
|
||||||
|
```
|
||||||
|
|
||||||
|
This uses the default publish policy of source files only and the default transfer policy.
|
||||||
|
|
||||||
|
## Production-oriented config
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /var/spool/distributor/reports
|
||||||
|
validation:
|
||||||
|
on_digest_mismatch: fail
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /srv/reports/archive
|
||||||
|
publish:
|
||||||
|
source: true
|
||||||
|
html: false
|
||||||
|
transfer:
|
||||||
|
on_destination_same: skip
|
||||||
|
on_destination_older: replace
|
||||||
|
on_destination_newer: skip
|
||||||
|
on_conflict: fail
|
||||||
|
```
|
||||||
|
|
||||||
|
## Reference
|
||||||
|
|
||||||
|
Top level:
|
||||||
|
|
||||||
|
- `pipelines`: required non-empty list.
|
||||||
|
|
||||||
|
Pipeline:
|
||||||
|
|
||||||
|
- `id`: required unique identifier.
|
||||||
|
- `source`: required backend config.
|
||||||
|
- `destinations`: required non-empty destination list.
|
||||||
|
- `validation.on_digest_mismatch`: optional, defaults to `fail`; only `fail` is supported.
|
||||||
|
|
||||||
|
Backends:
|
||||||
|
|
||||||
|
- `local`: requires `path`.
|
||||||
|
- `ssh`: requires `uri` and `path`.
|
||||||
|
- `s3`: requires `endpoint` and `bucket`; supports optional `prefix`, `region`, `force_path_style`, and `credentials`.
|
||||||
|
|
||||||
|
Destination policy:
|
||||||
|
|
||||||
|
- `publish.source`: publish source artifacts.
|
||||||
|
- `publish.html`: publish generated HTML artifacts from Markdown source files.
|
||||||
|
- `transfer.on_destination_same`: `skip` or `fail`, defaults to `skip`.
|
||||||
|
- `transfer.on_destination_older`: `replace` or `fail`, defaults to `replace`.
|
||||||
|
- `transfer.on_destination_newer`: `skip` or `fail`, defaults to `skip`.
|
||||||
|
- `transfer.on_conflict`: only `fail`, defaults to `fail`.
|
||||||
|
|
||||||
|
When `publish.html` is true, `transform.markdown_to_html.enabled: true` and `transform.markdown_to_html.mode: sidecar` are required.
|
||||||
|
|
||||||
|
Markdown-to-HTML sidecar generation writes `report.html` for `report.md` and does not mutate the source bundle.
|
||||||
|
|
||||||
|
## Secrets
|
||||||
|
|
||||||
|
Do not put literal secrets in config files. S3 credentials may refer to environment variable names with:
|
||||||
|
|
||||||
|
- `credentials.access_key_id_env`
|
||||||
|
- `credentials.secret_access_key_env`
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
Maintained examples live under [examples/](../examples/).
|
||||||
39
docs/internal/bundle.md
Normal file
39
docs/internal/bundle.md
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
# Bundles
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`internal/bundle` parses, discovers, and validates source bundles through the storage interface.
|
||||||
|
|
||||||
|
## Inputs and outputs
|
||||||
|
|
||||||
|
Input is a backend-rooted directory tree containing one or more `manifest.json` files. Output is a deterministic list of validated bundles with relative bundle paths and normalized manifest data.
|
||||||
|
|
||||||
|
## Manifest behavior
|
||||||
|
|
||||||
|
The source manifest requires:
|
||||||
|
|
||||||
|
- `schema_version: 1`
|
||||||
|
- `id`
|
||||||
|
- `digest`
|
||||||
|
- `created`
|
||||||
|
- non-empty `files`
|
||||||
|
|
||||||
|
Each file requires `path`, `sha256`, and `size`. Digests must use lowercase `sha256:<64 hex>` format. `created` must parse as RFC3339.
|
||||||
|
|
||||||
|
## Validation
|
||||||
|
|
||||||
|
Bundle validation checks source path safety, duplicate file paths, reserved paths, file existence, regular-file type, file size, per-file SHA-256, and the top-level bundle digest.
|
||||||
|
|
||||||
|
The bundle digest is SHA-256 of a deterministic JSON array of file records in manifest order with fields `path`, `sha256`, and `size`.
|
||||||
|
|
||||||
|
## Discovery
|
||||||
|
|
||||||
|
Discovery walks a storage backend beneath a source root, finds `manifest.json` files, sorts bundle paths lexically, and rejects nested manifests.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
Bundle code uses `internal/storage` and does not import local, SSH, or S3 adapters. CLI local path support is wired in `internal/app`.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
Before changing bundle behavior, inspect tests under `internal/bundle`.
|
||||||
13
docs/internal/notify.md
Normal file
13
docs/internal/notify.md
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
# Notify
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`internal/notify` defines the internal notification interface used by the application runner.
|
||||||
|
|
||||||
|
## Current behavior
|
||||||
|
|
||||||
|
The implemented notifier is a no-op. It is invoked only after a successful publish or replacement. Dry-run, skipped destinations, and failed destinations do not invoke it.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
No external notification adapters are implemented. Notification configuration is not part of the current user-facing config schema.
|
||||||
29
docs/internal/publish.md
Normal file
29
docs/internal/publish.md
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
# Publish
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`internal/publish` plans and executes publication for one validated source bundle and one destination.
|
||||||
|
|
||||||
|
## Inputs and outputs
|
||||||
|
|
||||||
|
Inputs are a source bundle, source backend, destination backend, pipeline id, destination id, publish policy, transfer policy, destination bundle path, and existing destination state.
|
||||||
|
|
||||||
|
Output is a plan with an action, reason, and selected source or generated outputs. Execution writes selected source files, generated files, and `.distributor.json` for publish or replacement actions.
|
||||||
|
|
||||||
|
## Actions
|
||||||
|
|
||||||
|
Supported actions are publish new, replace older destination, skip same source, skip newer destination, fail conflict, and fail unmanaged destination.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
The current implementation publishes source files and Markdown-to-HTML sidecar outputs. Remote backend execution is not implemented.
|
||||||
|
|
||||||
|
The package uses `internal/state` for destination comparison and `internal/storage` for IO. It does not parse CLI flags or load config files.
|
||||||
|
|
||||||
|
## Safety
|
||||||
|
|
||||||
|
Replacement deletes only outputs recorded in existing destination state plus `.distributor.json`. Failed local writes trigger cleanup of outputs written during the failed attempt.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
Before changing publish behavior, inspect tests under `internal/publish` and local run tests under `internal/app`.
|
||||||
40
docs/internal/state.md
Normal file
40
docs/internal/state.md
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
# Destination State
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`internal/state` parses, validates, writes, and compares `.distributor.json` destination state.
|
||||||
|
|
||||||
|
## Inputs and outputs
|
||||||
|
|
||||||
|
Input is JSON destination state plus the current source manifest, pipeline id, destination id, and whether the destination path has unmanaged content. Output is a deterministic comparison outcome and reason.
|
||||||
|
|
||||||
|
## State behavior
|
||||||
|
|
||||||
|
`.distributor.json` requires:
|
||||||
|
|
||||||
|
- `schema_version: 1`
|
||||||
|
- `pipeline_id`
|
||||||
|
- `destination_id`
|
||||||
|
- `published_at`
|
||||||
|
- `source.manifest`
|
||||||
|
- `outputs`
|
||||||
|
|
||||||
|
`distributor_version` is optional diagnostic metadata. `published_at` parses as RFC3339 and distributor-written state serializes it as RFC3339 UTC.
|
||||||
|
|
||||||
|
The embedded `source.manifest` is validated with the same source manifest rules used by `internal/bundle`.
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
|
||||||
|
Each output records `path`, `kind`, `source_path`, `sha256`, and `size`. Supported output kinds are `source` and `generated`. Generated outputs require `transform`.
|
||||||
|
|
||||||
|
## Comparison
|
||||||
|
|
||||||
|
Comparison outcomes cover absent destination state, unmanaged destination content, invalid state, pipeline or destination mismatch, same source manifest, older destination source, newer destination source, same-created digest conflict, and different source id conflict.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
This package does not publish files, delete files, inspect storage backends, or choose transfer policy actions. Publish planning consumes these comparison outcomes later.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
Before changing destination state behavior, inspect tests under `internal/state`.
|
||||||
37
docs/internal/storage.md
Normal file
37
docs/internal/storage.md
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
# Storage
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`internal/storage` defines backend-rooted logical file access for core packages. Callers use slash-separated paths relative to a configured backend root.
|
||||||
|
|
||||||
|
## Inputs and outputs
|
||||||
|
|
||||||
|
The storage interface supports byte reads, stream reads, byte writes, stream writes, exact metadata lookup, traversal, destination emptiness checks, and guarded managed deletion.
|
||||||
|
|
||||||
|
Entries report a logical path, type, and size when available. Entry types are `file`, `directory`, `symlink`, and `other`.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
Core packages should depend on `internal/storage`, not adapter packages. Adapter-specific path handling stays behind backend implementations.
|
||||||
|
|
||||||
|
The local adapter lives in `internal/adapters/local`. The fake backend lives in `internal/storage/fake` for tests and is not registered for runtime use.
|
||||||
|
|
||||||
|
## Paths
|
||||||
|
|
||||||
|
Logical file paths must be non-empty, relative, clean, slash-separated, and must not contain `.` or `..` segments or backslashes. Prefix paths follow the same rules, except an empty prefix means the backend root.
|
||||||
|
|
||||||
|
## Failure behavior
|
||||||
|
|
||||||
|
Storage errors use typed categories such as not found, already exists, invalid path, conflict, permission, temporary, unsupported, and unknown. Callers should use helper predicates rather than matching error strings.
|
||||||
|
|
||||||
|
## Deletion
|
||||||
|
|
||||||
|
Backends expose guarded managed deletion only. `DeleteManagedBundle` may delete listed managed outputs plus `.distributor.json`; it does not provide broad recursive deletion.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
Before changing storage behavior, inspect tests under:
|
||||||
|
|
||||||
|
- `internal/storage`
|
||||||
|
- `internal/storage/fake`
|
||||||
|
- `internal/adapters/local`
|
||||||
23
docs/internal/transform.md
Normal file
23
docs/internal/transform.md
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
# Transform
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`internal/transform` defines generated publication artifacts. `internal/transform/markdown` implements Markdown-to-HTML sidecar generation.
|
||||||
|
|
||||||
|
## Inputs and outputs
|
||||||
|
|
||||||
|
Inputs are a validated source bundle and source backend. Outputs include destination path, source path, transform id, generated bytes, SHA-256, and size.
|
||||||
|
|
||||||
|
## Markdown behavior
|
||||||
|
|
||||||
|
Markdown files ending in `.md` generate `.html` files in the same logical directory. Non-Markdown files do not generate outputs. Raw HTML embedded in Markdown is not passed through by the renderer.
|
||||||
|
|
||||||
|
Generated HTML is deterministic for the same source content and transform configuration.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
Transforms do not publish files, mutate source bundles, or write destination state. Publish planning selects and writes transform outputs.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
Before changing transform behavior, inspect tests under `internal/transform`.
|
||||||
49
docs/operations.md
Normal file
49
docs/operations.md
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
# Distributor Operations
|
||||||
|
|
||||||
|
## Normal workflow
|
||||||
|
|
||||||
|
Preview a local publication:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/local-publish.yml --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the local publication:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/local-publish.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the local HTML publication:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/distributor run --config examples/local-html.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
## Filesystem layout
|
||||||
|
|
||||||
|
Source bundles are discovered beneath the configured local source root. Destination bundle paths preserve the source bundle path relative to that source root.
|
||||||
|
|
||||||
|
The maintained example writes under `workspace/`, which is ignored by Git.
|
||||||
|
|
||||||
|
## Destination state
|
||||||
|
|
||||||
|
Each published destination bundle contains `.distributor.json`. This state file records the source manifest, copied source outputs, and generated outputs. It is the authoritative marker that a destination path is managed by `distributor`.
|
||||||
|
|
||||||
|
`manifest.json` from the source bundle is not copied as destination state.
|
||||||
|
|
||||||
|
## Retry behavior
|
||||||
|
|
||||||
|
If a destination already has matching `.distributor.json`, publication skips it as already published.
|
||||||
|
|
||||||
|
If destination state is older than the source manifest, publication replaces only managed outputs recorded in `.distributor.json` plus the state file.
|
||||||
|
|
||||||
|
If a write fails during local publication, `distributor` removes outputs written during that failed attempt where possible so a retry does not see an unmanaged destination.
|
||||||
|
|
||||||
|
If one destination fails, later destinations in the same fan-out are still planned and run where they are independent. The command exits non-zero after printing the final status when any destination fails.
|
||||||
|
|
||||||
|
After a successful publish or replacement, the internal notifier hook runs as a no-op. Skipped destinations do not invoke it.
|
||||||
|
|
||||||
|
## Caveats
|
||||||
|
|
||||||
|
Only local-to-local publication is implemented. SSH, S3, external notification adapters, and force overwrite behavior are not implemented.
|
||||||
32
examples/fan-out.yml
Normal file
32
examples/fan-out.yml
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /var/spool/distributor/reports
|
||||||
|
validation:
|
||||||
|
on_digest_mismatch: fail
|
||||||
|
destinations:
|
||||||
|
- id: markdown-archive
|
||||||
|
backend: s3
|
||||||
|
endpoint: https://s3.example.com
|
||||||
|
bucket: reports
|
||||||
|
prefix: archive
|
||||||
|
region: us-east-1
|
||||||
|
force_path_style: true
|
||||||
|
credentials:
|
||||||
|
access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
|
||||||
|
secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
|
||||||
|
publish:
|
||||||
|
source: true
|
||||||
|
html: false
|
||||||
|
- id: static-site
|
||||||
|
backend: ssh
|
||||||
|
uri: ssh://deploy@example.com:22
|
||||||
|
path: /srv/www/reports
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: sidecar
|
||||||
16
examples/local-html.yml
Normal file
16
examples/local-html.yml
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
pipelines:
|
||||||
|
- id: example-html-bundle
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: examples/source-bundle
|
||||||
|
destinations:
|
||||||
|
- id: local-html
|
||||||
|
backend: local
|
||||||
|
path: workspace/published/html-bundle
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: sidecar
|
||||||
12
examples/local-publish.yml
Normal file
12
examples/local-publish.yml
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
pipelines:
|
||||||
|
- id: example-source-bundle
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: examples/source-bundle
|
||||||
|
destinations:
|
||||||
|
- id: local-archive
|
||||||
|
backend: local
|
||||||
|
path: workspace/published/source-bundle
|
||||||
|
publish:
|
||||||
|
source: true
|
||||||
|
html: false
|
||||||
9
examples/local-to-local.yml
Normal file
9
examples/local-to-local.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /var/spool/distributor/reports
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /srv/reports/archive
|
||||||
18
examples/source-bundle/manifest.json
Normal file
18
examples/source-bundle/manifest.json
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"id": "weather.daily.brentwood.2026-05-30",
|
||||||
|
"digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe",
|
||||||
|
"created": "2026-05-30T11:10:00Z",
|
||||||
|
"files": [
|
||||||
|
{
|
||||||
|
"path": "report.md",
|
||||||
|
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
|
||||||
|
"size": 16
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "summary.txt",
|
||||||
|
"sha256": "sha256:3cbb36aca330b3bd113955dfbada0adb7a5f95ad9f678bd61f175406c6a37e95",
|
||||||
|
"size": 8
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
2
examples/source-bundle/report.md
Normal file
2
examples/source-bundle/report.md
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
# Report
|
||||||
|
Sunny.
|
||||||
1
examples/source-bundle/summary.txt
Normal file
1
examples/source-bundle/summary.txt
Normal file
@@ -0,0 +1 @@
|
|||||||
|
Summary
|
||||||
8
go.mod
Normal file
8
go.mod
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
module gitea.maximumdirect.net/eric/distributor
|
||||||
|
|
||||||
|
go 1.26
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/yuin/goldmark v1.8.2
|
||||||
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
|
)
|
||||||
6
go.sum
Normal file
6
go.sum
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
|
||||||
|
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
370
internal/adapters/local/backend.go
Normal file
370
internal/adapters/local/backend.go
Normal file
@@ -0,0 +1,370 @@
|
|||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
const backendName = "local"
|
||||||
|
|
||||||
|
type Backend struct {
|
||||||
|
root string
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(root string) (*Backend, error) {
|
||||||
|
if root == "" {
|
||||||
|
return nil, storage.NewError(storage.OpOpenBackend, backendName, "", storage.ErrInvalidPath, nil)
|
||||||
|
}
|
||||||
|
absRoot, err := filepath.Abs(root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, storage.NewError(storage.OpOpenBackend, backendName, root, storage.ErrInvalidPath, err)
|
||||||
|
}
|
||||||
|
return &Backend{root: filepath.Clean(absRoot)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) ReadFile(ctx context.Context, path string) ([]byte, error) {
|
||||||
|
reader, err := b.OpenReader(ctx, path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer reader.Close()
|
||||||
|
data, err := io.ReadAll(reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, storage.NewError(storage.OpReadFile, backendName, path, storage.ErrUnknown, err)
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) OpenReader(ctx context.Context, path string) (io.ReadCloser, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
nativePath, err := b.nativePath(path, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := b.rejectSymlinkAncestors(nativePath, true); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(nativePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, b.translateError(storage.OpOpenReader, path, err)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return nil, storage.NewError(storage.OpOpenReader, backendName, path, storage.ErrUnsupported, nil)
|
||||||
|
}
|
||||||
|
file, err := os.Open(nativePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, b.translateError(storage.OpOpenReader, path, err)
|
||||||
|
}
|
||||||
|
return file, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) WriteFile(ctx context.Context, path string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
|
||||||
|
opts.Size = int64(len(data))
|
||||||
|
opts.SizeKnown = true
|
||||||
|
return b.WriteFrom(ctx, path, bytes.NewReader(data), opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) WriteFrom(ctx context.Context, path string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return storage.Entry{}, err
|
||||||
|
}
|
||||||
|
nativePath, err := b.nativePath(path, false)
|
||||||
|
if err != nil {
|
||||||
|
return storage.Entry{}, err
|
||||||
|
}
|
||||||
|
parent := filepath.Dir(nativePath)
|
||||||
|
if err := b.rejectSymlinkAncestors(parent, true); err != nil {
|
||||||
|
return storage.Entry{}, err
|
||||||
|
}
|
||||||
|
if info, err := os.Lstat(nativePath); err == nil {
|
||||||
|
if !opts.Overwrite {
|
||||||
|
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrAlreadyExist, nil)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrConflict, nil)
|
||||||
|
}
|
||||||
|
} else if !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return storage.Entry{}, b.translateError(storage.OpWriteFrom, path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(parent, 0o755); err != nil {
|
||||||
|
return storage.Entry{}, b.translateError(storage.OpWriteFrom, path, err)
|
||||||
|
}
|
||||||
|
temp, err := os.CreateTemp(parent, ".distributor-write-*")
|
||||||
|
if err != nil {
|
||||||
|
return storage.Entry{}, b.translateError(storage.OpWriteFrom, path, err)
|
||||||
|
}
|
||||||
|
tempPath := temp.Name()
|
||||||
|
cleanup := true
|
||||||
|
defer func() {
|
||||||
|
if cleanup {
|
||||||
|
_ = os.Remove(tempPath)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
written, copyErr := io.Copy(temp, r)
|
||||||
|
closeErr := temp.Close()
|
||||||
|
if copyErr != nil {
|
||||||
|
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrUnknown, copyErr)
|
||||||
|
}
|
||||||
|
if closeErr != nil {
|
||||||
|
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrUnknown, closeErr)
|
||||||
|
}
|
||||||
|
if opts.SizeKnown && written != opts.Size {
|
||||||
|
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrConflict, fmt.Errorf("stream size %d does not match expected size %d", written, opts.Size))
|
||||||
|
}
|
||||||
|
if err := os.Rename(tempPath, nativePath); err != nil {
|
||||||
|
return storage.Entry{}, b.translateError(storage.OpWriteFrom, path, err)
|
||||||
|
}
|
||||||
|
cleanup = false
|
||||||
|
return b.Stat(ctx, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) Stat(ctx context.Context, path string) (storage.Entry, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return storage.Entry{}, err
|
||||||
|
}
|
||||||
|
nativePath, err := b.nativePath(path, true)
|
||||||
|
if err != nil {
|
||||||
|
return storage.Entry{}, err
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(nativePath)
|
||||||
|
if err != nil {
|
||||||
|
return storage.Entry{}, b.translateError(storage.OpStat, path, err)
|
||||||
|
}
|
||||||
|
return entryFromInfo(path, info), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOptions, fn storage.WalkFunc) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
nativePrefix, err := b.nativePath(prefix, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(nativePrefix)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return b.translateError(storage.OpWalk, prefix, err)
|
||||||
|
}
|
||||||
|
visited := 0
|
||||||
|
emit := func(entry storage.Entry) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if opts.Limit > 0 && visited >= opts.Limit {
|
||||||
|
return storage.ErrStopWalk
|
||||||
|
}
|
||||||
|
visited++
|
||||||
|
if err := fn(entry); err != nil {
|
||||||
|
if errors.Is(err, storage.ErrStopWalk) {
|
||||||
|
return storage.ErrStopWalk
|
||||||
|
}
|
||||||
|
return storage.NewError(storage.OpWalk, backendName, entry.Path, storage.ErrUnknown, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if !info.IsDir() {
|
||||||
|
if err := emit(entryFromInfo(prefix, info)); errors.Is(err, storage.ErrStopWalk) {
|
||||||
|
return nil
|
||||||
|
} else if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
walkErr := filepath.WalkDir(nativePrefix, func(nativePath string, dirEntry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return b.translateError(storage.OpWalk, prefix, err)
|
||||||
|
}
|
||||||
|
if nativePath == nativePrefix {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
relPath, err := filepath.Rel(b.root, nativePath)
|
||||||
|
if err != nil {
|
||||||
|
return b.translateError(storage.OpWalk, prefix, err)
|
||||||
|
}
|
||||||
|
logicalPath := filepath.ToSlash(relPath)
|
||||||
|
if !opts.Recursive && filepath.Dir(nativePath) != nativePrefix {
|
||||||
|
if dirEntry.IsDir() {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
info, err := dirEntry.Info()
|
||||||
|
if err != nil {
|
||||||
|
return b.translateError(storage.OpWalk, logicalPath, err)
|
||||||
|
}
|
||||||
|
return emit(entryFromInfo(logicalPath, info))
|
||||||
|
})
|
||||||
|
if errors.Is(walkErr, storage.ErrStopWalk) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
||||||
|
found := false
|
||||||
|
err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
|
||||||
|
found = true
|
||||||
|
return storage.ErrStopWalk
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return found, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := storage.ValidatePrefix(bundlePath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
targets := make([]string, 0, len(managedOutputPaths)+1)
|
||||||
|
for _, outputPath := range managedOutputPaths {
|
||||||
|
target, err := storage.Join(bundlePath, outputPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
targets = append(targets, target)
|
||||||
|
}
|
||||||
|
statePath, err := storage.StatePath(bundlePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
targets = append(targets, statePath)
|
||||||
|
|
||||||
|
for _, logicalPath := range targets {
|
||||||
|
nativePath, err := b.nativePath(logicalPath, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if nativePath == b.root {
|
||||||
|
return storage.NewError(storage.OpDeleteManagedBundle, backendName, logicalPath, storage.ErrInvalidPath, nil)
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(nativePath)
|
||||||
|
if err != nil {
|
||||||
|
if opts.IgnoreMissing && errors.Is(err, fs.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return b.translateError(storage.OpDeleteManagedBundle, logicalPath, err)
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
return storage.NewError(storage.OpDeleteManagedBundle, backendName, logicalPath, storage.ErrUnsupported, nil)
|
||||||
|
}
|
||||||
|
if err := os.Remove(nativePath); err != nil {
|
||||||
|
return b.translateError(storage.OpDeleteManagedBundle, logicalPath, err)
|
||||||
|
}
|
||||||
|
if opts.PruneEmptyDirs {
|
||||||
|
b.pruneEmptyParents(filepath.Dir(nativePath))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) nativePath(logicalPath string, allowEmpty bool) (string, error) {
|
||||||
|
if logicalPath == "" {
|
||||||
|
if !allowEmpty {
|
||||||
|
return "", storage.NewError(storage.OpValidatePath, backendName, logicalPath, storage.ErrInvalidPath, nil)
|
||||||
|
}
|
||||||
|
return b.root, nil
|
||||||
|
}
|
||||||
|
if err := storage.ValidatePath(logicalPath); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
nativePath := filepath.Clean(filepath.Join(b.root, filepath.FromSlash(logicalPath)))
|
||||||
|
rel, err := filepath.Rel(b.root, nativePath)
|
||||||
|
if err != nil {
|
||||||
|
return "", storage.NewError(storage.OpValidatePath, backendName, logicalPath, storage.ErrInvalidPath, err)
|
||||||
|
}
|
||||||
|
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
|
||||||
|
return "", storage.NewError(storage.OpValidatePath, backendName, logicalPath, storage.ErrInvalidPath, nil)
|
||||||
|
}
|
||||||
|
return nativePath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) rejectSymlinkAncestors(nativePath string, includeFinal bool) error {
|
||||||
|
rel, err := filepath.Rel(b.root, nativePath)
|
||||||
|
if err != nil {
|
||||||
|
return storage.NewError(storage.OpValidatePath, backendName, nativePath, storage.ErrInvalidPath, err)
|
||||||
|
}
|
||||||
|
if rel == "." {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
segments := strings.Split(rel, string(filepath.Separator))
|
||||||
|
limit := len(segments)
|
||||||
|
if !includeFinal {
|
||||||
|
limit--
|
||||||
|
}
|
||||||
|
current := b.root
|
||||||
|
for i := 0; i < limit; i++ {
|
||||||
|
current = filepath.Join(current, segments[i])
|
||||||
|
info, err := os.Lstat(current)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return b.translateError(storage.OpStat, filepath.ToSlash(filepath.Join(segments[:i+1]...)), err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return storage.NewError(storage.OpStat, backendName, filepath.ToSlash(filepath.Join(segments[:i+1]...)), storage.ErrUnsupported, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) pruneEmptyParents(start string) {
|
||||||
|
for current := start; current != b.root && strings.HasPrefix(current, b.root); current = filepath.Dir(current) {
|
||||||
|
err := os.Remove(current)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) translateError(op, path string, err error) error {
|
||||||
|
kind := storage.ErrUnknown
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, fs.ErrNotExist):
|
||||||
|
kind = storage.ErrNotFound
|
||||||
|
case errors.Is(err, fs.ErrExist):
|
||||||
|
kind = storage.ErrAlreadyExist
|
||||||
|
case errors.Is(err, fs.ErrPermission):
|
||||||
|
kind = storage.ErrPermission
|
||||||
|
}
|
||||||
|
return storage.NewError(op, backendName, path, kind, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func entryFromInfo(path string, info fs.FileInfo) storage.Entry {
|
||||||
|
entryType := storage.EntryTypeOther
|
||||||
|
switch {
|
||||||
|
case info.Mode()&os.ModeSymlink != 0:
|
||||||
|
entryType = storage.EntryTypeSymlink
|
||||||
|
case info.Mode().IsRegular():
|
||||||
|
entryType = storage.EntryTypeFile
|
||||||
|
case info.IsDir():
|
||||||
|
entryType = storage.EntryTypeDirectory
|
||||||
|
}
|
||||||
|
return storage.Entry{
|
||||||
|
Path: path,
|
||||||
|
Type: entryType,
|
||||||
|
Size: info.Size(),
|
||||||
|
}
|
||||||
|
}
|
||||||
233
internal/adapters/local/backend_test.go
Normal file
233
internal/adapters/local/backend_test.go
Normal file
@@ -0,0 +1,233 @@
|
|||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBackendRejectsTraversal(t *testing.T) {
|
||||||
|
backend := newBackend(t)
|
||||||
|
_, err := backend.ReadFile(context.Background(), "../outside")
|
||||||
|
if !storage.IsInvalidPath(err) {
|
||||||
|
t.Fatalf("ReadFile traversal error = %v, want invalid path", err)
|
||||||
|
}
|
||||||
|
_, err = backend.WriteFile(context.Background(), "/absolute", []byte("data"), storage.WriteOptions{})
|
||||||
|
if !storage.IsInvalidPath(err) {
|
||||||
|
t.Fatalf("WriteFile absolute path error = %v, want invalid path", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendReadWriteAndStream(t *testing.T) {
|
||||||
|
backend := newBackend(t)
|
||||||
|
|
||||||
|
entry, err := backend.WriteFile(context.Background(), "reports/report.md", []byte("hello"), storage.WriteOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if entry.Path != "reports/report.md" || entry.Type != storage.EntryTypeFile || entry.Size != 5 {
|
||||||
|
t.Fatalf("entry = %#v, want written file metadata", entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := backend.ReadFile(context.Background(), "reports/report.md")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if string(data) != "hello" {
|
||||||
|
t.Fatalf("ReadFile() = %q, want hello", data)
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, err := backend.OpenReader(context.Background(), "reports/report.md")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("OpenReader() error = %v", err)
|
||||||
|
}
|
||||||
|
streamed, err := io.ReadAll(reader)
|
||||||
|
closeErr := reader.Close()
|
||||||
|
if err != nil || closeErr != nil {
|
||||||
|
t.Fatalf("read stream error = %v close = %v", err, closeErr)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(streamed, data) {
|
||||||
|
t.Fatalf("streamed = %q, want %q", streamed, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = backend.WriteFile(context.Background(), "reports/report.md", []byte("again"), storage.WriteOptions{})
|
||||||
|
if !storage.IsAlreadyExists(err) {
|
||||||
|
t.Fatalf("WriteFile without overwrite error = %v, want already exists", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendStatWalkAndList(t *testing.T) {
|
||||||
|
backend := newBackend(t)
|
||||||
|
mustWrite(t, backend, "b/two.txt", "2")
|
||||||
|
mustWrite(t, backend, "a/one.txt", "1")
|
||||||
|
|
||||||
|
entry, err := backend.Stat(context.Background(), "a/one.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Stat() error = %v", err)
|
||||||
|
}
|
||||||
|
if entry.Type != storage.EntryTypeFile || entry.Size != 1 {
|
||||||
|
t.Fatalf("entry = %#v, want file size 1", entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := storage.List(context.Background(), backend, "", storage.WalkOptions{Recursive: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List() error = %v", err)
|
||||||
|
}
|
||||||
|
var paths []string
|
||||||
|
for _, entry := range entries {
|
||||||
|
paths = append(paths, entry.Path)
|
||||||
|
}
|
||||||
|
want := []string{"a", "a/one.txt", "b", "b/two.txt"}
|
||||||
|
if !reflect.DeepEqual(paths, want) {
|
||||||
|
t.Fatalf("paths = %v, want %v", paths, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err = storage.List(context.Background(), backend, "", storage.WalkOptions{Recursive: false})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List nonrecursive error = %v", err)
|
||||||
|
}
|
||||||
|
paths = paths[:0]
|
||||||
|
for _, entry := range entries {
|
||||||
|
paths = append(paths, entry.Path)
|
||||||
|
}
|
||||||
|
want = []string{"a", "b"}
|
||||||
|
if !reflect.DeepEqual(paths, want) {
|
||||||
|
t.Fatalf("nonrecursive paths = %v, want %v", paths, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendSymlinkReportingAndReadRejection(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
backend, err := New(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "target.txt"), []byte("target"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write target: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink("target.txt", filepath.Join(root, "link.txt")); err != nil {
|
||||||
|
t.Fatalf("symlink: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
entry, err := backend.Stat(context.Background(), "link.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Stat() error = %v", err)
|
||||||
|
}
|
||||||
|
if entry.Type != storage.EntryTypeSymlink {
|
||||||
|
t.Fatalf("entry type = %s, want symlink", entry.Type)
|
||||||
|
}
|
||||||
|
_, err = backend.ReadFile(context.Background(), "link.txt")
|
||||||
|
if !storage.IsUnsupported(err) {
|
||||||
|
t.Fatalf("ReadFile symlink error = %v, want unsupported", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendWriteFromSizeMismatchLeavesNoFinalFile(t *testing.T) {
|
||||||
|
backend := newBackend(t)
|
||||||
|
_, err := backend.WriteFrom(context.Background(), "out.txt", bytes.NewBufferString("short"), storage.WriteOptions{SizeKnown: true, Size: 99})
|
||||||
|
if !storage.IsConflict(err) {
|
||||||
|
t.Fatalf("WriteFrom size mismatch error = %v, want conflict", err)
|
||||||
|
}
|
||||||
|
_, err = backend.Stat(context.Background(), "out.txt")
|
||||||
|
if !storage.IsNotFound(err) {
|
||||||
|
t.Fatalf("Stat after failed write error = %v, want not found", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendManagedDeletion(t *testing.T) {
|
||||||
|
backend := newBackend(t)
|
||||||
|
mustWrite(t, backend, "bundle/report.html", "html")
|
||||||
|
mustWrite(t, backend, "bundle/keep.txt", "keep")
|
||||||
|
mustWrite(t, backend, "bundle/.distributor.json", "{}")
|
||||||
|
|
||||||
|
err := backend.DeleteManagedBundle(context.Background(), "bundle", []string{"report.html"}, storage.DeleteOptions{PruneEmptyDirs: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DeleteManagedBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := backend.Stat(context.Background(), "bundle/report.html"); !storage.IsNotFound(err) {
|
||||||
|
t.Fatalf("managed output stat error = %v, want not found", err)
|
||||||
|
}
|
||||||
|
if _, err := backend.Stat(context.Background(), "bundle/.distributor.json"); !storage.IsNotFound(err) {
|
||||||
|
t.Fatalf("state stat error = %v, want not found", err)
|
||||||
|
}
|
||||||
|
if _, err := backend.Stat(context.Background(), "bundle/keep.txt"); err != nil {
|
||||||
|
t.Fatalf("unlisted file stat error = %v", err)
|
||||||
|
}
|
||||||
|
if err := backend.DeleteManagedBundle(context.Background(), "bundle", []string{""}, storage.DeleteOptions{}); !storage.IsInvalidPath(err) {
|
||||||
|
t.Fatalf("DeleteManagedBundle invalid output error = %v, want invalid path", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendHasAny(t *testing.T) {
|
||||||
|
backend := newBackend(t)
|
||||||
|
found, err := backend.HasAny(context.Background(), "missing")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("HasAny missing error = %v", err)
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
t.Fatal("HasAny missing = true, want false")
|
||||||
|
}
|
||||||
|
mustWrite(t, backend, "bundle/report.md", "report")
|
||||||
|
found, err = backend.HasAny(context.Background(), "bundle")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("HasAny bundle error = %v", err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("HasAny bundle = false, want true")
|
||||||
|
}
|
||||||
|
found, err = backend.HasAny(context.Background(), "bund")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("HasAny sibling prefix error = %v", err)
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
t.Fatal("HasAny prefix sibling = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendWalkStops(t *testing.T) {
|
||||||
|
backend := newBackend(t)
|
||||||
|
mustWrite(t, backend, "a.txt", "a")
|
||||||
|
mustWrite(t, backend, "b.txt", "b")
|
||||||
|
visited := 0
|
||||||
|
err := backend.Walk(context.Background(), "", storage.WalkOptions{Recursive: true}, func(storage.Entry) error {
|
||||||
|
visited++
|
||||||
|
return storage.ErrStopWalk
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Walk() error = %v", err)
|
||||||
|
}
|
||||||
|
if visited != 1 {
|
||||||
|
t.Fatalf("visited = %d, want 1", visited)
|
||||||
|
}
|
||||||
|
|
||||||
|
errSentinel := errors.New("callback")
|
||||||
|
err = backend.Walk(context.Background(), "", storage.WalkOptions{Recursive: true}, func(storage.Entry) error {
|
||||||
|
return errSentinel
|
||||||
|
})
|
||||||
|
if !errors.Is(err, errSentinel) {
|
||||||
|
t.Fatalf("Walk callback error = %v, want sentinel", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBackend(t *testing.T) *Backend {
|
||||||
|
t.Helper()
|
||||||
|
backend, err := New(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("New() error = %v", err)
|
||||||
|
}
|
||||||
|
return backend
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustWrite(t *testing.T, backend *Backend, path, data string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{}); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
14
internal/app/app.go
Normal file
14
internal/app/app.go
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import "errors"
|
||||||
|
|
||||||
|
const Name = "distributor"
|
||||||
|
|
||||||
|
// Version can be replaced at build time with -ldflags "-X .../internal/app.Version=<value>".
|
||||||
|
var Version = "dev"
|
||||||
|
|
||||||
|
var ErrNotImplemented = errors.New("not implemented")
|
||||||
|
|
||||||
|
func VersionString() string {
|
||||||
|
return Name + " " + Version
|
||||||
|
}
|
||||||
16
internal/app/app_test.go
Normal file
16
internal/app/app_test.go
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestVersionString(t *testing.T) {
|
||||||
|
oldVersion := Version
|
||||||
|
t.Cleanup(func() {
|
||||||
|
Version = oldVersion
|
||||||
|
})
|
||||||
|
|
||||||
|
Version = "1.2.3"
|
||||||
|
|
||||||
|
if got, want := VersionString(), "distributor 1.2.3"; got != want {
|
||||||
|
t.Fatalf("VersionString() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
65
internal/app/inspect.go
Normal file
65
internal/app/inspect.go
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/adapters/local"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
type InspectOptions struct {
|
||||||
|
Path string
|
||||||
|
Stdout io.Writer
|
||||||
|
}
|
||||||
|
|
||||||
|
func Inspect(ctx context.Context, options InspectOptions) error {
|
||||||
|
if options.Path == "" {
|
||||||
|
return fmt.Errorf("inspect command requires a path")
|
||||||
|
}
|
||||||
|
backend, err := local.New(options.Path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
bundles, err := bundle.Discover(ctx, backend, "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return writeInspection(options.Stdout, bundles)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeInspection(w io.Writer, bundles []bundle.Bundle) error {
|
||||||
|
if w == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := fmt.Fprintf(w, "Bundles: %d\n", len(bundles)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, sourceBundle := range bundles {
|
||||||
|
if _, err := fmt.Fprintf(
|
||||||
|
w,
|
||||||
|
"- path=%s id=%s created=%s digest=%s files=%d\n",
|
||||||
|
displayBundlePath(sourceBundle.RootRelativePath),
|
||||||
|
sourceBundle.Manifest.ID,
|
||||||
|
sourceBundle.Manifest.Created.Format("2006-01-02T15:04:05Z07:00"),
|
||||||
|
sourceBundle.Manifest.Digest,
|
||||||
|
len(sourceBundle.Manifest.Files),
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, file := range sourceBundle.Manifest.Files {
|
||||||
|
if _, err := fmt.Fprintf(w, " - %s size=%d sha256=%s\n", file.Path, file.Size, file.SHA256); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func displayBundlePath(path string) string {
|
||||||
|
if path == "" {
|
||||||
|
return "."
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
40
internal/app/inspect_test.go
Normal file
40
internal/app/inspect_test.go
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestInspectPrintsBundleSummary(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Inspect(context.Background(), InspectOptions{
|
||||||
|
Path: filepath.Join("..", "bundle", "testdata", "valid_bundle"),
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Inspect() error = %v", err)
|
||||||
|
}
|
||||||
|
output := stdout.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"Bundles: 1",
|
||||||
|
"path=.",
|
||||||
|
"id=weather.daily.brentwood.2026-05-30",
|
||||||
|
"created=2026-05-30T11:10:00Z",
|
||||||
|
"report.md size=16",
|
||||||
|
"summary.txt size=8",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(output, want) {
|
||||||
|
t.Fatalf("Inspect() output = %q, want substring %q", output, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectRequiresPath(t *testing.T) {
|
||||||
|
err := Inspect(context.Background(), InspectOptions{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "requires a path") {
|
||||||
|
t.Fatalf("Inspect() error = %v, want required path", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
5
internal/app/pipeline.go
Normal file
5
internal/app/pipeline.go
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
type Pipeline struct {
|
||||||
|
ID string
|
||||||
|
}
|
||||||
282
internal/app/run.go
Normal file
282
internal/app/run.go
Normal file
@@ -0,0 +1,282 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/adapters/local"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/notify"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/publish"
|
||||||
|
)
|
||||||
|
|
||||||
|
type RunOptions struct {
|
||||||
|
ConfigPath string
|
||||||
|
DryRun bool
|
||||||
|
Stdout io.Writer
|
||||||
|
Notifier notify.Notifier
|
||||||
|
}
|
||||||
|
|
||||||
|
func Run(ctx context.Context, options RunOptions) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
configPath := options.ConfigPath
|
||||||
|
if configPath == "" {
|
||||||
|
configPath = config.DefaultConfigPath
|
||||||
|
}
|
||||||
|
cfg, err := config.LoadFile(configPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return runConfig(ctx, cfg, options)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runConfig(ctx context.Context, cfg config.Config, options RunOptions) error {
|
||||||
|
notifier := options.Notifier
|
||||||
|
if notifier == nil {
|
||||||
|
notifier = notify.Noop{}
|
||||||
|
}
|
||||||
|
summary := runSummary{dryRun: options.DryRun}
|
||||||
|
var failures runFailures
|
||||||
|
if options.Stdout != nil {
|
||||||
|
if _, err := fmt.Fprintf(options.Stdout, "Configured pipelines: %d\n", len(cfg.Pipelines)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, pipeline := range cfg.Pipelines {
|
||||||
|
if pipeline.Source.Backend != config.BackendLocal {
|
||||||
|
return fmt.Errorf("pipeline %s source backend %s is not implemented for execution", pipeline.ID, pipeline.Source.Backend)
|
||||||
|
}
|
||||||
|
sourceBackend, err := local.New(pipeline.Source.Path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
bundles, err := bundle.Discover(ctx, sourceBackend, "")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("pipeline %s discover source bundles: %w", pipeline.ID, err)
|
||||||
|
}
|
||||||
|
if options.Stdout != nil {
|
||||||
|
if _, err := fmt.Fprintf(options.Stdout, "- pipeline=%s source=%s bundles=%d destinations=%s\n", pipeline.ID, pipeline.Source.Backend, len(bundles), destinationSummary(pipeline.Destinations)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, sourceBundle := range bundles {
|
||||||
|
for _, destination := range pipeline.Destinations {
|
||||||
|
if destination.Backend != config.BackendLocal {
|
||||||
|
err := fmt.Errorf("backend %s is not implemented for execution", destination.Backend)
|
||||||
|
failures.add(pipeline.ID, destination.ID, displayBundlePath(sourceBundle.RootRelativePath), err)
|
||||||
|
summary.recordFailure()
|
||||||
|
if options.Stdout != nil {
|
||||||
|
writeErrorLine(options.Stdout, sourceBundle.RootRelativePath, destination.ID, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
destinationBackend, err := local.New(destination.Path)
|
||||||
|
if err != nil {
|
||||||
|
failures.add(pipeline.ID, destination.ID, displayBundlePath(sourceBundle.RootRelativePath), err)
|
||||||
|
summary.recordFailure()
|
||||||
|
if options.Stdout != nil {
|
||||||
|
writeErrorLine(options.Stdout, sourceBundle.RootRelativePath, destination.ID, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
req := publish.Request{
|
||||||
|
PipelineID: pipeline.ID,
|
||||||
|
DestinationID: destination.ID,
|
||||||
|
SourceBundle: sourceBundle,
|
||||||
|
SourceBackend: sourceBackend,
|
||||||
|
DestinationBackend: destinationBackend,
|
||||||
|
DestinationBundlePath: sourceBundle.RootRelativePath,
|
||||||
|
Publish: *destination.Publish,
|
||||||
|
Transform: destination.Transform,
|
||||||
|
Transfer: destination.Transfer,
|
||||||
|
DistributorVersion: Version,
|
||||||
|
}
|
||||||
|
plan, err := publish.Build(ctx, req)
|
||||||
|
if err != nil && plan.DestinationID == "" {
|
||||||
|
plan = publish.Plan{DestinationID: destination.ID, BundlePath: sourceBundle.RootRelativePath}
|
||||||
|
}
|
||||||
|
if options.Stdout != nil {
|
||||||
|
writePlanLine(options.Stdout, plan, err)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
failures.add(pipeline.ID, destination.ID, displayBundlePath(sourceBundle.RootRelativePath), err)
|
||||||
|
summary.recordFailure()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
summary.recordPlan(plan.Action)
|
||||||
|
if !options.DryRun {
|
||||||
|
if err := publish.Execute(ctx, req, plan); err != nil {
|
||||||
|
failures.add(pipeline.ID, destination.ID, displayBundlePath(sourceBundle.RootRelativePath), err)
|
||||||
|
summary.recordFailure()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if shouldNotify(plan.Action) {
|
||||||
|
if err := notifier.Notify(ctx, notifyEvent(plan)); err != nil {
|
||||||
|
failures.add(pipeline.ID, destination.ID, displayBundlePath(sourceBundle.RootRelativePath), err)
|
||||||
|
summary.recordFailure()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if options.Stdout != nil {
|
||||||
|
if _, err := fmt.Fprintln(options.Stdout, summary.Line()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(failures.items) > 0 {
|
||||||
|
return failures
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writePlanLine(w io.Writer, plan publish.Plan, planErr error) {
|
||||||
|
if w == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if planErr != nil {
|
||||||
|
destinationID := plan.DestinationID
|
||||||
|
if destinationID == "" {
|
||||||
|
destinationID = "unknown"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " - bundle=%s destination=%s action=error reason=%q\n", displayBundlePath(plan.BundlePath), destinationID, planErr.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " - bundle=%s destination=%s action=%s outputs=%s reason=%q\n", displayBundlePath(plan.BundlePath), plan.DestinationID, plan.Action, outputSummary(plan.Outputs), plan.Reason)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeErrorLine(w io.Writer, bundlePath, destinationID string, err error) {
|
||||||
|
if w == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " - bundle=%s destination=%s action=error reason=%q\n", displayBundlePath(bundlePath), destinationID, err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
func outputSummary(outputs []publish.Output) string {
|
||||||
|
if len(outputs) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
paths := make([]string, 0, len(outputs))
|
||||||
|
for _, output := range outputs {
|
||||||
|
paths = append(paths, output.DestinationPath)
|
||||||
|
}
|
||||||
|
return strings.Join(paths, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
func destinationSummary(destinations []config.Destination) string {
|
||||||
|
if len(destinations) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
ids := make([]string, 0, len(destinations))
|
||||||
|
for _, destination := range destinations {
|
||||||
|
ids = append(ids, destination.ID)
|
||||||
|
}
|
||||||
|
return strings.Join(ids, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
func shouldNotify(action publish.Action) bool {
|
||||||
|
return action == publish.ActionPublishNew || action == publish.ActionReplaceOlder
|
||||||
|
}
|
||||||
|
|
||||||
|
func notifyEvent(plan publish.Plan) notify.Event {
|
||||||
|
outputs := make([]notify.Output, 0, len(plan.Outputs))
|
||||||
|
for _, output := range plan.Outputs {
|
||||||
|
outputs = append(outputs, notify.Output{
|
||||||
|
Path: output.DestinationPath,
|
||||||
|
Kind: output.Kind,
|
||||||
|
SourcePath: output.SourcePath,
|
||||||
|
Transform: output.Transform,
|
||||||
|
SHA256: output.SHA256,
|
||||||
|
Size: output.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return notify.Event{
|
||||||
|
PipelineID: plan.PipelineID,
|
||||||
|
DestinationID: plan.DestinationID,
|
||||||
|
BundleID: plan.BundleID,
|
||||||
|
BundlePath: plan.BundlePath,
|
||||||
|
Action: string(plan.Action),
|
||||||
|
Outputs: outputs,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type runSummary struct {
|
||||||
|
dryRun bool
|
||||||
|
planned int
|
||||||
|
publishNew int
|
||||||
|
replaceOlder int
|
||||||
|
skipped int
|
||||||
|
failures int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *runSummary) recordPlan(action publish.Action) {
|
||||||
|
s.planned++
|
||||||
|
switch action {
|
||||||
|
case publish.ActionPublishNew:
|
||||||
|
s.publishNew++
|
||||||
|
case publish.ActionReplaceOlder:
|
||||||
|
s.replaceOlder++
|
||||||
|
case publish.ActionSkipSame, publish.ActionSkipDestinationNewer:
|
||||||
|
s.skipped++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *runSummary) recordFailure() {
|
||||||
|
s.failures++
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s runSummary) Line() string {
|
||||||
|
status := "ok"
|
||||||
|
if s.failures > 0 {
|
||||||
|
status = "failed"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("Final status: %s planned=%d publish_new=%d replace_older=%d skipped=%d failed=%d dry_run=%t", status, s.planned, s.publishNew, s.replaceOlder, s.skipped, s.failures, s.dryRun)
|
||||||
|
}
|
||||||
|
|
||||||
|
type runFailure struct {
|
||||||
|
pipelineID string
|
||||||
|
destinationID string
|
||||||
|
bundlePath string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type runFailures struct {
|
||||||
|
items []runFailure
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *runFailures) add(pipelineID, destinationID, bundlePath string, err error) {
|
||||||
|
f.items = append(f.items, runFailure{
|
||||||
|
pipelineID: pipelineID,
|
||||||
|
destinationID: destinationID,
|
||||||
|
bundlePath: bundlePath,
|
||||||
|
err: err,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f runFailures) Error() string {
|
||||||
|
if len(f.items) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
parts := make([]string, 0, len(f.items))
|
||||||
|
for _, item := range f.items {
|
||||||
|
parts = append(parts, fmt.Sprintf("pipeline %s destination %s bundle %s: %v", item.pipelineID, item.destinationID, item.bundlePath, item.err))
|
||||||
|
}
|
||||||
|
return "run failed: " + strings.Join(parts, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f runFailures) Unwrap() error {
|
||||||
|
errs := make([]error, 0, len(f.items))
|
||||||
|
for _, item := range f.items {
|
||||||
|
errs = append(errs, item.err)
|
||||||
|
}
|
||||||
|
return errors.Join(errs...)
|
||||||
|
}
|
||||||
624
internal/app/run_test.go
Normal file
624
internal/app/run_test.go
Normal file
@@ -0,0 +1,624 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/notify"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunDryRunPrintsConfigSummary(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
configPath := writeLocalConfig(t, sourceRoot, destinationRoot)
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: configPath,
|
||||||
|
DryRun: true,
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
output := stdout.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"Configured pipelines: 1",
|
||||||
|
"- pipeline=reports source=local bundles=1 destinations=archive",
|
||||||
|
"bundle=. destination=archive action=publish_new outputs=report.md,summary.txt",
|
||||||
|
"Final status: ok planned=1 publish_new=1 replace_older=0 skipped=0 failed=0 dry_run=true",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(output, want) {
|
||||||
|
t.Fatalf("Run() output = %q, want substring %q", output, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPublishesNewLocalBundle(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||||
|
assertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
||||||
|
if _, err := os.Stat(filepath.Join(destinationRoot, "manifest.json")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("destination manifest stat error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, ".distributor.json"))
|
||||||
|
if destinationState.PipelineID != "reports" || destinationState.DestinationID != "archive" {
|
||||||
|
t.Fatalf("state identity = %s/%s", destinationState.PipelineID, destinationState.DestinationID)
|
||||||
|
}
|
||||||
|
if destinationState.Source.Manifest.ID != manifest.ID {
|
||||||
|
t.Fatalf("state source id = %q, want %q", destinationState.Source.Manifest.ID, manifest.ID)
|
||||||
|
}
|
||||||
|
if got, want := len(destinationState.Outputs), 2; got != want {
|
||||||
|
t.Fatalf("state output count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunNotifiesAfterPublication(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
notifier := &recordingNotifier{
|
||||||
|
check: func() {
|
||||||
|
if _, err := os.Stat(filepath.Join(destinationRoot, ".distributor.json")); err != nil {
|
||||||
|
t.Fatalf("state stat during notify: %v", err)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
|
||||||
|
Notifier: notifier,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := len(notifier.events), 1; got != want {
|
||||||
|
t.Fatalf("notification count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
event := notifier.events[0]
|
||||||
|
if event.PipelineID != "reports" || event.DestinationID != "archive" || event.BundleID == "" || event.Action != "publish_new" {
|
||||||
|
t.Fatalf("notification event = %#v", event)
|
||||||
|
}
|
||||||
|
if got, want := len(event.Outputs), 2; got != want {
|
||||||
|
t.Fatalf("notification output count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunNotifiesAfterReplacement(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
older := manifest
|
||||||
|
older.Created = older.Created.Add(-time.Hour)
|
||||||
|
writeDestinationState(t, destinationRoot, "", older)
|
||||||
|
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write old output: %v", err)
|
||||||
|
}
|
||||||
|
notifier := &recordingNotifier{}
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
|
||||||
|
Notifier: notifier,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := len(notifier.events), 1; got != want {
|
||||||
|
t.Fatalf("notification count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
if notifier.events[0].Action != "replace_older" {
|
||||||
|
t.Fatalf("notification action = %q, want replace_older", notifier.events[0].Action)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDoesNotNotifyForSkippedDestination(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
configPath := writeLocalConfig(t, sourceRoot, destinationRoot)
|
||||||
|
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||||
|
t.Fatalf("first Run() error = %v", err)
|
||||||
|
}
|
||||||
|
notifier := &recordingNotifier{}
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: configPath, Notifier: notifier})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(notifier.events) != 0 {
|
||||||
|
t.Fatalf("notifications = %#v, want none", notifier.events)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDoesNotNotifyDuringDryRun(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
notifier := &recordingNotifier{}
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
|
||||||
|
DryRun: true,
|
||||||
|
Notifier: notifier,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(notifier.events) != 0 {
|
||||||
|
t.Fatalf("notifications = %#v, want none", notifier.events)
|
||||||
|
}
|
||||||
|
if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 {
|
||||||
|
t.Fatalf("destination entries = %v err=%v, want empty", entries, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunContinuesAfterDestinationFailure(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
firstDestination := t.TempDir()
|
||||||
|
secondDestination := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
if err := os.WriteFile(filepath.Join(firstDestination, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write unmanaged file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination),
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") {
|
||||||
|
t.Fatalf("Run() error = %v, want unmanaged failure", err)
|
||||||
|
}
|
||||||
|
output := stdout.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"destination=archive-one action=error",
|
||||||
|
"destination=archive-two action=publish_new",
|
||||||
|
"Final status: failed planned=1 publish_new=1 replace_older=0 skipped=0 failed=1 dry_run=false",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(output, want) {
|
||||||
|
t.Fatalf("stdout = %q, want substring %q", output, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertFile(t, filepath.Join(secondDestination, "report.md"), "# Report\nSunny.\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPublishesHTMLOnly(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, false, true)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
assertFileContains(t, filepath.Join(destinationRoot, "report.html"), "<h1>Report</h1>")
|
||||||
|
if _, err := os.Stat(filepath.Join(destinationRoot, "report.md")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("report.md stat error = %v, want not exist", err)
|
||||||
|
}
|
||||||
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, ".distributor.json"))
|
||||||
|
if got, want := len(destinationState.Outputs), 1; got != want {
|
||||||
|
t.Fatalf("state output count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
output := destinationState.Outputs[0]
|
||||||
|
if output.Kind != state.OutputKindGenerated || output.Transform != "markdown_to_html" || output.Path != "report.html" || output.SourcePath != "report.md" {
|
||||||
|
t.Fatalf("generated output metadata = %#v", output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunPublishesSourceAndHTML(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, true, true)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||||
|
assertFileContains(t, filepath.Join(destinationRoot, "report.html"), "<p>Sunny.</p>")
|
||||||
|
assertFile(t, filepath.Join(destinationRoot, "summary.txt"), "Summary\n")
|
||||||
|
destinationState := readStateFile(t, filepath.Join(destinationRoot, ".distributor.json"))
|
||||||
|
if got, want := len(destinationState.Outputs), 3; got != want {
|
||||||
|
t.Fatalf("state output count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDoesNotMutateSourceBundle(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
sourcePath := filepath.Join(sourceRoot, "report.md")
|
||||||
|
before, err := os.ReadFile(sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read source before: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, true, true)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
after, err := os.ReadFile(sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read source after: %v", err)
|
||||||
|
}
|
||||||
|
if string(after) != string(before) {
|
||||||
|
t.Fatalf("source changed from %q to %q", before, after)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFailsOnOutputPathCollision(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{ExtraFiles: []testFile{{Path: "report.html", Data: "<p>source html</p>\n"}}})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, true, true)})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "destination output path collision") {
|
||||||
|
t.Fatalf("Run() error = %v, want collision", err)
|
||||||
|
}
|
||||||
|
if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 {
|
||||||
|
t.Fatalf("destination entries = %v err=%v, want empty", entries, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDryRunReportsGeneratedOutputs(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, false, true),
|
||||||
|
DryRun: true,
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "outputs=report.html") {
|
||||||
|
t.Fatalf("stdout = %q, want generated output path", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunSkipsWhenDestinationStateMatches(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
configPath := writeLocalConfig(t, sourceRoot, destinationRoot)
|
||||||
|
if err := Run(context.Background(), RunOptions{ConfigPath: configPath}); err != nil {
|
||||||
|
t.Fatalf("first Run() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: configPath, Stdout: &stdout})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "action=skip_same") {
|
||||||
|
t.Fatalf("stdout = %q, want skip_same", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunReplacesOlderDestination(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
older := manifest
|
||||||
|
older.Created = older.Created.Add(-time.Hour)
|
||||||
|
writeDestinationState(t, destinationRoot, "", older)
|
||||||
|
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("old\n"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write old output: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "action=replace_older") {
|
||||||
|
t.Fatalf("stdout = %q, want replace_older", stdout.String())
|
||||||
|
}
|
||||||
|
assertFile(t, filepath.Join(destinationRoot, "report.md"), "# Report\nSunny.\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunSkipsNewerDestination(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
newer := manifest
|
||||||
|
newer.Created = newer.Created.Add(time.Hour)
|
||||||
|
writeDestinationState(t, destinationRoot, "", newer)
|
||||||
|
if err := os.WriteFile(filepath.Join(destinationRoot, "report.md"), []byte("newer\n"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write newer output: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot), Stdout: &stdout})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "action=skip_destination_newer") {
|
||||||
|
t.Fatalf("stdout = %q, want skip_destination_newer", stdout.String())
|
||||||
|
}
|
||||||
|
assertFile(t, filepath.Join(destinationRoot, "report.md"), "newer\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFailsOnConflict(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
manifest := writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
manifest.ID = "other.source"
|
||||||
|
writeDestinationState(t, destinationRoot, "", manifest)
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot)})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "fail_conflict") {
|
||||||
|
t.Fatalf("Run() error = %v, want fail_conflict", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFailsOnUnmanagedDestination(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
if err := os.WriteFile(filepath.Join(destinationRoot, "unmanaged.txt"), []byte("data"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write unmanaged file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot)})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "fail_unmanaged") {
|
||||||
|
t.Fatalf("Run() error = %v, want fail_unmanaged", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunFansOutToLocalDestinations(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
firstDestination := t.TempDir()
|
||||||
|
secondDestination := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "daily", testBundleOptions{})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{ConfigPath: writeFanoutConfig(t, sourceRoot, firstDestination, secondDestination)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
assertFile(t, filepath.Join(firstDestination, "daily", "report.md"), "# Report\nSunny.\n")
|
||||||
|
assertFile(t, filepath.Join(secondDestination, "daily", "summary.txt"), "Summary\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunDryRunDoesNotWrite(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeSourceBundle(t, sourceRoot, "", testBundleOptions{})
|
||||||
|
|
||||||
|
err := Run(context.Background(), RunOptions{
|
||||||
|
ConfigPath: writeLocalConfig(t, sourceRoot, destinationRoot),
|
||||||
|
DryRun: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run() error = %v", err)
|
||||||
|
}
|
||||||
|
if entries, err := os.ReadDir(destinationRoot); err != nil || len(entries) != 0 {
|
||||||
|
t.Fatalf("destination entries = %v err=%v, want empty", entries, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type testBundleOptions struct {
|
||||||
|
ID string
|
||||||
|
Created time.Time
|
||||||
|
ExtraFiles []testFile
|
||||||
|
}
|
||||||
|
|
||||||
|
type testFile struct {
|
||||||
|
Path string
|
||||||
|
Data string
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeSourceBundle(t *testing.T, root, relative string, opts testBundleOptions) bundle.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
if opts.ID == "" {
|
||||||
|
opts.ID = "weather.daily.brentwood.2026-05-30"
|
||||||
|
}
|
||||||
|
if opts.Created.IsZero() {
|
||||||
|
opts.Created = time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC)
|
||||||
|
}
|
||||||
|
bundleRoot := filepath.Join(root, filepath.FromSlash(relative))
|
||||||
|
if err := os.MkdirAll(bundleRoot, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir bundle: %v", err)
|
||||||
|
}
|
||||||
|
files := []struct {
|
||||||
|
path string
|
||||||
|
data string
|
||||||
|
}{
|
||||||
|
{path: "report.md", data: "# Report\nSunny.\n"},
|
||||||
|
{path: "summary.txt", data: "Summary\n"},
|
||||||
|
}
|
||||||
|
for _, extra := range opts.ExtraFiles {
|
||||||
|
files = append(files, struct {
|
||||||
|
path string
|
||||||
|
data string
|
||||||
|
}{path: extra.Path, data: extra.Data})
|
||||||
|
}
|
||||||
|
manifestFiles := make([]bundle.ManifestFile, 0, len(files))
|
||||||
|
for _, file := range files {
|
||||||
|
if err := os.WriteFile(filepath.Join(bundleRoot, filepath.FromSlash(file.path)), []byte(file.data), 0o600); err != nil {
|
||||||
|
t.Fatalf("write source file: %v", err)
|
||||||
|
}
|
||||||
|
manifestFiles = append(manifestFiles, bundle.ManifestFile{
|
||||||
|
Path: file.path,
|
||||||
|
SHA256: bundle.FileDigest([]byte(file.data)),
|
||||||
|
Size: int64(len(file.data)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
manifest := bundle.Manifest{
|
||||||
|
SchemaVersion: 1,
|
||||||
|
ID: opts.ID,
|
||||||
|
Created: opts.Created,
|
||||||
|
Files: manifestFiles,
|
||||||
|
}
|
||||||
|
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||||
|
data, err := json.MarshalIndent(manifest, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal manifest: %v", err)
|
||||||
|
}
|
||||||
|
data = append(data, '\n')
|
||||||
|
if err := os.WriteFile(filepath.Join(bundleRoot, "manifest.json"), data, 0o600); err != nil {
|
||||||
|
t.Fatalf("write manifest: %v", err)
|
||||||
|
}
|
||||||
|
return manifest
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeLocalConfig(t *testing.T, sourceRoot, destinationRoot string) string {
|
||||||
|
t.Helper()
|
||||||
|
return writeLocalConfigWithPolicy(t, sourceRoot, destinationRoot, true, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeLocalConfigWithPolicy(t *testing.T, sourceRoot, destinationRoot string, publishSource, publishHTML bool) string {
|
||||||
|
t.Helper()
|
||||||
|
transformConfig := ""
|
||||||
|
if publishHTML {
|
||||||
|
transformConfig = `
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: sidecar`
|
||||||
|
}
|
||||||
|
return writeConfigFile(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: `+destinationRoot+`
|
||||||
|
publish:
|
||||||
|
source: `+fmt.Sprintf("%t", publishSource)+`
|
||||||
|
html: `+fmt.Sprintf("%t", publishHTML)+transformConfig+`
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeFanoutConfig(t *testing.T, sourceRoot, firstDestination, secondDestination string) string {
|
||||||
|
t.Helper()
|
||||||
|
return writeConfigFile(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive-one
|
||||||
|
backend: local
|
||||||
|
path: `+firstDestination+`
|
||||||
|
- id: archive-two
|
||||||
|
backend: local
|
||||||
|
path: `+secondDestination+`
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeConfigFile(t *testing.T, body string) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
if err := os.WriteFile(path, []byte(strings.TrimSpace(body)+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeDestinationState(t *testing.T, root, relative string, manifest bundle.Manifest) {
|
||||||
|
t.Helper()
|
||||||
|
bundleRoot := filepath.Join(root, filepath.FromSlash(relative))
|
||||||
|
if err := os.MkdirAll(bundleRoot, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir destination: %v", err)
|
||||||
|
}
|
||||||
|
destinationState := state.DistributorState{
|
||||||
|
SchemaVersion: state.SchemaVersion,
|
||||||
|
PipelineID: "reports",
|
||||||
|
DestinationID: "archive",
|
||||||
|
PublishedAt: time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC),
|
||||||
|
Source: state.SourceState{Manifest: manifest},
|
||||||
|
Outputs: []state.OutputFile{
|
||||||
|
{Path: "report.md", Kind: state.OutputKindSource, SourcePath: "report.md", SHA256: manifest.Files[0].SHA256, Size: manifest.Files[0].Size},
|
||||||
|
{Path: "summary.txt", Kind: state.OutputKindSource, SourcePath: "summary.txt", SHA256: manifest.Files[1].SHA256, Size: manifest.Files[1].Size},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal state: %v", err)
|
||||||
|
}
|
||||||
|
data = append(data, '\n')
|
||||||
|
if err := os.WriteFile(filepath.Join(bundleRoot, ".distributor.json"), data, 0o600); err != nil {
|
||||||
|
t.Fatalf("write state: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readStateFile(t *testing.T, path string) state.DistributorState {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read state: %v", err)
|
||||||
|
}
|
||||||
|
destinationState, err := state.Parse(data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse state: %v", err)
|
||||||
|
}
|
||||||
|
return destinationState
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFile(t *testing.T, path, want string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read file %s: %v", path, err)
|
||||||
|
}
|
||||||
|
if got := string(data); got != want {
|
||||||
|
t.Fatalf("%s = %q, want %q", path, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFileContains(t *testing.T, path, want string) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read file %s: %v", path, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), want) {
|
||||||
|
t.Fatalf("%s = %q, want substring %q", path, data, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingNotifier struct {
|
||||||
|
events []notify.Event
|
||||||
|
check func()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *recordingNotifier) Notify(ctx context.Context, event notify.Event) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if n.check != nil {
|
||||||
|
n.check()
|
||||||
|
}
|
||||||
|
n.events = append(n.events, event)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
33
internal/app/validate.go
Normal file
33
internal/app/validate.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/adapters/local"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ValidateOptions struct {
|
||||||
|
Path string
|
||||||
|
Stdout io.Writer
|
||||||
|
}
|
||||||
|
|
||||||
|
func Validate(ctx context.Context, options ValidateOptions) error {
|
||||||
|
if options.Path == "" {
|
||||||
|
return fmt.Errorf("validate command requires a path")
|
||||||
|
}
|
||||||
|
backend, err := local.New(options.Path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
bundles, err := bundle.Discover(ctx, backend, "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if options.Stdout != nil {
|
||||||
|
_, err = fmt.Fprintf(options.Stdout, "Validated %d bundle(s)\n", len(bundles))
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
39
internal/app/validate_test.go
Normal file
39
internal/app/validate_test.go
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateLocalBundle(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
err := Validate(context.Background(), ValidateOptions{
|
||||||
|
Path: filepath.Join("..", "bundle", "testdata", "valid_bundle"),
|
||||||
|
Stdout: &stdout,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := stdout.String(), "Validated 1 bundle(s)\n"; got != want {
|
||||||
|
t.Fatalf("stdout = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateExampleSourceBundle(t *testing.T) {
|
||||||
|
err := Validate(context.Background(), ValidateOptions{
|
||||||
|
Path: filepath.Join("..", "..", "examples", "source-bundle"),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() example error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRequiresPath(t *testing.T) {
|
||||||
|
err := Validate(context.Background(), ValidateOptions{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "requires a path") {
|
||||||
|
t.Fatalf("Validate() error = %v, want required path", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
49
internal/bundle/digest.go
Normal file
49
internal/bundle/digest.go
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||||
|
|
||||||
|
func ValidateDigest(value string) error {
|
||||||
|
if !digestPattern.MatchString(value) {
|
||||||
|
return fmt.Errorf("must be lowercase sha256:<64 hex>")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func FileDigest(data []byte) string {
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
return "sha256:" + hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func BundleDigest(files []ManifestFile) string {
|
||||||
|
canonical := CanonicalFilePayload(files)
|
||||||
|
sum := sha256.Sum256([]byte(canonical))
|
||||||
|
return "sha256:" + hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func CanonicalFilePayload(files []ManifestFile) string {
|
||||||
|
var builder strings.Builder
|
||||||
|
builder.WriteByte('[')
|
||||||
|
for index, file := range files {
|
||||||
|
if index > 0 {
|
||||||
|
builder.WriteByte(',')
|
||||||
|
}
|
||||||
|
builder.WriteString(`{"path":`)
|
||||||
|
builder.WriteString(strconv.Quote(file.Path))
|
||||||
|
builder.WriteString(`,"sha256":`)
|
||||||
|
builder.WriteString(strconv.Quote(file.SHA256))
|
||||||
|
builder.WriteString(`,"size":`)
|
||||||
|
builder.WriteString(strconv.FormatInt(file.Size, 10))
|
||||||
|
builder.WriteByte('}')
|
||||||
|
}
|
||||||
|
builder.WriteByte(']')
|
||||||
|
return builder.String()
|
||||||
|
}
|
||||||
21
internal/bundle/digest_test.go
Normal file
21
internal/bundle/digest_test.go
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCanonicalBundleDigestReferenceFixture(t *testing.T) {
|
||||||
|
manifest, err := ParseManifest(readFixture(t, "testdata/valid_bundle/manifest.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
gotPayload := CanonicalFilePayload(manifest.Files)
|
||||||
|
wantPayload := strings.TrimSpace(string(readFixture(t, "testdata/canonical_payload.json")))
|
||||||
|
if gotPayload != wantPayload {
|
||||||
|
t.Fatalf("canonical payload = %q, want %q", gotPayload, wantPayload)
|
||||||
|
}
|
||||||
|
if got, want := BundleDigest(manifest.Files), manifest.Digest; got != want {
|
||||||
|
t.Fatalf("BundleDigest() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
82
internal/bundle/discover.go
Normal file
82
internal/bundle/discover.go
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func Discover(ctx context.Context, backend storage.Backend, sourceRoot string) ([]Bundle, error) {
|
||||||
|
if err := storage.ValidatePrefix(sourceRoot); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
entries, err := storage.List(ctx, backend, sourceRoot, storage.WalkOptions{Recursive: true})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var roots []string
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.Type == storage.EntryTypeDirectory {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if path.Base(entry.Path) == ManifestName {
|
||||||
|
roots = append(roots, path.Dir(entry.Path))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for index, root := range roots {
|
||||||
|
if root == "." {
|
||||||
|
roots[index] = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(roots)
|
||||||
|
if len(roots) == 0 {
|
||||||
|
return nil, fmt.Errorf("no bundles found under %q", displayRoot(sourceRoot))
|
||||||
|
}
|
||||||
|
if err := rejectNestedRoots(roots); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
bundles := make([]Bundle, 0, len(roots))
|
||||||
|
for _, root := range roots {
|
||||||
|
relativeRoot := relativeToSource(sourceRoot, root)
|
||||||
|
sourceBundle, err := validateAt(ctx, backend, root, relativeRoot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
bundles = append(bundles, sourceBundle)
|
||||||
|
}
|
||||||
|
return bundles, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func rejectNestedRoots(roots []string) error {
|
||||||
|
for index, root := range roots {
|
||||||
|
for _, candidate := range roots[index+1:] {
|
||||||
|
if isAncestor(root, candidate) {
|
||||||
|
return fmt.Errorf("nested manifest %q under bundle %q", displayRoot(candidate), displayRoot(root))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isAncestor(root, candidate string) bool {
|
||||||
|
if root == "" {
|
||||||
|
return candidate != ""
|
||||||
|
}
|
||||||
|
return strings.HasPrefix(candidate, root+"/")
|
||||||
|
}
|
||||||
|
|
||||||
|
func relativeToSource(sourceRoot, bundleRoot string) string {
|
||||||
|
if sourceRoot == "" {
|
||||||
|
return bundleRoot
|
||||||
|
}
|
||||||
|
if bundleRoot == sourceRoot {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimPrefix(bundleRoot, sourceRoot+"/")
|
||||||
|
}
|
||||||
70
internal/bundle/discover_test.go
Normal file
70
internal/bundle/discover_test.go
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDiscoverFindsBundlesInDeterministicOrder(t *testing.T) {
|
||||||
|
backend := fake.New()
|
||||||
|
addBundle(t, backend, "z/daily")
|
||||||
|
addBundle(t, backend, "a/daily")
|
||||||
|
|
||||||
|
bundles, err := Discover(context.Background(), backend, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Discover() error = %v", err)
|
||||||
|
}
|
||||||
|
var paths []string
|
||||||
|
for _, sourceBundle := range bundles {
|
||||||
|
paths = append(paths, sourceBundle.RootRelativePath)
|
||||||
|
}
|
||||||
|
want := []string{"a/daily", "z/daily"}
|
||||||
|
if !reflect.DeepEqual(paths, want) {
|
||||||
|
t.Fatalf("paths = %v, want %v", paths, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverFindsRootBundle(t *testing.T) {
|
||||||
|
backend := validFakeBundle(t)
|
||||||
|
bundles, err := Discover(context.Background(), backend, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Discover() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := len(bundles), 1; got != want {
|
||||||
|
t.Fatalf("bundle count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
if bundles[0].RootRelativePath != "" {
|
||||||
|
t.Fatalf("root = %q, want empty", bundles[0].RootRelativePath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverRejectsNestedManifests(t *testing.T) {
|
||||||
|
backend := fake.New()
|
||||||
|
addBundle(t, backend, "daily")
|
||||||
|
addBundle(t, backend, "daily/nested")
|
||||||
|
|
||||||
|
_, err := Discover(context.Background(), backend, "")
|
||||||
|
assertErrorContains(t, err, "nested manifest")
|
||||||
|
}
|
||||||
|
|
||||||
|
func addBundle(t *testing.T, backend *fake.Backend, root string) {
|
||||||
|
t.Helper()
|
||||||
|
writeFakeFile(t, backend, joinTestPath(root, "manifest.json"), string(readFixture(t, "testdata/valid_bundle/manifest.json")))
|
||||||
|
writeFakeFile(t, backend, joinTestPath(root, "report.md"), string(readFixture(t, "testdata/valid_bundle/report.md")))
|
||||||
|
writeFakeFile(t, backend, joinTestPath(root, "summary.txt"), string(readFixture(t, "testdata/valid_bundle/summary.txt")))
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinTestPath(root, file string) string {
|
||||||
|
if root == "" {
|
||||||
|
return file
|
||||||
|
}
|
||||||
|
joined, err := storage.Join(root, file)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return joined
|
||||||
|
}
|
||||||
127
internal/bundle/manifest.go
Normal file
127
internal/bundle/manifest.go
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const ManifestName = "manifest.json"
|
||||||
|
|
||||||
|
type Manifest struct {
|
||||||
|
SchemaVersion int `json:"schema_version"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
Created time.Time `json:"created"`
|
||||||
|
Files []ManifestFile `json:"files"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ManifestFile struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Bundle struct {
|
||||||
|
RootRelativePath string
|
||||||
|
Manifest Manifest
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawManifest struct {
|
||||||
|
SchemaVersion *int `json:"schema_version"`
|
||||||
|
ID *string `json:"id"`
|
||||||
|
Digest *string `json:"digest"`
|
||||||
|
Created *string `json:"created"`
|
||||||
|
Files []rawManifestFile `json:"files"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawManifestFile struct {
|
||||||
|
Path *string `json:"path"`
|
||||||
|
SHA256 *string `json:"sha256"`
|
||||||
|
Size *int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func ParseManifest(data []byte) (Manifest, error) {
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
var raw rawManifest
|
||||||
|
if err := decoder.Decode(&raw); err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("parse manifest: %w", err)
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if err := decoder.Decode(&extra); err != io.EOF {
|
||||||
|
return Manifest{}, fmt.Errorf("parse manifest: trailing data")
|
||||||
|
}
|
||||||
|
|
||||||
|
var manifest Manifest
|
||||||
|
if raw.SchemaVersion == nil {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest schema_version is required")
|
||||||
|
}
|
||||||
|
manifest.SchemaVersion = *raw.SchemaVersion
|
||||||
|
if manifest.SchemaVersion != 1 {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest schema_version must be 1")
|
||||||
|
}
|
||||||
|
if raw.ID == nil || *raw.ID == "" {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest id is required")
|
||||||
|
}
|
||||||
|
manifest.ID = *raw.ID
|
||||||
|
if raw.Digest == nil || *raw.Digest == "" {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest digest is required")
|
||||||
|
}
|
||||||
|
if err := ValidateDigest(*raw.Digest); err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest digest: %w", err)
|
||||||
|
}
|
||||||
|
manifest.Digest = *raw.Digest
|
||||||
|
if raw.Created == nil || *raw.Created == "" {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest created is required")
|
||||||
|
}
|
||||||
|
created, err := time.Parse(time.RFC3339, *raw.Created)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest created must be RFC3339: %w", err)
|
||||||
|
}
|
||||||
|
manifest.Created = created
|
||||||
|
if len(raw.Files) == 0 {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest files is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := make(map[string]struct{}, len(raw.Files))
|
||||||
|
for index, rawFile := range raw.Files {
|
||||||
|
file, err := parseManifestFile(index, rawFile)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, err
|
||||||
|
}
|
||||||
|
if _, exists := seen[file.Path]; exists {
|
||||||
|
return Manifest{}, fmt.Errorf("manifest files[%d].path duplicates %q", index, file.Path)
|
||||||
|
}
|
||||||
|
seen[file.Path] = struct{}{}
|
||||||
|
manifest.Files = append(manifest.Files, file)
|
||||||
|
}
|
||||||
|
return manifest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseManifestFile(index int, raw rawManifestFile) (ManifestFile, error) {
|
||||||
|
if raw.Path == nil || *raw.Path == "" {
|
||||||
|
return ManifestFile{}, fmt.Errorf("manifest files[%d].path is required", index)
|
||||||
|
}
|
||||||
|
if err := ValidateSourcePath(*raw.Path); err != nil {
|
||||||
|
return ManifestFile{}, fmt.Errorf("manifest files[%d].path: %w", index, err)
|
||||||
|
}
|
||||||
|
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
||||||
|
return ManifestFile{}, fmt.Errorf("manifest files[%d].sha256 is required", index)
|
||||||
|
}
|
||||||
|
if err := ValidateDigest(*raw.SHA256); err != nil {
|
||||||
|
return ManifestFile{}, fmt.Errorf("manifest files[%d].sha256: %w", index, err)
|
||||||
|
}
|
||||||
|
if raw.Size == nil {
|
||||||
|
return ManifestFile{}, fmt.Errorf("manifest files[%d].size is required", index)
|
||||||
|
}
|
||||||
|
if *raw.Size < 0 {
|
||||||
|
return ManifestFile{}, fmt.Errorf("manifest files[%d].size must be non-negative", index)
|
||||||
|
}
|
||||||
|
return ManifestFile{
|
||||||
|
Path: *raw.Path,
|
||||||
|
SHA256: *raw.SHA256,
|
||||||
|
Size: *raw.Size,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
119
internal/bundle/manifest_test.go
Normal file
119
internal/bundle/manifest_test.go
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseManifestValid(t *testing.T) {
|
||||||
|
data := readFixture(t, "testdata/valid_bundle/manifest.json")
|
||||||
|
manifest, err := ParseManifest(data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
if manifest.SchemaVersion != 1 {
|
||||||
|
t.Fatalf("schema version = %d, want 1", manifest.SchemaVersion)
|
||||||
|
}
|
||||||
|
if manifest.ID != "weather.daily.brentwood.2026-05-30" {
|
||||||
|
t.Fatalf("id = %q", manifest.ID)
|
||||||
|
}
|
||||||
|
if got, want := len(manifest.Files), 2; got != want {
|
||||||
|
t.Fatalf("file count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseManifestRejectsInvalidJSON(t *testing.T) {
|
||||||
|
_, err := ParseManifest([]byte(`{"schema_version":`))
|
||||||
|
assertErrorContains(t, err, "parse manifest")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseManifestRejectsMissingRequiredFields(t *testing.T) {
|
||||||
|
tests := map[string]string{
|
||||||
|
"schema_version": `{"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`,
|
||||||
|
"id": `{"schema_version":1,"digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`,
|
||||||
|
"digest": `{"schema_version":1,"id":"id","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`,
|
||||||
|
"created": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`,
|
||||||
|
"files": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z"}`,
|
||||||
|
"file path": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000","size":0}]}`,
|
||||||
|
"file digest": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","size":0}]}`,
|
||||||
|
"file size": `{"schema_version":1,"id":"id","digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","created":"2026-05-30T11:10:00Z","files":[{"path":"report.md","sha256":"sha256:0000000000000000000000000000000000000000000000000000000000000000"}]}`,
|
||||||
|
}
|
||||||
|
for name, body := range tests {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
_, err := ParseManifest([]byte(body))
|
||||||
|
assertErrorContains(t, err, "required")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseManifestRejectsInvalidSchemaVersion(t *testing.T) {
|
||||||
|
data := replaceFixture(t, `"schema_version": 1`, `"schema_version": 2`)
|
||||||
|
_, err := ParseManifest(data)
|
||||||
|
assertErrorContains(t, err, "schema_version must be 1")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseManifestRejectsInvalidTimestamp(t *testing.T) {
|
||||||
|
data := replaceFixture(t, `"created": "2026-05-30T11:10:00Z"`, `"created": "May 30"`)
|
||||||
|
_, err := ParseManifest(data)
|
||||||
|
assertErrorContains(t, err, "RFC3339")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseManifestRejectsInvalidDigestFormat(t *testing.T) {
|
||||||
|
data := replaceFixture(t, `"digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"`, `"digest": "SHA256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"`)
|
||||||
|
_, err := ParseManifest(data)
|
||||||
|
assertErrorContains(t, err, "lowercase")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseManifestRejectsUnsafeFilePaths(t *testing.T) {
|
||||||
|
tests := []string{
|
||||||
|
`"path": "../report.md"`,
|
||||||
|
`"path": "/report.md"`,
|
||||||
|
`"path": "nested/../report.md"`,
|
||||||
|
`"path": "manifest.json"`,
|
||||||
|
`"path": ".distributor.json"`,
|
||||||
|
}
|
||||||
|
for _, replacement := range tests {
|
||||||
|
t.Run(replacement, func(t *testing.T) {
|
||||||
|
data := replaceFixture(t, `"path": "report.md"`, replacement)
|
||||||
|
_, err := ParseManifest(data)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ParseManifest() error = nil, want error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseManifestRejectsDuplicatePaths(t *testing.T) {
|
||||||
|
data := replaceFixture(t, `"path": "summary.txt"`, `"path": "report.md"`)
|
||||||
|
_, err := ParseManifest(data)
|
||||||
|
assertErrorContains(t, err, "duplicates")
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFixture(t *testing.T, path string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read fixture %s: %v", path, err)
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func replaceFixture(t *testing.T, old, replacement string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
body := string(readFixture(t, "testdata/valid_bundle/manifest.json"))
|
||||||
|
if !strings.Contains(body, old) {
|
||||||
|
t.Fatalf("fixture does not contain %q", old)
|
||||||
|
}
|
||||||
|
return []byte(strings.Replace(body, old, replacement, 1))
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertErrorContains(t *testing.T, err error, want string) {
|
||||||
|
t.Helper()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("error = nil, want substring %q", want)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Fatalf("error = %q, want substring %q", err.Error(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
1
internal/bundle/testdata/canonical_payload.json
vendored
Normal file
1
internal/bundle/testdata/canonical_payload.json
vendored
Normal file
@@ -0,0 +1 @@
|
|||||||
|
[{"path":"report.md","sha256":"sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6","size":16},{"path":"summary.txt","sha256":"sha256:3cbb36aca330b3bd113955dfbada0adb7a5f95ad9f678bd61f175406c6a37e95","size":8}]
|
||||||
18
internal/bundle/testdata/valid_bundle/manifest.json
vendored
Normal file
18
internal/bundle/testdata/valid_bundle/manifest.json
vendored
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"id": "weather.daily.brentwood.2026-05-30",
|
||||||
|
"digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe",
|
||||||
|
"created": "2026-05-30T11:10:00Z",
|
||||||
|
"files": [
|
||||||
|
{
|
||||||
|
"path": "report.md",
|
||||||
|
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
|
||||||
|
"size": 16
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "summary.txt",
|
||||||
|
"sha256": "sha256:3cbb36aca330b3bd113955dfbada0adb7a5f95ad9f678bd61f175406c6a37e95",
|
||||||
|
"size": 8
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
2
internal/bundle/testdata/valid_bundle/report.md
vendored
Normal file
2
internal/bundle/testdata/valid_bundle/report.md
vendored
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
# Report
|
||||||
|
Sunny.
|
||||||
1
internal/bundle/testdata/valid_bundle/summary.txt
vendored
Normal file
1
internal/bundle/testdata/valid_bundle/summary.txt
vendored
Normal file
@@ -0,0 +1 @@
|
|||||||
|
Summary
|
||||||
85
internal/bundle/validate.go
Normal file
85
internal/bundle/validate.go
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ValidateSourcePath(path string) error {
|
||||||
|
if err := storage.ValidatePath(path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
switch path {
|
||||||
|
case ManifestName, ".distributor.json":
|
||||||
|
return fmt.Errorf("%q is reserved", path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Validate(ctx context.Context, backend storage.Backend, bundleRoot string) (Bundle, error) {
|
||||||
|
return validateAt(ctx, backend, bundleRoot, bundleRoot)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateAt(ctx context.Context, backend storage.Backend, bundleRoot, relativeRoot string) (Bundle, error) {
|
||||||
|
if err := storage.ValidatePrefix(bundleRoot); err != nil {
|
||||||
|
return Bundle{}, err
|
||||||
|
}
|
||||||
|
manifestPath, err := storage.Join(bundleRoot, ManifestName)
|
||||||
|
if err != nil {
|
||||||
|
return Bundle{}, err
|
||||||
|
}
|
||||||
|
manifestData, err := backend.ReadFile(ctx, manifestPath)
|
||||||
|
if err != nil {
|
||||||
|
return Bundle{}, fmt.Errorf("read manifest %q: %w", manifestPath, err)
|
||||||
|
}
|
||||||
|
manifest, err := ParseManifest(manifestData)
|
||||||
|
if err != nil {
|
||||||
|
return Bundle{}, fmt.Errorf("bundle %q: %w", displayRoot(relativeRoot), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for index, manifestFile := range manifest.Files {
|
||||||
|
filePath, err := storage.Join(bundleRoot, manifestFile.Path)
|
||||||
|
if err != nil {
|
||||||
|
return Bundle{}, fmt.Errorf("bundle %q file %q: %w", displayRoot(relativeRoot), manifestFile.Path, err)
|
||||||
|
}
|
||||||
|
entry, err := backend.Stat(ctx, filePath)
|
||||||
|
if err != nil {
|
||||||
|
return Bundle{}, fmt.Errorf("bundle %q file %q stat: %w", displayRoot(relativeRoot), manifestFile.Path, err)
|
||||||
|
}
|
||||||
|
if entry.Type != storage.EntryTypeFile {
|
||||||
|
return Bundle{}, fmt.Errorf("bundle %q file %q must be a regular file", displayRoot(relativeRoot), manifestFile.Path)
|
||||||
|
}
|
||||||
|
if entry.Size != manifestFile.Size {
|
||||||
|
return Bundle{}, fmt.Errorf("bundle %q file %q size mismatch: got %d want %d", displayRoot(relativeRoot), manifestFile.Path, entry.Size, manifestFile.Size)
|
||||||
|
}
|
||||||
|
data, err := backend.ReadFile(ctx, filePath)
|
||||||
|
if err != nil {
|
||||||
|
return Bundle{}, fmt.Errorf("bundle %q file %q read: %w", displayRoot(relativeRoot), manifestFile.Path, err)
|
||||||
|
}
|
||||||
|
actualDigest := FileDigest(data)
|
||||||
|
if actualDigest != manifestFile.SHA256 {
|
||||||
|
return Bundle{}, fmt.Errorf("bundle %q file %q sha256 mismatch: got %s want %s", displayRoot(relativeRoot), manifestFile.Path, actualDigest, manifestFile.SHA256)
|
||||||
|
}
|
||||||
|
manifest.Files[index].SHA256 = actualDigest
|
||||||
|
manifest.Files[index].Size = int64(len(data))
|
||||||
|
}
|
||||||
|
|
||||||
|
actualBundleDigest := BundleDigest(manifest.Files)
|
||||||
|
if actualBundleDigest != manifest.Digest {
|
||||||
|
return Bundle{}, fmt.Errorf("bundle %q digest mismatch: got %s want %s", displayRoot(relativeRoot), actualBundleDigest, manifest.Digest)
|
||||||
|
}
|
||||||
|
|
||||||
|
return Bundle{
|
||||||
|
RootRelativePath: relativeRoot,
|
||||||
|
Manifest: manifest,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func displayRoot(root string) string {
|
||||||
|
if root == "" {
|
||||||
|
return "."
|
||||||
|
}
|
||||||
|
return root
|
||||||
|
}
|
||||||
88
internal/bundle/validate_test.go
Normal file
88
internal/bundle/validate_test.go
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateValidBundle(t *testing.T) {
|
||||||
|
backend := validFakeBundle(t)
|
||||||
|
sourceBundle, err := Validate(context.Background(), backend, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v", err)
|
||||||
|
}
|
||||||
|
if sourceBundle.RootRelativePath != "" {
|
||||||
|
t.Fatalf("root = %q, want empty", sourceBundle.RootRelativePath)
|
||||||
|
}
|
||||||
|
if sourceBundle.Manifest.ID != "weather.daily.brentwood.2026-05-30" {
|
||||||
|
t.Fatalf("id = %q", sourceBundle.Manifest.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsMissingFile(t *testing.T) {
|
||||||
|
backend := validFakeBundle(t)
|
||||||
|
deleteFakeFile(t, backend, "summary.txt")
|
||||||
|
_, err := Validate(context.Background(), backend, "")
|
||||||
|
assertErrorContains(t, err, "stat")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsSizeMismatch(t *testing.T) {
|
||||||
|
backend := validFakeBundle(t)
|
||||||
|
manifest := strings.Replace(string(readFixture(t, "testdata/valid_bundle/manifest.json")), `"size": 8`, `"size": 9`, 1)
|
||||||
|
writeFakeFile(t, backend, "manifest.json", manifest)
|
||||||
|
_, err := Validate(context.Background(), backend, "")
|
||||||
|
assertErrorContains(t, err, "size mismatch")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsPerFileDigestMismatch(t *testing.T) {
|
||||||
|
backend := validFakeBundle(t)
|
||||||
|
writeFakeFile(t, backend, "report.md", "# Report\nCloud.\n")
|
||||||
|
_, err := Validate(context.Background(), backend, "")
|
||||||
|
assertErrorContains(t, err, "sha256 mismatch")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsBundleDigestMismatch(t *testing.T) {
|
||||||
|
backend := validFakeBundle(t)
|
||||||
|
manifest := strings.Replace(string(readFixture(t, "testdata/valid_bundle/manifest.json")), `"digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe"`, `"digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000"`, 1)
|
||||||
|
writeFakeFile(t, backend, "manifest.json", manifest)
|
||||||
|
_, err := Validate(context.Background(), backend, "")
|
||||||
|
assertErrorContains(t, err, "digest mismatch")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsSymlinkFile(t *testing.T) {
|
||||||
|
backend := validFakeBundle(t)
|
||||||
|
if err := backend.AddSymlink("summary.txt"); err != nil {
|
||||||
|
t.Fatalf("AddSymlink() error = %v", err)
|
||||||
|
}
|
||||||
|
_, err := Validate(context.Background(), backend, "")
|
||||||
|
assertErrorContains(t, err, "regular file")
|
||||||
|
}
|
||||||
|
|
||||||
|
func validFakeBundle(t *testing.T) *fake.Backend {
|
||||||
|
t.Helper()
|
||||||
|
backend := fake.New()
|
||||||
|
writeFakeFile(t, backend, "manifest.json", string(readFixture(t, "testdata/valid_bundle/manifest.json")))
|
||||||
|
writeFakeFile(t, backend, "report.md", string(readFixture(t, "testdata/valid_bundle/report.md")))
|
||||||
|
writeFakeFile(t, backend, "summary.txt", string(readFixture(t, "testdata/valid_bundle/summary.txt")))
|
||||||
|
return backend
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeFakeFile(t *testing.T, backend *fake.Backend, path, data string) {
|
||||||
|
t.Helper()
|
||||||
|
_, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{Overwrite: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteFakeFile(t *testing.T, backend *fake.Backend, path string) {
|
||||||
|
t.Helper()
|
||||||
|
err := backend.DeleteManagedBundle(context.Background(), "", []string{path}, storage.DeleteOptions{IgnoreMissing: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DeleteManagedBundle(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
36
internal/cli/inspect.go
Normal file
36
internal/cli/inspect.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||||
|
)
|
||||||
|
|
||||||
|
func inspectCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||||
|
if hasHelp(args) {
|
||||||
|
printInspectHelp(stdout)
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
if len(args) > 1 {
|
||||||
|
fmt.Fprintf(stderr, "%s: inspect accepts at most one path\n", app.Name)
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
var path string
|
||||||
|
if len(args) == 1 {
|
||||||
|
path = args[0]
|
||||||
|
}
|
||||||
|
if err := app.Inspect(ctx, app.InspectOptions{Path: path, Stdout: stdout}); err != nil {
|
||||||
|
return fail(stderr, err)
|
||||||
|
}
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func printInspectHelp(w io.Writer) {
|
||||||
|
fmt.Fprint(w, `Usage:
|
||||||
|
distributor inspect <path>
|
||||||
|
|
||||||
|
Print a normalized summary of local source bundles.
|
||||||
|
`)
|
||||||
|
}
|
||||||
84
internal/cli/root.go
Normal file
84
internal/cli/root.go
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
exitOK = 0
|
||||||
|
exitError = 1
|
||||||
|
exitUsage = 2
|
||||||
|
)
|
||||||
|
|
||||||
|
func Execute(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||||
|
if len(args) == 0 {
|
||||||
|
printRootHelp(stdout)
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
switch args[0] {
|
||||||
|
case "-h", "--help", "help":
|
||||||
|
printRootHelp(stdout)
|
||||||
|
return exitOK
|
||||||
|
case "version":
|
||||||
|
return versionCommand(ctx, args[1:], stdout, stderr)
|
||||||
|
case "run":
|
||||||
|
return runCommand(ctx, args[1:], stdout, stderr)
|
||||||
|
case "validate":
|
||||||
|
return validateCommand(ctx, args[1:], stdout, stderr)
|
||||||
|
case "inspect":
|
||||||
|
return inspectCommand(ctx, args[1:], stdout, stderr)
|
||||||
|
default:
|
||||||
|
fmt.Fprintf(stderr, "%s: unknown command %q\n\n", app.Name, args[0])
|
||||||
|
printRootHelp(stderr)
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func printRootHelp(w io.Writer) {
|
||||||
|
fmt.Fprintf(w, `%s validates and publishes manifested report bundles.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
%s <command> [options]
|
||||||
|
|
||||||
|
Commands:
|
||||||
|
version Print version information
|
||||||
|
run Run configured distribution pipelines
|
||||||
|
validate Validate a source bundle or bundle tree
|
||||||
|
inspect Inspect bundles or distributor state
|
||||||
|
|
||||||
|
Use "%s <command> --help" for command-specific help.
|
||||||
|
`, app.Name, app.Name, app.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasHelp(args []string) bool {
|
||||||
|
for _, arg := range args {
|
||||||
|
if arg == "-h" || arg == "--help" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func fail(stderr io.Writer, err error) int {
|
||||||
|
if errors.Is(err, app.ErrNotImplemented) {
|
||||||
|
fmt.Fprintf(stderr, "%s: %s\n", app.Name, err)
|
||||||
|
return exitError
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stderr, "%s: %s\n", app.Name, err)
|
||||||
|
return exitError
|
||||||
|
}
|
||||||
|
|
||||||
|
func rejectExtraArgs(stderr io.Writer, command string, args []string) bool {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stderr, "%s: %s does not accept arguments: %s\n", app.Name, command, strings.Join(args, " "))
|
||||||
|
return true
|
||||||
|
}
|
||||||
181
internal/cli/root_test.go
Normal file
181
internal/cli/root_test.go
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecuteRootHelp(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"--help"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d", code, exitOK)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "Usage:") {
|
||||||
|
t.Fatalf("stdout = %q, want help text", stdout.String())
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteVersion(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"version"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d", code, exitOK)
|
||||||
|
}
|
||||||
|
if got, want := stdout.String(), "distributor dev\n"; got != want {
|
||||||
|
t.Fatalf("stdout = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteValidate(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"validate", filepath.Join("..", "bundle", "testdata", "valid_bundle")}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
if got, want := stdout.String(), "Validated 1 bundle(s)\n"; got != want {
|
||||||
|
t.Fatalf("stdout = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteInspect(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"inspect", filepath.Join("..", "bundle", "testdata", "valid_bundle")}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "id=weather.daily.brentwood.2026-05-30") {
|
||||||
|
t.Fatalf("stdout = %q, want bundle summary", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunDryRun(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
writeCLIBundle(t, sourceRoot)
|
||||||
|
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
err := os.WriteFile(configPath, []byte(`
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: `+t.TempDir()+`
|
||||||
|
`), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"run", "--config", configPath, "--dry-run"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "action=publish_new") {
|
||||||
|
t.Fatalf("stdout = %q, want config summary", stdout.String())
|
||||||
|
}
|
||||||
|
if stderr.Len() != 0 {
|
||||||
|
t.Fatalf("stderr = %q, want empty", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteRunPublishes(t *testing.T) {
|
||||||
|
sourceRoot := t.TempDir()
|
||||||
|
destinationRoot := t.TempDir()
|
||||||
|
writeCLIBundle(t, sourceRoot)
|
||||||
|
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
err := os.WriteFile(configPath, []byte(`
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: `+sourceRoot+`
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: `+destinationRoot+`
|
||||||
|
`), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"run", "--config", configPath}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitOK {
|
||||||
|
t.Fatalf("exit code = %d, want %d; stderr = %q", code, exitOK, stderr.String())
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(destinationRoot, ".distributor.json")); err != nil {
|
||||||
|
t.Fatalf("state stat error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnknownCommandIsUsageError(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := Execute(context.Background(), []string{"nope"}, &stdout, &stderr)
|
||||||
|
|
||||||
|
if code != exitUsage {
|
||||||
|
t.Fatalf("exit code = %d, want %d", code, exitUsage)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "unknown command") {
|
||||||
|
t.Fatalf("stderr = %q, want unknown command error", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeCLIBundle(t *testing.T, root string) {
|
||||||
|
t.Helper()
|
||||||
|
for _, file := range []struct {
|
||||||
|
path string
|
||||||
|
data string
|
||||||
|
}{
|
||||||
|
{"manifest.json", `{
|
||||||
|
"schema_version": 1,
|
||||||
|
"id": "weather.daily.brentwood.2026-05-30",
|
||||||
|
"digest": "sha256:099b205780d2b050024868399961b05731729a548d5d6329c7b06a6740dd75fe",
|
||||||
|
"created": "2026-05-30T11:10:00Z",
|
||||||
|
"files": [
|
||||||
|
{
|
||||||
|
"path": "report.md",
|
||||||
|
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
|
||||||
|
"size": 16
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "summary.txt",
|
||||||
|
"sha256": "sha256:3cbb36aca330b3bd113955dfbada0adb7a5f95ad9f678bd61f175406c6a37e95",
|
||||||
|
"size": 8
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
`},
|
||||||
|
{"report.md", "# Report\nSunny.\n"},
|
||||||
|
{"summary.txt", "Summary\n"},
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(root, file.path), []byte(file.data), 0o600); err != nil {
|
||||||
|
t.Fatalf("write bundle file: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
51
internal/cli/run.go
Normal file
51
internal/cli/run.go
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||||
|
)
|
||||||
|
|
||||||
|
func runCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||||
|
if hasHelp(args) {
|
||||||
|
printRunHelp(stdout)
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
flags := flag.NewFlagSet("run", flag.ContinueOnError)
|
||||||
|
flags.SetOutput(stderr)
|
||||||
|
configPath := flags.String("config", "", "path to config file")
|
||||||
|
dryRun := flags.Bool("dry-run", false, "load and validate config without publishing")
|
||||||
|
if err := flags.Parse(args); err != nil {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
if flags.NArg() > 0 {
|
||||||
|
fmt.Fprintf(stderr, "%s: run does not accept positional arguments: %v\n", app.Name, flags.Args())
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := app.Run(ctx, app.RunOptions{
|
||||||
|
ConfigPath: *configPath,
|
||||||
|
DryRun: *dryRun,
|
||||||
|
Stdout: stdout,
|
||||||
|
}); err != nil {
|
||||||
|
return fail(stderr, err)
|
||||||
|
}
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func printRunHelp(w io.Writer) {
|
||||||
|
fmt.Fprint(w, `Usage:
|
||||||
|
distributor run --config <path> --dry-run
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--config <path> Path to config file
|
||||||
|
--dry-run Load and validate config without publishing
|
||||||
|
|
||||||
|
Run discovers local source bundles, plans each configured destination, publishes
|
||||||
|
selected outputs unless --dry-run is set, and prints a final status summary.
|
||||||
|
`)
|
||||||
|
}
|
||||||
36
internal/cli/validate.go
Normal file
36
internal/cli/validate.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||||
|
)
|
||||||
|
|
||||||
|
func validateCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||||
|
if hasHelp(args) {
|
||||||
|
printValidateHelp(stdout)
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
if len(args) > 1 {
|
||||||
|
fmt.Fprintf(stderr, "%s: validate accepts at most one path\n", app.Name)
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
var path string
|
||||||
|
if len(args) == 1 {
|
||||||
|
path = args[0]
|
||||||
|
}
|
||||||
|
if err := app.Validate(ctx, app.ValidateOptions{Path: path, Stdout: stdout}); err != nil {
|
||||||
|
return fail(stderr, err)
|
||||||
|
}
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func printValidateHelp(w io.Writer) {
|
||||||
|
fmt.Fprint(w, `Usage:
|
||||||
|
distributor validate <path>
|
||||||
|
|
||||||
|
Validate a local source bundle directory or a tree containing source bundles.
|
||||||
|
`)
|
||||||
|
}
|
||||||
29
internal/cli/version.go
Normal file
29
internal/cli/version.go
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/app"
|
||||||
|
)
|
||||||
|
|
||||||
|
func versionCommand(_ context.Context, args []string, stdout, stderr io.Writer) int {
|
||||||
|
if hasHelp(args) {
|
||||||
|
printVersionHelp(stdout)
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
if rejectExtraArgs(stderr, "version", args) {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
fmt.Fprintln(stdout, app.VersionString())
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func printVersionHelp(w io.Writer) {
|
||||||
|
fmt.Fprint(w, `Usage:
|
||||||
|
distributor version
|
||||||
|
|
||||||
|
Print version information.
|
||||||
|
`)
|
||||||
|
}
|
||||||
70
internal/config/config.go
Normal file
70
internal/config/config.go
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
Pipelines []Pipeline `yaml:"pipelines"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Pipeline struct {
|
||||||
|
ID string `yaml:"id"`
|
||||||
|
Source Backend `yaml:"source"`
|
||||||
|
Validation ValidationPolicy `yaml:"validation"`
|
||||||
|
Destinations []Destination `yaml:"destinations"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Destination struct {
|
||||||
|
ID string `yaml:"id"`
|
||||||
|
Backend string `yaml:"backend"`
|
||||||
|
Path string `yaml:"path"`
|
||||||
|
URI string `yaml:"uri"`
|
||||||
|
Endpoint string `yaml:"endpoint"`
|
||||||
|
Bucket string `yaml:"bucket"`
|
||||||
|
Prefix string `yaml:"prefix"`
|
||||||
|
Region string `yaml:"region"`
|
||||||
|
ForcePath bool `yaml:"force_path_style"`
|
||||||
|
Creds Credentials `yaml:"credentials"`
|
||||||
|
Publish *PublishPolicy `yaml:"publish"`
|
||||||
|
Transform Transform `yaml:"transform"`
|
||||||
|
Transfer TransferPolicy `yaml:"transfer"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Backend struct {
|
||||||
|
Backend string `yaml:"backend"`
|
||||||
|
Path string `yaml:"path"`
|
||||||
|
URI string `yaml:"uri"`
|
||||||
|
Endpoint string `yaml:"endpoint"`
|
||||||
|
Bucket string `yaml:"bucket"`
|
||||||
|
Prefix string `yaml:"prefix"`
|
||||||
|
Region string `yaml:"region"`
|
||||||
|
ForcePath bool `yaml:"force_path_style"`
|
||||||
|
Creds Credentials `yaml:"credentials"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Credentials struct {
|
||||||
|
AccessKeyIDEnv string `yaml:"access_key_id_env"`
|
||||||
|
SecretAccessKeyEnv string `yaml:"secret_access_key_env"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ValidationPolicy struct {
|
||||||
|
OnDigestMismatch string `yaml:"on_digest_mismatch"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type PublishPolicy struct {
|
||||||
|
Source bool `yaml:"source"`
|
||||||
|
HTML bool `yaml:"html"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Transform struct {
|
||||||
|
MarkdownToHTML *MarkdownToHTML `yaml:"markdown_to_html"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type MarkdownToHTML struct {
|
||||||
|
Enabled bool `yaml:"enabled"`
|
||||||
|
Mode string `yaml:"mode"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type TransferPolicy struct {
|
||||||
|
OnDestinationSame string `yaml:"on_destination_same"`
|
||||||
|
OnDestinationOlder string `yaml:"on_destination_older"`
|
||||||
|
OnDestinationNewer string `yaml:"on_destination_newer"`
|
||||||
|
OnConflict string `yaml:"on_conflict"`
|
||||||
|
}
|
||||||
50
internal/config/defaults.go
Normal file
50
internal/config/defaults.go
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
const DefaultConfigPath = "/usr/local/etc/distributor/config.yml"
|
||||||
|
|
||||||
|
const (
|
||||||
|
BackendLocal = "local"
|
||||||
|
BackendSSH = "ssh"
|
||||||
|
BackendS3 = "s3"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ValidationActionFail = "fail"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
TransferActionSkip = "skip"
|
||||||
|
TransferActionReplace = "replace"
|
||||||
|
TransferActionFail = "fail"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
TransformModeSidecar = "sidecar"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ApplyDefaults(cfg *Config) {
|
||||||
|
for pipelineIndex := range cfg.Pipelines {
|
||||||
|
pipeline := &cfg.Pipelines[pipelineIndex]
|
||||||
|
if pipeline.Validation.OnDigestMismatch == "" {
|
||||||
|
pipeline.Validation.OnDigestMismatch = ValidationActionFail
|
||||||
|
}
|
||||||
|
for destinationIndex := range pipeline.Destinations {
|
||||||
|
destination := &pipeline.Destinations[destinationIndex]
|
||||||
|
if destination.Publish == nil {
|
||||||
|
destination.Publish = &PublishPolicy{Source: true}
|
||||||
|
}
|
||||||
|
if destination.Transfer.OnDestinationSame == "" {
|
||||||
|
destination.Transfer.OnDestinationSame = TransferActionSkip
|
||||||
|
}
|
||||||
|
if destination.Transfer.OnDestinationOlder == "" {
|
||||||
|
destination.Transfer.OnDestinationOlder = TransferActionReplace
|
||||||
|
}
|
||||||
|
if destination.Transfer.OnDestinationNewer == "" {
|
||||||
|
destination.Transfer.OnDestinationNewer = TransferActionSkip
|
||||||
|
}
|
||||||
|
if destination.Transfer.OnConflict == "" {
|
||||||
|
destination.Transfer.OnConflict = TransferActionFail
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
29
internal/config/load.go
Normal file
29
internal/config/load.go
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func LoadFile(path string) (Config, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return Config{}, fmt.Errorf("load config %q: %w", path, err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
var cfg Config
|
||||||
|
decoder := yaml.NewDecoder(file)
|
||||||
|
decoder.KnownFields(true)
|
||||||
|
if err := decoder.Decode(&cfg); err != nil {
|
||||||
|
return Config{}, fmt.Errorf("parse config %q: %w", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ApplyDefaults(&cfg)
|
||||||
|
if err := Validate(cfg); err != nil {
|
||||||
|
return Config{}, fmt.Errorf("validate config %q: %w", path, err)
|
||||||
|
}
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
308
internal/config/load_test.go
Normal file
308
internal/config/load_test.go
Normal file
@@ -0,0 +1,308 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadFileValidMinimalLocalToLocalConfig(t *testing.T) {
|
||||||
|
cfg := loadConfig(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: local-copy
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /var/spool/reports
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /srv/archive
|
||||||
|
`)
|
||||||
|
|
||||||
|
if got, want := len(cfg.Pipelines), 1; got != want {
|
||||||
|
t.Fatalf("pipeline count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
destination := cfg.Pipelines[0].Destinations[0]
|
||||||
|
if !destination.Publish.Source || destination.Publish.HTML {
|
||||||
|
t.Fatalf("publish defaults = source:%t html:%t, want source:true html:false", destination.Publish.Source, destination.Publish.HTML)
|
||||||
|
}
|
||||||
|
if got, want := cfg.Pipelines[0].Validation.OnDigestMismatch, ValidationActionFail; got != want {
|
||||||
|
t.Fatalf("validation default = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if got, want := destination.Transfer.OnDestinationOlder, TransferActionReplace; got != want {
|
||||||
|
t.Fatalf("transfer default = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileValidFanOutConfig(t *testing.T) {
|
||||||
|
cfg := loadConfig(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: fan-out
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /var/spool/reports
|
||||||
|
destinations:
|
||||||
|
- id: markdown-archive
|
||||||
|
backend: s3
|
||||||
|
endpoint: https://s3.example.com
|
||||||
|
bucket: reports
|
||||||
|
prefix: archive
|
||||||
|
publish:
|
||||||
|
source: true
|
||||||
|
html: false
|
||||||
|
- id: static-site
|
||||||
|
backend: ssh
|
||||||
|
uri: ssh://deploy@example.com:22
|
||||||
|
path: /srv/www/reports
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
transform:
|
||||||
|
markdown_to_html:
|
||||||
|
enabled: true
|
||||||
|
mode: sidecar
|
||||||
|
`)
|
||||||
|
|
||||||
|
if got, want := len(cfg.Pipelines[0].Destinations), 2; got != want {
|
||||||
|
t.Fatalf("destination count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileValidBackendConfigs(t *testing.T) {
|
||||||
|
tests := map[string]string{
|
||||||
|
"local": `
|
||||||
|
pipelines:
|
||||||
|
- id: local-backend
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /source
|
||||||
|
destinations:
|
||||||
|
- id: local-destination
|
||||||
|
backend: local
|
||||||
|
path: /destination
|
||||||
|
`,
|
||||||
|
"ssh": `
|
||||||
|
pipelines:
|
||||||
|
- id: ssh-backend
|
||||||
|
source:
|
||||||
|
backend: ssh
|
||||||
|
uri: ssh://reports@example.com:22
|
||||||
|
path: /source
|
||||||
|
destinations:
|
||||||
|
- id: ssh-destination
|
||||||
|
backend: ssh
|
||||||
|
uri: ssh://deploy@example.com:22
|
||||||
|
path: /destination
|
||||||
|
`,
|
||||||
|
"s3": `
|
||||||
|
pipelines:
|
||||||
|
- id: s3-backend
|
||||||
|
source:
|
||||||
|
backend: s3
|
||||||
|
endpoint: https://s3.example.com
|
||||||
|
bucket: source
|
||||||
|
prefix: incoming
|
||||||
|
region: us-east-1
|
||||||
|
force_path_style: true
|
||||||
|
credentials:
|
||||||
|
access_key_id_env: DISTRIBUTOR_S3_ACCESS_KEY_ID
|
||||||
|
secret_access_key_env: DISTRIBUTOR_S3_SECRET_ACCESS_KEY
|
||||||
|
destinations:
|
||||||
|
- id: s3-destination
|
||||||
|
backend: s3
|
||||||
|
endpoint: https://s3.example.com
|
||||||
|
bucket: destination
|
||||||
|
prefix: archive
|
||||||
|
`,
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, body := range tests {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
loadConfig(t, body)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileRejectsDuplicatePipelineIDs(t *testing.T) {
|
||||||
|
assertLoadError(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: duplicate
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /one
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /archive
|
||||||
|
- id: duplicate
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /two
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /archive
|
||||||
|
`, "pipeline id duplicate is duplicated")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileRejectsDuplicateDestinationIDs(t *testing.T) {
|
||||||
|
assertLoadError(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /source
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /archive-one
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /archive-two
|
||||||
|
`, "destination id archive is duplicated")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileRejectsMissingRequiredFields(t *testing.T) {
|
||||||
|
tests := map[string]string{
|
||||||
|
"pipelines": ``,
|
||||||
|
"pipeline id": `pipelines: [{source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||||
|
"source backend": `pipelines: [{id: reports, source: {path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||||
|
"destinations": `pipelines: [{id: reports, source: {backend: local, path: /source}}]`,
|
||||||
|
"destination id": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{backend: local, path: /archive}]}]`,
|
||||||
|
"local path": `pipelines: [{id: reports, source: {backend: local}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||||
|
"ssh uri": `pipelines: [{id: reports, source: {backend: ssh, path: /source}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||||
|
"s3 bucket": `pipelines: [{id: reports, source: {backend: s3, endpoint: "https://s3.example.com"}, destinations: [{id: archive, backend: local, path: /archive}]}]`,
|
||||||
|
"publish outputs": `pipelines: [{id: reports, source: {backend: local, path: /source}, destinations: [{id: archive, backend: local, path: /archive, publish: {source: false, html: false}}]}]`,
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, body := range tests {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
assertLoadError(t, body, "")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileRejectsUnsupportedBackend(t *testing.T) {
|
||||||
|
assertLoadError(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: ftp
|
||||||
|
path: /source
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /archive
|
||||||
|
`, "backend ftp is unsupported")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileRejectsInvalidTransferAction(t *testing.T) {
|
||||||
|
assertLoadError(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /source
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /archive
|
||||||
|
transfer:
|
||||||
|
on_destination_older: overwrite
|
||||||
|
`, "on_destination_older must be replace or fail")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileRejectsInvalidValidationAction(t *testing.T) {
|
||||||
|
assertLoadError(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /source
|
||||||
|
validation:
|
||||||
|
on_digest_mismatch: warn
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /archive
|
||||||
|
`, "on_digest_mismatch must be fail")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileRejectsHTMLPublishWithoutTransform(t *testing.T) {
|
||||||
|
assertLoadError(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /source
|
||||||
|
destinations:
|
||||||
|
- id: static
|
||||||
|
backend: local
|
||||||
|
path: /srv/www
|
||||||
|
publish:
|
||||||
|
source: false
|
||||||
|
html: true
|
||||||
|
`, "markdown_to_html is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadFileRejectsUnknownFields(t *testing.T) {
|
||||||
|
assertLoadError(t, `
|
||||||
|
pipelines:
|
||||||
|
- id: reports
|
||||||
|
surprise: true
|
||||||
|
source:
|
||||||
|
backend: local
|
||||||
|
path: /source
|
||||||
|
destinations:
|
||||||
|
- id: archive
|
||||||
|
backend: local
|
||||||
|
path: /archive
|
||||||
|
`, "field surprise not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExampleConfigsLoad(t *testing.T) {
|
||||||
|
for _, path := range []string{
|
||||||
|
"../../examples/local-to-local.yml",
|
||||||
|
"../../examples/local-publish.yml",
|
||||||
|
"../../examples/local-html.yml",
|
||||||
|
"../../examples/fan-out.yml",
|
||||||
|
} {
|
||||||
|
t.Run(path, func(t *testing.T) {
|
||||||
|
if _, err := LoadFile(path); err != nil {
|
||||||
|
t.Fatalf("LoadFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadConfig(t *testing.T, body string) Config {
|
||||||
|
t.Helper()
|
||||||
|
path := writeConfig(t, body)
|
||||||
|
cfg, err := LoadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadFile() error = %v", err)
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertLoadError(t *testing.T, body, want string) {
|
||||||
|
t.Helper()
|
||||||
|
path := writeConfig(t, body)
|
||||||
|
_, err := LoadFile(path)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("LoadFile() error = nil, want error")
|
||||||
|
}
|
||||||
|
if want != "" && !strings.Contains(err.Error(), want) {
|
||||||
|
t.Fatalf("LoadFile() error = %q, want substring %q", err.Error(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeConfig(t *testing.T, body string) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
if err := os.WriteFile(path, []byte(strings.TrimSpace(body)+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatalf("write config: %v", err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
153
internal/config/validate.go
Normal file
153
internal/config/validate.go
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
var idPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*$`)
|
||||||
|
|
||||||
|
type ValidationErrors []string
|
||||||
|
|
||||||
|
func (e ValidationErrors) Error() string {
|
||||||
|
if len(e) == 1 {
|
||||||
|
return e[0]
|
||||||
|
}
|
||||||
|
return strings.Join(e, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func Validate(cfg Config) error {
|
||||||
|
var errs ValidationErrors
|
||||||
|
|
||||||
|
if len(cfg.Pipelines) == 0 {
|
||||||
|
errs = append(errs, "pipelines is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
pipelineIDs := make(map[string]struct{}, len(cfg.Pipelines))
|
||||||
|
for pipelineIndex, pipeline := range cfg.Pipelines {
|
||||||
|
pipelineContext := fmt.Sprintf("pipelines[%d]", pipelineIndex)
|
||||||
|
if pipeline.ID == "" {
|
||||||
|
errs = append(errs, pipelineContext+".id is required")
|
||||||
|
} else if !idPattern.MatchString(pipeline.ID) {
|
||||||
|
errs = append(errs, pipelineContext+".id must be a slug-like identifier")
|
||||||
|
} else if _, exists := pipelineIDs[pipeline.ID]; exists {
|
||||||
|
errs = append(errs, "pipeline id "+pipeline.ID+" is duplicated")
|
||||||
|
} else {
|
||||||
|
pipelineIDs[pipeline.ID] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
errs = validateBackend(errs, pipelineContext+".source", pipeline.Source.Backend, pipeline.Source.Path, pipeline.Source.URI, pipeline.Source.Endpoint, pipeline.Source.Bucket)
|
||||||
|
errs = validateValidationPolicy(errs, pipelineContext+".validation", pipeline.Validation)
|
||||||
|
if len(pipeline.Destinations) == 0 {
|
||||||
|
errs = append(errs, pipelineContext+".destinations is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
destinationIDs := make(map[string]struct{}, len(pipeline.Destinations))
|
||||||
|
for destinationIndex, destination := range pipeline.Destinations {
|
||||||
|
destinationContext := fmt.Sprintf("%s.destinations[%d]", pipelineContext, destinationIndex)
|
||||||
|
if destination.ID == "" {
|
||||||
|
errs = append(errs, destinationContext+".id is required")
|
||||||
|
} else if !idPattern.MatchString(destination.ID) {
|
||||||
|
errs = append(errs, destinationContext+".id must be a slug-like identifier")
|
||||||
|
} else if _, exists := destinationIDs[destination.ID]; exists {
|
||||||
|
errs = append(errs, "destination id "+destination.ID+" is duplicated in pipeline "+pipeline.ID)
|
||||||
|
} else {
|
||||||
|
destinationIDs[destination.ID] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
errs = validateBackend(errs, destinationContext, destination.Backend, destination.Path, destination.URI, destination.Endpoint, destination.Bucket)
|
||||||
|
errs = validatePublishPolicy(errs, destinationContext+".publish", destination.Publish)
|
||||||
|
errs = validateTransform(errs, destinationContext+".transform", destination.Publish, destination.Transform)
|
||||||
|
errs = validateTransferPolicy(errs, destinationContext+".transfer", destination.Transfer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(errs) > 0 {
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateBackend(errs ValidationErrors, context, backend, path, uri, endpoint, bucket string) ValidationErrors {
|
||||||
|
switch backend {
|
||||||
|
case "":
|
||||||
|
errs = append(errs, context+".backend is required")
|
||||||
|
case BackendLocal:
|
||||||
|
if path == "" {
|
||||||
|
errs = append(errs, context+".path is required for local backend")
|
||||||
|
}
|
||||||
|
case BackendSSH:
|
||||||
|
if uri == "" {
|
||||||
|
errs = append(errs, context+".uri is required for ssh backend")
|
||||||
|
}
|
||||||
|
if path == "" {
|
||||||
|
errs = append(errs, context+".path is required for ssh backend")
|
||||||
|
}
|
||||||
|
case BackendS3:
|
||||||
|
if endpoint == "" {
|
||||||
|
errs = append(errs, context+".endpoint is required for s3 backend")
|
||||||
|
}
|
||||||
|
if bucket == "" {
|
||||||
|
errs = append(errs, context+".bucket is required for s3 backend")
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
errs = append(errs, context+".backend "+backend+" is unsupported")
|
||||||
|
}
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateValidationPolicy(errs ValidationErrors, context string, policy ValidationPolicy) ValidationErrors {
|
||||||
|
if policy.OnDigestMismatch != ValidationActionFail {
|
||||||
|
errs = append(errs, context+".on_digest_mismatch must be "+ValidationActionFail)
|
||||||
|
}
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
|
func validatePublishPolicy(errs ValidationErrors, context string, policy *PublishPolicy) ValidationErrors {
|
||||||
|
if policy == nil {
|
||||||
|
errs = append(errs, context+" is required")
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
if !policy.Source && !policy.HTML {
|
||||||
|
errs = append(errs, context+" must enable source or html")
|
||||||
|
}
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateTransform(errs ValidationErrors, context string, publish *PublishPolicy, transform Transform) ValidationErrors {
|
||||||
|
publishesHTML := publish != nil && publish.HTML
|
||||||
|
if transform.MarkdownToHTML == nil {
|
||||||
|
if publishesHTML {
|
||||||
|
errs = append(errs, context+".markdown_to_html is required when publish.html is true")
|
||||||
|
}
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
|
if publishesHTML && !transform.MarkdownToHTML.Enabled {
|
||||||
|
errs = append(errs, context+".markdown_to_html.enabled must be true when publish.html is true")
|
||||||
|
}
|
||||||
|
if transform.MarkdownToHTML.Enabled && transform.MarkdownToHTML.Mode != TransformModeSidecar {
|
||||||
|
errs = append(errs, context+".markdown_to_html.mode must be "+TransformModeSidecar)
|
||||||
|
}
|
||||||
|
if !transform.MarkdownToHTML.Enabled && transform.MarkdownToHTML.Mode != "" && transform.MarkdownToHTML.Mode != TransformModeSidecar {
|
||||||
|
errs = append(errs, context+".markdown_to_html.mode must be "+TransformModeSidecar)
|
||||||
|
}
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateTransferPolicy(errs ValidationErrors, context string, policy TransferPolicy) ValidationErrors {
|
||||||
|
if policy.OnDestinationSame != TransferActionSkip && policy.OnDestinationSame != TransferActionFail {
|
||||||
|
errs = append(errs, context+".on_destination_same must be skip or fail")
|
||||||
|
}
|
||||||
|
if policy.OnDestinationOlder != TransferActionReplace && policy.OnDestinationOlder != TransferActionFail {
|
||||||
|
errs = append(errs, context+".on_destination_older must be replace or fail")
|
||||||
|
}
|
||||||
|
if policy.OnDestinationNewer != TransferActionSkip && policy.OnDestinationNewer != TransferActionFail {
|
||||||
|
errs = append(errs, context+".on_destination_newer must be skip or fail")
|
||||||
|
}
|
||||||
|
if policy.OnConflict != TransferActionFail {
|
||||||
|
errs = append(errs, context+".on_conflict must be fail")
|
||||||
|
}
|
||||||
|
return errs
|
||||||
|
}
|
||||||
7
internal/logging/logging.go
Normal file
7
internal/logging/logging.go
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
package logging
|
||||||
|
|
||||||
|
import "io"
|
||||||
|
|
||||||
|
func Configure(io.Writer) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
9
internal/notify/noop.go
Normal file
9
internal/notify/noop.go
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
package notify
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
type Noop struct{}
|
||||||
|
|
||||||
|
func (Noop) Notify(ctx context.Context, event Event) error {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
25
internal/notify/notify.go
Normal file
25
internal/notify/notify.go
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
package notify
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
type Event struct {
|
||||||
|
PipelineID string
|
||||||
|
DestinationID string
|
||||||
|
BundleID string
|
||||||
|
BundlePath string
|
||||||
|
Action string
|
||||||
|
Outputs []Output
|
||||||
|
}
|
||||||
|
|
||||||
|
type Output struct {
|
||||||
|
Path string
|
||||||
|
Kind string
|
||||||
|
SourcePath string
|
||||||
|
Transform string
|
||||||
|
SHA256 string
|
||||||
|
Size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type Notifier interface {
|
||||||
|
Notify(ctx context.Context, event Event) error
|
||||||
|
}
|
||||||
101
internal/publish/execute.go
Normal file
101
internal/publish/execute.go
Normal file
@@ -0,0 +1,101 @@
|
|||||||
|
package publish
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func Execute(ctx context.Context, req Request, plan Plan) error {
|
||||||
|
switch plan.Action {
|
||||||
|
case ActionSkipSame, ActionSkipDestinationNewer:
|
||||||
|
return nil
|
||||||
|
case ActionPublishNew, ActionReplaceOlder:
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("cannot execute action %s: %s", plan.Action, plan.Reason)
|
||||||
|
}
|
||||||
|
|
||||||
|
if plan.Action == ActionReplaceOlder {
|
||||||
|
if plan.ExistingState == nil {
|
||||||
|
return fmt.Errorf("replace requires existing destination state")
|
||||||
|
}
|
||||||
|
if err := req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, existingManagedOutputPaths(*plan.ExistingState), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := ensureDestinationEmpty(ctx, req.DestinationBackend, req.DestinationBundlePath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
writtenOutputs := make([]Output, 0, len(plan.Outputs))
|
||||||
|
cleanup := func() {
|
||||||
|
_ = req.DestinationBackend.DeleteManagedBundle(ctx, req.DestinationBundlePath, managedOutputPaths(writtenOutputs), storage.DeleteOptions{IgnoreMissing: true, PruneEmptyDirs: true})
|
||||||
|
}
|
||||||
|
for _, output := range plan.Outputs {
|
||||||
|
destinationPath, err := storage.Join(req.DestinationBundlePath, output.DestinationPath)
|
||||||
|
if err != nil {
|
||||||
|
cleanup()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data := output.Data
|
||||||
|
if output.Kind == state.OutputKindSource {
|
||||||
|
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, output.SourcePath)
|
||||||
|
if err != nil {
|
||||||
|
cleanup()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err = req.SourceBackend.ReadFile(ctx, sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
cleanup()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := req.DestinationBackend.WriteFile(ctx, destinationPath, data, storage.WriteOptions{Overwrite: false, PreferAtomic: true}); err != nil {
|
||||||
|
cleanup()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
writtenOutputs = append(writtenOutputs, output)
|
||||||
|
}
|
||||||
|
|
||||||
|
destinationState := state.DistributorState{
|
||||||
|
SchemaVersion: state.SchemaVersion,
|
||||||
|
DistributorVersion: req.DistributorVersion,
|
||||||
|
PipelineID: req.PipelineID,
|
||||||
|
DestinationID: req.DestinationID,
|
||||||
|
PublishedAt: time.Now().UTC(),
|
||||||
|
Source: state.SourceState{Manifest: req.SourceBundle.Manifest},
|
||||||
|
Outputs: stateOutputs(plan.Outputs),
|
||||||
|
}
|
||||||
|
if err := state.Validate(destinationState); err != nil {
|
||||||
|
cleanup()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err := json.MarshalIndent(destinationState, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
cleanup()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data = append(data, '\n')
|
||||||
|
statePath, err := storage.StatePath(req.DestinationBundlePath)
|
||||||
|
if err != nil {
|
||||||
|
cleanup()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := req.DestinationBackend.WriteFile(ctx, statePath, data, storage.WriteOptions{Overwrite: false, PreferAtomic: true}); err != nil {
|
||||||
|
cleanup()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func existingManagedOutputPaths(destinationState state.DistributorState) []string {
|
||||||
|
paths := make([]string, 0, len(destinationState.Outputs))
|
||||||
|
for _, output := range destinationState.Outputs {
|
||||||
|
paths = append(paths, output.Path)
|
||||||
|
}
|
||||||
|
return paths
|
||||||
|
}
|
||||||
91
internal/publish/execute_test.go
Normal file
91
internal/publish/execute_test.go
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
package publish
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecuteCleansUpAfterWriteFailure(t *testing.T) {
|
||||||
|
sourceBackend := fake.New()
|
||||||
|
destinationBackend := &failingBackend{Backend: fake.New(), failPath: "summary.txt"}
|
||||||
|
sourceBundle := writeFakeSourceBundle(t, sourceBackend)
|
||||||
|
req := Request{
|
||||||
|
PipelineID: "reports",
|
||||||
|
DestinationID: "archive",
|
||||||
|
SourceBundle: sourceBundle,
|
||||||
|
SourceBackend: sourceBackend,
|
||||||
|
DestinationBackend: destinationBackend,
|
||||||
|
DestinationBundlePath: "",
|
||||||
|
Publish: config.PublishPolicy{Source: true},
|
||||||
|
Transfer: config.TransferPolicy{OnDestinationSame: config.TransferActionSkip, OnDestinationOlder: config.TransferActionReplace, OnDestinationNewer: config.TransferActionSkip, OnConflict: config.TransferActionFail},
|
||||||
|
DistributorVersion: "test",
|
||||||
|
}
|
||||||
|
plan, err := Build(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Build() error = %v", err)
|
||||||
|
}
|
||||||
|
err = Execute(context.Background(), req, plan)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Execute() error = nil, want error")
|
||||||
|
}
|
||||||
|
found, err := destinationBackend.HasAny(context.Background(), "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("HasAny() error = %v", err)
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
t.Fatal("destination has content after failed execution")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type failingBackend struct {
|
||||||
|
*fake.Backend
|
||||||
|
failPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *failingBackend) WriteFile(ctx context.Context, path string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
|
||||||
|
if path == b.failPath {
|
||||||
|
return storage.Entry{}, fmt.Errorf("injected write failure")
|
||||||
|
}
|
||||||
|
return b.Backend.WriteFile(ctx, path, data, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *failingBackend) WriteFrom(ctx context.Context, path string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
|
||||||
|
if path == b.failPath {
|
||||||
|
return storage.Entry{}, fmt.Errorf("injected write failure")
|
||||||
|
}
|
||||||
|
return b.Backend.WriteFrom(ctx, path, r, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeFakeSourceBundle(t *testing.T, backend *fake.Backend) bundle.Bundle {
|
||||||
|
t.Helper()
|
||||||
|
files := []struct {
|
||||||
|
path string
|
||||||
|
data string
|
||||||
|
}{
|
||||||
|
{path: "report.md", data: "# Report\nSunny.\n"},
|
||||||
|
{path: "summary.txt", data: "Summary\n"},
|
||||||
|
}
|
||||||
|
manifestFiles := make([]bundle.ManifestFile, 0, len(files))
|
||||||
|
for _, file := range files {
|
||||||
|
if _, err := backend.WriteFile(context.Background(), file.path, []byte(file.data), storage.WriteOptions{}); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
manifestFiles = append(manifestFiles, bundle.ManifestFile{Path: file.path, SHA256: bundle.FileDigest([]byte(file.data)), Size: int64(len(file.data))})
|
||||||
|
}
|
||||||
|
manifest := bundle.Manifest{
|
||||||
|
SchemaVersion: 1,
|
||||||
|
ID: "weather.daily.brentwood.2026-05-30",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Files: manifestFiles,
|
||||||
|
}
|
||||||
|
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||||
|
return bundle.Bundle{Manifest: manifest}
|
||||||
|
}
|
||||||
102
internal/publish/output.go
Normal file
102
internal/publish/output.go
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
package publish
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
func PlanOutputs(ctx context.Context, req Request) ([]Output, error) {
|
||||||
|
var outputs []Output
|
||||||
|
if req.Publish.Source {
|
||||||
|
sourceOutputs, err := PlanSourceOutputs(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
outputs = append(outputs, sourceOutputs...)
|
||||||
|
}
|
||||||
|
if req.Publish.HTML {
|
||||||
|
generatedOutputs, err := markdownTransformer().Generate(ctx, transform.Request{
|
||||||
|
SourceBundle: req.SourceBundle,
|
||||||
|
SourceBackend: req.SourceBackend,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(generatedOutputs) == 0 {
|
||||||
|
return nil, fmt.Errorf("publish html requested but no markdown source files were found")
|
||||||
|
}
|
||||||
|
for _, generated := range generatedOutputs {
|
||||||
|
outputs = append(outputs, Output{
|
||||||
|
SourcePath: generated.SourcePath,
|
||||||
|
DestinationPath: generated.Path,
|
||||||
|
Kind: state.OutputKindGenerated,
|
||||||
|
Transform: generated.Transform,
|
||||||
|
Data: generated.Data,
|
||||||
|
SHA256: generated.SHA256,
|
||||||
|
Size: generated.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := rejectOutputCollisions(outputs); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return outputs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func PlanSourceOutputs(req Request) ([]Output, error) {
|
||||||
|
outputs := make([]Output, 0, len(req.SourceBundle.Manifest.Files))
|
||||||
|
for _, file := range req.SourceBundle.Manifest.Files {
|
||||||
|
if err := storage.ValidatePath(file.Path); err != nil {
|
||||||
|
return nil, fmt.Errorf("destination output path %q: %w", file.Path, err)
|
||||||
|
}
|
||||||
|
outputs = append(outputs, Output{
|
||||||
|
SourcePath: file.Path,
|
||||||
|
DestinationPath: file.Path,
|
||||||
|
Kind: state.OutputKindSource,
|
||||||
|
SHA256: file.SHA256,
|
||||||
|
Size: file.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return outputs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func rejectOutputCollisions(outputs []Output) error {
|
||||||
|
seen := make(map[string]struct{}, len(outputs))
|
||||||
|
for _, output := range outputs {
|
||||||
|
if err := storage.ValidatePath(output.DestinationPath); err != nil {
|
||||||
|
return fmt.Errorf("destination output path %q: %w", output.DestinationPath, err)
|
||||||
|
}
|
||||||
|
if _, exists := seen[output.DestinationPath]; exists {
|
||||||
|
return fmt.Errorf("destination output path collision: %s", output.DestinationPath)
|
||||||
|
}
|
||||||
|
seen[output.DestinationPath] = struct{}{}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateOutputs(outputs []Output) []state.OutputFile {
|
||||||
|
files := make([]state.OutputFile, 0, len(outputs))
|
||||||
|
for _, output := range outputs {
|
||||||
|
files = append(files, state.OutputFile{
|
||||||
|
Path: output.DestinationPath,
|
||||||
|
Kind: output.Kind,
|
||||||
|
SourcePath: output.SourcePath,
|
||||||
|
Transform: output.Transform,
|
||||||
|
SHA256: output.SHA256,
|
||||||
|
Size: output.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return files
|
||||||
|
}
|
||||||
|
|
||||||
|
func managedOutputPaths(outputs []Output) []string {
|
||||||
|
paths := make([]string, 0, len(outputs))
|
||||||
|
for _, output := range outputs {
|
||||||
|
paths = append(paths, output.DestinationPath)
|
||||||
|
}
|
||||||
|
return paths
|
||||||
|
}
|
||||||
101
internal/publish/output_test.go
Normal file
101
internal/publish/output_test.go
Normal file
@@ -0,0 +1,101 @@
|
|||||||
|
package publish
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPlanOutputsRejectsCollision(t *testing.T) {
|
||||||
|
sourceBackend := fake.New()
|
||||||
|
if _, err := sourceBackend.WriteFile(context.Background(), "report.md", []byte("# Report\n"), storage.WriteOptions{}); err != nil {
|
||||||
|
t.Fatalf("WriteFile report.md error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := sourceBackend.WriteFile(context.Background(), "report.html", []byte("<p>source html</p>\n"), storage.WriteOptions{}); err != nil {
|
||||||
|
t.Fatalf("WriteFile report.html error = %v", err)
|
||||||
|
}
|
||||||
|
reportDigest := bundle.FileDigest([]byte("# Report\n"))
|
||||||
|
htmlDigest := bundle.FileDigest([]byte("<p>source html</p>\n"))
|
||||||
|
files := []bundle.ManifestFile{
|
||||||
|
{Path: "report.md", SHA256: reportDigest, Size: 9},
|
||||||
|
{Path: "report.html", SHA256: htmlDigest, Size: 19},
|
||||||
|
}
|
||||||
|
_, err := PlanOutputs(context.Background(), Request{
|
||||||
|
SourceBackend: sourceBackend,
|
||||||
|
SourceBundle: bundle.Bundle{
|
||||||
|
Manifest: bundle.Manifest{
|
||||||
|
SchemaVersion: 1,
|
||||||
|
ID: "bundle",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Digest: bundle.BundleDigest(files),
|
||||||
|
Files: files,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Publish: config.PublishPolicy{Source: true, HTML: true},
|
||||||
|
Transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("PlanSourceOutputs() error = nil, want collision")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlanOutputsRejectsHTMLWithoutMarkdown(t *testing.T) {
|
||||||
|
sourceBackend := fake.New()
|
||||||
|
if _, err := sourceBackend.WriteFile(context.Background(), "summary.txt", []byte("Summary\n"), storage.WriteOptions{}); err != nil {
|
||||||
|
t.Fatalf("WriteFile summary.txt error = %v", err)
|
||||||
|
}
|
||||||
|
files := []bundle.ManifestFile{{Path: "summary.txt", SHA256: bundle.FileDigest([]byte("Summary\n")), Size: 8}}
|
||||||
|
_, err := PlanOutputs(context.Background(), Request{
|
||||||
|
SourceBackend: sourceBackend,
|
||||||
|
SourceBundle: bundle.Bundle{Manifest: bundle.Manifest{
|
||||||
|
SchemaVersion: 1,
|
||||||
|
ID: "bundle",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Digest: bundle.BundleDigest(files),
|
||||||
|
Files: files,
|
||||||
|
}},
|
||||||
|
Publish: config.PublishPolicy{HTML: true},
|
||||||
|
Transform: config.Transform{MarkdownToHTML: &config.MarkdownToHTML{Enabled: true, Mode: config.TransformModeSidecar}},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("PlanOutputs() error = nil, want no markdown failure")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildRejectsHTMLWithoutTransform(t *testing.T) {
|
||||||
|
sourceBackend := fake.New()
|
||||||
|
destinationBackend := fake.New()
|
||||||
|
if _, err := sourceBackend.WriteFile(context.Background(), "report.md", []byte("# Report\n"), storage.WriteOptions{}); err != nil {
|
||||||
|
t.Fatalf("WriteFile report.md error = %v", err)
|
||||||
|
}
|
||||||
|
files := []bundle.ManifestFile{{Path: "report.md", SHA256: bundle.FileDigest([]byte("# Report\n")), Size: 9}}
|
||||||
|
_, err := Build(context.Background(), Request{
|
||||||
|
PipelineID: "reports",
|
||||||
|
DestinationID: "archive",
|
||||||
|
SourceBackend: sourceBackend,
|
||||||
|
DestinationBackend: destinationBackend,
|
||||||
|
DestinationBundlePath: "",
|
||||||
|
SourceBundle: bundle.Bundle{Manifest: bundle.Manifest{
|
||||||
|
SchemaVersion: 1,
|
||||||
|
ID: "bundle",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Digest: bundle.BundleDigest(files),
|
||||||
|
Files: files,
|
||||||
|
}},
|
||||||
|
Publish: config.PublishPolicy{HTML: true},
|
||||||
|
Transfer: config.TransferPolicy{
|
||||||
|
OnDestinationSame: config.TransferActionSkip,
|
||||||
|
OnDestinationOlder: config.TransferActionReplace,
|
||||||
|
OnDestinationNewer: config.TransferActionSkip,
|
||||||
|
OnConflict: config.TransferActionFail,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Build() error = nil, want missing transform error")
|
||||||
|
}
|
||||||
|
}
|
||||||
153
internal/publish/plan.go
Normal file
153
internal/publish/plan.go
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
package publish
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||||
|
markdowntransform "gitea.maximumdirect.net/eric/distributor/internal/transform/markdown"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Action string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ActionPublishNew Action = "publish_new"
|
||||||
|
ActionReplaceOlder Action = "replace_older"
|
||||||
|
ActionSkipSame Action = "skip_same"
|
||||||
|
ActionSkipDestinationNewer Action = "skip_destination_newer"
|
||||||
|
ActionFailConflict Action = "fail_conflict"
|
||||||
|
ActionFailUnmanaged Action = "fail_unmanaged"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Request struct {
|
||||||
|
PipelineID string
|
||||||
|
DestinationID string
|
||||||
|
SourceBundle bundle.Bundle
|
||||||
|
SourceBackend storage.Backend
|
||||||
|
DestinationBackend storage.Backend
|
||||||
|
DestinationBundlePath string
|
||||||
|
Publish config.PublishPolicy
|
||||||
|
Transform config.Transform
|
||||||
|
Transfer config.TransferPolicy
|
||||||
|
DistributorVersion string
|
||||||
|
}
|
||||||
|
|
||||||
|
type Plan struct {
|
||||||
|
PipelineID string
|
||||||
|
DestinationID string
|
||||||
|
BundleID string
|
||||||
|
BundlePath string
|
||||||
|
DestinationBundlePath string
|
||||||
|
Action Action
|
||||||
|
Reason string
|
||||||
|
Outputs []Output
|
||||||
|
ExistingState *state.DistributorState
|
||||||
|
}
|
||||||
|
|
||||||
|
type Output struct {
|
||||||
|
SourcePath string
|
||||||
|
DestinationPath string
|
||||||
|
Kind string
|
||||||
|
Transform string
|
||||||
|
Data []byte
|
||||||
|
SHA256 string
|
||||||
|
Size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func Build(ctx context.Context, req Request) (Plan, error) {
|
||||||
|
if err := validateRequest(req); err != nil {
|
||||||
|
return Plan{}, err
|
||||||
|
}
|
||||||
|
outputs, err := PlanOutputs(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
return Plan{}, err
|
||||||
|
}
|
||||||
|
status, err := inspectDestination(ctx, req.DestinationBackend, req.DestinationBundlePath)
|
||||||
|
if err != nil {
|
||||||
|
return Plan{}, err
|
||||||
|
}
|
||||||
|
comparison := state.Compare(req.SourceBundle.Manifest, req.PipelineID, req.DestinationID, status)
|
||||||
|
action, reason := actionForComparison(comparison, req.Transfer)
|
||||||
|
plan := Plan{
|
||||||
|
PipelineID: req.PipelineID,
|
||||||
|
DestinationID: req.DestinationID,
|
||||||
|
BundleID: req.SourceBundle.Manifest.ID,
|
||||||
|
BundlePath: req.SourceBundle.RootRelativePath,
|
||||||
|
DestinationBundlePath: req.DestinationBundlePath,
|
||||||
|
Action: action,
|
||||||
|
Reason: reason,
|
||||||
|
Outputs: outputs,
|
||||||
|
ExistingState: status.State,
|
||||||
|
}
|
||||||
|
if action == ActionFailConflict || action == ActionFailUnmanaged {
|
||||||
|
return plan, fmt.Errorf("%s: %s", action, reason)
|
||||||
|
}
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRequest(req Request) error {
|
||||||
|
if req.PipelineID == "" {
|
||||||
|
return fmt.Errorf("pipeline id is required")
|
||||||
|
}
|
||||||
|
if req.DestinationID == "" {
|
||||||
|
return fmt.Errorf("destination id is required")
|
||||||
|
}
|
||||||
|
if req.SourceBackend == nil {
|
||||||
|
return fmt.Errorf("source backend is required")
|
||||||
|
}
|
||||||
|
if req.DestinationBackend == nil {
|
||||||
|
return fmt.Errorf("destination backend is required")
|
||||||
|
}
|
||||||
|
if !req.Publish.Source && !req.Publish.HTML {
|
||||||
|
return fmt.Errorf("publish source or html must be enabled")
|
||||||
|
}
|
||||||
|
if req.Publish.HTML {
|
||||||
|
if req.Transform.MarkdownToHTML == nil || !req.Transform.MarkdownToHTML.Enabled || req.Transform.MarkdownToHTML.Mode != config.TransformModeSidecar {
|
||||||
|
return fmt.Errorf("publish html requires markdown_to_html transform enabled with sidecar mode")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func markdownTransformer() transform.Transformer {
|
||||||
|
return markdowntransform.New()
|
||||||
|
}
|
||||||
|
|
||||||
|
func actionForComparison(comparison state.Comparison, transfer config.TransferPolicy) (Action, string) {
|
||||||
|
switch comparison.Outcome {
|
||||||
|
case state.OutcomeDestinationAbsent:
|
||||||
|
return ActionPublishNew, comparison.Reason
|
||||||
|
case state.OutcomeDestinationUnmanaged:
|
||||||
|
return ActionFailUnmanaged, comparison.Reason
|
||||||
|
case state.OutcomeInvalidState, state.OutcomeIdentityMismatch, state.OutcomeSameCreatedConflict, state.OutcomeDifferentSourceConflict:
|
||||||
|
return ActionFailConflict, comparison.Reason
|
||||||
|
case state.OutcomeSameSource:
|
||||||
|
if transfer.OnDestinationSame == config.TransferActionFail {
|
||||||
|
return ActionFailConflict, "destination matches source and transfer policy requires failure"
|
||||||
|
}
|
||||||
|
return ActionSkipSame, comparison.Reason
|
||||||
|
case state.OutcomeDestinationOlder:
|
||||||
|
if transfer.OnDestinationOlder == config.TransferActionFail {
|
||||||
|
return ActionFailConflict, "destination is older and transfer policy requires failure"
|
||||||
|
}
|
||||||
|
return ActionReplaceOlder, comparison.Reason
|
||||||
|
case state.OutcomeDestinationNewer:
|
||||||
|
if transfer.OnDestinationNewer == config.TransferActionFail {
|
||||||
|
return ActionFailConflict, "destination is newer and transfer policy requires failure"
|
||||||
|
}
|
||||||
|
return ActionSkipDestinationNewer, comparison.Reason
|
||||||
|
default:
|
||||||
|
return ActionFailConflict, "unsupported comparison outcome"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func displayPath(path string) string {
|
||||||
|
if path == "" {
|
||||||
|
return "."
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
31
internal/publish/reconcile.go
Normal file
31
internal/publish/reconcile.go
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
package publish
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func inspectDestination(ctx context.Context, backend storage.Backend, bundlePath string) (state.DestinationStatus, error) {
|
||||||
|
statePath, err := storage.StatePath(bundlePath)
|
||||||
|
if err != nil {
|
||||||
|
return state.DestinationStatus{}, err
|
||||||
|
}
|
||||||
|
data, err := backend.ReadFile(ctx, statePath)
|
||||||
|
if err == nil {
|
||||||
|
destinationState, parseErr := state.Parse(data)
|
||||||
|
if parseErr != nil {
|
||||||
|
return state.DestinationStatus{StateErr: parseErr}, nil
|
||||||
|
}
|
||||||
|
return state.DestinationStatus{State: &destinationState, HasContents: true}, nil
|
||||||
|
}
|
||||||
|
if !storage.IsNotFound(err) {
|
||||||
|
return state.DestinationStatus{}, err
|
||||||
|
}
|
||||||
|
hasContents, err := backend.HasAny(ctx, bundlePath)
|
||||||
|
if err != nil {
|
||||||
|
return state.DestinationStatus{}, err
|
||||||
|
}
|
||||||
|
return state.DestinationStatus{HasContents: hasContents}, nil
|
||||||
|
}
|
||||||
19
internal/publish/safety.go
Normal file
19
internal/publish/safety.go
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
package publish
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ensureDestinationEmpty(ctx context.Context, backend storage.Backend, bundlePath string) error {
|
||||||
|
hasAny, err := backend.HasAny(ctx, bundlePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if hasAny {
|
||||||
|
return fmt.Errorf("destination bundle path %q is not empty after managed cleanup", displayPath(bundlePath))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
89
internal/state/compare.go
Normal file
89
internal/state/compare.go
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
package state
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Outcome string
|
||||||
|
|
||||||
|
const (
|
||||||
|
OutcomeDestinationAbsent Outcome = "destination_absent"
|
||||||
|
OutcomeDestinationUnmanaged Outcome = "destination_unmanaged"
|
||||||
|
OutcomeInvalidState Outcome = "invalid_destination_state"
|
||||||
|
OutcomeIdentityMismatch Outcome = "destination_identity_mismatch"
|
||||||
|
OutcomeSameSource Outcome = "same_source_manifest"
|
||||||
|
OutcomeDestinationOlder Outcome = "destination_older"
|
||||||
|
OutcomeDestinationNewer Outcome = "destination_newer"
|
||||||
|
OutcomeSameCreatedConflict Outcome = "same_created_digest_conflict"
|
||||||
|
OutcomeDifferentSourceConflict Outcome = "different_source_conflict"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DestinationStatus struct {
|
||||||
|
State *DistributorState
|
||||||
|
StateErr error
|
||||||
|
HasContents bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type Comparison struct {
|
||||||
|
Outcome Outcome
|
||||||
|
Reason string
|
||||||
|
}
|
||||||
|
|
||||||
|
func Compare(source bundle.Manifest, pipelineID, destinationID string, status DestinationStatus) Comparison {
|
||||||
|
if status.StateErr != nil {
|
||||||
|
return Comparison{Outcome: OutcomeInvalidState, Reason: status.StateErr.Error()}
|
||||||
|
}
|
||||||
|
if status.State == nil {
|
||||||
|
if status.HasContents {
|
||||||
|
return Comparison{Outcome: OutcomeDestinationUnmanaged, Reason: "destination has content but no distributor state"}
|
||||||
|
}
|
||||||
|
return Comparison{Outcome: OutcomeDestinationAbsent, Reason: "destination state is absent"}
|
||||||
|
}
|
||||||
|
|
||||||
|
destinationState := *status.State
|
||||||
|
if err := Validate(destinationState); err != nil {
|
||||||
|
return Comparison{Outcome: OutcomeInvalidState, Reason: err.Error()}
|
||||||
|
}
|
||||||
|
if destinationState.PipelineID != pipelineID {
|
||||||
|
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("pipeline id %q does not match %q", destinationState.PipelineID, pipelineID)}
|
||||||
|
}
|
||||||
|
if destinationState.DestinationID != destinationID {
|
||||||
|
return Comparison{Outcome: OutcomeIdentityMismatch, Reason: fmt.Sprintf("destination id %q does not match %q", destinationState.DestinationID, destinationID)}
|
||||||
|
}
|
||||||
|
|
||||||
|
destinationManifest := destinationState.Source.Manifest
|
||||||
|
if manifestsEqual(source, destinationManifest) {
|
||||||
|
return Comparison{Outcome: OutcomeSameSource, Reason: "destination source manifest matches source"}
|
||||||
|
}
|
||||||
|
if destinationManifest.ID != source.ID {
|
||||||
|
return Comparison{Outcome: OutcomeDifferentSourceConflict, Reason: "destination source id differs from source"}
|
||||||
|
}
|
||||||
|
if destinationManifest.Created.Before(source.Created) {
|
||||||
|
return Comparison{Outcome: OutcomeDestinationOlder, Reason: "destination source is older than source"}
|
||||||
|
}
|
||||||
|
if destinationManifest.Created.After(source.Created) {
|
||||||
|
return Comparison{Outcome: OutcomeDestinationNewer, Reason: "destination source is newer than source"}
|
||||||
|
}
|
||||||
|
if destinationManifest.Digest != source.Digest {
|
||||||
|
return Comparison{Outcome: OutcomeSameCreatedConflict, Reason: "destination source has same id and created time but different digest"}
|
||||||
|
}
|
||||||
|
return Comparison{Outcome: OutcomeInvalidState, Reason: "destination source differs from source without a supported comparison outcome"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifestsEqual(a, b bundle.Manifest) bool {
|
||||||
|
if a.SchemaVersion != b.SchemaVersion ||
|
||||||
|
a.ID != b.ID ||
|
||||||
|
a.Digest != b.Digest ||
|
||||||
|
!a.Created.Equal(b.Created) ||
|
||||||
|
len(a.Files) != len(b.Files) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for index := range a.Files {
|
||||||
|
if a.Files[index] != b.Files[index] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
121
internal/state/compare_test.go
Normal file
121
internal/state/compare_test.go
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
package state
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCompareOutcomes(t *testing.T) {
|
||||||
|
source := validManifest(t)
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
status DestinationStatus
|
||||||
|
want Outcome
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "destination absent",
|
||||||
|
status: DestinationStatus{},
|
||||||
|
want: OutcomeDestinationAbsent,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "destination unmanaged",
|
||||||
|
status: DestinationStatus{HasContents: true},
|
||||||
|
want: OutcomeDestinationUnmanaged,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid destination state",
|
||||||
|
status: DestinationStatus{StateErr: errors.New("invalid json")},
|
||||||
|
want: OutcomeInvalidState,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "pipeline mismatch",
|
||||||
|
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.PipelineID = "other" })},
|
||||||
|
want: OutcomeIdentityMismatch,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "destination mismatch",
|
||||||
|
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) { s.DestinationID = "other" })},
|
||||||
|
want: OutcomeIdentityMismatch,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "same source manifest",
|
||||||
|
status: DestinationStatus{State: withState(t, source, nil)},
|
||||||
|
want: OutcomeSameSource,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "destination older",
|
||||||
|
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||||
|
s.Source.Manifest.Created = source.Created.Add(-time.Hour)
|
||||||
|
})},
|
||||||
|
want: OutcomeDestinationOlder,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "destination newer",
|
||||||
|
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||||
|
s.Source.Manifest.Created = source.Created.Add(time.Hour)
|
||||||
|
})},
|
||||||
|
want: OutcomeDestinationNewer,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "same created digest conflict",
|
||||||
|
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||||
|
s.Source.Manifest.Files[0].SHA256 = "sha256:3333333333333333333333333333333333333333333333333333333333333333"
|
||||||
|
s.Source.Manifest.Digest = bundle.BundleDigest(s.Source.Manifest.Files)
|
||||||
|
})},
|
||||||
|
want: OutcomeSameCreatedConflict,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "different source id",
|
||||||
|
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||||
|
s.Source.Manifest.ID = "other.source"
|
||||||
|
})},
|
||||||
|
want: OutcomeDifferentSourceConflict,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid state object",
|
||||||
|
status: DestinationStatus{State: withState(t, source, func(s *DistributorState) {
|
||||||
|
s.Outputs[0].Kind = "other"
|
||||||
|
})},
|
||||||
|
want: OutcomeInvalidState,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := Compare(source, "reports", "archive", tt.status)
|
||||||
|
if got.Outcome != tt.want {
|
||||||
|
t.Fatalf("Compare() outcome = %s reason=%q, want %s", got.Outcome, got.Reason, tt.want)
|
||||||
|
}
|
||||||
|
if got.Reason == "" {
|
||||||
|
t.Fatal("Compare() reason is empty")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func withState(t *testing.T, source bundle.Manifest, mutate func(*DistributorState)) *DistributorState {
|
||||||
|
t.Helper()
|
||||||
|
stateManifest := source
|
||||||
|
stateManifest.Files = append([]bundle.ManifestFile(nil), source.Files...)
|
||||||
|
state := DistributorState{
|
||||||
|
SchemaVersion: SchemaVersion,
|
||||||
|
PipelineID: "reports",
|
||||||
|
DestinationID: "archive",
|
||||||
|
PublishedAt: time.Date(2026, 5, 30, 11, 12, 0, 0, time.UTC),
|
||||||
|
Source: SourceState{Manifest: stateManifest},
|
||||||
|
Outputs: []OutputFile{{
|
||||||
|
Path: "report.md",
|
||||||
|
Kind: OutputKindSource,
|
||||||
|
SourcePath: "report.md",
|
||||||
|
SHA256: source.Files[0].SHA256,
|
||||||
|
Size: source.Files[0].Size,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
if mutate != nil {
|
||||||
|
mutate(&state)
|
||||||
|
}
|
||||||
|
return &state
|
||||||
|
}
|
||||||
214
internal/state/distributor.go
Normal file
214
internal/state/distributor.go
Normal file
@@ -0,0 +1,214 @@
|
|||||||
|
package state
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
const SchemaVersion = 1
|
||||||
|
|
||||||
|
type DistributorState struct {
|
||||||
|
SchemaVersion int
|
||||||
|
DistributorVersion string
|
||||||
|
PipelineID string
|
||||||
|
DestinationID string
|
||||||
|
PublishedAt time.Time
|
||||||
|
Source SourceState
|
||||||
|
Outputs []OutputFile
|
||||||
|
}
|
||||||
|
|
||||||
|
type SourceState struct {
|
||||||
|
Manifest bundle.Manifest
|
||||||
|
}
|
||||||
|
|
||||||
|
type OutputFile struct {
|
||||||
|
Path string
|
||||||
|
Kind string
|
||||||
|
SourcePath string
|
||||||
|
Transform string
|
||||||
|
SHA256 string
|
||||||
|
Size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawDistributorState struct {
|
||||||
|
SchemaVersion *int `json:"schema_version"`
|
||||||
|
DistributorVersion string `json:"distributor_version"`
|
||||||
|
PipelineID *string `json:"pipeline_id"`
|
||||||
|
DestinationID *string `json:"destination_id"`
|
||||||
|
PublishedAt *string `json:"published_at"`
|
||||||
|
Source *rawSourceState `json:"source"`
|
||||||
|
Outputs []rawOutputFile `json:"outputs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawSourceState struct {
|
||||||
|
Manifest json.RawMessage `json:"manifest"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rawOutputFile struct {
|
||||||
|
Path *string `json:"path"`
|
||||||
|
Kind *string `json:"kind"`
|
||||||
|
SourcePath *string `json:"source_path"`
|
||||||
|
Transform string `json:"transform"`
|
||||||
|
SHA256 *string `json:"sha256"`
|
||||||
|
Size *int64 `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func Parse(data []byte) (DistributorState, error) {
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
var raw rawDistributorState
|
||||||
|
if err := decoder.Decode(&raw); err != nil {
|
||||||
|
return DistributorState{}, fmt.Errorf("parse distributor state: %w", err)
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if err := decoder.Decode(&extra); err != io.EOF {
|
||||||
|
return DistributorState{}, fmt.Errorf("parse distributor state: trailing data")
|
||||||
|
}
|
||||||
|
state, err := parseRaw(raw)
|
||||||
|
if err != nil {
|
||||||
|
return DistributorState{}, err
|
||||||
|
}
|
||||||
|
if err := Validate(state); err != nil {
|
||||||
|
return DistributorState{}, err
|
||||||
|
}
|
||||||
|
return state, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRaw(raw rawDistributorState) (DistributorState, error) {
|
||||||
|
var state DistributorState
|
||||||
|
if raw.SchemaVersion == nil {
|
||||||
|
return DistributorState{}, fmt.Errorf("state schema_version is required")
|
||||||
|
}
|
||||||
|
state.SchemaVersion = *raw.SchemaVersion
|
||||||
|
if state.SchemaVersion != SchemaVersion {
|
||||||
|
return DistributorState{}, fmt.Errorf("state schema_version must be %d", SchemaVersion)
|
||||||
|
}
|
||||||
|
state.DistributorVersion = raw.DistributorVersion
|
||||||
|
if raw.PipelineID == nil || *raw.PipelineID == "" {
|
||||||
|
return DistributorState{}, fmt.Errorf("state pipeline_id is required")
|
||||||
|
}
|
||||||
|
state.PipelineID = *raw.PipelineID
|
||||||
|
if raw.DestinationID == nil || *raw.DestinationID == "" {
|
||||||
|
return DistributorState{}, fmt.Errorf("state destination_id is required")
|
||||||
|
}
|
||||||
|
state.DestinationID = *raw.DestinationID
|
||||||
|
if raw.PublishedAt == nil || *raw.PublishedAt == "" {
|
||||||
|
return DistributorState{}, fmt.Errorf("state published_at is required")
|
||||||
|
}
|
||||||
|
publishedAt, err := time.Parse(time.RFC3339, *raw.PublishedAt)
|
||||||
|
if err != nil {
|
||||||
|
return DistributorState{}, fmt.Errorf("state published_at must be RFC3339: %w", err)
|
||||||
|
}
|
||||||
|
state.PublishedAt = publishedAt.UTC()
|
||||||
|
if raw.Source == nil || len(raw.Source.Manifest) == 0 {
|
||||||
|
return DistributorState{}, fmt.Errorf("state source.manifest is required")
|
||||||
|
}
|
||||||
|
manifest, err := bundle.ParseManifest(raw.Source.Manifest)
|
||||||
|
if err != nil {
|
||||||
|
return DistributorState{}, fmt.Errorf("state source.manifest: %w", err)
|
||||||
|
}
|
||||||
|
state.Source.Manifest = manifest
|
||||||
|
if raw.Outputs == nil {
|
||||||
|
return DistributorState{}, fmt.Errorf("state outputs is required")
|
||||||
|
}
|
||||||
|
outputs, err := parseOutputs(raw.Outputs)
|
||||||
|
if err != nil {
|
||||||
|
return DistributorState{}, err
|
||||||
|
}
|
||||||
|
state.Outputs = outputs
|
||||||
|
return state, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseOutputs(rawOutputs []rawOutputFile) ([]OutputFile, error) {
|
||||||
|
outputs := make([]OutputFile, 0, len(rawOutputs))
|
||||||
|
seen := make(map[string]struct{}, len(rawOutputs))
|
||||||
|
for index, raw := range rawOutputs {
|
||||||
|
output, err := parseOutput(index, raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, exists := seen[output.Path]; exists {
|
||||||
|
return nil, fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
||||||
|
}
|
||||||
|
seen[output.Path] = struct{}{}
|
||||||
|
outputs = append(outputs, output)
|
||||||
|
}
|
||||||
|
return outputs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseOutput(index int, raw rawOutputFile) (OutputFile, error) {
|
||||||
|
if raw.Path == nil || *raw.Path == "" {
|
||||||
|
return OutputFile{}, fmt.Errorf("state outputs[%d].path is required", index)
|
||||||
|
}
|
||||||
|
if raw.Kind == nil || *raw.Kind == "" {
|
||||||
|
return OutputFile{}, fmt.Errorf("state outputs[%d].kind is required", index)
|
||||||
|
}
|
||||||
|
if raw.SourcePath == nil || *raw.SourcePath == "" {
|
||||||
|
return OutputFile{}, fmt.Errorf("state outputs[%d].source_path is required", index)
|
||||||
|
}
|
||||||
|
if raw.SHA256 == nil || *raw.SHA256 == "" {
|
||||||
|
return OutputFile{}, fmt.Errorf("state outputs[%d].sha256 is required", index)
|
||||||
|
}
|
||||||
|
if raw.Size == nil {
|
||||||
|
return OutputFile{}, fmt.Errorf("state outputs[%d].size is required", index)
|
||||||
|
}
|
||||||
|
return OutputFile{
|
||||||
|
Path: *raw.Path,
|
||||||
|
Kind: *raw.Kind,
|
||||||
|
SourcePath: *raw.SourcePath,
|
||||||
|
Transform: raw.Transform,
|
||||||
|
SHA256: *raw.SHA256,
|
||||||
|
Size: *raw.Size,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s DistributorState) PublishedAtString() string {
|
||||||
|
return s.PublishedAt.UTC().Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s DistributorState) MarshalJSON() ([]byte, error) {
|
||||||
|
type sourceJSON struct {
|
||||||
|
Manifest bundle.Manifest `json:"manifest"`
|
||||||
|
}
|
||||||
|
type stateJSON struct {
|
||||||
|
SchemaVersion int `json:"schema_version"`
|
||||||
|
DistributorVersion string `json:"distributor_version,omitempty"`
|
||||||
|
PipelineID string `json:"pipeline_id"`
|
||||||
|
DestinationID string `json:"destination_id"`
|
||||||
|
PublishedAt string `json:"published_at"`
|
||||||
|
Source sourceJSON `json:"source"`
|
||||||
|
Outputs []OutputFile `json:"outputs"`
|
||||||
|
}
|
||||||
|
return json.Marshal(stateJSON{
|
||||||
|
SchemaVersion: s.SchemaVersion,
|
||||||
|
DistributorVersion: s.DistributorVersion,
|
||||||
|
PipelineID: s.PipelineID,
|
||||||
|
DestinationID: s.DestinationID,
|
||||||
|
PublishedAt: s.PublishedAtString(),
|
||||||
|
Source: sourceJSON{Manifest: s.Source.Manifest},
|
||||||
|
Outputs: s.Outputs,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o OutputFile) MarshalJSON() ([]byte, error) {
|
||||||
|
type outputJSON struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
SourcePath string `json:"source_path"`
|
||||||
|
Transform string `json:"transform,omitempty"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
}
|
||||||
|
return json.Marshal(outputJSON{
|
||||||
|
Path: o.Path,
|
||||||
|
Kind: o.Kind,
|
||||||
|
SourcePath: o.SourcePath,
|
||||||
|
Transform: o.Transform,
|
||||||
|
SHA256: o.SHA256,
|
||||||
|
Size: o.Size,
|
||||||
|
})
|
||||||
|
}
|
||||||
191
internal/state/distributor_test.go
Normal file
191
internal/state/distributor_test.go
Normal file
@@ -0,0 +1,191 @@
|
|||||||
|
package state
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseValidState(t *testing.T) {
|
||||||
|
state, err := Parse([]byte(validStateJSON(t)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse() error = %v", err)
|
||||||
|
}
|
||||||
|
if state.SchemaVersion != SchemaVersion {
|
||||||
|
t.Fatalf("schema version = %d, want %d", state.SchemaVersion, SchemaVersion)
|
||||||
|
}
|
||||||
|
if state.PipelineID != "reports" || state.DestinationID != "archive" {
|
||||||
|
t.Fatalf("identity = %q/%q", state.PipelineID, state.DestinationID)
|
||||||
|
}
|
||||||
|
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||||
|
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if got, want := len(state.Outputs), 1; got != want {
|
||||||
|
t.Fatalf("output count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNormalizesPublishedAtOffset(t *testing.T) {
|
||||||
|
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "2026-05-30T13:12:00+02:00"`, 1)
|
||||||
|
state, err := Parse([]byte(body))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := state.PublishedAtString(), "2026-05-30T11:12:00Z"; got != want {
|
||||||
|
t.Fatalf("PublishedAtString() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseRejectsMissingFields(t *testing.T) {
|
||||||
|
tests := map[string]string{
|
||||||
|
"schema_version": `"schema_version"`,
|
||||||
|
"pipeline_id": `"pipeline_id"`,
|
||||||
|
"destination_id": `"destination_id"`,
|
||||||
|
"published_at": `"published_at"`,
|
||||||
|
"source": `"source"`,
|
||||||
|
"outputs": `"outputs"`,
|
||||||
|
}
|
||||||
|
for name, field := range tests {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
body := strings.Replace(validStateJSON(t), field, `"missing_`+name+`"`, 1)
|
||||||
|
_, err := Parse([]byte(body))
|
||||||
|
assertStateErrorContains(t, err, "required")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseRejectsInvalidSchemaVersion(t *testing.T) {
|
||||||
|
body := strings.Replace(validStateJSON(t), `"schema_version": 1`, `"schema_version": 2`, 1)
|
||||||
|
_, err := Parse([]byte(body))
|
||||||
|
assertStateErrorContains(t, err, "schema_version must be 1")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseRejectsInvalidEmbeddedManifest(t *testing.T) {
|
||||||
|
body := validStateWithManifestJSON(t, strings.Replace(manifestJSON(t), `"schema_version": 1`, `"schema_version": 2`, 1))
|
||||||
|
_, err := Parse([]byte(body))
|
||||||
|
assertStateErrorContains(t, err, "source.manifest")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseRejectsInvalidOutputMetadata(t *testing.T) {
|
||||||
|
source := validManifest(t)
|
||||||
|
tests := map[string]func(*DistributorState){
|
||||||
|
"unsafe path": func(s *DistributorState) {
|
||||||
|
s.Outputs[0].Path = "../report.md"
|
||||||
|
},
|
||||||
|
"invalid kind": func(s *DistributorState) {
|
||||||
|
s.Outputs[0].Kind = "other"
|
||||||
|
},
|
||||||
|
"invalid source": func(s *DistributorState) {
|
||||||
|
s.Outputs[0].SourcePath = "../report.md"
|
||||||
|
},
|
||||||
|
"generated missing": func(s *DistributorState) {
|
||||||
|
s.Outputs[0].Kind = OutputKindGenerated
|
||||||
|
},
|
||||||
|
"invalid digest": func(s *DistributorState) {
|
||||||
|
s.Outputs[0].SHA256 = "SHA256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6"
|
||||||
|
},
|
||||||
|
"negative size": func(s *DistributorState) {
|
||||||
|
s.Outputs[0].Size = -1
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for name, mutate := range tests {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
state := *withState(t, source, mutate)
|
||||||
|
err := Validate(state)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Validate() error = nil, want error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseRejectsMalformedPublishedTimestamp(t *testing.T) {
|
||||||
|
body := strings.Replace(validStateJSON(t), `"published_at": "2026-05-30T11:12:00Z"`, `"published_at": "May 30"`, 1)
|
||||||
|
_, err := Parse([]byte(body))
|
||||||
|
assertStateErrorContains(t, err, "published_at must be RFC3339")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalNormalizesPublishedAtUTC(t *testing.T) {
|
||||||
|
source := validManifest(t)
|
||||||
|
state := DistributorState{
|
||||||
|
SchemaVersion: SchemaVersion,
|
||||||
|
PipelineID: "reports",
|
||||||
|
DestinationID: "archive",
|
||||||
|
PublishedAt: time.Date(2026, 5, 30, 13, 12, 0, 0, time.FixedZone("offset", 2*60*60)),
|
||||||
|
Source: SourceState{Manifest: source},
|
||||||
|
Outputs: []OutputFile{{
|
||||||
|
Path: "report.md",
|
||||||
|
Kind: OutputKindSource,
|
||||||
|
SourcePath: "report.md",
|
||||||
|
SHA256: source.Files[0].SHA256,
|
||||||
|
Size: source.Files[0].Size,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(state)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal() error = %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(data), `"published_at":"2026-05-30T11:12:00Z"`) {
|
||||||
|
t.Fatalf("json = %s, want UTC RFC3339 published_at", data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validStateJSON(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
return validStateWithManifestJSON(t, manifestJSON(t))
|
||||||
|
}
|
||||||
|
|
||||||
|
func validStateWithManifestJSON(t *testing.T, manifest string) string {
|
||||||
|
t.Helper()
|
||||||
|
return `{
|
||||||
|
"schema_version": 1,
|
||||||
|
"distributor_version": "dev",
|
||||||
|
"pipeline_id": "reports",
|
||||||
|
"destination_id": "archive",
|
||||||
|
"published_at": "2026-05-30T11:12:00Z",
|
||||||
|
"source": {
|
||||||
|
"manifest": ` + manifest + `
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"path": "report.md",
|
||||||
|
"kind": "source",
|
||||||
|
"source_path": "report.md",
|
||||||
|
"sha256": "sha256:3640fd37140ee4d2e0e93e78834f232ea67a50e7bc6279203690cc7de1975fa6",
|
||||||
|
"size": 16
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifestJSON(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile("../bundle/testdata/valid_bundle/manifest.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read manifest fixture: %v", err)
|
||||||
|
}
|
||||||
|
return string(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validManifest(t *testing.T) bundle.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
manifest, err := bundle.ParseManifest([]byte(manifestJSON(t)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseManifest() error = %v", err)
|
||||||
|
}
|
||||||
|
return manifest
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertStateErrorContains(t *testing.T, err error, want string) {
|
||||||
|
t.Helper()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("error = nil, want substring %q", want)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Fatalf("error = %q, want substring %q", err.Error(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
107
internal/state/validate.go
Normal file
107
internal/state/validate.go
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
package state
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
OutputKindSource = "source"
|
||||||
|
OutputKindGenerated = "generated"
|
||||||
|
)
|
||||||
|
|
||||||
|
func Validate(s DistributorState) error {
|
||||||
|
if s.SchemaVersion != SchemaVersion {
|
||||||
|
return fmt.Errorf("state schema_version must be %d", SchemaVersion)
|
||||||
|
}
|
||||||
|
if s.PipelineID == "" {
|
||||||
|
return fmt.Errorf("state pipeline_id is required")
|
||||||
|
}
|
||||||
|
if s.DestinationID == "" {
|
||||||
|
return fmt.Errorf("state destination_id is required")
|
||||||
|
}
|
||||||
|
if s.PublishedAt.IsZero() {
|
||||||
|
return fmt.Errorf("state published_at is required")
|
||||||
|
}
|
||||||
|
if err := validateEmbeddedManifest(s.Source.Manifest); err != nil {
|
||||||
|
return fmt.Errorf("state source.manifest: %w", err)
|
||||||
|
}
|
||||||
|
if s.Outputs == nil {
|
||||||
|
return fmt.Errorf("state outputs is required")
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(s.Outputs))
|
||||||
|
for index, output := range s.Outputs {
|
||||||
|
if err := validateOutput(index, output); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, exists := seen[output.Path]; exists {
|
||||||
|
return fmt.Errorf("state outputs[%d].path duplicates %q", index, output.Path)
|
||||||
|
}
|
||||||
|
seen[output.Path] = struct{}{}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateEmbeddedManifest(manifest bundle.Manifest) error {
|
||||||
|
if manifest.SchemaVersion != 1 {
|
||||||
|
return fmt.Errorf("schema_version must be 1")
|
||||||
|
}
|
||||||
|
if manifest.ID == "" {
|
||||||
|
return fmt.Errorf("id is required")
|
||||||
|
}
|
||||||
|
if err := bundle.ValidateDigest(manifest.Digest); err != nil {
|
||||||
|
return fmt.Errorf("digest: %w", err)
|
||||||
|
}
|
||||||
|
if manifest.Created.IsZero() {
|
||||||
|
return fmt.Errorf("created is required")
|
||||||
|
}
|
||||||
|
if len(manifest.Files) == 0 {
|
||||||
|
return fmt.Errorf("files is required")
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(manifest.Files))
|
||||||
|
for index, file := range manifest.Files {
|
||||||
|
if err := bundle.ValidateSourcePath(file.Path); err != nil {
|
||||||
|
return fmt.Errorf("files[%d].path: %w", index, err)
|
||||||
|
}
|
||||||
|
if err := bundle.ValidateDigest(file.SHA256); err != nil {
|
||||||
|
return fmt.Errorf("files[%d].sha256: %w", index, err)
|
||||||
|
}
|
||||||
|
if file.Size < 0 {
|
||||||
|
return fmt.Errorf("files[%d].size must be non-negative", index)
|
||||||
|
}
|
||||||
|
if _, exists := seen[file.Path]; exists {
|
||||||
|
return fmt.Errorf("files[%d].path duplicates %q", index, file.Path)
|
||||||
|
}
|
||||||
|
seen[file.Path] = struct{}{}
|
||||||
|
}
|
||||||
|
if actual := bundle.BundleDigest(manifest.Files); actual != manifest.Digest {
|
||||||
|
return fmt.Errorf("digest mismatch: got %s want %s", actual, manifest.Digest)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateOutput(index int, output OutputFile) error {
|
||||||
|
if err := storage.ValidatePath(output.Path); err != nil {
|
||||||
|
return fmt.Errorf("state outputs[%d].path: %w", index, err)
|
||||||
|
}
|
||||||
|
switch output.Kind {
|
||||||
|
case OutputKindSource, OutputKindGenerated:
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("state outputs[%d].kind must be source or generated", index)
|
||||||
|
}
|
||||||
|
if err := storage.ValidatePath(output.SourcePath); err != nil {
|
||||||
|
return fmt.Errorf("state outputs[%d].source_path: %w", index, err)
|
||||||
|
}
|
||||||
|
if output.Kind == OutputKindGenerated && output.Transform == "" {
|
||||||
|
return fmt.Errorf("state outputs[%d].transform is required for generated output", index)
|
||||||
|
}
|
||||||
|
if err := bundle.ValidateDigest(output.SHA256); err != nil {
|
||||||
|
return fmt.Errorf("state outputs[%d].sha256: %w", index, err)
|
||||||
|
}
|
||||||
|
if output.Size < 0 {
|
||||||
|
return fmt.Errorf("state outputs[%d].size must be non-negative", index)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
68
internal/storage/backend.go
Normal file
68
internal/storage/backend.go
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
type EntryType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
EntryTypeFile EntryType = "file"
|
||||||
|
EntryTypeDirectory EntryType = "directory"
|
||||||
|
EntryTypeSymlink EntryType = "symlink"
|
||||||
|
EntryTypeOther EntryType = "other"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Entry struct {
|
||||||
|
Path string
|
||||||
|
Type EntryType
|
||||||
|
Size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type Backend interface {
|
||||||
|
ReadFile(ctx context.Context, path string) ([]byte, error)
|
||||||
|
OpenReader(ctx context.Context, path string) (io.ReadCloser, error)
|
||||||
|
WriteFile(ctx context.Context, path string, data []byte, opts WriteOptions) (Entry, error)
|
||||||
|
WriteFrom(ctx context.Context, path string, r io.Reader, opts WriteOptions) (Entry, error)
|
||||||
|
Stat(ctx context.Context, path string) (Entry, error)
|
||||||
|
Walk(ctx context.Context, prefix string, opts WalkOptions, fn WalkFunc) error
|
||||||
|
HasAny(ctx context.Context, prefix string) (bool, error)
|
||||||
|
DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts DeleteOptions) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type WalkOptions struct {
|
||||||
|
Recursive bool
|
||||||
|
Limit int
|
||||||
|
}
|
||||||
|
|
||||||
|
type WalkFunc func(Entry) error
|
||||||
|
|
||||||
|
var ErrStopWalk = errors.New("stop walk")
|
||||||
|
|
||||||
|
type WriteOptions struct {
|
||||||
|
ContentType string
|
||||||
|
Overwrite bool
|
||||||
|
PreferAtomic bool
|
||||||
|
Size int64
|
||||||
|
SizeKnown bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type DeleteOptions struct {
|
||||||
|
IgnoreMissing bool
|
||||||
|
PruneEmptyDirs bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func List(ctx context.Context, backend Backend, prefix string, opts WalkOptions) ([]Entry, error) {
|
||||||
|
var entries []Entry
|
||||||
|
err := backend.Walk(ctx, prefix, opts, func(entry Entry) error {
|
||||||
|
entries = append(entries, entry)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
SortEntries(entries)
|
||||||
|
return entries, nil
|
||||||
|
}
|
||||||
128
internal/storage/errors.go
Normal file
128
internal/storage/errors.go
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ErrorKind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ErrNotFound ErrorKind = "not_found"
|
||||||
|
ErrAlreadyExist ErrorKind = "already_exists"
|
||||||
|
ErrNotEmpty ErrorKind = "not_empty"
|
||||||
|
ErrInvalidPath ErrorKind = "invalid_path"
|
||||||
|
ErrConflict ErrorKind = "conflict"
|
||||||
|
ErrPermission ErrorKind = "permission"
|
||||||
|
ErrTemporary ErrorKind = "temporary"
|
||||||
|
ErrUnsupported ErrorKind = "unsupported"
|
||||||
|
ErrUnknown ErrorKind = "unknown"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
OpValidatePath = "validate path"
|
||||||
|
OpReadFile = "read file"
|
||||||
|
OpOpenReader = "open reader"
|
||||||
|
OpWriteFile = "write file"
|
||||||
|
OpWriteFrom = "write stream"
|
||||||
|
OpStat = "stat"
|
||||||
|
OpWalk = "walk"
|
||||||
|
OpHasAny = "has any"
|
||||||
|
OpDeleteManagedBundle = "delete managed bundle"
|
||||||
|
OpRegisterBackend = "register backend"
|
||||||
|
OpOpenBackend = "open backend"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Error struct {
|
||||||
|
Op string
|
||||||
|
Backend string
|
||||||
|
Path string
|
||||||
|
Kind ErrorKind
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewError(op, backend, path string, kind ErrorKind, err error) *Error {
|
||||||
|
return &Error{
|
||||||
|
Op: op,
|
||||||
|
Backend: backend,
|
||||||
|
Path: path,
|
||||||
|
Kind: kind,
|
||||||
|
Err: err,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Error) Error() string {
|
||||||
|
if e == nil {
|
||||||
|
return "<nil>"
|
||||||
|
}
|
||||||
|
message := e.Op
|
||||||
|
if e.Backend != "" {
|
||||||
|
message += " " + e.Backend
|
||||||
|
}
|
||||||
|
if e.Path != "" {
|
||||||
|
message += " " + e.Path
|
||||||
|
}
|
||||||
|
if e.Kind != "" {
|
||||||
|
message += ": " + string(e.Kind)
|
||||||
|
}
|
||||||
|
if e.Err != nil {
|
||||||
|
message += ": " + e.Err.Error()
|
||||||
|
}
|
||||||
|
return message
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Error) Unwrap() error {
|
||||||
|
if e == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return e.Err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Error) Is(target error) bool {
|
||||||
|
var targetError *Error
|
||||||
|
if !errors.As(target, &targetError) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return targetError.Kind == "" || e.Kind == targetError.Kind
|
||||||
|
}
|
||||||
|
|
||||||
|
func Errorf(op, backend, path string, kind ErrorKind, format string, args ...any) *Error {
|
||||||
|
return NewError(op, backend, path, kind, fmt.Errorf(format, args...))
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsKind(err error, kind ErrorKind) bool {
|
||||||
|
var storageErr *Error
|
||||||
|
return errors.As(err, &storageErr) && storageErr.Kind == kind
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsNotFound(err error) bool {
|
||||||
|
return IsKind(err, ErrNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsAlreadyExists(err error) bool {
|
||||||
|
return IsKind(err, ErrAlreadyExist)
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsNotEmpty(err error) bool {
|
||||||
|
return IsKind(err, ErrNotEmpty)
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsInvalidPath(err error) bool {
|
||||||
|
return IsKind(err, ErrInvalidPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsConflict(err error) bool {
|
||||||
|
return IsKind(err, ErrConflict)
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsPermission(err error) bool {
|
||||||
|
return IsKind(err, ErrPermission)
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsTemporary(err error) bool {
|
||||||
|
return IsKind(err, ErrTemporary)
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsUnsupported(err error) bool {
|
||||||
|
return IsKind(err, ErrUnsupported)
|
||||||
|
}
|
||||||
316
internal/storage/fake/backend.go
Normal file
316
internal/storage/fake/backend.go
Normal file
@@ -0,0 +1,316 @@
|
|||||||
|
package fake
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
const backendName = "fake"
|
||||||
|
|
||||||
|
type Backend struct {
|
||||||
|
files map[string][]byte
|
||||||
|
dirs map[string]struct{}
|
||||||
|
symlinks map[string]struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func New() *Backend {
|
||||||
|
return &Backend{
|
||||||
|
files: make(map[string][]byte),
|
||||||
|
dirs: map[string]struct{}{"": {}},
|
||||||
|
symlinks: make(map[string]struct{}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) AddDirectory(path string) error {
|
||||||
|
if err := storage.ValidatePrefix(path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.ensureParents(path)
|
||||||
|
b.dirs[path] = struct{}{}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) AddSymlink(path string) error {
|
||||||
|
if err := storage.ValidatePath(path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.ensureParents(path)
|
||||||
|
delete(b.files, path)
|
||||||
|
delete(b.dirs, path)
|
||||||
|
b.symlinks[path] = struct{}{}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) ReadFile(ctx context.Context, path string) ([]byte, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := storage.ValidatePath(path); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
data, ok := b.files[path]
|
||||||
|
if !ok {
|
||||||
|
if b.exists(path) {
|
||||||
|
return nil, storage.NewError(storage.OpReadFile, backendName, path, storage.ErrUnsupported, nil)
|
||||||
|
}
|
||||||
|
return nil, storage.NewError(storage.OpReadFile, backendName, path, storage.ErrNotFound, nil)
|
||||||
|
}
|
||||||
|
return append([]byte(nil), data...), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) OpenReader(ctx context.Context, path string) (io.ReadCloser, error) {
|
||||||
|
data, err := b.ReadFile(ctx, path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return io.NopCloser(bytes.NewReader(data)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) WriteFile(ctx context.Context, path string, data []byte, opts storage.WriteOptions) (storage.Entry, error) {
|
||||||
|
opts.Size = int64(len(data))
|
||||||
|
opts.SizeKnown = true
|
||||||
|
return b.WriteFrom(ctx, path, bytes.NewReader(data), opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) WriteFrom(ctx context.Context, path string, r io.Reader, opts storage.WriteOptions) (storage.Entry, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return storage.Entry{}, err
|
||||||
|
}
|
||||||
|
if err := storage.ValidatePath(path); err != nil {
|
||||||
|
return storage.Entry{}, err
|
||||||
|
}
|
||||||
|
if b.exists(path) && !opts.Overwrite {
|
||||||
|
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrAlreadyExist, nil)
|
||||||
|
}
|
||||||
|
if _, ok := b.dirs[path]; ok {
|
||||||
|
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrConflict, nil)
|
||||||
|
}
|
||||||
|
if _, ok := b.symlinks[path]; ok {
|
||||||
|
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrConflict, nil)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(r)
|
||||||
|
if err != nil {
|
||||||
|
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrUnknown, err)
|
||||||
|
}
|
||||||
|
if opts.SizeKnown && int64(len(data)) != opts.Size {
|
||||||
|
return storage.Entry{}, storage.NewError(storage.OpWriteFrom, backendName, path, storage.ErrConflict, nil)
|
||||||
|
}
|
||||||
|
b.ensureParents(path)
|
||||||
|
b.files[path] = append([]byte(nil), data...)
|
||||||
|
delete(b.symlinks, path)
|
||||||
|
return storage.Entry{Path: path, Type: storage.EntryTypeFile, Size: int64(len(data))}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) Stat(ctx context.Context, path string) (storage.Entry, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return storage.Entry{}, err
|
||||||
|
}
|
||||||
|
if err := storage.ValidatePrefix(path); err != nil {
|
||||||
|
return storage.Entry{}, err
|
||||||
|
}
|
||||||
|
if data, ok := b.files[path]; ok {
|
||||||
|
return storage.Entry{Path: path, Type: storage.EntryTypeFile, Size: int64(len(data))}, nil
|
||||||
|
}
|
||||||
|
if _, ok := b.symlinks[path]; ok {
|
||||||
|
return storage.Entry{Path: path, Type: storage.EntryTypeSymlink}, nil
|
||||||
|
}
|
||||||
|
if _, ok := b.dirs[path]; ok {
|
||||||
|
return storage.Entry{Path: path, Type: storage.EntryTypeDirectory}, nil
|
||||||
|
}
|
||||||
|
return storage.Entry{}, storage.NewError(storage.OpStat, backendName, path, storage.ErrNotFound, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) Walk(ctx context.Context, prefix string, opts storage.WalkOptions, fn storage.WalkFunc) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := storage.ValidatePrefix(prefix); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if entry, err := b.Stat(ctx, prefix); err == nil && entry.Type != storage.EntryTypeDirectory {
|
||||||
|
return emit(ctx, entry, opts, fn)
|
||||||
|
} else if err != nil && !storage.IsNotFound(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
entries := b.entries()
|
||||||
|
visited := 0
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.Path == "" || !entryBelow(prefix, entry.Path) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !opts.Recursive && !isImmediateChild(prefix, entry.Path) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if opts.Limit > 0 && visited >= opts.Limit {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
visited++
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := fn(entry); err != nil {
|
||||||
|
if errors.Is(err, storage.ErrStopWalk) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return storage.NewError(storage.OpWalk, backendName, entry.Path, storage.ErrUnknown, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) HasAny(ctx context.Context, prefix string) (bool, error) {
|
||||||
|
found := false
|
||||||
|
err := b.Walk(ctx, prefix, storage.WalkOptions{Recursive: false, Limit: 1}, func(storage.Entry) error {
|
||||||
|
found = true
|
||||||
|
return storage.ErrStopWalk
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return found, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) DeleteManagedBundle(ctx context.Context, bundlePath string, managedOutputPaths []string, opts storage.DeleteOptions) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := storage.ValidatePrefix(bundlePath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
targets := make([]string, 0, len(managedOutputPaths)+1)
|
||||||
|
for _, outputPath := range managedOutputPaths {
|
||||||
|
target, err := storage.Join(bundlePath, outputPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
targets = append(targets, target)
|
||||||
|
}
|
||||||
|
statePath, err := storage.StatePath(bundlePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
targets = append(targets, statePath)
|
||||||
|
|
||||||
|
for _, target := range targets {
|
||||||
|
if _, ok := b.dirs[target]; ok {
|
||||||
|
return storage.NewError(storage.OpDeleteManagedBundle, backendName, target, storage.ErrUnsupported, nil)
|
||||||
|
}
|
||||||
|
if !b.exists(target) {
|
||||||
|
if opts.IgnoreMissing {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return storage.NewError(storage.OpDeleteManagedBundle, backendName, target, storage.ErrNotFound, nil)
|
||||||
|
}
|
||||||
|
delete(b.files, target)
|
||||||
|
delete(b.symlinks, target)
|
||||||
|
if opts.PruneEmptyDirs {
|
||||||
|
b.pruneEmptyParents(parentOf(target))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) ensureParents(path string) {
|
||||||
|
parent := parentOf(path)
|
||||||
|
for parent != "" {
|
||||||
|
b.dirs[parent] = struct{}{}
|
||||||
|
parent = parentOf(parent)
|
||||||
|
}
|
||||||
|
b.dirs[""] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) pruneEmptyParents(path string) {
|
||||||
|
for path != "" {
|
||||||
|
if b.hasChild(path) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
delete(b.dirs, path)
|
||||||
|
path = parentOf(path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) hasChild(path string) bool {
|
||||||
|
for candidate := range b.files {
|
||||||
|
if entryBelow(path, candidate) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for candidate := range b.symlinks {
|
||||||
|
if entryBelow(path, candidate) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for candidate := range b.dirs {
|
||||||
|
if candidate != path && entryBelow(path, candidate) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) exists(path string) bool {
|
||||||
|
_, file := b.files[path]
|
||||||
|
_, dir := b.dirs[path]
|
||||||
|
_, symlink := b.symlinks[path]
|
||||||
|
return file || dir || symlink
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Backend) entries() []storage.Entry {
|
||||||
|
entries := make([]storage.Entry, 0, len(b.files)+len(b.dirs)+len(b.symlinks))
|
||||||
|
for path, data := range b.files {
|
||||||
|
entries = append(entries, storage.Entry{Path: path, Type: storage.EntryTypeFile, Size: int64(len(data))})
|
||||||
|
}
|
||||||
|
for path := range b.dirs {
|
||||||
|
entries = append(entries, storage.Entry{Path: path, Type: storage.EntryTypeDirectory})
|
||||||
|
}
|
||||||
|
for path := range b.symlinks {
|
||||||
|
entries = append(entries, storage.Entry{Path: path, Type: storage.EntryTypeSymlink})
|
||||||
|
}
|
||||||
|
sort.Slice(entries, func(i, j int) bool {
|
||||||
|
return entries[i].Path < entries[j].Path
|
||||||
|
})
|
||||||
|
return entries
|
||||||
|
}
|
||||||
|
|
||||||
|
func emit(ctx context.Context, entry storage.Entry, opts storage.WalkOptions, fn storage.WalkFunc) error {
|
||||||
|
if opts.Limit > 0 && opts.Limit < 1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := fn(entry); err != nil && !errors.Is(err, storage.ErrStopWalk) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func entryBelow(prefix, path string) bool {
|
||||||
|
if prefix == "" {
|
||||||
|
return path != ""
|
||||||
|
}
|
||||||
|
return strings.HasPrefix(path, prefix+"/")
|
||||||
|
}
|
||||||
|
|
||||||
|
func isImmediateChild(prefix, path string) bool {
|
||||||
|
remainder := path
|
||||||
|
if prefix != "" {
|
||||||
|
remainder = strings.TrimPrefix(path, prefix+"/")
|
||||||
|
}
|
||||||
|
return !strings.Contains(remainder, "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
func parentOf(path string) string {
|
||||||
|
index := strings.LastIndex(path, "/")
|
||||||
|
if index == -1 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return path[:index]
|
||||||
|
}
|
||||||
187
internal/storage/fake/backend_test.go
Normal file
187
internal/storage/fake/backend_test.go
Normal file
@@ -0,0 +1,187 @@
|
|||||||
|
package fake
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBackendReadWriteAndStream(t *testing.T) {
|
||||||
|
backend := New()
|
||||||
|
entry, err := backend.WriteFile(context.Background(), "reports/report.md", []byte("hello"), storage.WriteOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if entry.Path != "reports/report.md" || entry.Type != storage.EntryTypeFile || entry.Size != 5 {
|
||||||
|
t.Fatalf("entry = %#v, want file metadata", entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := backend.ReadFile(context.Background(), "reports/report.md")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile() error = %v", err)
|
||||||
|
}
|
||||||
|
reader, err := backend.OpenReader(context.Background(), "reports/report.md")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("OpenReader() error = %v", err)
|
||||||
|
}
|
||||||
|
streamed, err := io.ReadAll(reader)
|
||||||
|
closeErr := reader.Close()
|
||||||
|
if err != nil || closeErr != nil {
|
||||||
|
t.Fatalf("stream read error = %v close = %v", err, closeErr)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(data, streamed) {
|
||||||
|
t.Fatalf("streamed = %q, want %q", streamed, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = backend.WriteFile(context.Background(), "reports/report.md", []byte("again"), storage.WriteOptions{})
|
||||||
|
if !storage.IsAlreadyExists(err) {
|
||||||
|
t.Fatalf("WriteFile existing error = %v, want already exists", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendStatWalkAndList(t *testing.T) {
|
||||||
|
backend := New()
|
||||||
|
mustWrite(t, backend, "b/two.txt", "2")
|
||||||
|
mustWrite(t, backend, "a/one.txt", "1")
|
||||||
|
|
||||||
|
entry, err := backend.Stat(context.Background(), "a")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Stat directory error = %v", err)
|
||||||
|
}
|
||||||
|
if entry.Type != storage.EntryTypeDirectory {
|
||||||
|
t.Fatalf("entry type = %s, want directory", entry.Type)
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := storage.List(context.Background(), backend, "", storage.WalkOptions{Recursive: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List() error = %v", err)
|
||||||
|
}
|
||||||
|
var paths []string
|
||||||
|
for _, entry := range entries {
|
||||||
|
paths = append(paths, entry.Path)
|
||||||
|
}
|
||||||
|
want := []string{"a", "a/one.txt", "b", "b/two.txt"}
|
||||||
|
if !reflect.DeepEqual(paths, want) {
|
||||||
|
t.Fatalf("paths = %v, want %v", paths, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendRejectsInvalidPaths(t *testing.T) {
|
||||||
|
backend := New()
|
||||||
|
_, err := backend.WriteFile(context.Background(), "../outside", []byte("data"), storage.WriteOptions{})
|
||||||
|
if !storage.IsInvalidPath(err) {
|
||||||
|
t.Fatalf("WriteFile traversal error = %v, want invalid path", err)
|
||||||
|
}
|
||||||
|
_, err = backend.ReadFile(context.Background(), `bad\path`)
|
||||||
|
if !storage.IsInvalidPath(err) {
|
||||||
|
t.Fatalf("ReadFile backslash error = %v, want invalid path", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendSymlinkReportingAndReadRejection(t *testing.T) {
|
||||||
|
backend := New()
|
||||||
|
if err := backend.AddSymlink("link.txt"); err != nil {
|
||||||
|
t.Fatalf("AddSymlink() error = %v", err)
|
||||||
|
}
|
||||||
|
entry, err := backend.Stat(context.Background(), "link.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Stat() error = %v", err)
|
||||||
|
}
|
||||||
|
if entry.Type != storage.EntryTypeSymlink {
|
||||||
|
t.Fatalf("entry type = %s, want symlink", entry.Type)
|
||||||
|
}
|
||||||
|
_, err = backend.ReadFile(context.Background(), "link.txt")
|
||||||
|
if !storage.IsUnsupported(err) {
|
||||||
|
t.Fatalf("ReadFile symlink error = %v, want unsupported", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendWriteFromSizeMismatch(t *testing.T) {
|
||||||
|
backend := New()
|
||||||
|
_, err := backend.WriteFrom(context.Background(), "out.txt", bytes.NewBufferString("short"), storage.WriteOptions{SizeKnown: true, Size: 99})
|
||||||
|
if !storage.IsConflict(err) {
|
||||||
|
t.Fatalf("WriteFrom size mismatch error = %v, want conflict", err)
|
||||||
|
}
|
||||||
|
if _, err := backend.Stat(context.Background(), "out.txt"); !storage.IsNotFound(err) {
|
||||||
|
t.Fatalf("Stat after failed write error = %v, want not found", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendManagedDeletion(t *testing.T) {
|
||||||
|
backend := New()
|
||||||
|
mustWrite(t, backend, "bundle/report.html", "html")
|
||||||
|
mustWrite(t, backend, "bundle/keep.txt", "keep")
|
||||||
|
mustWrite(t, backend, "bundle/.distributor.json", "{}")
|
||||||
|
|
||||||
|
err := backend.DeleteManagedBundle(context.Background(), "bundle", []string{"report.html"}, storage.DeleteOptions{PruneEmptyDirs: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DeleteManagedBundle() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := backend.Stat(context.Background(), "bundle/report.html"); !storage.IsNotFound(err) {
|
||||||
|
t.Fatalf("managed output stat error = %v, want not found", err)
|
||||||
|
}
|
||||||
|
if _, err := backend.Stat(context.Background(), "bundle/.distributor.json"); !storage.IsNotFound(err) {
|
||||||
|
t.Fatalf("state stat error = %v, want not found", err)
|
||||||
|
}
|
||||||
|
if _, err := backend.Stat(context.Background(), "bundle/keep.txt"); err != nil {
|
||||||
|
t.Fatalf("unlisted file stat error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendHasAnyAndWalkStop(t *testing.T) {
|
||||||
|
backend := New()
|
||||||
|
found, err := backend.HasAny(context.Background(), "missing")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("HasAny missing error = %v", err)
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
t.Fatal("HasAny missing = true, want false")
|
||||||
|
}
|
||||||
|
mustWrite(t, backend, "bundle/report.md", "report")
|
||||||
|
found, err = backend.HasAny(context.Background(), "bundle")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("HasAny bundle error = %v", err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("HasAny bundle = false, want true")
|
||||||
|
}
|
||||||
|
found, err = backend.HasAny(context.Background(), "bund")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("HasAny sibling prefix error = %v", err)
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
t.Fatal("HasAny sibling prefix = true, want false")
|
||||||
|
}
|
||||||
|
|
||||||
|
visited := 0
|
||||||
|
err = backend.Walk(context.Background(), "", storage.WalkOptions{Recursive: true}, func(storage.Entry) error {
|
||||||
|
visited++
|
||||||
|
return storage.ErrStopWalk
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Walk stop error = %v", err)
|
||||||
|
}
|
||||||
|
if visited != 1 {
|
||||||
|
t.Fatalf("visited = %d, want 1", visited)
|
||||||
|
}
|
||||||
|
|
||||||
|
errSentinel := errors.New("callback")
|
||||||
|
err = backend.Walk(context.Background(), "", storage.WalkOptions{Recursive: true}, func(storage.Entry) error {
|
||||||
|
return errSentinel
|
||||||
|
})
|
||||||
|
if !errors.Is(err, errSentinel) {
|
||||||
|
t.Fatalf("Walk callback error = %v, want sentinel", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustWrite(t *testing.T, backend *Backend, path, data string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := backend.WriteFile(context.Background(), path, []byte(data), storage.WriteOptions{}); err != nil {
|
||||||
|
t.Fatalf("WriteFile(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
64
internal/storage/path.go
Normal file
64
internal/storage/path.go
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const stateFileName = ".distributor.json"
|
||||||
|
|
||||||
|
func ValidatePath(value string) error {
|
||||||
|
if value == "" {
|
||||||
|
return NewError(OpValidatePath, "", value, ErrInvalidPath, nil)
|
||||||
|
}
|
||||||
|
return validateLogicalPath(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ValidatePrefix(value string) error {
|
||||||
|
if value == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return validateLogicalPath(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func Join(base, child string) (string, error) {
|
||||||
|
if err := ValidatePrefix(base); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := ValidatePath(child); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if base == "" {
|
||||||
|
return child, nil
|
||||||
|
}
|
||||||
|
return base + "/" + child, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func StatePath(bundlePath string) (string, error) {
|
||||||
|
if bundlePath == "" {
|
||||||
|
return stateFileName, nil
|
||||||
|
}
|
||||||
|
return Join(bundlePath, stateFileName)
|
||||||
|
}
|
||||||
|
|
||||||
|
func SortEntries(entries []Entry) {
|
||||||
|
sort.Slice(entries, func(i, j int) bool {
|
||||||
|
return entries[i].Path < entries[j].Path
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateLogicalPath(value string) error {
|
||||||
|
if strings.Contains(value, "\\") || strings.HasPrefix(value, "/") {
|
||||||
|
return NewError(OpValidatePath, "", value, ErrInvalidPath, nil)
|
||||||
|
}
|
||||||
|
if path.Clean(value) != value {
|
||||||
|
return NewError(OpValidatePath, "", value, ErrInvalidPath, nil)
|
||||||
|
}
|
||||||
|
for _, segment := range strings.Split(value, "/") {
|
||||||
|
if segment == "" || segment == "." || segment == ".." {
|
||||||
|
return NewError(OpValidatePath, "", value, ErrInvalidPath, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
133
internal/storage/path_test.go
Normal file
133
internal/storage/path_test.go
Normal file
@@ -0,0 +1,133 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidatePath(t *testing.T) {
|
||||||
|
valid := []string{
|
||||||
|
"report.md",
|
||||||
|
"daily/report.md",
|
||||||
|
"a-b_1.2/report.html",
|
||||||
|
}
|
||||||
|
for _, path := range valid {
|
||||||
|
t.Run("valid "+path, func(t *testing.T) {
|
||||||
|
if err := ValidatePath(path); err != nil {
|
||||||
|
t.Fatalf("ValidatePath(%q) error = %v", path, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
invalid := []string{
|
||||||
|
"",
|
||||||
|
"/absolute",
|
||||||
|
"../outside",
|
||||||
|
"nested/../outside",
|
||||||
|
"nested/./file",
|
||||||
|
"nested//file",
|
||||||
|
`nested\file`,
|
||||||
|
}
|
||||||
|
for _, path := range invalid {
|
||||||
|
t.Run("invalid "+path, func(t *testing.T) {
|
||||||
|
err := ValidatePath(path)
|
||||||
|
if !IsInvalidPath(err) {
|
||||||
|
t.Fatalf("ValidatePath(%q) error = %v, want invalid path", path, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatePrefixAllowsRoot(t *testing.T) {
|
||||||
|
if err := ValidatePrefix(""); err != nil {
|
||||||
|
t.Fatalf("ValidatePrefix(\"\") error = %v", err)
|
||||||
|
}
|
||||||
|
if err := ValidatePrefix("a/.."); !IsInvalidPath(err) {
|
||||||
|
t.Fatalf("ValidatePrefix traversal error = %v, want invalid path", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListSortsEntries(t *testing.T) {
|
||||||
|
backend := walkBackend{
|
||||||
|
entries: []Entry{
|
||||||
|
{Path: "z.txt", Type: EntryTypeFile},
|
||||||
|
{Path: "a.txt", Type: EntryTypeFile},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := List(context.Background(), backend, "", WalkOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := []string{entries[0].Path, entries[1].Path}, []string{"a.txt", "z.txt"}; got[0] != want[0] || got[1] != want[1] {
|
||||||
|
t.Fatalf("paths = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTypedErrorPredicates(t *testing.T) {
|
||||||
|
err := NewError(OpReadFile, "test", "missing", ErrNotFound, errors.New("missing"))
|
||||||
|
if !IsNotFound(err) {
|
||||||
|
t.Fatalf("IsNotFound(%v) = false, want true", err)
|
||||||
|
}
|
||||||
|
if IsInvalidPath(err) {
|
||||||
|
t.Fatalf("IsInvalidPath(%v) = true, want false", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegistry(t *testing.T) {
|
||||||
|
registry := NewRegistry()
|
||||||
|
if err := registry.Register("test", func(context.Context, OpenConfig) (Backend, error) {
|
||||||
|
return walkBackend{}, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Register() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := registry.Open(context.Background(), "test", nil); err != nil {
|
||||||
|
t.Fatalf("Open() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := registry.Open(context.Background(), "missing", nil); !IsUnsupported(err) {
|
||||||
|
t.Fatalf("Open() error = %v, want unsupported", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type walkBackend struct {
|
||||||
|
entries []Entry
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b walkBackend) ReadFile(context.Context, string) ([]byte, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b walkBackend) OpenReader(context.Context, string) (io.ReadCloser, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b walkBackend) WriteFile(context.Context, string, []byte, WriteOptions) (Entry, error) {
|
||||||
|
return Entry{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b walkBackend) WriteFrom(context.Context, string, io.Reader, WriteOptions) (Entry, error) {
|
||||||
|
return Entry{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b walkBackend) Stat(context.Context, string) (Entry, error) {
|
||||||
|
return Entry{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b walkBackend) Walk(_ context.Context, _ string, _ WalkOptions, fn WalkFunc) error {
|
||||||
|
for _, entry := range b.entries {
|
||||||
|
if err := fn(entry); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b walkBackend) HasAny(context.Context, string) (bool, error) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b walkBackend) DeleteManagedBundle(context.Context, string, []string, DeleteOptions) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
46
internal/storage/registry.go
Normal file
46
internal/storage/registry.go
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
type OpenConfig map[string]string
|
||||||
|
|
||||||
|
type Opener func(context.Context, OpenConfig) (Backend, error)
|
||||||
|
|
||||||
|
type Registry struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
openers map[string]Opener
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewRegistry() *Registry {
|
||||||
|
return &Registry{openers: make(map[string]Opener)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) Register(name string, opener Opener) error {
|
||||||
|
if name == "" || opener == nil {
|
||||||
|
return NewError(OpRegisterBackend, name, "", ErrInvalidPath, nil)
|
||||||
|
}
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
if _, exists := r.openers[name]; exists {
|
||||||
|
return NewError(OpRegisterBackend, name, "", ErrAlreadyExist, nil)
|
||||||
|
}
|
||||||
|
r.openers[name] = opener
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) Open(ctx context.Context, name string, cfg OpenConfig) (Backend, error) {
|
||||||
|
r.mu.RLock()
|
||||||
|
opener, ok := r.openers[name]
|
||||||
|
r.mu.RUnlock()
|
||||||
|
if !ok {
|
||||||
|
return nil, NewError(OpOpenBackend, name, "", ErrUnsupported, nil)
|
||||||
|
}
|
||||||
|
backend, err := opener(ctx, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return backend, nil
|
||||||
|
}
|
||||||
57
internal/transform/markdown/markdown.go
Normal file
57
internal/transform/markdown/markdown.go
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
package markdown
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/yuin/goldmark"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Transformer struct {
|
||||||
|
renderer goldmark.Markdown
|
||||||
|
}
|
||||||
|
|
||||||
|
func New() *Transformer {
|
||||||
|
return &Transformer{renderer: goldmark.New()}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *Transformer) Generate(ctx context.Context, req transform.Request) ([]transform.Output, error) {
|
||||||
|
if t.renderer == nil {
|
||||||
|
t.renderer = goldmark.New()
|
||||||
|
}
|
||||||
|
var outputs []transform.Output
|
||||||
|
for _, file := range req.SourceBundle.Manifest.Files {
|
||||||
|
if !strings.HasSuffix(file.Path, ".md") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sourcePath, err := storage.Join(req.SourceBundle.RootRelativePath, file.Path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
data, err := req.SourceBackend.ReadFile(ctx, sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read markdown source %q: %w", file.Path, err)
|
||||||
|
}
|
||||||
|
var rendered bytes.Buffer
|
||||||
|
if err := t.renderer.Convert(data, &rendered); err != nil {
|
||||||
|
return nil, fmt.Errorf("render markdown source %q: %w", file.Path, err)
|
||||||
|
}
|
||||||
|
html := wrapHTML(rendered.Bytes())
|
||||||
|
outputPath := strings.TrimSuffix(file.Path, ".md") + ".html"
|
||||||
|
outputs = append(outputs, transform.Output{
|
||||||
|
Path: outputPath,
|
||||||
|
SourcePath: file.Path,
|
||||||
|
Transform: transform.MarkdownToHTML,
|
||||||
|
Data: html,
|
||||||
|
SHA256: bundle.FileDigest(html),
|
||||||
|
Size: int64(len(html)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return outputs, nil
|
||||||
|
}
|
||||||
113
internal/transform/markdown/markdown_test.go
Normal file
113
internal/transform/markdown/markdown_test.go
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
package markdown
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage/fake"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/transform"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGenerateMarkdownSidecar(t *testing.T) {
|
||||||
|
backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n")
|
||||||
|
outputs, err := New().Generate(context.Background(), transform.Request{SourceBackend: backend, SourceBundle: sourceBundle})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
if got, want := len(outputs), 1; got != want {
|
||||||
|
t.Fatalf("output count = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
output := outputs[0]
|
||||||
|
if output.Path != "report.html" {
|
||||||
|
t.Fatalf("path = %q, want report.html", output.Path)
|
||||||
|
}
|
||||||
|
if output.SourcePath != "report.md" || output.Transform != transform.MarkdownToHTML {
|
||||||
|
t.Fatalf("metadata = %#v", output)
|
||||||
|
}
|
||||||
|
html := string(output.Data)
|
||||||
|
for _, want := range []string{"<!doctype html>", "<h1>Title</h1>", "<p>Hello.</p>"} {
|
||||||
|
if !strings.Contains(html, want) {
|
||||||
|
t.Fatalf("html = %q, want substring %q", html, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if output.SHA256 != bundle.FileDigest(output.Data) || output.Size != int64(len(output.Data)) {
|
||||||
|
t.Fatalf("digest/size metadata = %s/%d", output.SHA256, output.Size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateIgnoresNonMarkdown(t *testing.T) {
|
||||||
|
backend := fake.New()
|
||||||
|
if _, err := backend.WriteFile(context.Background(), "summary.txt", []byte("Summary\n"), storage.WriteOptions{}); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
sourceBundle := bundle.Bundle{Manifest: bundle.Manifest{
|
||||||
|
SchemaVersion: 1,
|
||||||
|
ID: "bundle",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Files: []bundle.ManifestFile{{Path: "summary.txt", SHA256: bundle.FileDigest([]byte("Summary\n")), Size: 8}},
|
||||||
|
}}
|
||||||
|
outputs, err := New().Generate(context.Background(), transform.Request{SourceBackend: backend, SourceBundle: sourceBundle})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(outputs) != 0 {
|
||||||
|
t.Fatalf("outputs = %#v, want none", outputs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateDoesNotPassRawHTML(t *testing.T) {
|
||||||
|
backend, sourceBundle := markdownFixture(t, "# Title\n\n<script>alert('x')</script>\n")
|
||||||
|
outputs, err := New().Generate(context.Background(), transform.Request{SourceBackend: backend, SourceBundle: sourceBundle})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
html := string(outputs[0].Data)
|
||||||
|
if strings.Contains(html, "<script>") {
|
||||||
|
t.Fatalf("html contains raw script: %q", html)
|
||||||
|
}
|
||||||
|
if !strings.Contains(html, "raw HTML omitted") && !strings.Contains(html, "<script>") {
|
||||||
|
t.Fatalf("html = %q, want raw HTML disabled or escaped", html)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateDeterministicOutput(t *testing.T) {
|
||||||
|
backend, sourceBundle := markdownFixture(t, "# Title\n\nHello.\n")
|
||||||
|
first, err := New().Generate(context.Background(), transform.Request{SourceBackend: backend, SourceBundle: sourceBundle})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
second, err := New().Generate(context.Background(), transform.Request{SourceBackend: backend, SourceBundle: sourceBundle})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second Generate() error = %v", err)
|
||||||
|
}
|
||||||
|
if string(first[0].Data) != string(second[0].Data) {
|
||||||
|
t.Fatalf("outputs differ:\n%s\n%s", first[0].Data, second[0].Data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func markdownFixture(t *testing.T, markdown string) (*fake.Backend, bundle.Bundle) {
|
||||||
|
t.Helper()
|
||||||
|
backend := fake.New()
|
||||||
|
if _, err := backend.WriteFile(context.Background(), "report.md", []byte(markdown), storage.WriteOptions{}); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := backend.WriteFile(context.Background(), "summary.txt", []byte("Summary\n"), storage.WriteOptions{}); err != nil {
|
||||||
|
t.Fatalf("WriteFile() error = %v", err)
|
||||||
|
}
|
||||||
|
files := []bundle.ManifestFile{
|
||||||
|
{Path: "report.md", SHA256: bundle.FileDigest([]byte(markdown)), Size: int64(len(markdown))},
|
||||||
|
{Path: "summary.txt", SHA256: bundle.FileDigest([]byte("Summary\n")), Size: 8},
|
||||||
|
}
|
||||||
|
manifest := bundle.Manifest{
|
||||||
|
SchemaVersion: 1,
|
||||||
|
ID: "bundle",
|
||||||
|
Created: time.Date(2026, 5, 30, 11, 10, 0, 0, time.UTC),
|
||||||
|
Files: files,
|
||||||
|
}
|
||||||
|
manifest.Digest = bundle.BundleDigest(manifest.Files)
|
||||||
|
return backend, bundle.Bundle{Manifest: manifest}
|
||||||
|
}
|
||||||
11
internal/transform/markdown/template.go
Normal file
11
internal/transform/markdown/template.go
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
package markdown
|
||||||
|
|
||||||
|
import "bytes"
|
||||||
|
|
||||||
|
func wrapHTML(body []byte) []byte {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
buf.WriteString("<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n<title></title>\n</head>\n<body>\n")
|
||||||
|
buf.Write(body)
|
||||||
|
buf.WriteString("</body>\n</html>\n")
|
||||||
|
return buf.Bytes()
|
||||||
|
}
|
||||||
3
internal/transform/plan.go
Normal file
3
internal/transform/plan.go
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
package transform
|
||||||
|
|
||||||
|
const MarkdownToHTML = "markdown_to_html"
|
||||||
30
internal/transform/registry.go
Normal file
30
internal/transform/registry.go
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
package transform
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
type Registry struct {
|
||||||
|
transformers map[string]Transformer
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewRegistry() *Registry {
|
||||||
|
return &Registry{transformers: make(map[string]Transformer)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) Register(name string, transformer Transformer) error {
|
||||||
|
if name == "" {
|
||||||
|
return fmt.Errorf("transform name is required")
|
||||||
|
}
|
||||||
|
if transformer == nil {
|
||||||
|
return fmt.Errorf("transformer %s is nil", name)
|
||||||
|
}
|
||||||
|
if _, exists := r.transformers[name]; exists {
|
||||||
|
return fmt.Errorf("transform %s is already registered", name)
|
||||||
|
}
|
||||||
|
r.transformers[name] = transformer
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) Get(name string) (Transformer, bool) {
|
||||||
|
transformer, ok := r.transformers[name]
|
||||||
|
return transformer, ok
|
||||||
|
}
|
||||||
26
internal/transform/transform.go
Normal file
26
internal/transform/transform.go
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
package transform
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/bundle"
|
||||||
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Output struct {
|
||||||
|
Path string
|
||||||
|
SourcePath string
|
||||||
|
Transform string
|
||||||
|
Data []byte
|
||||||
|
SHA256 string
|
||||||
|
Size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type Request struct {
|
||||||
|
SourceBundle bundle.Bundle
|
||||||
|
SourceBackend storage.Backend
|
||||||
|
}
|
||||||
|
|
||||||
|
type Transformer interface {
|
||||||
|
Generate(ctx context.Context, req Request) ([]Output, error)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user