Update documentation relating to the public packages and http_upload API
This commit is contained in:
@@ -1,9 +1,104 @@
|
||||
// Package bundle provides producer-facing helpers for distributor source
|
||||
// bundle manifests.
|
||||
// bundles.
|
||||
//
|
||||
// A source bundle is a local directory containing a manifest.json file and the
|
||||
// files listed by that manifest. This package owns the public manifest model,
|
||||
// digest calculation, path validation, manifest parsing, manifest building,
|
||||
// local bundle writing, and local bundle validation used by Go producer
|
||||
// applications.
|
||||
// A source bundle is a local directory containing manifest.json and the files
|
||||
// listed by that manifest. Producer applications use this package when they
|
||||
// need to generate manifests, validate bundles locally, or write complete
|
||||
// bundle directories for distributor to discover, upload, or publish.
|
||||
//
|
||||
// # Bundle Contract
|
||||
//
|
||||
// The source manifest is the producer-to-distributor contract. It is named by
|
||||
// ManifestName, currently "manifest.json", and uses SchemaVersion, currently 1.
|
||||
// A Manifest contains:
|
||||
//
|
||||
// - SchemaVersion: the source manifest schema version.
|
||||
// - ID: the producer's stable bundle identifier.
|
||||
// - Digest: the canonical digest of the ordered file records.
|
||||
// - Created: an RFC3339 timestamp when marshaled to JSON.
|
||||
// - Files: an ordered list of ManifestFile records.
|
||||
//
|
||||
// Each ManifestFile records a slash-separated bundle-relative Path, a lowercase
|
||||
// sha256:<64 hex> SHA256 digest, and a byte Size. File order is significant for
|
||||
// the bundle digest and should be chosen deliberately by the producer. Explicit
|
||||
// file lists preserve caller order; scan mode sorts by slash-separated path.
|
||||
//
|
||||
// # Path Rules
|
||||
//
|
||||
// Public bundle paths are always slash-separated and relative to the bundle
|
||||
// root. ValidateSourcePath rejects empty paths, absolute paths, path traversal,
|
||||
// dot segments, backslashes, and reserved manifest/state paths. Source files
|
||||
// must be regular files; symlinks and other special files are rejected.
|
||||
//
|
||||
// BuildManifest with Scan true recursively scans Root, includes regular files
|
||||
// including dotfiles, excludes manifest.json and .distributor.json, rejects
|
||||
// symlinks, and sorts paths lexically. BuildManifest with Files uses exactly
|
||||
// the caller-provided paths and preserves their order. Exactly one selection
|
||||
// mode must be used.
|
||||
//
|
||||
// # Manifest Workflows
|
||||
//
|
||||
// BuildManifest reads existing files under a local root, calculates each
|
||||
// ManifestFile, defaults a zero Created value to the current UTC time, calculates
|
||||
// the bundle digest, and validates the result. WriteManifest writes
|
||||
// manifest.json and fails if it already exists unless WriteManifestOptions has
|
||||
// Overwrite set. LoadManifest reads and parses manifest.json. ParseManifest and
|
||||
// MarshalManifest are useful when an application stores or transmits manifest
|
||||
// bytes directly; MarshalManifest validates before writing deterministic,
|
||||
// indented JSON with a trailing newline.
|
||||
//
|
||||
// ValidateManifest checks manifest-only semantics, including schema version,
|
||||
// required fields, path safety, duplicate file paths, digest syntax, file sizes,
|
||||
// and bundle digest. ValidateBundle checks a supplied Manifest against local
|
||||
// files under a root, including existence, regular-file type, size, SHA-256
|
||||
// digest, path safety, and bundle digest.
|
||||
//
|
||||
// # Complete Bundle Writing
|
||||
//
|
||||
// WriteBundle is the most convenient producer workflow when source files live
|
||||
// outside the final bundle directory. It copies each BundleFile.SourcePath into
|
||||
// a staged bundle at BundleFile.Path, builds and writes a compliant manifest,
|
||||
// validates the staged bundle, and promotes it to WriteBundleOptions.Root.
|
||||
// Overwrite permits replacement of an existing bundle root using a best-effort
|
||||
// sibling temporary and backup strategy.
|
||||
//
|
||||
// # Digest Helpers
|
||||
//
|
||||
// FileDigest returns the sha256:<64 hex> digest for file bytes. BundleDigest
|
||||
// returns the canonical bundle digest for an ordered []ManifestFile.
|
||||
// CanonicalFilePayload returns the JSON payload used by BundleDigest, which is
|
||||
// mainly useful for tests and diagnostics. ValidateDigest checks digest syntax.
|
||||
//
|
||||
// Example: build and write a manifest for files already under a bundle root.
|
||||
//
|
||||
// root := "/var/lib/reports/daily-2026-06-06"
|
||||
// manifest, err := bundle.BuildManifest(bundle.BuildOptions{
|
||||
// Root: root,
|
||||
// ID: "reports.daily.2026-06-06",
|
||||
// Files: []string{"report.md", "summary.txt"},
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// if err := bundle.WriteManifest(root, manifest, bundle.WriteManifestOptions{}); err != nil {
|
||||
// return err
|
||||
// }
|
||||
// if err := bundle.ValidateBundle(root, manifest); err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// Example: create a complete bundle from producer-generated files.
|
||||
//
|
||||
// manifest, err := bundle.WriteBundle(bundle.WriteBundleOptions{
|
||||
// Root: "/var/lib/distributor-source/daily-2026-06-06",
|
||||
// ID: "reports.daily.2026-06-06",
|
||||
// Files: []bundle.BundleFile{
|
||||
// {SourcePath: "/tmp/report.md", Path: "report.md"},
|
||||
// {SourcePath: "/tmp/summary.txt", Path: "summary.txt"},
|
||||
// },
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// _ = manifest
|
||||
package bundle
|
||||
|
||||
125
pkg/upload/doc.go
Normal file
125
pkg/upload/doc.go
Normal file
@@ -0,0 +1,125 @@
|
||||
// Package upload provides producer-facing helpers for submitting distributor
|
||||
// source bundles to the HTTP upload API.
|
||||
//
|
||||
// The package is intended for Go producer applications that already create
|
||||
// reports or other Markdown bundle contents and want to hand those bundles to a
|
||||
// running distributor server. It builds on pkg/bundle for manifest generation,
|
||||
// path validation, digest calculation, local bundle writing, and local bundle
|
||||
// validation. It does not expose distributor internals, server configuration,
|
||||
// storage backends, destination state, or publish behavior.
|
||||
//
|
||||
// # Client Construction
|
||||
//
|
||||
// NewClient creates a Client from ClientOptions. Endpoint is required and must
|
||||
// be an http or https distributor server base URL without userinfo, query, or
|
||||
// fragment. The client derives /upload for submissions and /runs/<run-id> for
|
||||
// status checks. Token is required and is sent as Authorization: Bearer <token>.
|
||||
// Token values are redacted from errors produced by the client.
|
||||
//
|
||||
// HTTPClient is optional. When omitted, the package uses a client with a
|
||||
// conservative timeout. Retry is optional; zero values select safe defaults.
|
||||
// RetryOptions.MaxAttempts, BaseDelay, and MaxDelay must not be negative, and
|
||||
// MaxDelay must be greater than or equal to BaseDelay.
|
||||
//
|
||||
// # Upload Workflows
|
||||
//
|
||||
// UploadBundle uploads an existing local source bundle root. The root must
|
||||
// contain manifest.json. By default, UploadBundle loads the manifest and
|
||||
// validates the complete local bundle with pkg/bundle before making any HTTP
|
||||
// request. The generated gzip-compressed tar archive contains manifest.json and
|
||||
// exactly the manifest-listed files; unlisted files are not uploaded.
|
||||
//
|
||||
// UploadFiles is the convenience workflow for producer applications that have
|
||||
// generated files but have not yet assembled a bundle directory. It uses
|
||||
// pkg/bundle to create a temporary complete bundle from explicit
|
||||
// bundle.BundleFile values, validates it by default, archives it, uploads it,
|
||||
// and removes temporary files when the call returns. UploadFiles does not write
|
||||
// into producer source directories. A zero Created timestamp follows
|
||||
// pkg/bundle defaulting behavior.
|
||||
//
|
||||
// Validation is enabled by default. Set DisableValidation when the application
|
||||
// has already performed equivalent local validation and wants to skip the
|
||||
// package's validation step. Validate and DisableValidation must not both be
|
||||
// true.
|
||||
//
|
||||
// # Idempotency And Retry
|
||||
//
|
||||
// Every upload request includes Idempotency-Key. If UploadBundleOptions or
|
||||
// UploadFilesOptions provides IdempotencyKey, the client validates and uses
|
||||
// that value. Otherwise, it generates a random 128-bit lowercase hexadecimal
|
||||
// key once for that upload operation and reuses it for all retries from that
|
||||
// call.
|
||||
//
|
||||
// Generated idempotency keys are useful for retrying transient failures within
|
||||
// a single process call. Producers that need cross-process retry safety should
|
||||
// provide their own stable key, such as a key derived from the producer job id
|
||||
// or report id. Valid keys are non-empty ASCII strings up to 128 bytes using
|
||||
// letters, digits, '.', '_', '-', and ':'.
|
||||
//
|
||||
// The client retries only safe cases: 503 Service Unavailable, temporary
|
||||
// network errors, and ambiguous mid-upload failures. Retries use the same
|
||||
// idempotency key and replayable gzip archive body. The client does not retry
|
||||
// 400, 401, 409, 413, 415, or any response after 202 Accepted. Context
|
||||
// cancellation is honored before each attempt and while waiting between
|
||||
// retries.
|
||||
//
|
||||
// # Results And Errors
|
||||
//
|
||||
// Result represents upload admission. A successful UploadBundle or UploadFiles
|
||||
// call means the server accepted the upload and returned a run id; it does not
|
||||
// mean the asynchronous distribution run has finished successfully.
|
||||
//
|
||||
// Status fetches the current server status for a run id and returns RunStatus.
|
||||
// This is a separate polling helper; upload calls do not wait for publication
|
||||
// completion.
|
||||
//
|
||||
// Non-2xx upload and status responses return *HTTPError when the server status
|
||||
// can be represented as an HTTP failure. HTTPError includes the numeric status
|
||||
// code, HTTP status string, response message, and server retryable flag when
|
||||
// present. A 409 Conflict response is returned as *IdempotencyConflictError,
|
||||
// which wraps HTTPError and can be detected with errors.As.
|
||||
//
|
||||
// Example: upload producer files with a stable idempotency key.
|
||||
//
|
||||
// ctx := context.Background()
|
||||
// client, err := upload.NewClient(upload.ClientOptions{
|
||||
// Endpoint: "https://distributor.example.com",
|
||||
// Token: os.Getenv("DISTRIBUTOR_UPLOAD_TOKEN"),
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// result, err := client.UploadFiles(ctx, upload.UploadFilesOptions{
|
||||
// ID: "reports.daily.2026-06-06",
|
||||
// IdempotencyKey: "reports.daily.2026-06-06",
|
||||
// Files: []bundle.BundleFile{
|
||||
// {SourcePath: "/tmp/report.md", Path: "report.md"},
|
||||
// {SourcePath: "/tmp/summary.txt", Path: "summary.txt"},
|
||||
// },
|
||||
// })
|
||||
// if err != nil {
|
||||
// var conflict *upload.IdempotencyConflictError
|
||||
// if errors.As(err, &conflict) {
|
||||
// return fmt.Errorf("upload conflicts with an earlier different bundle: %w", err)
|
||||
// }
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// status, err := client.Status(ctx, result.RunID)
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// _ = status
|
||||
//
|
||||
// Example: upload an existing bundle root.
|
||||
//
|
||||
// result, err := client.UploadBundle(ctx, upload.UploadBundleOptions{
|
||||
// Root: "/var/lib/reports/daily-2026-06-06",
|
||||
// IdempotencyKey: "reports.daily.2026-06-06",
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// _ = result
|
||||
package upload
|
||||
Reference in New Issue
Block a user