Files
weatherreporter/internal/comparison/publish.go

758 lines
24 KiB
Go

package comparison
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
)
// DestinationErrorKind identifies a comparison destination safety failure.
type DestinationErrorKind string
const (
DestinationInvalidPath DestinationErrorKind = "invalid_path"
DestinationFilesystemRoot DestinationErrorKind = "filesystem_root"
DestinationWorkingDirectory DestinationErrorKind = "working_directory"
DestinationSymlink DestinationErrorKind = "symlink"
DestinationNotDirectory DestinationErrorKind = "not_directory"
DestinationNotEmpty DestinationErrorKind = "not_empty"
DestinationUnrecognized DestinationErrorKind = "unrecognized"
DestinationInspection DestinationErrorKind = "inspection"
maxComparisonComponentBytes = 255
temporaryRandomBytes = 10
stagingDirectoryPattern = ".weatherreporter-staging-*"
backupDirectoryPattern = ".weatherreporter-backup-*"
)
// DestinationError provides inspectable context without making filesystem
// implementation details part of a manifest or command-result schema.
type DestinationError struct {
Kind DestinationErrorKind
Target string
Err error
}
func (err *DestinationError) Error() string {
if err.Err == nil {
return fmt.Sprintf("comparison destination %q is %s", err.Target, err.Kind)
}
return fmt.Sprintf("comparison destination %q is %s: %v", err.Target, err.Kind, err.Err)
}
func (err *DestinationError) Unwrap() error {
return err.Err
}
// DestinationPlan records a preflighted, exact bundle directory. Callers must
// pass it to Publish rather than recreating destination policy themselves.
type DestinationPlan struct {
WorkingDirectory string
Target string
Replace bool
state destinationState
}
type destinationState uint8
const (
destinationAbsent destinationState = iota
destinationEmpty
destinationBundle
)
// ErrUnrecognizedBundle marks a directory that is not a valid current-schema
// Weatherreporter comparison bundle.
var ErrUnrecognizedBundle = errors.New("unrecognized comparison bundle")
// PlanDestination performs the read-only comparison destination preflight.
func PlanDestination(workingDirectory, target string, replace bool) (DestinationPlan, error) {
return planDestination(workingDirectory, target, replace, RecognizeBundle)
}
func planDestination(workingDirectory, target string, replace bool, recognize func(string) (Manifest, error)) (DestinationPlan, error) {
plan, err := newDestinationPlan(workingDirectory, target, replace)
if err != nil {
return DestinationPlan{}, err
}
return inspectDestination(plan, recognize)
}
func newDestinationPlan(workingDirectory, target string, replace bool) (DestinationPlan, error) {
workingDirectory, err := absoluteCleanPath(workingDirectory)
if err != nil {
return DestinationPlan{}, newDestinationError(DestinationInvalidPath, workingDirectory, err)
}
target, err = absoluteCleanPath(target)
if err != nil {
return DestinationPlan{}, newDestinationError(DestinationInvalidPath, target, err)
}
if filepath.Dir(target) == target {
return DestinationPlan{}, newDestinationError(DestinationFilesystemRoot, target, nil)
}
if target == workingDirectory {
return DestinationPlan{}, newDestinationError(DestinationWorkingDirectory, target, nil)
}
if err := validateTransactionSiblingNames(target); err != nil {
return DestinationPlan{}, newDestinationError(DestinationInvalidPath, target, err)
}
return DestinationPlan{WorkingDirectory: workingDirectory, Target: target, Replace: replace}, nil
}
func reauthorizeDestination(plan DestinationPlan) (DestinationPlan, error) {
plan, err := newDestinationPlan(plan.WorkingDirectory, plan.Target, plan.Replace)
if err != nil {
return DestinationPlan{}, err
}
return inspectDestination(plan, nil)
}
func inspectDestination(plan DestinationPlan, recognize func(string) (Manifest, error)) (DestinationPlan, error) {
target := plan.Target
info, err := os.Lstat(target)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
return DestinationPlan{}, newDestinationError(DestinationInspection, target, err)
}
if err := inspectMissingDestinationParent(target); err != nil {
return DestinationPlan{}, err
}
return plan, nil
}
if info.Mode()&os.ModeSymlink != 0 {
return DestinationPlan{}, newDestinationError(DestinationSymlink, target, nil)
}
if !info.IsDir() {
return DestinationPlan{}, newDestinationError(DestinationNotDirectory, target, nil)
}
entries, err := os.ReadDir(target)
if err != nil {
return DestinationPlan{}, newDestinationError(DestinationInspection, target, err)
}
if len(entries) == 0 {
plan.state = destinationEmpty
return plan, nil
}
if !plan.Replace {
return DestinationPlan{}, newDestinationError(DestinationNotEmpty, target, nil)
}
if recognize != nil {
if _, err := recognize(target); err != nil {
return DestinationPlan{}, newDestinationError(DestinationUnrecognized, target, err)
}
}
plan.state = destinationBundle
return plan, nil
}
func validateTransactionSiblingNames(target string) error {
if len(filepath.Base(target)) > maxComparisonComponentBytes {
return fmt.Errorf("destination name exceeds the %d-byte limit", maxComparisonComponentBytes)
}
for _, pattern := range []string{stagingDirectoryPattern, backupDirectoryPattern} {
if !temporaryPatternFitsComponentLimit(pattern) {
return fmt.Errorf("comparison transaction sibling pattern exceeds the %d-byte limit", maxComparisonComponentBytes)
}
}
return nil
}
func temporaryPatternFitsComponentLimit(pattern string) bool {
return len(strings.Replace(pattern, "*", strings.Repeat("0", temporaryRandomBytes), 1)) <= maxComparisonComponentBytes
}
func absoluteCleanPath(value string) (string, error) {
if strings.TrimSpace(value) == "" {
return "", fmt.Errorf("path is required")
}
if !filepath.IsAbs(value) {
return "", fmt.Errorf("path %q must be absolute", value)
}
return filepath.Clean(value), nil
}
func inspectMissingDestinationParent(target string) error {
for parent := filepath.Dir(target); ; parent = filepath.Dir(parent) {
info, err := os.Lstat(parent)
if err == nil {
if info.Mode()&os.ModeSymlink != 0 {
resolved, statErr := os.Stat(parent)
if statErr != nil {
return newDestinationError(DestinationInspection, target, statErr)
}
if !resolved.IsDir() {
return newDestinationError(DestinationNotDirectory, target, nil)
}
return nil
}
if !info.IsDir() {
return newDestinationError(DestinationNotDirectory, target, nil)
}
return nil
}
if !errors.Is(err, os.ErrNotExist) {
return newDestinationError(DestinationInspection, target, err)
}
if filepath.Dir(parent) == parent {
return newDestinationError(DestinationInspection, target, fmt.Errorf("no existing directory ancestor"))
}
}
}
func newDestinationError(kind DestinationErrorKind, target string, err error) error {
return &DestinationError{Kind: kind, Target: target, Err: err}
}
// RecognizeBundle verifies that directory contains exactly one valid current
// comparison bundle. It never follows bundle entries through symlinks.
func RecognizeBundle(directory string) (Manifest, error) {
return recognizeBundle(directory, defaultRecognitionOperations)
}
type recognitionOperations struct {
lstat func(string) (os.FileInfo, error)
readDir func(string) ([]os.DirEntry, error)
open func(string) (io.ReadCloser, error)
}
var defaultRecognitionOperations = recognitionOperations{
lstat: os.Lstat,
readDir: os.ReadDir,
open: func(path string) (io.ReadCloser, error) {
return os.Open(path)
},
}
func recognizeBundle(directory string, operations recognitionOperations) (Manifest, error) {
info, err := operations.lstat(directory)
if err != nil {
return Manifest{}, unrecognizedBundleError("inspect directory", err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
return Manifest{}, unrecognizedBundleError("directory is not a real directory", nil)
}
entries, err := operations.readDir(directory)
if err != nil {
return Manifest{}, unrecognizedBundleError("read directory", err)
}
manifestData, err := readBundleFile(directory, ManifestFilename, operations)
if err != nil {
return Manifest{}, err
}
manifest, err := decodeManifest(manifestData)
if err != nil {
return Manifest{}, err
}
if err := manifest.Validate(); err != nil {
return Manifest{}, unrecognizedBundleError("validate manifest", err)
}
expected := map[string]struct{}{
ManifestFilename: {},
DataPackageFilename: {},
}
for _, result := range manifest.Results {
if result.Status == StatusSucceeded {
expected[result.ReportPath] = struct{}{}
}
}
if len(entries) != len(expected) {
return Manifest{}, unrecognizedBundleError("directory entries do not match manifest", nil)
}
for _, entry := range entries {
if _, ok := expected[entry.Name()]; !ok {
return Manifest{}, unrecognizedBundleError("directory has an undeclared entry", nil)
}
if entry.Name() == ManifestFilename || entry.Name() == DataPackageFilename {
continue
}
if err := inspectBundleFile(directory, entry.Name(), operations); err != nil {
return Manifest{}, err
}
}
dataPackage, err := readBundleFile(directory, DataPackageFilename, operations)
if err != nil {
return Manifest{}, err
}
if SHA256(dataPackage) != manifest.DataPackage.SHA256 {
return Manifest{}, unrecognizedBundleError("data package digest does not match manifest", nil)
}
return manifest, nil
}
func readBundleFile(directory, name string, operations recognitionOperations) ([]byte, error) {
file, err := openBundleFile(directory, name, operations)
if err != nil {
return nil, err
}
defer file.Close()
data, err := io.ReadAll(file)
if err != nil {
return nil, unrecognizedBundleError("read bundle entry", err)
}
return data, nil
}
func inspectBundleFile(directory, name string, operations recognitionOperations) error {
file, err := openBundleFile(directory, name, operations)
if err != nil {
return err
}
if err := file.Close(); err != nil {
return unrecognizedBundleError("close bundle entry", err)
}
return nil
}
func openBundleFile(directory, name string, operations recognitionOperations) (io.ReadCloser, error) {
if !isArtifactBasename(name) {
return nil, unrecognizedBundleError("bundle entry name is unsafe", nil)
}
filePath := filepath.Join(directory, name)
info, err := operations.lstat(filePath)
if err != nil {
return nil, unrecognizedBundleError("inspect bundle entry", err)
}
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return nil, unrecognizedBundleError("bundle entry is not a regular file", nil)
}
file, err := operations.open(filePath)
if err != nil {
return nil, unrecognizedBundleError("read bundle entry", err)
}
return file, nil
}
func decodeManifest(data []byte) (Manifest, error) {
if err := validateManifestJSON(data); err != nil {
return Manifest{}, unrecognizedBundleError("decode manifest", err)
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
var manifest Manifest
if err := decoder.Decode(&manifest); err != nil {
return Manifest{}, unrecognizedBundleError("decode manifest", err)
}
var trailing any
if err := decoder.Decode(&trailing); err != io.EOF {
if err == nil {
return Manifest{}, unrecognizedBundleError("manifest has multiple JSON values", nil)
}
return Manifest{}, unrecognizedBundleError("decode manifest", err)
}
return manifest, nil
}
var manifestFields = map[string]jsonValueValidator{
"schemaVersion": nil,
"comparisonId": nil,
"startedAt": nil,
"finishedAt": nil,
"reportId": nil,
"validPeriod": validateValidPeriodJSON,
"timezone": nil,
"promptId": nil,
"promptVersion": nil,
"promptHash": nil,
"dataPackage": validateDataPackageJSON,
"total": nil,
"succeeded": nil,
"failed": nil,
"results": validateResultsJSON,
}
var validPeriodFields = map[string]jsonValueValidator{
"start": nil,
"end": nil,
}
var dataPackageFields = map[string]jsonValueValidator{
"path": nil,
"sha256": nil,
}
var resultFields = map[string]jsonValueValidator{
"position": nil,
"profileId": nil,
"backendId": nil,
"modelName": nil,
"status": nil,
"validationStatus": nil,
"reportPath": nil,
"error": validateSafeErrorJSON,
}
var safeErrorFields = map[string]jsonValueValidator{
"category": nil,
"message": nil,
}
type jsonValueValidator func(*json.Decoder) error
func validateManifestJSON(data []byte) error {
decoder := json.NewDecoder(bytes.NewReader(data))
if err := validateJSONObject(decoder, manifestFields); err != nil {
return err
}
if _, err := decoder.Token(); err != io.EOF {
if err == nil {
return fmt.Errorf("manifest has multiple JSON values")
}
return err
}
return nil
}
func validateValidPeriodJSON(decoder *json.Decoder) error {
return validateJSONObject(decoder, validPeriodFields)
}
func validateDataPackageJSON(decoder *json.Decoder) error {
return validateJSONObject(decoder, dataPackageFields)
}
func validateResultsJSON(decoder *json.Decoder) error {
token, err := decoder.Token()
if err != nil {
return err
}
if delimiter, ok := token.(json.Delim); !ok || delimiter != '[' {
return fmt.Errorf("results must be an array")
}
for decoder.More() {
if err := validateJSONObject(decoder, resultFields); err != nil {
return err
}
}
token, err = decoder.Token()
if err != nil {
return err
}
if delimiter, ok := token.(json.Delim); !ok || delimiter != ']' {
return fmt.Errorf("results has an invalid array terminator")
}
return nil
}
func validateSafeErrorJSON(decoder *json.Decoder) error {
token, err := decoder.Token()
if err != nil {
return err
}
if token == nil {
return nil
}
if delimiter, ok := token.(json.Delim); !ok || delimiter != '{' {
return fmt.Errorf("error must be an object")
}
return validateJSONObjectBody(decoder, safeErrorFields)
}
func validateJSONObject(decoder *json.Decoder, fields map[string]jsonValueValidator) error {
token, err := decoder.Token()
if err != nil {
return err
}
if delimiter, ok := token.(json.Delim); !ok || delimiter != '{' {
return fmt.Errorf("manifest value must be an object")
}
return validateJSONObjectBody(decoder, fields)
}
func validateJSONObjectBody(decoder *json.Decoder, fields map[string]jsonValueValidator) error {
seen := make(map[string]struct{}, len(fields))
for decoder.More() {
token, err := decoder.Token()
if err != nil {
return err
}
name, ok := token.(string)
if !ok {
return fmt.Errorf("manifest field name is invalid")
}
validator, known := fields[name]
if !known {
return fmt.Errorf("manifest field %q is not canonical", name)
}
if _, duplicate := seen[name]; duplicate {
return fmt.Errorf("manifest field %q is duplicated", name)
}
seen[name] = struct{}{}
if validator == nil {
var value json.RawMessage
if err := decoder.Decode(&value); err != nil {
return err
}
continue
}
if err := validator(decoder); err != nil {
return err
}
}
token, err := decoder.Token()
if err != nil {
return err
}
if delimiter, ok := token.(json.Delim); !ok || delimiter != '}' {
return fmt.Errorf("manifest object has an invalid terminator")
}
return nil
}
func unrecognizedBundleError(action string, err error) error {
if err == nil {
return fmt.Errorf("%w: %s", ErrUnrecognizedBundle, action)
}
return fmt.Errorf("%w: %s: %v", ErrUnrecognizedBundle, action, err)
}
// BackupRecoveryState describes what remains of a prior bundle after cleanup
// reports an error.
type BackupRecoveryState string
const (
BackupRecoveryComplete BackupRecoveryState = "complete"
BackupRecoveryPartial BackupRecoveryState = "partial"
BackupRecoveryAbsent BackupRecoveryState = "absent"
BackupRecoveryUnknown BackupRecoveryState = "unknown"
)
// PublicationResult describes the durable state of a publication attempt.
type PublicationResult struct {
Committed bool
RecoveryState BackupRecoveryState
RecoveryPath string
}
// PublicationCleanupError reports that a committed bundle could not completely
// remove its prior sibling backup. The new bundle remains installed; recovery
// fields describe the state observed after cleanup failed.
type PublicationCleanupError struct {
RecoveryState BackupRecoveryState
RecoveryPath string
Err error
}
func (err *PublicationCleanupError) Error() string {
switch err.RecoveryState {
case BackupRecoveryComplete:
return fmt.Sprintf("remove comparison backup %q: %v; complete recovery bundle remains", err.RecoveryPath, err.Err)
case BackupRecoveryPartial:
return fmt.Sprintf("remove comparison backup %q: %v; partial remnants remain", err.RecoveryPath, err.Err)
case BackupRecoveryAbsent:
return fmt.Sprintf("remove comparison backup: %v; no recovery bundle remains", err.Err)
default:
return fmt.Sprintf("remove comparison backup: %v; recovery state is unknown", err.Err)
}
}
func (err *PublicationCleanupError) Unwrap() error {
return err.Err
}
// Publish writes a complete logical bundle through a private sibling directory
// and atomically installs it at a preflighted destination.
func Publish(ctx context.Context, plan DestinationPlan, bundle LogicalBundle) (PublicationResult, error) {
return publish(ctx, plan, bundle, publishOperations{rename: os.Rename, removeAll: os.RemoveAll})
}
type publishOperations struct {
rename func(string, string) error
removeAll func(string) error
beforeCommit func()
recognize func(string) (Manifest, error)
}
func publish(ctx context.Context, plan DestinationPlan, bundle LogicalBundle, operations publishOperations) (PublicationResult, error) {
if operations.rename == nil {
operations.rename = os.Rename
}
if operations.removeAll == nil {
operations.removeAll = os.RemoveAll
}
if operations.recognize == nil {
operations.recognize = RecognizeBundle
}
if err := bundle.Validate(); err != nil {
return PublicationResult{}, fmt.Errorf("validate comparison bundle: %w", err)
}
manifestData, err := EncodeManifest(bundle.Manifest)
if err != nil {
return PublicationResult{}, err
}
if err := ctx.Err(); err != nil {
return PublicationResult{}, err
}
plan, err = reauthorizeDestination(plan)
if err != nil {
return PublicationResult{}, err
}
if err := os.MkdirAll(filepath.Dir(plan.Target), 0o755); err != nil {
return PublicationResult{}, fmt.Errorf("create comparison destination parent %q: %w", filepath.Dir(plan.Target), err)
}
temporaryDirectory, err := os.MkdirTemp(filepath.Dir(plan.Target), stagingDirectoryPattern)
if err != nil {
return PublicationResult{}, fmt.Errorf("create comparison staging directory: %w", err)
}
if err := os.Chmod(temporaryDirectory, 0o700); err != nil {
os.RemoveAll(temporaryDirectory)
return PublicationResult{}, fmt.Errorf("secure comparison staging directory: %w", err)
}
defer func() {
if temporaryDirectory != "" {
_ = os.RemoveAll(temporaryDirectory)
}
}()
if err := writeLogicalBundle(ctx, temporaryDirectory, bundle, manifestData); err != nil {
return PublicationResult{}, err
}
currentPlan, err := reauthorizeDestination(plan)
if err != nil {
return PublicationResult{}, err
}
if operations.beforeCommit != nil {
operations.beforeCommit()
}
if err := ctx.Err(); err != nil {
return PublicationResult{}, err
}
if currentPlan.state == destinationAbsent {
if err := operations.rename(temporaryDirectory, currentPlan.Target); err != nil {
return PublicationResult{}, fmt.Errorf("publish comparison bundle to %q: %w", currentPlan.Target, err)
}
temporaryDirectory = ""
return PublicationResult{Committed: true}, nil
}
backupDirectory, err := uniqueSiblingPath(filepath.Dir(currentPlan.Target), backupDirectoryPattern)
if err != nil {
return PublicationResult{}, err
}
if err := operations.rename(currentPlan.Target, backupDirectory); err != nil {
return PublicationResult{}, fmt.Errorf("back up comparison destination %q: %w", currentPlan.Target, err)
}
if err := authorizeMovedDestination(currentPlan, backupDirectory, operations.recognize); err != nil {
return PublicationResult{}, restoreMovedDestination(operations, backupDirectory, currentPlan.Target, err)
}
if err := ctx.Err(); err != nil {
return PublicationResult{}, restoreMovedDestination(operations, backupDirectory, currentPlan.Target, err)
}
if err := operations.rename(temporaryDirectory, currentPlan.Target); err != nil {
return PublicationResult{}, restoreMovedDestination(operations, backupDirectory, currentPlan.Target, fmt.Errorf("replace comparison destination %q: %w", currentPlan.Target, err))
}
temporaryDirectory = ""
if err := operations.removeAll(backupDirectory); err != nil {
recoveryState, recoveryPath := inspectBackupRecovery(backupDirectory)
cleanupErr := &PublicationCleanupError{RecoveryState: recoveryState, RecoveryPath: recoveryPath, Err: err}
return PublicationResult{Committed: true, RecoveryState: recoveryState, RecoveryPath: recoveryPath}, cleanupErr
}
return PublicationResult{Committed: true}, nil
}
func inspectBackupRecovery(backupDirectory string) (BackupRecoveryState, string) {
if _, err := os.Lstat(backupDirectory); err != nil {
if errors.Is(err, os.ErrNotExist) {
return BackupRecoveryAbsent, ""
}
return BackupRecoveryUnknown, ""
}
if _, err := RecognizeBundle(backupDirectory); err == nil {
return BackupRecoveryComplete, backupDirectory
}
return BackupRecoveryPartial, backupDirectory
}
func authorizeMovedDestination(plan DestinationPlan, backupDirectory string, recognize func(string) (Manifest, error)) error {
backupPlan, err := planDestination(plan.WorkingDirectory, backupDirectory, plan.Replace, recognize)
if err != nil {
return fmt.Errorf("authorize moved comparison destination %q: %w", backupDirectory, err)
}
if backupPlan.state != destinationEmpty && backupPlan.state != destinationBundle {
return fmt.Errorf("authorize moved comparison destination %q: destination disappeared", backupDirectory)
}
return nil
}
func restoreMovedDestination(operations publishOperations, backupDirectory, target string, cause error) error {
if _, err := os.Lstat(target); err == nil {
return errors.Join(
cause,
fmt.Errorf("restore prior comparison destination from %q: destination %q reappeared", backupDirectory, target),
)
} else if !errors.Is(err, os.ErrNotExist) {
return errors.Join(
cause,
fmt.Errorf("inspect comparison destination %q before restoring from %q: %w", target, backupDirectory, err),
)
}
if restoreErr := operations.rename(backupDirectory, target); restoreErr != nil {
return errors.Join(
cause,
fmt.Errorf("restore prior comparison destination from %q: %w", backupDirectory, restoreErr),
)
}
return cause
}
func writeLogicalBundle(ctx context.Context, directory string, bundle LogicalBundle, manifestData []byte) error {
if err := writeBundleFile(ctx, filepath.Join(directory, DataPackageFilename), bundle.DataPackage); err != nil {
return err
}
for _, report := range bundle.Reports {
if err := writeBundleFile(ctx, filepath.Join(directory, report.Path), report.Markdown); err != nil {
return err
}
}
if err := writeBundleFile(ctx, filepath.Join(directory, ManifestFilename), manifestData); err != nil {
return err
}
return nil
}
func writeBundleFile(ctx context.Context, filePath string, content []byte) error {
if err := ctx.Err(); err != nil {
return err
}
file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return fmt.Errorf("create comparison bundle file %q: %w", filePath, err)
}
if _, err := file.Write(content); err != nil {
file.Close()
return fmt.Errorf("write comparison bundle file %q: %w", filePath, err)
}
if err := file.Close(); err != nil {
return fmt.Errorf("close comparison bundle file %q: %w", filePath, err)
}
return nil
}
func uniqueSiblingPath(parent, prefix string) (string, error) {
file, err := os.CreateTemp(parent, prefix)
if err != nil {
return "", fmt.Errorf("reserve comparison backup path: %w", err)
}
path := file.Name()
if err := file.Close(); err != nil {
os.Remove(path)
return "", fmt.Errorf("close comparison backup reservation: %w", err)
}
if err := os.Remove(path); err != nil {
return "", fmt.Errorf("release comparison backup reservation: %w", err)
}
return path, nil
}