396 lines
13 KiB
Go
396 lines
13 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"sort"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
|
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
|
)
|
|
|
|
const reconcileStateWalkLimit = 10000
|
|
|
|
type ReconcileStateOptions struct {
|
|
ConfigPath string
|
|
PipelineID string
|
|
DestinationID string
|
|
AllOwners bool
|
|
DryRun bool
|
|
Stdout io.Writer
|
|
OutputFormat OutputFormat
|
|
}
|
|
|
|
type ReconcileStateReport struct {
|
|
PipelineID string `json:"pipeline_id"`
|
|
DestinationID string `json:"destination_id"`
|
|
Backend string `json:"backend"`
|
|
RootPath string `json:"root_path"`
|
|
StateSchema int `json:"state_schema"`
|
|
OwnerScope *ReconcileStateOwnerScope `json:"owner_scope,omitempty"`
|
|
CheckedCount int `json:"checked_count"`
|
|
MissingManagedOutputs []ReconcileStatePath `json:"missing_managed_outputs"`
|
|
UnmanagedEntries []ReconcileStateEntry `json:"unmanaged_entries"`
|
|
Changed bool `json:"changed"`
|
|
WouldChange bool `json:"would_change"`
|
|
DryRun bool `json:"dry_run"`
|
|
}
|
|
|
|
type ReconcileStateOwnerScope struct {
|
|
PipelineID string `json:"pipeline_id"`
|
|
DestinationID string `json:"destination_id"`
|
|
AllOwners bool `json:"all_owners,omitempty"`
|
|
}
|
|
|
|
type ReconcileStatePath struct {
|
|
Path string `json:"path"`
|
|
OwnerScope *ReconcileStateOwnerScope `json:"owner_scope,omitempty"`
|
|
StorageStatus string `json:"storage_status"`
|
|
}
|
|
|
|
type ReconcileStateEntry struct {
|
|
Path string `json:"path"`
|
|
Type string `json:"type"`
|
|
Size int64 `json:"size,omitempty"`
|
|
}
|
|
|
|
func ReconcileState(ctx context.Context, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
|
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
|
|
setup, err := loadRuntimeSetup(options.ConfigPath)
|
|
if err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
return reconcileStateSetup(ctx, setup, options)
|
|
}
|
|
|
|
func reconcileStateConfigWithBackendFactory(ctx context.Context, cfg config.Config, options ReconcileStateOptions, provider backendFactoryProvider) (ReconcileStateReport, error) {
|
|
setup, err := runtimeSetupFromConfig("", cfg)
|
|
if err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
return reconcileStateSetupWithBackendFactory(ctx, setup, options, provider)
|
|
}
|
|
|
|
func reconcileStateSetup(ctx context.Context, setup runtimeSetup, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
|
return reconcileStateSetupWithBackendFactory(ctx, setup, options, newBackendFactoryWithEnvironment)
|
|
}
|
|
|
|
func reconcileStateSetupWithBackendFactory(ctx context.Context, setup runtimeSetup, options ReconcileStateOptions, provider backendFactoryProvider) (ReconcileStateReport, error) {
|
|
if err := requireReconcileStateScope(options); err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
pipeline, ok := findPipeline(setup.Config, options.PipelineID)
|
|
if !ok {
|
|
return ReconcileStateReport{}, PipelineNotFoundError{ID: options.PipelineID}
|
|
}
|
|
destination, ok := findDestination(pipeline, options.DestinationID)
|
|
if !ok {
|
|
return ReconcileStateReport{}, fmt.Errorf("pipeline %s destination %s not found", options.PipelineID, options.DestinationID)
|
|
}
|
|
|
|
backends := provider(setup.Environment)
|
|
destinationBackend, err := backends.openDestination(ctx, destination)
|
|
if err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
defer closeBackend(destinationBackend)
|
|
|
|
report, err := buildReconcileStateReport(ctx, destinationBackend, pipeline, destination, options)
|
|
if err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
if err := WriteReconcileStateReport(options.Stdout, options.OutputFormat, report); err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
return report, nil
|
|
}
|
|
|
|
func requireReconcileStateScope(options ReconcileStateOptions) error {
|
|
if options.PipelineID == "" {
|
|
return fmt.Errorf("pipeline id is required")
|
|
}
|
|
if options.DestinationID == "" {
|
|
return fmt.Errorf("destination id is required")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func findDestination(pipeline config.Pipeline, id string) (config.Destination, bool) {
|
|
for _, destination := range pipeline.Destinations {
|
|
if destination.ID == id {
|
|
return destination, true
|
|
}
|
|
}
|
|
return config.Destination{}, false
|
|
}
|
|
|
|
func buildReconcileStateReport(ctx context.Context, backend storage.Backend, pipeline config.Pipeline, destination config.Destination, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
|
statePath, err := storage.StatePath("")
|
|
if err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
data, err := backend.ReadFile(ctx, statePath)
|
|
if err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
document, err := state.ParseDocument(data)
|
|
if err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
|
|
report := ReconcileStateReport{
|
|
PipelineID: pipeline.ID,
|
|
DestinationID: destination.ID,
|
|
Backend: destination.Backend,
|
|
RootPath: destinationRootPath(destination),
|
|
MissingManagedOutputs: []ReconcileStatePath{},
|
|
UnmanagedEntries: []ReconcileStateEntry{},
|
|
DryRun: options.DryRun,
|
|
}
|
|
scope := state.CurrentOwnerScope(pipeline.ID, destination.ID)
|
|
if document.SingleOwner != nil {
|
|
return reconcileSingleOwnerState(ctx, backend, statePath, *document.SingleOwner, scope, report, options)
|
|
}
|
|
return reconcileSharedRootState(ctx, backend, statePath, *document.SharedRoot, scope, report, options)
|
|
}
|
|
|
|
func reconcileSingleOwnerState(ctx context.Context, backend storage.Backend, statePath string, destinationState state.DistributorState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
|
if destinationState.PipelineID != scope.PipelineID || destinationState.DestinationID != scope.DestinationID {
|
|
return ReconcileStateReport{}, fmt.Errorf("state owner is %s/%s, not %s/%s", destinationState.PipelineID, destinationState.DestinationID, scope.PipelineID, scope.DestinationID)
|
|
}
|
|
report.StateSchema = destinationState.SchemaVersion
|
|
report.OwnerScope = &ReconcileStateOwnerScope{PipelineID: scope.PipelineID, DestinationID: scope.DestinationID}
|
|
managed := state.ManagedOutputPaths(destinationState)
|
|
missing, err := missingSingleOwnerOutputs(ctx, backend, destinationState.Outputs)
|
|
if err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
report.CheckedCount = len(managed)
|
|
report.MissingManagedOutputs = missing
|
|
unmanaged, err := unmanagedEntries(ctx, backend, managed)
|
|
if err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
report.UnmanagedEntries = unmanaged
|
|
report.WouldChange = options.DryRun && len(missing) > 0
|
|
|
|
if !options.DryRun && len(missing) > 0 {
|
|
missingPaths := missingReportPaths(missing)
|
|
next, changed := state.RemoveMissingOutputs(destinationState, missingPaths)
|
|
report.Changed = changed
|
|
if changed {
|
|
next.UpdatedAt = time.Now().UTC()
|
|
if err := state.Validate(next); err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
}
|
|
}
|
|
return report, nil
|
|
}
|
|
|
|
func reconcileSharedRootState(ctx context.Context, backend storage.Backend, statePath string, sharedRoot state.SharedRootState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
|
report.StateSchema = sharedRoot.SchemaVersion
|
|
report.OwnerScope = &ReconcileStateOwnerScope{
|
|
PipelineID: scope.PipelineID,
|
|
DestinationID: scope.DestinationID,
|
|
AllOwners: options.AllOwners,
|
|
}
|
|
managed := sharedRoot.AllManagedOutputPaths()
|
|
outputs := sharedRoot.Outputs
|
|
if !options.AllOwners {
|
|
outputs = sharedRootOutputsForOwner(sharedRoot.Outputs, scope)
|
|
}
|
|
missing, err := missingSharedRootOutputs(ctx, backend, outputs)
|
|
if err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
report.CheckedCount = len(outputs)
|
|
report.MissingManagedOutputs = missing
|
|
unmanaged, err := unmanagedEntries(ctx, backend, managed)
|
|
if err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
report.UnmanagedEntries = unmanaged
|
|
report.WouldChange = options.DryRun && len(missing) > 0
|
|
|
|
if !options.DryRun && len(missing) > 0 {
|
|
missingPaths := missingReportPaths(missing)
|
|
var next state.SharedRootState
|
|
var changed bool
|
|
if options.AllOwners {
|
|
next, changed = state.RemoveMissingSharedRootOutputs(sharedRoot, missingPaths)
|
|
} else {
|
|
next, changed = state.RemoveMissingSharedRootOwnerOutputs(sharedRoot, scope, missingPaths)
|
|
}
|
|
report.Changed = changed
|
|
if changed {
|
|
next.UpdatedAt = time.Now().UTC()
|
|
if err := state.ValidateSharedRoot(next); err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
|
return ReconcileStateReport{}, err
|
|
}
|
|
}
|
|
}
|
|
return report, nil
|
|
}
|
|
|
|
func missingSingleOwnerOutputs(ctx context.Context, backend storage.Backend, outputs []state.OutputFile) ([]ReconcileStatePath, error) {
|
|
missing := make([]ReconcileStatePath, 0)
|
|
for _, output := range outputs {
|
|
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
|
if storage.IsNotFound(err) {
|
|
missing = append(missing, ReconcileStatePath{Path: output.Path, StorageStatus: "missing"})
|
|
continue
|
|
}
|
|
return nil, err
|
|
}
|
|
}
|
|
return missing, nil
|
|
}
|
|
|
|
func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outputs []state.SharedRootOutputFile) ([]ReconcileStatePath, error) {
|
|
missing := make([]ReconcileStatePath, 0)
|
|
for _, output := range outputs {
|
|
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
|
if storage.IsNotFound(err) {
|
|
missing = append(missing, ReconcileStatePath{
|
|
Path: output.Path,
|
|
OwnerScope: &ReconcileStateOwnerScope{
|
|
PipelineID: output.Owner.PipelineID,
|
|
DestinationID: output.Owner.DestinationID,
|
|
},
|
|
StorageStatus: "missing",
|
|
})
|
|
continue
|
|
}
|
|
return nil, err
|
|
}
|
|
}
|
|
return missing, nil
|
|
}
|
|
|
|
func checkManagedOutput(ctx context.Context, backend storage.Backend, path string) error {
|
|
_, err := backend.Stat(ctx, path)
|
|
return err
|
|
}
|
|
|
|
func unmanagedEntries(ctx context.Context, backend storage.Backend, managedPaths []string) ([]ReconcileStateEntry, error) {
|
|
managed := make(map[string]struct{}, len(managedPaths)+1)
|
|
for _, path := range managedPaths {
|
|
managed[path] = struct{}{}
|
|
}
|
|
managed[storage.StateFileName] = struct{}{}
|
|
|
|
entries := make([]ReconcileStateEntry, 0)
|
|
err := backend.Walk(ctx, "", storage.WalkOptions{Recursive: true, Limit: reconcileStateWalkLimit}, func(entry storage.Entry) error {
|
|
if entry.Type == storage.EntryTypeDirectory {
|
|
return nil
|
|
}
|
|
if _, ok := managed[entry.Path]; ok {
|
|
return nil
|
|
}
|
|
entries = append(entries, ReconcileStateEntry{
|
|
Path: entry.Path,
|
|
Type: string(entry.Type),
|
|
Size: entry.Size,
|
|
})
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sort.Slice(entries, func(i, j int) bool {
|
|
return entries[i].Path < entries[j].Path
|
|
})
|
|
return entries, nil
|
|
}
|
|
|
|
func sharedRootOutputsForOwner(outputs []state.SharedRootOutputFile, scope state.OwnerScope) []state.SharedRootOutputFile {
|
|
selected := make([]state.SharedRootOutputFile, 0, len(outputs))
|
|
for _, output := range outputs {
|
|
if output.Owner == scope {
|
|
selected = append(selected, output)
|
|
}
|
|
}
|
|
return selected
|
|
}
|
|
|
|
func missingReportPaths(missing []ReconcileStatePath) []string {
|
|
paths := make([]string, 0, len(missing))
|
|
for _, item := range missing {
|
|
paths = append(paths, item.Path)
|
|
}
|
|
return paths
|
|
}
|
|
|
|
func writeRepairedState(ctx context.Context, backend storage.Backend, path string, value any) error {
|
|
data, err := json.MarshalIndent(value, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
data = append(data, '\n')
|
|
_, err = backend.WriteFile(ctx, path, data, storage.WriteOptions{Overwrite: true, PreferAtomic: true})
|
|
return err
|
|
}
|
|
|
|
func destinationRootPath(destination config.Destination) string {
|
|
switch destination.Backend {
|
|
case config.BackendS3:
|
|
if destination.Prefix == "" {
|
|
return "."
|
|
}
|
|
return destination.Prefix
|
|
default:
|
|
if destination.Path == "" {
|
|
return "."
|
|
}
|
|
return destination.Path
|
|
}
|
|
}
|
|
|
|
func WriteReconcileStateReport(w io.Writer, format OutputFormat, report ReconcileStateReport) error {
|
|
if IsJSONOutput(format) {
|
|
return WriteJSONEnvelope(w, "reconcile-state", true, nil, report, nil)
|
|
}
|
|
return writeReconcileStateReportText(w, report)
|
|
}
|
|
|
|
func writeReconcileStateReportText(w io.Writer, report ReconcileStateReport) error {
|
|
if w == nil {
|
|
return nil
|
|
}
|
|
status := "unchanged"
|
|
if report.Changed {
|
|
status = "changed"
|
|
} else if report.WouldChange {
|
|
status = "would_change"
|
|
}
|
|
_, err := fmt.Fprintf(w, "Reconcile state: pipeline=%s destination=%s backend=%s root=%s status=%s checked=%d missing=%d unmanaged=%d dry_run=%t\n",
|
|
report.PipelineID,
|
|
report.DestinationID,
|
|
report.Backend,
|
|
report.RootPath,
|
|
status,
|
|
report.CheckedCount,
|
|
len(report.MissingManagedOutputs),
|
|
len(report.UnmanagedEntries),
|
|
report.DryRun,
|
|
)
|
|
return err
|
|
}
|