Add reconcile state planning core
This commit is contained in:
395
internal/app/reconcile_state.go
Normal file
395
internal/app/reconcile_state.go
Normal file
@@ -0,0 +1,395 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/config"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/state"
|
||||
"gitea.maximumdirect.net/eric/distributor/internal/storage"
|
||||
)
|
||||
|
||||
const reconcileStateWalkLimit = 10000
|
||||
|
||||
type ReconcileStateOptions struct {
|
||||
ConfigPath string
|
||||
PipelineID string
|
||||
DestinationID string
|
||||
AllOwners bool
|
||||
DryRun bool
|
||||
Stdout io.Writer
|
||||
OutputFormat OutputFormat
|
||||
}
|
||||
|
||||
type ReconcileStateReport struct {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
Backend string `json:"backend"`
|
||||
RootPath string `json:"root_path"`
|
||||
StateSchema int `json:"state_schema"`
|
||||
OwnerScope *ReconcileStateOwnerScope `json:"owner_scope,omitempty"`
|
||||
CheckedCount int `json:"checked_count"`
|
||||
MissingManagedOutputs []ReconcileStatePath `json:"missing_managed_outputs"`
|
||||
UnmanagedEntries []ReconcileStateEntry `json:"unmanaged_entries"`
|
||||
Changed bool `json:"changed"`
|
||||
WouldChange bool `json:"would_change"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
}
|
||||
|
||||
type ReconcileStateOwnerScope struct {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
DestinationID string `json:"destination_id"`
|
||||
AllOwners bool `json:"all_owners,omitempty"`
|
||||
}
|
||||
|
||||
type ReconcileStatePath struct {
|
||||
Path string `json:"path"`
|
||||
OwnerScope *ReconcileStateOwnerScope `json:"owner_scope,omitempty"`
|
||||
StorageStatus string `json:"storage_status"`
|
||||
}
|
||||
|
||||
type ReconcileStateEntry struct {
|
||||
Path string `json:"path"`
|
||||
Type string `json:"type"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
}
|
||||
|
||||
func ReconcileState(ctx context.Context, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
if err := ValidateOutputFormat(options.OutputFormat); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
|
||||
setup, err := loadRuntimeSetup(options.ConfigPath)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
return reconcileStateSetup(ctx, setup, options)
|
||||
}
|
||||
|
||||
func reconcileStateConfigWithBackendFactory(ctx context.Context, cfg config.Config, options ReconcileStateOptions, provider backendFactoryProvider) (ReconcileStateReport, error) {
|
||||
setup, err := runtimeSetupFromConfig("", cfg)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
return reconcileStateSetupWithBackendFactory(ctx, setup, options, provider)
|
||||
}
|
||||
|
||||
func reconcileStateSetup(ctx context.Context, setup runtimeSetup, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
return reconcileStateSetupWithBackendFactory(ctx, setup, options, newBackendFactoryWithEnvironment)
|
||||
}
|
||||
|
||||
func reconcileStateSetupWithBackendFactory(ctx context.Context, setup runtimeSetup, options ReconcileStateOptions, provider backendFactoryProvider) (ReconcileStateReport, error) {
|
||||
if err := requireReconcileStateScope(options); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
pipeline, ok := findPipeline(setup.Config, options.PipelineID)
|
||||
if !ok {
|
||||
return ReconcileStateReport{}, PipelineNotFoundError{ID: options.PipelineID}
|
||||
}
|
||||
destination, ok := findDestination(pipeline, options.DestinationID)
|
||||
if !ok {
|
||||
return ReconcileStateReport{}, fmt.Errorf("pipeline %s destination %s not found", options.PipelineID, options.DestinationID)
|
||||
}
|
||||
|
||||
backends := provider(setup.Environment)
|
||||
destinationBackend, err := backends.openDestination(ctx, destination)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
defer closeBackend(destinationBackend)
|
||||
|
||||
report, err := buildReconcileStateReport(ctx, destinationBackend, pipeline, destination, options)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := WriteReconcileStateReport(options.Stdout, options.OutputFormat, report); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func requireReconcileStateScope(options ReconcileStateOptions) error {
|
||||
if options.PipelineID == "" {
|
||||
return fmt.Errorf("pipeline id is required")
|
||||
}
|
||||
if options.DestinationID == "" {
|
||||
return fmt.Errorf("destination id is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func findDestination(pipeline config.Pipeline, id string) (config.Destination, bool) {
|
||||
for _, destination := range pipeline.Destinations {
|
||||
if destination.ID == id {
|
||||
return destination, true
|
||||
}
|
||||
}
|
||||
return config.Destination{}, false
|
||||
}
|
||||
|
||||
func buildReconcileStateReport(ctx context.Context, backend storage.Backend, pipeline config.Pipeline, destination config.Destination, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
statePath, err := storage.StatePath("")
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
data, err := backend.ReadFile(ctx, statePath)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
document, err := state.ParseDocument(data)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
|
||||
report := ReconcileStateReport{
|
||||
PipelineID: pipeline.ID,
|
||||
DestinationID: destination.ID,
|
||||
Backend: destination.Backend,
|
||||
RootPath: destinationRootPath(destination),
|
||||
MissingManagedOutputs: []ReconcileStatePath{},
|
||||
UnmanagedEntries: []ReconcileStateEntry{},
|
||||
DryRun: options.DryRun,
|
||||
}
|
||||
scope := state.CurrentOwnerScope(pipeline.ID, destination.ID)
|
||||
if document.SingleOwner != nil {
|
||||
return reconcileSingleOwnerState(ctx, backend, statePath, *document.SingleOwner, scope, report, options)
|
||||
}
|
||||
return reconcileSharedRootState(ctx, backend, statePath, *document.SharedRoot, scope, report, options)
|
||||
}
|
||||
|
||||
func reconcileSingleOwnerState(ctx context.Context, backend storage.Backend, statePath string, destinationState state.DistributorState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
if destinationState.PipelineID != scope.PipelineID || destinationState.DestinationID != scope.DestinationID {
|
||||
return ReconcileStateReport{}, fmt.Errorf("state owner is %s/%s, not %s/%s", destinationState.PipelineID, destinationState.DestinationID, scope.PipelineID, scope.DestinationID)
|
||||
}
|
||||
report.StateSchema = destinationState.SchemaVersion
|
||||
report.OwnerScope = &ReconcileStateOwnerScope{PipelineID: scope.PipelineID, DestinationID: scope.DestinationID}
|
||||
managed := state.ManagedOutputPaths(destinationState)
|
||||
missing, err := missingSingleOwnerOutputs(ctx, backend, destinationState.Outputs)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.CheckedCount = len(managed)
|
||||
report.MissingManagedOutputs = missing
|
||||
unmanaged, err := unmanagedEntries(ctx, backend, managed)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.UnmanagedEntries = unmanaged
|
||||
report.WouldChange = options.DryRun && len(missing) > 0
|
||||
|
||||
if !options.DryRun && len(missing) > 0 {
|
||||
missingPaths := missingReportPaths(missing)
|
||||
next, changed := state.RemoveMissingOutputs(destinationState, missingPaths)
|
||||
report.Changed = changed
|
||||
if changed {
|
||||
next.UpdatedAt = time.Now().UTC()
|
||||
if err := state.Validate(next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func reconcileSharedRootState(ctx context.Context, backend storage.Backend, statePath string, sharedRoot state.SharedRootState, scope state.OwnerScope, report ReconcileStateReport, options ReconcileStateOptions) (ReconcileStateReport, error) {
|
||||
report.StateSchema = sharedRoot.SchemaVersion
|
||||
report.OwnerScope = &ReconcileStateOwnerScope{
|
||||
PipelineID: scope.PipelineID,
|
||||
DestinationID: scope.DestinationID,
|
||||
AllOwners: options.AllOwners,
|
||||
}
|
||||
managed := sharedRoot.AllManagedOutputPaths()
|
||||
outputs := sharedRoot.Outputs
|
||||
if !options.AllOwners {
|
||||
outputs = sharedRootOutputsForOwner(sharedRoot.Outputs, scope)
|
||||
}
|
||||
missing, err := missingSharedRootOutputs(ctx, backend, outputs)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.CheckedCount = len(outputs)
|
||||
report.MissingManagedOutputs = missing
|
||||
unmanaged, err := unmanagedEntries(ctx, backend, managed)
|
||||
if err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
report.UnmanagedEntries = unmanaged
|
||||
report.WouldChange = options.DryRun && len(missing) > 0
|
||||
|
||||
if !options.DryRun && len(missing) > 0 {
|
||||
missingPaths := missingReportPaths(missing)
|
||||
var next state.SharedRootState
|
||||
var changed bool
|
||||
if options.AllOwners {
|
||||
next, changed = state.RemoveMissingSharedRootOutputs(sharedRoot, missingPaths)
|
||||
} else {
|
||||
next, changed = state.RemoveMissingSharedRootOwnerOutputs(sharedRoot, scope, missingPaths)
|
||||
}
|
||||
report.Changed = changed
|
||||
if changed {
|
||||
next.UpdatedAt = time.Now().UTC()
|
||||
if err := state.ValidateSharedRoot(next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
if err := writeRepairedState(ctx, backend, statePath, next); err != nil {
|
||||
return ReconcileStateReport{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func missingSingleOwnerOutputs(ctx context.Context, backend storage.Backend, outputs []state.OutputFile) ([]ReconcileStatePath, error) {
|
||||
missing := make([]ReconcileStatePath, 0)
|
||||
for _, output := range outputs {
|
||||
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
||||
if storage.IsNotFound(err) {
|
||||
missing = append(missing, ReconcileStatePath{Path: output.Path, StorageStatus: "missing"})
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
func missingSharedRootOutputs(ctx context.Context, backend storage.Backend, outputs []state.SharedRootOutputFile) ([]ReconcileStatePath, error) {
|
||||
missing := make([]ReconcileStatePath, 0)
|
||||
for _, output := range outputs {
|
||||
if err := checkManagedOutput(ctx, backend, output.Path); err != nil {
|
||||
if storage.IsNotFound(err) {
|
||||
missing = append(missing, ReconcileStatePath{
|
||||
Path: output.Path,
|
||||
OwnerScope: &ReconcileStateOwnerScope{
|
||||
PipelineID: output.Owner.PipelineID,
|
||||
DestinationID: output.Owner.DestinationID,
|
||||
},
|
||||
StorageStatus: "missing",
|
||||
})
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
func checkManagedOutput(ctx context.Context, backend storage.Backend, path string) error {
|
||||
_, err := backend.Stat(ctx, path)
|
||||
return err
|
||||
}
|
||||
|
||||
func unmanagedEntries(ctx context.Context, backend storage.Backend, managedPaths []string) ([]ReconcileStateEntry, error) {
|
||||
managed := make(map[string]struct{}, len(managedPaths)+1)
|
||||
for _, path := range managedPaths {
|
||||
managed[path] = struct{}{}
|
||||
}
|
||||
managed[storage.StateFileName] = struct{}{}
|
||||
|
||||
entries := make([]ReconcileStateEntry, 0)
|
||||
err := backend.Walk(ctx, "", storage.WalkOptions{Recursive: true, Limit: reconcileStateWalkLimit}, func(entry storage.Entry) error {
|
||||
if entry.Type == storage.EntryTypeDirectory {
|
||||
return nil
|
||||
}
|
||||
if _, ok := managed[entry.Path]; ok {
|
||||
return nil
|
||||
}
|
||||
entries = append(entries, ReconcileStateEntry{
|
||||
Path: entry.Path,
|
||||
Type: string(entry.Type),
|
||||
Size: entry.Size,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
return entries[i].Path < entries[j].Path
|
||||
})
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func sharedRootOutputsForOwner(outputs []state.SharedRootOutputFile, scope state.OwnerScope) []state.SharedRootOutputFile {
|
||||
selected := make([]state.SharedRootOutputFile, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
if output.Owner == scope {
|
||||
selected = append(selected, output)
|
||||
}
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
func missingReportPaths(missing []ReconcileStatePath) []string {
|
||||
paths := make([]string, 0, len(missing))
|
||||
for _, item := range missing {
|
||||
paths = append(paths, item.Path)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func writeRepairedState(ctx context.Context, backend storage.Backend, path string, value any) error {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
_, err = backend.WriteFile(ctx, path, data, storage.WriteOptions{Overwrite: true, PreferAtomic: true})
|
||||
return err
|
||||
}
|
||||
|
||||
func destinationRootPath(destination config.Destination) string {
|
||||
switch destination.Backend {
|
||||
case config.BackendS3:
|
||||
if destination.Prefix == "" {
|
||||
return "."
|
||||
}
|
||||
return destination.Prefix
|
||||
default:
|
||||
if destination.Path == "" {
|
||||
return "."
|
||||
}
|
||||
return destination.Path
|
||||
}
|
||||
}
|
||||
|
||||
func WriteReconcileStateReport(w io.Writer, format OutputFormat, report ReconcileStateReport) error {
|
||||
if IsJSONOutput(format) {
|
||||
return WriteJSONEnvelope(w, "reconcile-state", true, nil, report, nil)
|
||||
}
|
||||
return writeReconcileStateReportText(w, report)
|
||||
}
|
||||
|
||||
func writeReconcileStateReportText(w io.Writer, report ReconcileStateReport) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
status := "unchanged"
|
||||
if report.Changed {
|
||||
status = "changed"
|
||||
} else if report.WouldChange {
|
||||
status = "would_change"
|
||||
}
|
||||
_, err := fmt.Fprintf(w, "Reconcile state: pipeline=%s destination=%s backend=%s root=%s status=%s checked=%d missing=%d unmanaged=%d dry_run=%t\n",
|
||||
report.PipelineID,
|
||||
report.DestinationID,
|
||||
report.Backend,
|
||||
report.RootPath,
|
||||
status,
|
||||
report.CheckedCount,
|
||||
len(report.MissingManagedOutputs),
|
||||
len(report.UnmanagedEntries),
|
||||
report.DryRun,
|
||||
)
|
||||
return err
|
||||
}
|
||||
Reference in New Issue
Block a user