422 lines
12 KiB
Go
422 lines
12 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/previouscache"
|
|
)
|
|
|
|
// RestoreActionKind identifies one restore planner action.
|
|
type RestoreActionKind string
|
|
|
|
const (
|
|
RestoreActionDownload RestoreActionKind = "download"
|
|
RestoreActionSkipSame RestoreActionKind = "skip_same"
|
|
RestoreActionConflict RestoreActionKind = "conflict"
|
|
)
|
|
|
|
// RestoreAction is one deterministic planner action.
|
|
type RestoreAction struct {
|
|
Kind RestoreActionKind
|
|
RemoteKey string
|
|
LocalRelativePath string
|
|
LocalPath string
|
|
Size int64
|
|
ETag string
|
|
ExistsLocal bool
|
|
SameLocal bool
|
|
Conflict bool
|
|
Reason string
|
|
}
|
|
|
|
// RestorePlan is the deterministic output of restore planning.
|
|
type RestorePlan struct {
|
|
Actions []RestoreAction
|
|
DownloadCount int
|
|
SkipSameCount int
|
|
ConflictCount int
|
|
}
|
|
|
|
// RestorePlanOptions control restore planning scope and classification.
|
|
type RestorePlanOptions struct {
|
|
IncludeAudio bool
|
|
Force bool
|
|
DryRun bool
|
|
}
|
|
|
|
func buildRestorePlan(ctx context.Context, cfg *config.Config, current *RemoteCurrentState, store storage.ObjectStore, opts RestorePlanOptions) (*RestorePlan, error) {
|
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
|
return nil, fmt.Errorf("resolved config with pipeline/session is required")
|
|
}
|
|
if current == nil {
|
|
return nil, fmt.Errorf("remote current state is required")
|
|
}
|
|
if store == nil {
|
|
return nil, fmt.Errorf("remote object store is required")
|
|
}
|
|
prefix := normalizeRemoteKey(current.SessionPrefix)
|
|
if strings.TrimSpace(prefix) == "" {
|
|
return nil, fmt.Errorf("remote session prefix is required")
|
|
}
|
|
if !strings.HasSuffix(prefix, "/") {
|
|
prefix += "/"
|
|
}
|
|
|
|
sessionPaths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
|
|
objects, err := store.List(ctx, prefix)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list remote session objects under %q: %w", prefix, err)
|
|
}
|
|
|
|
candidates := make(map[string]storage.ObjectInfo, len(objects)+1)
|
|
for _, obj := range objects {
|
|
key := normalizeRemoteKey(obj.Key)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
obj.Key = key
|
|
candidates[key] = obj
|
|
}
|
|
if strings.TrimSpace(current.CurrentManifestKey) != "" {
|
|
key := normalizeRemoteKey(current.CurrentManifestKey)
|
|
if _, ok := candidates[key]; !ok {
|
|
candidates[key] = storage.ObjectInfo{Key: key}
|
|
}
|
|
}
|
|
|
|
actions := make([]RestoreAction, 0, len(candidates))
|
|
for key, obj := range candidates {
|
|
rel, include, err := restoreLocalRelativePathForKey(prefix, normalizeRemoteKey(current.CurrentManifestKey), key, opts.IncludeAudio)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("map remote key %q: %w", key, err)
|
|
}
|
|
if !include {
|
|
continue
|
|
}
|
|
localPath, err := joinWithinSessionRoot(sessionPaths.Root, rel)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("map remote key %q: %w", key, err)
|
|
}
|
|
|
|
action, err := classifyRestoreAction(ctx, store, obj, rel, localPath, opts.Force)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("classify remote key %q: %w", key, err)
|
|
}
|
|
actions = append(actions, action)
|
|
}
|
|
|
|
previousActions, err := buildPreviousCacheRestoreActions(ctx, cfg, sessionPaths, store, opts.Force)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
actions = append(actions, previousActions...)
|
|
|
|
sort.Slice(actions, func(i, j int) bool {
|
|
if actions[i].LocalRelativePath == actions[j].LocalRelativePath {
|
|
return actions[i].RemoteKey < actions[j].RemoteKey
|
|
}
|
|
return actions[i].LocalRelativePath < actions[j].LocalRelativePath
|
|
})
|
|
|
|
plan := &RestorePlan{Actions: actions}
|
|
for _, action := range actions {
|
|
switch action.Kind {
|
|
case RestoreActionDownload:
|
|
plan.DownloadCount++
|
|
case RestoreActionSkipSame:
|
|
plan.SkipSameCount++
|
|
case RestoreActionConflict:
|
|
plan.ConflictCount++
|
|
}
|
|
}
|
|
|
|
_ = opts.DryRun
|
|
return plan, nil
|
|
}
|
|
|
|
func normalizeRemoteKey(v string) string {
|
|
return strings.Trim(strings.ReplaceAll(strings.TrimSpace(v), "\\", "/"), "/")
|
|
}
|
|
|
|
func restoreLocalRelativePathForKey(sessionPrefix, currentManifestKey, key string, includeAudio bool) (string, bool, error) {
|
|
if key == "" {
|
|
return "", false, nil
|
|
}
|
|
if key == currentManifestKey {
|
|
return config.PathManifestFile, true, nil
|
|
}
|
|
if !strings.HasPrefix(key, sessionPrefix) {
|
|
return "", false, fmt.Errorf("key is outside resolved session prefix %q", sessionPrefix)
|
|
}
|
|
|
|
rel := strings.TrimPrefix(key, sessionPrefix)
|
|
rel = strings.TrimSpace(rel)
|
|
if rel == "" {
|
|
return "", false, nil
|
|
}
|
|
|
|
cleanRel := path.Clean(rel)
|
|
if cleanRel == "." || cleanRel == "" {
|
|
return "", false, nil
|
|
}
|
|
if cleanRel == ".." || strings.HasPrefix(cleanRel, "../") || strings.HasPrefix(cleanRel, "/") {
|
|
return "", false, fmt.Errorf("key relative path %q escapes session scope", rel)
|
|
}
|
|
|
|
if cleanRel == config.PathManifestFile {
|
|
return config.PathManifestFile, true, nil
|
|
}
|
|
if strings.HasPrefix(cleanRel, config.S3CurrentSegment+"/") {
|
|
return "", false, nil
|
|
}
|
|
if strings.HasPrefix(cleanRel, config.S3RunsSegment+"/") {
|
|
return "", false, nil
|
|
}
|
|
|
|
excludedRoots := []string{
|
|
config.PathLogsDirSegment,
|
|
config.PathReportsDirSegment,
|
|
config.PathConfigDirSegment,
|
|
config.PathInputsDirSegment,
|
|
}
|
|
for _, root := range excludedRoots {
|
|
if cleanRel == root || strings.HasPrefix(cleanRel, root+"/") {
|
|
return "", false, nil
|
|
}
|
|
}
|
|
|
|
if cleanRel == config.PathTranscriptsSegment || strings.HasPrefix(cleanRel, config.PathTranscriptsSegment+"/") {
|
|
return cleanRel, true, nil
|
|
}
|
|
if cleanRel == config.PathArtifactsDirSegment || strings.HasPrefix(cleanRel, config.PathArtifactsDirSegment+"/") {
|
|
return cleanRel, true, nil
|
|
}
|
|
if cleanRel == config.PathPreviousDirSegment || strings.HasPrefix(cleanRel, config.PathPreviousDirSegment+"/") {
|
|
return "", false, nil
|
|
}
|
|
if includeAudio && (cleanRel == config.PathAudioDirSegment || strings.HasPrefix(cleanRel, config.PathAudioDirSegment+"/")) {
|
|
return cleanRel, true, nil
|
|
}
|
|
|
|
return "", false, nil
|
|
}
|
|
|
|
func joinWithinSessionRoot(sessionRoot, relative string) (string, error) {
|
|
if strings.TrimSpace(sessionRoot) == "" {
|
|
return "", fmt.Errorf("session root is required")
|
|
}
|
|
joined, err := pathsafe.JoinSlashRelativeUnderRoot(sessionRoot, filepath.ToSlash(strings.TrimSpace(relative)))
|
|
if err != nil {
|
|
if errors.Is(err, pathsafe.ErrRelativePathRequired) {
|
|
return "", fmt.Errorf("relative path is required")
|
|
}
|
|
if errors.Is(err, pathsafe.ErrRelativePathEscape) || errors.Is(err, pathsafe.ErrRelativePathAbsolute) {
|
|
return "", fmt.Errorf("relative path escapes session root")
|
|
}
|
|
return "", fmt.Errorf("join relative path under session root: %w", err)
|
|
}
|
|
return joined, nil
|
|
}
|
|
|
|
func buildPreviousCacheRestoreActions(
|
|
ctx context.Context,
|
|
cfg *config.Config,
|
|
sessionPaths artifacts.SessionPaths,
|
|
store storage.ObjectStore,
|
|
force bool,
|
|
) ([]RestoreAction, error) {
|
|
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Scriptorium == nil {
|
|
return nil, nil
|
|
}
|
|
requirements := artifacts.CollectPreviousArtifactRequirements(cfg.Pipeline.Scriptorium.Artifacts)
|
|
if len(requirements) == 0 {
|
|
return nil, nil
|
|
}
|
|
plan, err := previouscache.BuildPlan(ctx, cfg, sessionPaths, requirements, store)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("plan previous-session cache restore: %w", err)
|
|
}
|
|
actions := make([]RestoreAction, 0, len(plan.Records))
|
|
for _, record := range plan.Records {
|
|
action, err := classifyRestoreAction(ctx, store, storage.ObjectInfo{Key: record.RemoteKey}, record.LocalRelativePath, record.LocalPath, force)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("classify previous-session cache object %q: %w", record.RemoteKey, err)
|
|
}
|
|
actions = append(actions, action)
|
|
}
|
|
return actions, nil
|
|
}
|
|
|
|
func classifyRestoreAction(
|
|
ctx context.Context,
|
|
store storage.ObjectStore,
|
|
object storage.ObjectInfo,
|
|
localRelPath string,
|
|
localPath string,
|
|
force bool,
|
|
) (RestoreAction, error) {
|
|
action := RestoreAction{
|
|
RemoteKey: normalizeRemoteKey(object.Key),
|
|
LocalRelativePath: localRelPath,
|
|
LocalPath: localPath,
|
|
Size: object.Size,
|
|
ETag: object.ETag,
|
|
}
|
|
|
|
info, err := os.Stat(localPath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
action.Kind = RestoreActionDownload
|
|
action.Reason = "local file missing"
|
|
return action, nil
|
|
}
|
|
return RestoreAction{}, fmt.Errorf("stat local file: %w", err)
|
|
}
|
|
|
|
action.ExistsLocal = true
|
|
if info.IsDir() {
|
|
action.Kind = RestoreActionConflict
|
|
action.Conflict = true
|
|
action.Reason = "local path is a directory"
|
|
return action, nil
|
|
}
|
|
|
|
if restoreRelativePathIsAudio(localRelPath) {
|
|
if object.Size > 0 {
|
|
if info.Size() == object.Size {
|
|
action.Kind = RestoreActionSkipSame
|
|
action.SameLocal = true
|
|
action.Reason = "local audio size matches remote content"
|
|
return action, nil
|
|
}
|
|
if force {
|
|
action.Kind = RestoreActionDownload
|
|
action.Reason = "local audio differs (size mismatch); overwrite with --force"
|
|
return action, nil
|
|
}
|
|
action.Kind = RestoreActionConflict
|
|
action.Conflict = true
|
|
action.Reason = "local audio differs (size mismatch)"
|
|
return action, nil
|
|
}
|
|
if force {
|
|
action.Kind = RestoreActionDownload
|
|
action.Reason = "local audio exists; remote size unavailable; overwrite with --force"
|
|
return action, nil
|
|
}
|
|
action.Kind = RestoreActionConflict
|
|
action.Conflict = true
|
|
action.Reason = "local audio exists; remote size unavailable"
|
|
return action, nil
|
|
}
|
|
|
|
if object.Size > 0 && info.Size() != object.Size {
|
|
if force {
|
|
action.Kind = RestoreActionDownload
|
|
action.Reason = "local file differs (size mismatch); overwrite with --force"
|
|
return action, nil
|
|
}
|
|
action.Kind = RestoreActionConflict
|
|
action.Conflict = true
|
|
action.Reason = "local file differs (size mismatch)"
|
|
return action, nil
|
|
}
|
|
|
|
localDigest, err := artifacts.SHA256File(localPath)
|
|
if err != nil {
|
|
return RestoreAction{}, fmt.Errorf("checksum local file: %w", err)
|
|
}
|
|
remotePath, err := storage.DownloadObjectToTemp(ctx, store, action.RemoteKey, "narratio-restore-plan-remote-*.tmp")
|
|
if err != nil {
|
|
return RestoreAction{}, fmt.Errorf("download remote object: %w", err)
|
|
}
|
|
defer func() { _ = os.Remove(remotePath) }()
|
|
|
|
remoteDigest, err := artifacts.SHA256File(remotePath)
|
|
if err != nil {
|
|
return RestoreAction{}, fmt.Errorf("checksum remote object: %w", err)
|
|
}
|
|
|
|
if remoteDigest == localDigest {
|
|
action.Kind = RestoreActionSkipSame
|
|
action.SameLocal = true
|
|
action.Reason = "local file matches remote content"
|
|
return action, nil
|
|
}
|
|
|
|
if force {
|
|
action.Kind = RestoreActionDownload
|
|
action.Reason = "local file differs; overwrite with --force"
|
|
return action, nil
|
|
}
|
|
|
|
action.Kind = RestoreActionConflict
|
|
action.Conflict = true
|
|
action.Reason = "local file differs"
|
|
return action, nil
|
|
}
|
|
|
|
func restoreActionIsAudio(action RestoreAction) bool {
|
|
return restoreRelativePathIsAudio(action.LocalRelativePath)
|
|
}
|
|
|
|
func restoreRelativePathIsAudio(rel string) bool {
|
|
cleanRel := path.Clean(strings.TrimSpace(rel))
|
|
return cleanRel == config.PathAudioDirSegment || strings.HasPrefix(cleanRel, config.PathAudioDirSegment+"/")
|
|
}
|
|
|
|
func writeRestorePlan(out io.Writer, current *RemoteCurrentState, plan *RestorePlan, opts RestorePlanOptions) error {
|
|
if out == nil {
|
|
return fmt.Errorf("output writer is required")
|
|
}
|
|
if current == nil {
|
|
return fmt.Errorf("remote current state is required")
|
|
}
|
|
if plan == nil {
|
|
return fmt.Errorf("restore plan is required")
|
|
}
|
|
|
|
if _, err := fmt.Fprintf(
|
|
out,
|
|
"restore plan: session %s/%s run=%s actions=%d download=%d skip_same=%d conflict=%d dry_run=%t force=%t include_audio=%t\n",
|
|
current.Campaign,
|
|
current.SessionID,
|
|
current.RunID,
|
|
len(plan.Actions),
|
|
plan.DownloadCount,
|
|
plan.SkipSameCount,
|
|
plan.ConflictCount,
|
|
opts.DryRun,
|
|
opts.Force,
|
|
opts.IncludeAudio,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
for _, action := range plan.Actions {
|
|
if _, err := fmt.Fprintf(out, "%s %s <- %s", action.Kind, action.LocalRelativePath, action.RemoteKey); err != nil {
|
|
return err
|
|
}
|
|
if strings.TrimSpace(action.Reason) != "" {
|
|
if _, err := fmt.Fprintf(out, " (%s)", action.Reason); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if _, err := fmt.Fprintln(out); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|