283 lines
9.2 KiB
Go
283 lines
9.2 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
|
)
|
|
|
|
// Clean removes local workspace/spool state while preserving durable cache
|
|
// state unless cache cleanup is explicitly requested.
|
|
func Clean(ctx context.Context, args []string, out io.Writer) error {
|
|
fs := flag.NewFlagSet("clean", flag.ContinueOnError)
|
|
fs.SetOutput(io.Discard)
|
|
var flags commonConfigFlags
|
|
var all bool
|
|
var dryRun bool
|
|
var clearCache bool
|
|
addCommonConfigFlags(fs, &flags)
|
|
fs.BoolVar(&all, "all", false, "clean all local session work/spool state")
|
|
fs.BoolVar(&dryRun, "dry-run", false, "print cleanup targets without deleting")
|
|
fs.BoolVar(&clearCache, "clear-cache", false, "also clear durable S3 audio cache entries")
|
|
if err := parseSessionAwareFlags("clean", fs, args, &flags.sessionID); err != nil {
|
|
return err
|
|
}
|
|
if all {
|
|
return cleanAllLocal(flags, dryRun, clearCache, out)
|
|
}
|
|
return cleanSession(ctx, flags, dryRun, clearCache, out)
|
|
}
|
|
|
|
func cleanSession(ctx context.Context, flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
|
if strings.TrimSpace(flags.sessionID) == "" {
|
|
return fmt.Errorf("clean: session_id is required unless --all is set")
|
|
}
|
|
cfg, err := loadCommandConfig(ctx, flags.pipelinePath, flags.campaignPath, flags.campaignFilePath, flags.sessionPath, flags.sessionOptions())
|
|
if err != nil {
|
|
return fmt.Errorf("clean: %w", err)
|
|
}
|
|
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
|
return fmt.Errorf("clean: resolved pipeline and session config are required")
|
|
}
|
|
campaign := strings.TrimSpace(cfg.Session.Campaign)
|
|
sessionID := strings.TrimSpace(cfg.Session.SessionID)
|
|
if campaign == "" || sessionID == "" {
|
|
return fmt.Errorf("clean: campaign and session_id are required")
|
|
}
|
|
|
|
if dryRun {
|
|
fmt.Fprintf(out, "Clean plan for %s/%s\n", campaign, sessionID)
|
|
} else {
|
|
fmt.Fprintf(out, "Cleaned %s/%s\n", campaign, sessionID)
|
|
}
|
|
|
|
workDir := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, campaign, sessionID)
|
|
spoolDir := artifacts.SessionSpoolDir(cfg.Pipeline.Spool.Root, campaign, sessionID)
|
|
if err := reportCleanScopedDir(out, cfg.Pipeline.Workspace.Root, workDir, "clean.workspace.session", dryRun); err != nil {
|
|
return fmt.Errorf("clean: %w", err)
|
|
}
|
|
if err := reportCleanScopedDir(out, cfg.Pipeline.Spool.Root, spoolDir, "clean.spool.session", dryRun); err != nil {
|
|
return fmt.Errorf("clean: %w", err)
|
|
}
|
|
|
|
if clearCache {
|
|
if err := cleanSessionAudioCache(ctx, cfg, dryRun, out); err != nil {
|
|
return fmt.Errorf("clean: %w", err)
|
|
}
|
|
} else {
|
|
fmt.Fprintln(out, "Cache: preserved")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func cleanAllLocal(flags commonConfigFlags, dryRun, clearCache bool, out io.Writer) error {
|
|
if strings.TrimSpace(flags.campaignPath) != "" ||
|
|
strings.TrimSpace(flags.campaignFilePath) != "" ||
|
|
strings.TrimSpace(flags.sessionPath) != "" ||
|
|
strings.TrimSpace(flags.sessionID) != "" ||
|
|
strings.TrimSpace(flags.previousSessionID) != "" {
|
|
return fmt.Errorf("clean: --all cannot be combined with --campaign, --campaign-file, --session, a session_id, or --previous-session-id")
|
|
}
|
|
resolvedPipelinePath, err := resolvePipelineConfigPath(flags.pipelinePath)
|
|
if err != nil {
|
|
return fmt.Errorf("clean: %w", err)
|
|
}
|
|
pipelineCfg, err := config.LoadPipeline(resolvedPipelinePath)
|
|
if err != nil {
|
|
return fmt.Errorf("clean: %w", err)
|
|
}
|
|
|
|
if dryRun {
|
|
fmt.Fprintln(out, "Clean plan for all local sessions")
|
|
} else {
|
|
fmt.Fprintln(out, "Cleaned all local sessions")
|
|
}
|
|
|
|
workRoot := filepath.Join(pipelineCfg.Workspace.Root, config.PathWorkDirSegment)
|
|
if err := reportCleanScopedDir(out, pipelineCfg.Workspace.Root, workRoot, "clean.workspace.all", dryRun); err != nil {
|
|
return fmt.Errorf("clean: %w", err)
|
|
}
|
|
if err := reportCleanRootChildren(out, pipelineCfg.Spool.Root, "clean.spool.all", dryRun); err != nil {
|
|
return fmt.Errorf("clean: %w", err)
|
|
}
|
|
|
|
if clearCache {
|
|
if err := cleanAllAudioCache(pipelineCfg, dryRun, out); err != nil {
|
|
return fmt.Errorf("clean: %w", err)
|
|
}
|
|
} else {
|
|
fmt.Fprintln(out, "Cache: preserved")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func reportCleanScopedDir(out io.Writer, root, target, policy string, dryRun bool) error {
|
|
dir, err := validateScopedDir(root, target, policy)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if dryRun {
|
|
if dir.Exists {
|
|
fmt.Fprintf(out, "Would delete: %s\n", dir.TargetAbs)
|
|
} else {
|
|
fmt.Fprintf(out, "Would skip missing: %s\n", dir.TargetAbs)
|
|
}
|
|
return nil
|
|
}
|
|
if !dir.Exists {
|
|
fmt.Fprintf(out, "Missing: %s\n", dir.TargetAbs)
|
|
return nil
|
|
}
|
|
if err := fileops.RemoveAllUnderRoot(dir.RootAbs, dir.TargetAbs); err != nil {
|
|
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, dir.TargetAbs, err)
|
|
}
|
|
fmt.Fprintf(out, "Deleted: %s\n", dir.TargetAbs)
|
|
return nil
|
|
}
|
|
|
|
func reportCleanRootChildren(out io.Writer, root, policy string, dryRun bool) error {
|
|
rootAbs, entries, err := cleanableRootChildren(root, policy)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(entries) == 0 {
|
|
if dryRun {
|
|
fmt.Fprintf(out, "Would skip empty: %s\n", rootAbs)
|
|
} else {
|
|
fmt.Fprintf(out, "Empty: %s\n", rootAbs)
|
|
}
|
|
return nil
|
|
}
|
|
for _, entry := range entries {
|
|
if dryRun {
|
|
fmt.Fprintf(out, "Would delete: %s\n", entry)
|
|
continue
|
|
}
|
|
if err := fileops.RemoveAllUnderRoot(rootAbs, entry); err != nil {
|
|
return fmt.Errorf("cleanup policy %s: remove %q: %w", policy, entry, err)
|
|
}
|
|
fmt.Fprintf(out, "Deleted: %s\n", entry)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func cleanableRootChildren(root, policy string) (string, []string, error) {
|
|
rootAbs, exists, err := validateCleanRoot(root, policy)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
if !exists {
|
|
return rootAbs, nil, nil
|
|
}
|
|
entries, err := os.ReadDir(rootAbs)
|
|
if err != nil {
|
|
return "", nil, fmt.Errorf("cleanup policy %s: read root %q: %w", policy, rootAbs, err)
|
|
}
|
|
out := make([]string, 0, len(entries))
|
|
for _, entry := range entries {
|
|
path := filepath.Join(rootAbs, entry.Name())
|
|
info, err := os.Lstat(path)
|
|
if err != nil {
|
|
return "", nil, fmt.Errorf("cleanup policy %s: stat child %q: %w", policy, path, err)
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 {
|
|
return "", nil, fmt.Errorf("cleanup policy %s: refusing to delete symlink path %q", policy, path)
|
|
}
|
|
out = append(out, path)
|
|
}
|
|
return rootAbs, out, nil
|
|
}
|
|
|
|
func cleanSessionAudioCache(ctx context.Context, cfg *config.Config, dryRun bool, out io.Writer) error {
|
|
if cfg.Session.Inputs.AudioS3 == nil {
|
|
fmt.Fprintln(out, "Cache: skipped (session does not use audio_s3)")
|
|
return nil
|
|
}
|
|
if cfg.Pipeline.Storage.S3 == nil || strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket) == "" {
|
|
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
|
}
|
|
store, err := newCommandObjectStore(ctx, cfg, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("initialize object store for cache cleanup: %w", err)
|
|
}
|
|
sessionPrefix := artifacts.S3SessionPrefix(cfg.Pipeline.Storage.S3.RootPrefix, cfg.Session.Campaign, cfg.Session.SessionID)
|
|
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, cfg.Session.Inputs.AudioS3.Prefix)
|
|
objects, err := store.List(ctx, audioPrefix)
|
|
if err != nil {
|
|
return fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
|
}
|
|
count := 0
|
|
for _, obj := range objects {
|
|
key := strings.TrimSpace(obj.Key)
|
|
if key == "" || strings.HasSuffix(key, "/") || !cleanIsFlac(key) {
|
|
continue
|
|
}
|
|
cachePath, err := artifacts.S3AudioCachePath(cfg.Pipeline.Cache.Root, cfg.Pipeline.Storage.S3.Bucket, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
deleted, err := reportCleanScopedFile(out, cfg.Pipeline.Cache.Root, cachePath, "clean.cache.session", dryRun)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if deleted {
|
|
count++
|
|
}
|
|
}
|
|
if count == 0 {
|
|
fmt.Fprintf(out, "Cache: no cached S3 audio files found for %s\n", audioPrefix)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func cleanAllAudioCache(cfg *config.PipelineConfig, dryRun bool, out io.Writer) error {
|
|
if cfg.Storage.S3 == nil || strings.TrimSpace(cfg.Storage.S3.Bucket) == "" {
|
|
return fmt.Errorf("clear cache requires pipeline.storage.s3.bucket")
|
|
}
|
|
namespaceDir, err := artifacts.S3AudioCacheNamespaceDir(cfg.Cache.Root, cfg.Storage.S3.Bucket, cfg.Storage.S3.RootPrefix)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return reportCleanScopedDir(out, cfg.Cache.Root, namespaceDir, "clean.cache.all", dryRun)
|
|
}
|
|
|
|
func reportCleanScopedFile(out io.Writer, root, target, policy string, dryRun bool) (bool, error) {
|
|
file, err := validateScopedFile(root, target, policy)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if dryRun {
|
|
if file.Exists {
|
|
fmt.Fprintf(out, "Would delete cache file: %s\n", file.TargetAbs)
|
|
return true, nil
|
|
}
|
|
fmt.Fprintf(out, "Would skip missing cache file: %s\n", file.TargetAbs)
|
|
return false, nil
|
|
}
|
|
if !file.Exists {
|
|
fmt.Fprintf(out, "Missing cache file: %s\n", file.TargetAbs)
|
|
return false, nil
|
|
}
|
|
if err := fileops.RemoveAllUnderRoot(file.RootAbs, file.TargetAbs); err != nil {
|
|
return false, fmt.Errorf("cleanup policy %s: remove %q: %w", policy, file.TargetAbs, err)
|
|
}
|
|
fmt.Fprintf(out, "Deleted cache file: %s\n", file.TargetAbs)
|
|
return true, nil
|
|
}
|
|
|
|
func validateScopedFile(root, target, policy string) (scopedDir, error) {
|
|
return validateScopedTarget(root, target, policy, false)
|
|
}
|
|
|
|
func cleanIsFlac(path string) bool {
|
|
return strings.EqualFold(filepath.Ext(path), ".flac")
|
|
}
|