eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Persist retryable post-publish cleanup obligations

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-10 20:29:00 +00:00
parent 0cf2cbfeb3
commit eac7e155a5
12 changed files with 438 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

178
internal/app/post_publish_cleanup.go
View File

@@ -12,104 +12,104 @@ import (
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
var removeRunScopedDirFn = removeRunScopedDir
func runPostPublishCleanup(ctx context.Context, env *Env, manifestPath string, m *manifest.Manifest, executed []string) error {
if env == nil || env.Config == nil || env.Config.Pipeline == nil || m == nil {
return nil
}
spoolRequested := env.Config.Pipeline.Spool.DeleteAudioAfterPublish
workRequested := env.Config.Pipeline.Workspace.CleanupAfterPublish
if !spoolRequested && !workRequested {
return nil
}
sr := publishStageRecordForCleanup(m, executed)
if sr == nil {
return nil
}
if sr.Metadata == nil {
sr.Metadata = map[string]any{}
}
sr.Metadata["spool_cleanup_requested"] = spoolRequested
sr.Metadata["workdir_cleanup_requested"] = workRequested
eligible, reason := publishCleanupEligible(env.Config, sr)
if !eligible {
sr.Metadata["cleanup_skipped"] = true
sr.Metadata["cleanup_skipped_reason"] = reason
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return fmt.Errorf("save manifest cleanup skip metadata %q: %w", manifestPath, err)
cleanup := m.PostPublishCleanup
if cleanup == nil {
var err error
cleanup, err = createPostPublishCleanup(env.Config, m, executed)
if err != nil {
return err
}
return nil
if cleanup == nil {
return nil
}
m.PostPublishCleanup = cleanup
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return fmt.Errorf("persist post-publish cleanup obligation %q: %w", manifestPath, err)
}
}
for index := range cleanup.Targets {
target := &cleanup.Targets[index]
if target.Completed {
continue
}
if err := removeRunScopedDirFn(target.Root, target.Path, target.Policy); err != nil {
return fmt.Errorf("complete post-publish cleanup for %q: %w", target.Path, err)
}
target.Completed = true
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
target.Completed = false
return fmt.Errorf("persist post-publish cleanup completion %q: %w", manifestPath, err)
}
}
return nil
}
func createPostPublishCleanup(cfg *config.Config, m *manifest.Manifest, executed []string) (*manifest.PostPublishCleanup, error) {
spoolRequested := cfg.Pipeline.Spool.DeleteAudioAfterPublish
workRequested := cfg.Pipeline.Workspace.CleanupAfterPublish
if !spoolRequested && !workRequested {
return nil, nil
}
sr := publishStageRecordForCleanup(m)
publishedRunID, eligible, _ := publishCleanupEligible(cfg, m, sr, executed)
if !eligible {
return nil, nil
}
spoolDir := strings.TrimSpace(m.LocalSpoolDir)
if spoolDir == "" {
spoolDir = artifacts.SessionSpoolAudioDir(
env.Config.Pipeline.Spool.Root,
strings.TrimSpace(env.Config.Session.Campaign),
strings.TrimSpace(env.Config.Session.SessionID),
strings.TrimSpace(m.RunID),
cfg.Pipeline.Spool.Root,
strings.TrimSpace(m.Campaign),
strings.TrimSpace(m.SessionID),
publishedRunID,
)
}
workDir := strings.TrimSpace(m.LocalWorkDir)
if workDir == "" {
workDir = artifacts.SessionRunRootForCampaign(
env.Config.Pipeline.Workspace.Root,
strings.TrimSpace(env.Config.Session.Campaign),
strings.TrimSpace(env.Config.Session.SessionID),
strings.TrimSpace(m.RunID),
cfg.Pipeline.Workspace.Root,
strings.TrimSpace(m.Campaign),
strings.TrimSpace(m.SessionID),
publishedRunID,
)
}
cleanup := &manifest.PostPublishCleanup{
CommittedRunID: publishedRunID,
RemoteCommitKey: strings.TrimSpace(asString(sr.Metadata["remote_commit_key"])),
CurrentCommitPointerKey: strings.TrimSpace(asString(sr.Metadata["current_commit_pointer_key"])),
}
if spoolRequested {
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Spool.Root), spoolDir, "pipeline.spool.delete_audio_after_publish"); err != nil {
sr.Metadata["cleanup_failed"] = true
sr.Metadata["cleanup_failed_policy"] = "pipeline.spool.delete_audio_after_publish"
sr.Metadata["cleanup_failed_path"] = spoolDir
_ = env.ManifestStore.Save(ctx, manifestPath, m)
return err
}
sr.Metadata["spool_cleanup_deleted"] = filepath.Clean(spoolDir)
cleanup.Targets = append(cleanup.Targets, manifest.CleanupTarget{
Policy: "pipeline.spool.delete_audio_after_publish",
Root: strings.TrimSpace(cfg.Pipeline.Spool.Root),
Path: filepath.Clean(spoolDir),
})
}
if !workRequested {
sr.Metadata["cleanup_completed"] = true
sr.Metadata["cleanup_skipped"] = false
if err := env.ManifestStore.Save(ctx, manifestPath, m); err != nil {
return fmt.Errorf("save manifest cleanup metadata %q: %w", manifestPath, err)
}
return nil
if workRequested {
cleanup.Targets = append(cleanup.Targets, manifest.CleanupTarget{
Policy: "pipeline.workspace.cleanup_after_publish",
Root: strings.TrimSpace(cfg.Pipeline.Workspace.Root),
Path: filepath.Clean(workDir),
})
}
if err := removeRunScopedDir(strings.TrimSpace(env.Config.Pipeline.Workspace.Root), workDir, "pipeline.workspace.cleanup_after_publish"); err != nil {
sr.Metadata["cleanup_failed"] = true
sr.Metadata["cleanup_failed_policy"] = "pipeline.workspace.cleanup_after_publish"
sr.Metadata["cleanup_failed_path"] = workDir
_ = env.ManifestStore.Save(ctx, manifestPath, m)
return err
}
sr.Metadata["workdir_cleanup_deleted"] = filepath.Clean(workDir)
sr.Metadata["cleanup_completed"] = true
sr.Metadata["cleanup_skipped"] = false
return nil
return cleanup, nil
}
func publishStageRecordForCleanup(m *manifest.Manifest, executed []string) *manifest.StageRecord {
func publishStageRecordForCleanup(m *manifest.Manifest) *manifest.StageRecord {
if m == nil {
return nil
}
publishRan := false
for _, name := range executed {
if name == "publish" {
publishRan = true
break
}
}
if !publishRan {
return nil
}
sr := m.Stages["publish"]
if sr == nil || sr.Status != manifest.StatusSucceeded {
return nil
@@ -117,40 +117,56 @@ func publishStageRecordForCleanup(m *manifest.Manifest, executed []string) *mani
return sr
}
func publishCleanupEligible(cfg *config.Config, sr *manifest.StageRecord) (bool, string) {
func publishCleanupEligible(cfg *config.Config, m *manifest.Manifest, sr *manifest.StageRecord, executed []string) (string, bool, string) {
if cfg == nil || cfg.Pipeline == nil || cfg.Pipeline.Publish == nil {
return false, "publish configuration is missing"
return "", false, "publish configuration is missing"
}
enabled := true
if cfg.Pipeline.Publish.Enabled != nil {
enabled = *cfg.Pipeline.Publish.Enabled
}
if !enabled {
return false, "publish.enabled is false"
return "", false, "publish.enabled is false"
}
uploadRun := true
if cfg.Pipeline.Publish.UploadRun != nil {
uploadRun = *cfg.Pipeline.Publish.UploadRun
}
if !uploadRun {
return false, "publish.upload_run is false"
return "", false, "publish.upload_run is false"
}
if sr == nil || sr.Metadata == nil {
return false, "publish metadata is missing"
return "", false, "publish metadata is missing"
}
if skipped, _ := sr.Metadata["skipped"].(bool); skipped {
return false, "publish stage was skipped"
return "", false, "publish stage was skipped"
}
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
return false, "publish did not upload run record"
return "", false, "publish did not upload run record"
}
if strings.TrimSpace(asString(sr.Metadata["remote_commit_key"])) == "" {
return false, "publish remote commit key is missing"
return "", false, "publish remote commit key is missing"
}
if strings.TrimSpace(asString(sr.Metadata["current_commit_pointer_key"])) == "" {
return false, "publish current commit pointer key is missing"
return "", false, "publish current commit pointer key is missing"
}
return true, ""
publishedRunID := strings.TrimSpace(asString(sr.Metadata["published_run_id"]))
if publishedRunID == "" && containsStage(executed, "publish") && m != nil {
publishedRunID = strings.TrimSpace(m.RunID)
}
if publishedRunID == "" {
return "", false, "publish run id is missing"
}
return publishedRunID, true, ""
}
func containsStage(names []string, target string) bool {
for _, name := range names {
if name == target {
return true
}
}
return false
}
type scopedDir struct {