package stage import ( "context" "fmt" "os" "path/filepath" "strings" "gitea.maximumdirect.net/eric/narratio/internal/adapters/storage" "gitea.maximumdirect.net/eric/narratio/internal/artifacts" "gitea.maximumdirect.net/eric/narratio/internal/manifest" ) type publishStage struct{} var publishPrerequisiteStages = []string{ "prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", } func (publishStage) Name() string { return "publish" } func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) { if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil { return nil, fmt.Errorf("publish: resolved config must include pipeline and session") } if publishDisabled(env) { return &StageResult{ Metadata: map[string]any{ "stage": "publish", "skipped": true, "publish_enabled": false, "audio_upload_skipped": true, }, }, nil } if publishRunUploadDisabled(env) { return &StageResult{ Metadata: map[string]any{ "stage": "publish", "skipped": true, "upload_run_enabled": false, "audio_upload_skipped": true, }, }, nil } if err := validatePublishPrerequisites(m); err != nil { return nil, fmt.Errorf("publish: %w", err) } if env.ObjectStore == nil { return nil, fmt.Errorf("publish: remote object store backend is required when publish run upload is enabled") } runRoot, err := resolvePublishRunRoot(env, m) if err != nil { return nil, fmt.Errorf("publish: resolve run root: %w", err) } runPrefix, err := artifacts.ResolvePublishRunPrefix(env.Config, m) if err != nil { return nil, fmt.Errorf("publish: resolve s3 run prefix: %w", err) } sessionPrefix, err := artifacts.ResolvePublishSessionPrefix(env.Config, m) if err != nil { return nil, fmt.Errorf("publish: resolve s3 session prefix: %w", err) } bucket := artifacts.ResolvePublishBucket(env.Config, m) if bucket == "" { return nil, fmt.Errorf("publish: resolve s3 bucket: bucket is required") } runID := strings.TrimSpace(m.RunID) if runID == "" { return nil, fmt.Errorf("publish: run id is required") } runFiles, err := collectPublishRunFiles(runRoot, m) if err != nil { return nil, fmt.Errorf("publish: collect run files: %w", err) } runSources, err := openPublishRunFiles(runRoot, runFiles) if err != nil { return nil, fmt.Errorf("publish: verify run files: %w", err) } defer closePublishSources(runSources) uploader, ok := env.ObjectStore.(storage.ReaderUploader) if !ok { return nil, fmt.Errorf("publish: object store does not support verified source uploads") } sessionPaths := publishSessionPaths(env, m) previousFiles, err := collectPublishPreviousFiles(ctx, sessionPaths.PreviousDir) if err != nil { return nil, fmt.Errorf("publish: collect previous files: %w", err) } previousSources, err := openPublishRunFiles(sessionPaths.PreviousDir, previousFiles) if err != nil { return nil, fmt.Errorf("publish: verify previous files: %w", err) } defer closePublishSources(previousSources) runtimeCatalog, err := buildPublishRuntimeArtifactCatalog( sessionPaths, m, env.Config.Pipeline.Scriptorium, env.Config.Pipeline.Notarius, ) if err != nil { return nil, fmt.Errorf("publish: build runtime artifact catalog: %w", err) } publishOutputs, skippedOptionalOutputs, skippedUnselectedOutputs, lockedOutputs, err := resolvePublishOutputs( sessionPaths, m, runtimeCatalog, env.Config.Pipeline.Publish.Outputs, env.Config.Pipeline.Publish.Locks, env.SelectedArtifactKeys, runPrefix, ) if err != nil { return nil, fmt.Errorf("publish: resolve publish output rules: %w", err) } runUploaded := publishUploadPaths(runFiles) publishedUploaded := publishOutputPaths(publishOutputs) previousUploaded := publishUploadPaths(previousFiles) commitKey := artifacts.S3RunCommitKey(sessionPrefix, runID) pointerKey := artifacts.S3CurrentCommitPointerKey(sessionPrefix) transfers, err := buildPublishTransferPlan(runPrefix, runFiles, runSources, previousFiles, previousSources, publishOutputs) if err != nil { return nil, fmt.Errorf("publish: build immutable upload mapping: %w", err) } defer func() { closePublishTransfers(transfers) }() manifestTempPath, err := writeCommittedSessionManifest(m, publishMetadata( bucket, runPrefix, runID, runUploaded, publishedUploaded, previousUploaded, skippedOptionalOutputs, skippedUnselectedOutputs, lockedOutputs, commitKey, pointerKey, )) if err != nil { return nil, fmt.Errorf("publish: build committed session manifest: %w", err) } if err := appendCommittedSessionManifest(&transfers, manifestTempPath, artifacts.S3RunSessionManifestKey(sessionPrefix, runID)); err != nil { return nil, fmt.Errorf("publish: add committed session manifest: %w", err) } if err := validatePublishTransferPlan(transfers, commitKey); err != nil { return nil, fmt.Errorf("publish: validate immutable upload mapping: %w", err) } // Durability boundary: transfer every declared immutable object, write the // immutable commit, perform pre-pointer checks, then write the sole mutable // current pointer exactly once. declarations, err := uploadPublishTransfers(ctx, env.ObjectStore, uploader, transfers) if err != nil { return nil, fmt.Errorf("publish: %w", err) } committed, err := writeImmutablePublishCommit( ctx, env.ObjectStore, uploader, sessionPrefix, m.Campaign, m.SessionID, runID, declarations, ) if err != nil { return nil, fmt.Errorf("publish: %w", err) } if err := revalidatePublishCommitLocks(ctx, env, publishOutputs); err != nil { return nil, fmt.Errorf("publish: revalidate locks before current commit selection: %w", err) } if err := writePublishCurrentPointer(ctx, uploader, committed); err != nil { return nil, fmt.Errorf("publish: %w", err) } return &StageResult{ Metadata: map[string]any{ "stage": "publish", "uploaded": true, "s3_bucket": bucket, "s3_run_prefix": runPrefix, "run_files_uploaded": len(runUploaded), "run_uploaded_paths": runUploaded, "published_files_uploaded": len(publishedUploaded), "published_paths": publishedUploaded, "previous_files_uploaded": len(previousUploaded), "previous_uploaded_paths": previousUploaded, "skipped_optional_outputs": skippedOptionalOutputs, "skipped_unselected_outputs": skippedUnselectedOutputMetadata(skippedUnselectedOutputs), "locked_output_count": len(lockedOutputs), "locked_outputs": lockedOutputMetadata(lockedOutputs), "remote_commit_key": commitKey, "current_commit_pointer_key": pointerKey, "audio_upload_skipped": true, }, }, nil } func revalidatePublishCommitLocks(ctx context.Context, env *Env, outputs []publishOutput) error { if env == nil || env.RevalidatePublishLocks == nil || len(outputs) == 0 { return nil } locks, err := env.RevalidatePublishLocks(ctx) if err != nil { return err } lockSet := publishLockSet(locks) for _, output := range outputs { if lock, locked := lockSet[output.Source]; locked { return fmt.Errorf("source %q is locked: %s", output.Source, strings.TrimSpace(lock.Reason)) } } return nil } func publishDisabled(env *Env) bool { cfg := env.Config.Pipeline.Publish if cfg == nil { return true } return cfg.Enabled != nil && !*cfg.Enabled } func publishRunUploadDisabled(env *Env) bool { cfg := env.Config.Pipeline.Publish if cfg == nil { return true } return cfg.UploadRun != nil && !*cfg.UploadRun } func validatePublishPrerequisites(m *manifest.Manifest) error { if m == nil { return fmt.Errorf("manifest is required") } for _, stageName := range publishPrerequisiteStages { sr := m.Stages[stageName] if sr == nil { return fmt.Errorf("prerequisite stage %q has not succeeded", stageName) } if sr.Status != manifest.StatusSucceeded { return fmt.Errorf("prerequisite stage %q status is %q (want %q)", stageName, sr.Status, manifest.StatusSucceeded) } } return nil } func resolvePublishRunRoot(env *Env, m *manifest.Manifest) (string, error) { sessionID := strings.TrimSpace(env.Config.Session.SessionID) if sessionID == "" && m != nil { sessionID = strings.TrimSpace(m.SessionID) } campaign := strings.TrimSpace(env.Config.Session.Campaign) if campaign == "" && m != nil { campaign = strings.TrimSpace(m.Campaign) } runID := "" if m != nil { runID = strings.TrimSpace(m.RunID) } if sessionID == "" || campaign == "" { return "", fmt.Errorf("campaign and session id are required") } if runID == "" { return "", fmt.Errorf("run id is required") } canonical := filepath.Clean(artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, campaign, sessionID, runID)) info, err := os.Lstat(canonical) if err != nil { return "", fmt.Errorf("inspect canonical run root %q: %w", canonical, err) } if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { return "", fmt.Errorf("canonical run root %q is not a directory", canonical) } return canonical, nil } func publishSessionPaths(env *Env, m *manifest.Manifest) artifacts.SessionPaths { sessionID := strings.TrimSpace(env.Config.Session.SessionID) if sessionID == "" && m != nil { sessionID = strings.TrimSpace(m.SessionID) } campaign := strings.TrimSpace(env.Config.Session.Campaign) if campaign == "" && m != nil { campaign = strings.TrimSpace(m.Campaign) } store := artifacts.NewLocalStore(env.Config.Pipeline.Workspace.Root) return store.SessionPathsFor(campaign, sessionID) } func publishMetadata( bucket, runPrefix, runID string, runUploaded []string, publishedUploaded []string, previousUploaded []string, skippedOptionalOutputs []string, skippedUnselectedOutputs []publishSkippedUnselectedOutput, lockedOutputs []publishLockedOutput, commitKey string, pointerKey string, ) map[string]any { return map[string]any{ "stage": "publish", "uploaded": true, "s3_bucket": bucket, "s3_run_prefix": runPrefix, "published_run_id": runID, "run_files_uploaded": len(runUploaded), "run_uploaded_paths": append([]string(nil), runUploaded...), "published_files_uploaded": len(publishedUploaded), "published_paths": append([]string(nil), publishedUploaded...), "previous_files_uploaded": len(previousUploaded), "previous_uploaded_paths": append([]string(nil), previousUploaded...), "skipped_optional_outputs": append([]string(nil), skippedOptionalOutputs...), "skipped_unselected_outputs": skippedUnselectedOutputMetadata(skippedUnselectedOutputs), "locked_output_count": len(lockedOutputs), "locked_outputs": lockedOutputMetadata(lockedOutputs), "remote_commit_key": commitKey, "current_commit_pointer_key": pointerKey, "audio_upload_skipped": true, } } func skippedUnselectedOutputMetadata(skipped []publishSkippedUnselectedOutput) []map[string]any { out := make([]map[string]any, 0, len(skipped)) for _, item := range skipped { out = append(out, map[string]any{ "source": item.Source, "dest": item.Dest, "required": item.Required, }) } return out } func lockedOutputMetadata(locked []publishLockedOutput) []map[string]any { out := make([]map[string]any, 0, len(locked)) for _, item := range locked { out = append(out, map[string]any{ "source": item.Source, "dest": item.Dest, "remote_key": item.RemoteKey, "reason": item.Reason, "required": item.Required, "local_path": item.LocalPath, "provenance": item.Provenance, }) } return out }