Add archive promotion locks
This commit is contained in:
@@ -126,7 +126,14 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("archive: build runtime artifact catalog: %w", err)
|
||||
}
|
||||
promotions, skippedOptional, err := resolveArchivePromotions(sessionPaths, m, runtimeCatalog, env.Config.Pipeline.Archive.PromoteArtifacts)
|
||||
promotions, skippedOptional, lockedPromotions, err := resolveArchivePromotions(
|
||||
sessionPaths,
|
||||
m,
|
||||
runtimeCatalog,
|
||||
env.Config.Pipeline.Archive.PromoteArtifacts,
|
||||
env.Config.Pipeline.Archive.Locks,
|
||||
sessionPrefix,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("archive: resolve promotion rules: %w", err)
|
||||
}
|
||||
@@ -166,6 +173,7 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
promotedUploaded,
|
||||
previousUploaded,
|
||||
skippedOptional,
|
||||
lockedPromotions,
|
||||
currentManifestKey,
|
||||
))
|
||||
if err != nil {
|
||||
@@ -204,6 +212,8 @@ func (archiveStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
"previous_files_uploaded": len(previousUploaded),
|
||||
"previous_uploaded_paths": previousUploaded,
|
||||
"skipped_optional_promotions": skippedOptional,
|
||||
"locked_promotion_count": len(lockedPromotions),
|
||||
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
|
||||
"current_manifest_key": currentManifestKey,
|
||||
"current_run_id_key": currentRunPointerKey,
|
||||
"current_pointer_written": true,
|
||||
@@ -220,6 +230,16 @@ type archivePromotion struct {
|
||||
Provenance string
|
||||
}
|
||||
|
||||
type archiveLockedPromotion struct {
|
||||
Source string
|
||||
Dest string
|
||||
RemoteKey string
|
||||
Reason string
|
||||
Required bool
|
||||
LocalPath string
|
||||
Provenance string
|
||||
}
|
||||
|
||||
func archiveDisabled(env *Env) bool {
|
||||
cfg := env.Config.Pipeline.Archive
|
||||
if cfg == nil {
|
||||
@@ -319,26 +339,53 @@ func resolveArchivePromotions(
|
||||
m *manifest.Manifest,
|
||||
catalog *artifacts.ArtifactCatalog,
|
||||
rules []config.ArchivePromotionRule,
|
||||
) ([]archivePromotion, []string, error) {
|
||||
locks []config.ArchiveLockRule,
|
||||
sessionPrefix string,
|
||||
) ([]archivePromotion, []string, []archiveLockedPromotion, error) {
|
||||
out := make([]archivePromotion, 0, len(rules))
|
||||
skippedOptional := make([]string, 0)
|
||||
lockedPromotions := make([]archiveLockedPromotion, 0)
|
||||
lockSet := archiveLockSet(locks)
|
||||
for _, rule := range rules {
|
||||
source := strings.TrimSpace(rule.Source)
|
||||
required := rule.Required == nil || *rule.Required
|
||||
dest, err := resolveArchivePromotionDest(rule, catalog)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("source %q: %w", source, err)
|
||||
return nil, nil, nil, fmt.Errorf("source %q: %w", source, err)
|
||||
}
|
||||
lock, locked := lockSet[source]
|
||||
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(paths, m, source, catalog)
|
||||
if err != nil {
|
||||
if locked {
|
||||
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
|
||||
Source: source,
|
||||
Dest: dest,
|
||||
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
|
||||
Reason: strings.TrimSpace(lock.Reason),
|
||||
Required: required,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) && !required {
|
||||
skippedOptional = append(skippedOptional, dest)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
|
||||
return nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
|
||||
return nil, nil, nil, fmt.Errorf("required promotion source unavailable: %q", source)
|
||||
}
|
||||
return nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
|
||||
return nil, nil, nil, fmt.Errorf("resolve source %q: %w", source, err)
|
||||
}
|
||||
if locked {
|
||||
lockedPromotions = append(lockedPromotions, archiveLockedPromotion{
|
||||
Source: source,
|
||||
Dest: dest,
|
||||
RemoteKey: artifacts.S3PromotedArtifactKey(sessionPrefix, dest),
|
||||
Reason: strings.TrimSpace(lock.Reason),
|
||||
Required: required,
|
||||
LocalPath: resolved.Path,
|
||||
Provenance: resolved.Provenance,
|
||||
})
|
||||
continue
|
||||
}
|
||||
out = append(out, archivePromotion{
|
||||
Source: source,
|
||||
@@ -348,7 +395,21 @@ func resolveArchivePromotions(
|
||||
Provenance: resolved.Provenance,
|
||||
})
|
||||
}
|
||||
return out, skippedOptional, nil
|
||||
return out, skippedOptional, lockedPromotions, nil
|
||||
}
|
||||
|
||||
func archiveLockSet(locks []config.ArchiveLockRule) map[string]config.ArchiveLockRule {
|
||||
out := make(map[string]config.ArchiveLockRule, len(locks))
|
||||
for _, lock := range locks {
|
||||
source := strings.TrimSpace(lock.Source)
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
lock.Source = source
|
||||
lock.Reason = strings.TrimSpace(lock.Reason)
|
||||
out[source] = lock
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func resolveArchivePromotionDest(rule config.ArchivePromotionRule, catalog *artifacts.ArtifactCatalog) (string, error) {
|
||||
@@ -663,6 +724,7 @@ func archiveMetadataPreview(
|
||||
promotedUploaded []string,
|
||||
previousUploaded []string,
|
||||
skippedOptional []string,
|
||||
lockedPromotions []archiveLockedPromotion,
|
||||
currentManifestKey string,
|
||||
) map[string]any {
|
||||
return map[string]any{
|
||||
@@ -677,9 +739,27 @@ func archiveMetadataPreview(
|
||||
"previous_files_uploaded": len(previousUploaded),
|
||||
"previous_uploaded_paths": append([]string(nil), previousUploaded...),
|
||||
"skipped_optional_promotions": append([]string(nil), skippedOptional...),
|
||||
"locked_promotion_count": len(lockedPromotions),
|
||||
"locked_promotions": lockedPromotionMetadata(lockedPromotions),
|
||||
"current_manifest_key": currentManifestKey,
|
||||
"current_run_id_key": artifacts.S3CurrentRunPointerKey(sessionPrefix),
|
||||
"current_pointer_written": false,
|
||||
"audio_upload_skipped": true,
|
||||
}
|
||||
}
|
||||
|
||||
func lockedPromotionMetadata(locked []archiveLockedPromotion) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(locked))
|
||||
for _, item := range locked {
|
||||
out = append(out, map[string]any{
|
||||
"source": item.Source,
|
||||
"dest": item.Dest,
|
||||
"remote_key": item.RemoteKey,
|
||||
"reason": item.Reason,
|
||||
"required": item.Required,
|
||||
"local_path": item.LocalPath,
|
||||
"provenance": item.Provenance,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user