Remove retired cleanup manifest state

This commit is contained in:
2026-09-05 15:30:31 +00:00
parent adc60aa588
commit 078f661e3a
10 changed files with 84 additions and 74 deletions

View File

@@ -78,7 +78,9 @@ Execution order and safety:
- restored manifest local references are rebased beneath the selected local - restored manifest local references are rebased beneath the selected local
session root. Unsafe relative references and producer-machine absolute paths session root. Unsafe relative references and producer-machine absolute paths
outside the manifest's producer session root are rejected; producer-local outside the manifest's producer session root are rejected; producer-local
spool/cache and cleanup locations are not restored as authority. spool paths and run-local diagnostic references are not restored as
authority. Cleanup remains ordinary restored stage state and is not reset or
synthesized by restore.
Audio restore path: Audio restore path:

View File

@@ -16,16 +16,18 @@ manifest placement belongs in
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`) - remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
- `inputs` records - `inputs` records
- durable `artifacts` records - durable `artifacts` records
- per-stage `stages` map - per-stage `stages` map, including the ordinary cleanup stage record
- an optional `post_publish_cleanup` obligation, which binds a committed run,
remote commit identity, and each exact root-confined local target to its
completion evidence
Session, campaign, and run identities in local and downloaded manifests must be Session, campaign, and run identities in local and downloaded manifests must be
portable opaque segments. Unsafe legacy identities are rejected with migration portable opaque segments. Unsafe legacy identities are rejected with migration
guidance rather than being normalized into a different workspace or remote guidance rather than being normalized into a different workspace or remote
namespace. namespace.
Decoding is strict at every typed manifest field. Unknown fields are rejected;
the schema version remains `narratio.session-manifest.v1` and there is no
compatibility reader that silently drops retired state. Cleanup progress uses
only the cleanup entry in `stages`.
Every persisted session manifest has a non-nil `stages` map. Each entry is Every persisted session manifest has a non-nil `stages` map. Each entry is
non-nil and its embedded `name` exactly matches its map key; loading never non-nil and its embedded `name` exactly matches its map key; loading never
invents either marker. A missing or unsupported schema version, or incomplete invents either marker. A missing or unsupported schema version, or incomplete
@@ -145,6 +147,7 @@ establish a current immutable snapshot.
`manifest.LocalStore`: `manifest.LocalStore`:
- accepts only the exact supported session-manifest schema version; - accepts only the exact supported session-manifest schema version;
- rejects unknown fields and multiple JSON values;
- validates stage-map identity without synthesizing missing maps, records, or - validates stage-map identity without synthesizing missing maps, records, or
names; names;
- retains only representation-preserving collection normalization; - retains only representation-preserving collection normalization;

View File

@@ -610,6 +610,14 @@ also available when the manifest has valid publish evidence. A range ending at
`notify` or earlier and the standalone `publish` command never delete local `notify` or earlier and the standalone `publish` command never delete local
state. state.
Older local manifests that contain the retired cleanup-obligation field are
intentionally incompatible with strict manifest decoding. If such an
obligation is still outstanding, complete it with the pre-upgrade binary.
Otherwise verify that the remote publication is committed, move aside or
regenerate the incompatible local manifest (or restore the verified remote
snapshot), and then use the cleanup stage. Manual JSON editing is not the
normal migration path.
## Operational Caveats ## Operational Caveats
- Local and S3 audio modes are mutually exclusive. - Local and S3 audio modes are mutually exclusive.

View File

@@ -246,7 +246,7 @@ post-loop deletion path.
## Stage 4 — Remove The Cleanup Obligation From Manifest State ## Stage 4 — Remove The Cleanup Obligation From Manifest State
**Status:** Pending **Status:** Completed
### Goal ### Goal

View File

@@ -73,6 +73,11 @@ Safe fix:
regenerate it; restore a verified current committed snapshot or republish regenerate it; restore a verified current committed snapshot or republish
healthy regenerated state when appropriate. Do not add version or stage healthy regenerated state when appropriate. Do not add version or stage
markers by hand. markers by hand.
- if an older manifest is rejected for its retired cleanup-obligation field,
finish outstanding cleanup with the pre-upgrade binary. Otherwise verify the
remote commit, move aside or regenerate the incompatible manifest, and use
the cleanup stage. Do not treat manual JSON editing as the normal migration
path.
Relevant reference: [Configuration](./config.md). Relevant reference: [Configuration](./config.md).

View File

@@ -122,9 +122,6 @@ func rebaseRestoredManifestPaths(cfg *config.Config, m *manifest.Manifest, desti
m.LocalWorkDir = destinationRoot m.LocalWorkDir = destinationRoot
} }
m.LocalSpoolDir = "" m.LocalSpoolDir = ""
// A post-publish cleanup record is authority to delete producer-local
// directories. It must never cross a restore boundary.
m.PostPublishCleanup = nil
return nil return nil
} }

View File

@@ -168,6 +168,7 @@ func TestRebaseRestoredManifestPathsRejectsUnsafeReferencesAndRebasesPortablePat
m.RunID = "20260519T010203Z-a1b2c3d4" m.RunID = "20260519T010203Z-a1b2c3d4"
m.LocalWorkDir = test.workDir m.LocalWorkDir = test.workDir
m.Stages["analyze"] = &manifest.StageRecord{Outputs: []manifest.ArtifactRecord{{Kind: "recap", LocalPath: test.output}}} m.Stages["analyze"] = &manifest.StageRecord{Outputs: []manifest.ArtifactRecord{{Kind: "recap", LocalPath: test.output}}}
m.MarkStageFailed("cleanup", nowUTC(), "retry local deletion")
destination := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID) destination := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
err := rebaseRestoredManifestPaths(cfg, m, destination) err := rebaseRestoredManifestPaths(cfg, m, destination)
@@ -183,6 +184,9 @@ func TestRebaseRestoredManifestPathsRejectsUnsafeReferencesAndRebasesPortablePat
if got, want := m.Stages["analyze"].Outputs[0].LocalPath, filepath.Join(destination, test.wantPath); got != want { if got, want := m.Stages["analyze"].Outputs[0].LocalPath, filepath.Join(destination, test.wantPath); got != want {
t.Fatalf("rebased local path = %q, want %q", got, want) t.Fatalf("rebased local path = %q, want %q", got, want)
} }
if got := m.Stages["cleanup"].Status; got != manifest.StatusFailed {
t.Fatalf("cleanup status = %q, want preserved failed state", got)
}
}) })
} }
} }

View File

@@ -66,25 +66,6 @@ type StageRecord struct {
AnalyzeArtifacts map[string]AnalyzeArtifactRecord `json:"analyze_artifacts,omitempty"` AnalyzeArtifacts map[string]AnalyzeArtifactRecord `json:"analyze_artifacts,omitempty"`
} }
// CleanupTarget records one root-confined local deletion requested by a
// committed publication.
type CleanupTarget struct {
Policy string `json:"policy"`
Root string `json:"root"`
Path string `json:"path"`
Completed bool `json:"completed"`
}
// PostPublishCleanup records the durable deletion work left by a committed
// publish. It remains after completion as evidence of the exact committed run
// and local paths involved.
type PostPublishCleanup struct {
CommittedRunID string `json:"committed_run_id"`
RemoteCommitKey string `json:"remote_commit_key"`
CurrentCommitPointerKey string `json:"current_commit_pointer_key"`
Targets []CleanupTarget `json:"targets"`
}
// EffectiveConfigProvenance records bounded non-secret configuration identity // EffectiveConfigProvenance records bounded non-secret configuration identity
// for one resolved invocation. // for one resolved invocation.
type EffectiveConfigProvenance struct { type EffectiveConfigProvenance struct {
@@ -101,24 +82,23 @@ type SelectedProfileProvenance struct {
// Manifest is the durable run-state record for a session execution. // Manifest is the durable run-state record for a session execution.
type Manifest struct { type Manifest struct {
SchemaVersion string `json:"schema_version"` SchemaVersion string `json:"schema_version"`
SessionID string `json:"session_id"` SessionID string `json:"session_id"`
Campaign string `json:"campaign,omitempty"` Campaign string `json:"campaign,omitempty"`
RunID string `json:"run_id,omitempty"` RunID string `json:"run_id,omitempty"`
LocalWorkDir string `json:"local_workdir,omitempty"` LocalWorkDir string `json:"local_workdir,omitempty"`
LocalSpoolDir string `json:"local_spool_dir,omitempty"` LocalSpoolDir string `json:"local_spool_dir,omitempty"`
S3Bucket string `json:"s3_bucket,omitempty"` S3Bucket string `json:"s3_bucket,omitempty"`
S3SessionPrefix string `json:"s3_session_prefix,omitempty"` S3SessionPrefix string `json:"s3_session_prefix,omitempty"`
S3RunPrefix string `json:"s3_run_prefix,omitempty"` S3RunPrefix string `json:"s3_run_prefix,omitempty"`
PipelineVersion string `json:"pipeline_version,omitempty"` PipelineVersion string `json:"pipeline_version,omitempty"`
EffectiveConfig *EffectiveConfigProvenance `json:"effective_config,omitempty"` EffectiveConfig *EffectiveConfigProvenance `json:"effective_config,omitempty"`
CreatedAt time.Time `json:"created_at"` CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"` UpdatedAt time.Time `json:"updated_at"`
LastError *ErrorRecord `json:"last_error,omitempty"` LastError *ErrorRecord `json:"last_error,omitempty"`
Inputs []InputRecord `json:"inputs,omitempty"` Inputs []InputRecord `json:"inputs,omitempty"`
Artifacts []ArtifactRecord `json:"artifacts,omitempty"` Artifacts []ArtifactRecord `json:"artifacts,omitempty"`
Stages map[string]*StageRecord `json:"stages"` Stages map[string]*StageRecord `json:"stages"`
PostPublishCleanup *PostPublishCleanup `json:"post_publish_cleanup,omitempty"`
} }
// New constructs a new manifest with deterministic timestamps. // New constructs a new manifest with deterministic timestamps.

View File

@@ -1,6 +1,7 @@
package manifest package manifest
import ( import (
"bytes"
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
@@ -69,7 +70,16 @@ func (s *LocalStore) LoadReader(ctx context.Context, source io.Reader) (*Manifes
} }
var m Manifest var m Manifest
if err := json.Unmarshal(data, &m); err != nil { decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&m); err != nil {
return nil, fmt.Errorf("decode manifest: %w", err)
}
var trailing any
if err := decoder.Decode(&trailing); err != io.EOF {
if err == nil {
return nil, fmt.Errorf("decode manifest: multiple JSON values are not allowed")
}
return nil, fmt.Errorf("decode manifest: %w", err) return nil, fmt.Errorf("decode manifest: %w", err)
} }
@@ -104,9 +114,6 @@ func (s *LocalStore) Save(ctx context.Context, path string, m *Manifest) error {
if err := validateManifestIdentities(m.SessionID, m.Campaign, m.RunID); err != nil { if err := validateManifestIdentities(m.SessionID, m.Campaign, m.RunID); err != nil {
return fmt.Errorf("save manifest: %w", err) return fmt.Errorf("save manifest: %w", err)
} }
if err := validatePostPublishCleanup(m.PostPublishCleanup); err != nil {
return fmt.Errorf("save manifest: %w", err)
}
if err := validateSessionStages(m.Stages); err != nil { if err := validateSessionStages(m.Stages); err != nil {
return fmt.Errorf("save manifest: %w", err) return fmt.Errorf("save manifest: %w", err)
} }
@@ -149,9 +156,6 @@ func validateLoadedManifest(m *Manifest) error {
if err := validateManifestIdentities(m.SessionID, m.Campaign, m.RunID); err != nil { if err := validateManifestIdentities(m.SessionID, m.Campaign, m.RunID); err != nil {
return err return err
} }
if err := validatePostPublishCleanup(m.PostPublishCleanup); err != nil {
return err
}
if err := validateSessionStages(m.Stages); err != nil { if err := validateSessionStages(m.Stages); err != nil {
return err return err
} }
@@ -175,27 +179,6 @@ func normalizeManifest(m *Manifest) {
} }
} }
func validatePostPublishCleanup(cleanup *PostPublishCleanup) error {
if cleanup == nil {
return nil
}
if err := pathsafe.ValidateOpaqueSegment(cleanup.CommittedRunID); err != nil {
return fmt.Errorf("post_publish_cleanup.committed_run_id is not a portable opaque identifier: %w", err)
}
if strings.TrimSpace(cleanup.RemoteCommitKey) == "" || strings.TrimSpace(cleanup.CurrentCommitPointerKey) == "" {
return fmt.Errorf("post_publish_cleanup commit identity is required")
}
if len(cleanup.Targets) == 0 {
return fmt.Errorf("post_publish_cleanup.targets is required")
}
for index, target := range cleanup.Targets {
if strings.TrimSpace(target.Policy) == "" || strings.TrimSpace(target.Root) == "" || strings.TrimSpace(target.Path) == "" {
return fmt.Errorf("post_publish_cleanup.targets[%d] policy, root, and path are required", index)
}
}
return nil
}
func validateManifestIdentities(sessionID, campaign, runID string) error { func validateManifestIdentities(sessionID, campaign, runID string) error {
identities := []struct { identities := []struct {
field string field string

View File

@@ -6,6 +6,7 @@ import (
"errors" "errors"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -256,6 +257,33 @@ func TestLoadRejectsInvalidManifest(t *testing.T) {
} }
} }
func TestLocalStoreRejectsRetiredCleanupFieldAndRoundTripsCanonicalV1(t *testing.T) {
store := &LocalStore{}
canonical := `{"schema_version":"narratio.session-manifest.v1","session_id":"2026-05-03","created_at":"2026-05-03T10:00:00Z","updated_at":"2026-05-03T10:01:00Z","stages":{}}`
loaded, err := store.LoadReader(context.Background(), strings.NewReader(canonical))
if err != nil {
t.Fatalf("load canonical manifest: %v", err)
}
encoded, err := json.Marshal(loaded)
if err != nil {
t.Fatalf("marshal canonical manifest: %v", err)
}
roundTripped, err := store.LoadReader(context.Background(), strings.NewReader(string(encoded)))
if err != nil {
t.Fatalf("reload canonical manifest: %v", err)
}
if !reflect.DeepEqual(roundTripped, loaded) {
t.Fatalf("round-tripped manifest = %#v, want %#v", roundTripped, loaded)
}
retired := strings.TrimSuffix(canonical, "}") + `,"post_publish_cleanup":{"committed_run_id":"20260905T010203Z-a1b2c3d4"}}`
_, err = store.LoadReader(context.Background(), strings.NewReader(retired))
if err == nil || !strings.Contains(err.Error(), `unknown field "post_publish_cleanup"`) {
t.Fatalf("load retired cleanup field error = %v, want strict unknown-field rejection", err)
}
}
func TestLocalStoreRejectsUnsupportedSchemaVersions(t *testing.T) { func TestLocalStoreRejectsUnsupportedSchemaVersions(t *testing.T) {
store := &LocalStore{} store := &LocalStore{}
now := time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC) now := time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC)