Remove retired cleanup manifest state
This commit is contained in:
@@ -78,7 +78,9 @@ Execution order and safety:
|
|||||||
- restored manifest local references are rebased beneath the selected local
|
- restored manifest local references are rebased beneath the selected local
|
||||||
session root. Unsafe relative references and producer-machine absolute paths
|
session root. Unsafe relative references and producer-machine absolute paths
|
||||||
outside the manifest's producer session root are rejected; producer-local
|
outside the manifest's producer session root are rejected; producer-local
|
||||||
spool/cache and cleanup locations are not restored as authority.
|
spool paths and run-local diagnostic references are not restored as
|
||||||
|
authority. Cleanup remains ordinary restored stage state and is not reset or
|
||||||
|
synthesized by restore.
|
||||||
|
|
||||||
Audio restore path:
|
Audio restore path:
|
||||||
|
|
||||||
|
|||||||
@@ -16,16 +16,18 @@ manifest placement belongs in
|
|||||||
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
|
- remote identity metadata (`s3_bucket`, `s3_session_prefix`, `s3_run_prefix`)
|
||||||
- `inputs` records
|
- `inputs` records
|
||||||
- durable `artifacts` records
|
- durable `artifacts` records
|
||||||
- per-stage `stages` map
|
- per-stage `stages` map, including the ordinary cleanup stage record
|
||||||
- an optional `post_publish_cleanup` obligation, which binds a committed run,
|
|
||||||
remote commit identity, and each exact root-confined local target to its
|
|
||||||
completion evidence
|
|
||||||
|
|
||||||
Session, campaign, and run identities in local and downloaded manifests must be
|
Session, campaign, and run identities in local and downloaded manifests must be
|
||||||
portable opaque segments. Unsafe legacy identities are rejected with migration
|
portable opaque segments. Unsafe legacy identities are rejected with migration
|
||||||
guidance rather than being normalized into a different workspace or remote
|
guidance rather than being normalized into a different workspace or remote
|
||||||
namespace.
|
namespace.
|
||||||
|
|
||||||
|
Decoding is strict at every typed manifest field. Unknown fields are rejected;
|
||||||
|
the schema version remains `narratio.session-manifest.v1` and there is no
|
||||||
|
compatibility reader that silently drops retired state. Cleanup progress uses
|
||||||
|
only the cleanup entry in `stages`.
|
||||||
|
|
||||||
Every persisted session manifest has a non-nil `stages` map. Each entry is
|
Every persisted session manifest has a non-nil `stages` map. Each entry is
|
||||||
non-nil and its embedded `name` exactly matches its map key; loading never
|
non-nil and its embedded `name` exactly matches its map key; loading never
|
||||||
invents either marker. A missing or unsupported schema version, or incomplete
|
invents either marker. A missing or unsupported schema version, or incomplete
|
||||||
@@ -145,6 +147,7 @@ establish a current immutable snapshot.
|
|||||||
`manifest.LocalStore`:
|
`manifest.LocalStore`:
|
||||||
|
|
||||||
- accepts only the exact supported session-manifest schema version;
|
- accepts only the exact supported session-manifest schema version;
|
||||||
|
- rejects unknown fields and multiple JSON values;
|
||||||
- validates stage-map identity without synthesizing missing maps, records, or
|
- validates stage-map identity without synthesizing missing maps, records, or
|
||||||
names;
|
names;
|
||||||
- retains only representation-preserving collection normalization;
|
- retains only representation-preserving collection normalization;
|
||||||
|
|||||||
@@ -610,6 +610,14 @@ also available when the manifest has valid publish evidence. A range ending at
|
|||||||
`notify` or earlier and the standalone `publish` command never delete local
|
`notify` or earlier and the standalone `publish` command never delete local
|
||||||
state.
|
state.
|
||||||
|
|
||||||
|
Older local manifests that contain the retired cleanup-obligation field are
|
||||||
|
intentionally incompatible with strict manifest decoding. If such an
|
||||||
|
obligation is still outstanding, complete it with the pre-upgrade binary.
|
||||||
|
Otherwise verify that the remote publication is committed, move aside or
|
||||||
|
regenerate the incompatible local manifest (or restore the verified remote
|
||||||
|
snapshot), and then use the cleanup stage. Manual JSON editing is not the
|
||||||
|
normal migration path.
|
||||||
|
|
||||||
## Operational Caveats
|
## Operational Caveats
|
||||||
|
|
||||||
- Local and S3 audio modes are mutually exclusive.
|
- Local and S3 audio modes are mutually exclusive.
|
||||||
|
|||||||
@@ -246,7 +246,7 @@ post-loop deletion path.
|
|||||||
|
|
||||||
## Stage 4 — Remove The Cleanup Obligation From Manifest State
|
## Stage 4 — Remove The Cleanup Obligation From Manifest State
|
||||||
|
|
||||||
**Status:** Pending
|
**Status:** Completed
|
||||||
|
|
||||||
### Goal
|
### Goal
|
||||||
|
|
||||||
|
|||||||
@@ -73,6 +73,11 @@ Safe fix:
|
|||||||
regenerate it; restore a verified current committed snapshot or republish
|
regenerate it; restore a verified current committed snapshot or republish
|
||||||
healthy regenerated state when appropriate. Do not add version or stage
|
healthy regenerated state when appropriate. Do not add version or stage
|
||||||
markers by hand.
|
markers by hand.
|
||||||
|
- if an older manifest is rejected for its retired cleanup-obligation field,
|
||||||
|
finish outstanding cleanup with the pre-upgrade binary. Otherwise verify the
|
||||||
|
remote commit, move aside or regenerate the incompatible manifest, and use
|
||||||
|
the cleanup stage. Do not treat manual JSON editing as the normal migration
|
||||||
|
path.
|
||||||
|
|
||||||
Relevant reference: [Configuration](./config.md).
|
Relevant reference: [Configuration](./config.md).
|
||||||
|
|
||||||
|
|||||||
@@ -122,9 +122,6 @@ func rebaseRestoredManifestPaths(cfg *config.Config, m *manifest.Manifest, desti
|
|||||||
m.LocalWorkDir = destinationRoot
|
m.LocalWorkDir = destinationRoot
|
||||||
}
|
}
|
||||||
m.LocalSpoolDir = ""
|
m.LocalSpoolDir = ""
|
||||||
// A post-publish cleanup record is authority to delete producer-local
|
|
||||||
// directories. It must never cross a restore boundary.
|
|
||||||
m.PostPublishCleanup = nil
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -168,6 +168,7 @@ func TestRebaseRestoredManifestPathsRejectsUnsafeReferencesAndRebasesPortablePat
|
|||||||
m.RunID = "20260519T010203Z-a1b2c3d4"
|
m.RunID = "20260519T010203Z-a1b2c3d4"
|
||||||
m.LocalWorkDir = test.workDir
|
m.LocalWorkDir = test.workDir
|
||||||
m.Stages["analyze"] = &manifest.StageRecord{Outputs: []manifest.ArtifactRecord{{Kind: "recap", LocalPath: test.output}}}
|
m.Stages["analyze"] = &manifest.StageRecord{Outputs: []manifest.ArtifactRecord{{Kind: "recap", LocalPath: test.output}}}
|
||||||
|
m.MarkStageFailed("cleanup", nowUTC(), "retry local deletion")
|
||||||
destination := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
destination := artifacts.SessionWorkDirForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID)
|
||||||
|
|
||||||
err := rebaseRestoredManifestPaths(cfg, m, destination)
|
err := rebaseRestoredManifestPaths(cfg, m, destination)
|
||||||
@@ -183,6 +184,9 @@ func TestRebaseRestoredManifestPathsRejectsUnsafeReferencesAndRebasesPortablePat
|
|||||||
if got, want := m.Stages["analyze"].Outputs[0].LocalPath, filepath.Join(destination, test.wantPath); got != want {
|
if got, want := m.Stages["analyze"].Outputs[0].LocalPath, filepath.Join(destination, test.wantPath); got != want {
|
||||||
t.Fatalf("rebased local path = %q, want %q", got, want)
|
t.Fatalf("rebased local path = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
if got := m.Stages["cleanup"].Status; got != manifest.StatusFailed {
|
||||||
|
t.Fatalf("cleanup status = %q, want preserved failed state", got)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -66,25 +66,6 @@ type StageRecord struct {
|
|||||||
AnalyzeArtifacts map[string]AnalyzeArtifactRecord `json:"analyze_artifacts,omitempty"`
|
AnalyzeArtifacts map[string]AnalyzeArtifactRecord `json:"analyze_artifacts,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanupTarget records one root-confined local deletion requested by a
|
|
||||||
// committed publication.
|
|
||||||
type CleanupTarget struct {
|
|
||||||
Policy string `json:"policy"`
|
|
||||||
Root string `json:"root"`
|
|
||||||
Path string `json:"path"`
|
|
||||||
Completed bool `json:"completed"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// PostPublishCleanup records the durable deletion work left by a committed
|
|
||||||
// publish. It remains after completion as evidence of the exact committed run
|
|
||||||
// and local paths involved.
|
|
||||||
type PostPublishCleanup struct {
|
|
||||||
CommittedRunID string `json:"committed_run_id"`
|
|
||||||
RemoteCommitKey string `json:"remote_commit_key"`
|
|
||||||
CurrentCommitPointerKey string `json:"current_commit_pointer_key"`
|
|
||||||
Targets []CleanupTarget `json:"targets"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// EffectiveConfigProvenance records bounded non-secret configuration identity
|
// EffectiveConfigProvenance records bounded non-secret configuration identity
|
||||||
// for one resolved invocation.
|
// for one resolved invocation.
|
||||||
type EffectiveConfigProvenance struct {
|
type EffectiveConfigProvenance struct {
|
||||||
@@ -101,24 +82,23 @@ type SelectedProfileProvenance struct {
|
|||||||
|
|
||||||
// Manifest is the durable run-state record for a session execution.
|
// Manifest is the durable run-state record for a session execution.
|
||||||
type Manifest struct {
|
type Manifest struct {
|
||||||
SchemaVersion string `json:"schema_version"`
|
SchemaVersion string `json:"schema_version"`
|
||||||
SessionID string `json:"session_id"`
|
SessionID string `json:"session_id"`
|
||||||
Campaign string `json:"campaign,omitempty"`
|
Campaign string `json:"campaign,omitempty"`
|
||||||
RunID string `json:"run_id,omitempty"`
|
RunID string `json:"run_id,omitempty"`
|
||||||
LocalWorkDir string `json:"local_workdir,omitempty"`
|
LocalWorkDir string `json:"local_workdir,omitempty"`
|
||||||
LocalSpoolDir string `json:"local_spool_dir,omitempty"`
|
LocalSpoolDir string `json:"local_spool_dir,omitempty"`
|
||||||
S3Bucket string `json:"s3_bucket,omitempty"`
|
S3Bucket string `json:"s3_bucket,omitempty"`
|
||||||
S3SessionPrefix string `json:"s3_session_prefix,omitempty"`
|
S3SessionPrefix string `json:"s3_session_prefix,omitempty"`
|
||||||
S3RunPrefix string `json:"s3_run_prefix,omitempty"`
|
S3RunPrefix string `json:"s3_run_prefix,omitempty"`
|
||||||
PipelineVersion string `json:"pipeline_version,omitempty"`
|
PipelineVersion string `json:"pipeline_version,omitempty"`
|
||||||
EffectiveConfig *EffectiveConfigProvenance `json:"effective_config,omitempty"`
|
EffectiveConfig *EffectiveConfigProvenance `json:"effective_config,omitempty"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
UpdatedAt time.Time `json:"updated_at"`
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
LastError *ErrorRecord `json:"last_error,omitempty"`
|
LastError *ErrorRecord `json:"last_error,omitempty"`
|
||||||
Inputs []InputRecord `json:"inputs,omitempty"`
|
Inputs []InputRecord `json:"inputs,omitempty"`
|
||||||
Artifacts []ArtifactRecord `json:"artifacts,omitempty"`
|
Artifacts []ArtifactRecord `json:"artifacts,omitempty"`
|
||||||
Stages map[string]*StageRecord `json:"stages"`
|
Stages map[string]*StageRecord `json:"stages"`
|
||||||
PostPublishCleanup *PostPublishCleanup `json:"post_publish_cleanup,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// New constructs a new manifest with deterministic timestamps.
|
// New constructs a new manifest with deterministic timestamps.
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package manifest
|
package manifest
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -69,7 +70,16 @@ func (s *LocalStore) LoadReader(ctx context.Context, source io.Reader) (*Manifes
|
|||||||
}
|
}
|
||||||
|
|
||||||
var m Manifest
|
var m Manifest
|
||||||
if err := json.Unmarshal(data, &m); err != nil {
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
decoder.DisallowUnknownFields()
|
||||||
|
if err := decoder.Decode(&m); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode manifest: %w", err)
|
||||||
|
}
|
||||||
|
var trailing any
|
||||||
|
if err := decoder.Decode(&trailing); err != io.EOF {
|
||||||
|
if err == nil {
|
||||||
|
return nil, fmt.Errorf("decode manifest: multiple JSON values are not allowed")
|
||||||
|
}
|
||||||
return nil, fmt.Errorf("decode manifest: %w", err)
|
return nil, fmt.Errorf("decode manifest: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -104,9 +114,6 @@ func (s *LocalStore) Save(ctx context.Context, path string, m *Manifest) error {
|
|||||||
if err := validateManifestIdentities(m.SessionID, m.Campaign, m.RunID); err != nil {
|
if err := validateManifestIdentities(m.SessionID, m.Campaign, m.RunID); err != nil {
|
||||||
return fmt.Errorf("save manifest: %w", err)
|
return fmt.Errorf("save manifest: %w", err)
|
||||||
}
|
}
|
||||||
if err := validatePostPublishCleanup(m.PostPublishCleanup); err != nil {
|
|
||||||
return fmt.Errorf("save manifest: %w", err)
|
|
||||||
}
|
|
||||||
if err := validateSessionStages(m.Stages); err != nil {
|
if err := validateSessionStages(m.Stages); err != nil {
|
||||||
return fmt.Errorf("save manifest: %w", err)
|
return fmt.Errorf("save manifest: %w", err)
|
||||||
}
|
}
|
||||||
@@ -149,9 +156,6 @@ func validateLoadedManifest(m *Manifest) error {
|
|||||||
if err := validateManifestIdentities(m.SessionID, m.Campaign, m.RunID); err != nil {
|
if err := validateManifestIdentities(m.SessionID, m.Campaign, m.RunID); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := validatePostPublishCleanup(m.PostPublishCleanup); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := validateSessionStages(m.Stages); err != nil {
|
if err := validateSessionStages(m.Stages); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -175,27 +179,6 @@ func normalizeManifest(m *Manifest) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func validatePostPublishCleanup(cleanup *PostPublishCleanup) error {
|
|
||||||
if cleanup == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := pathsafe.ValidateOpaqueSegment(cleanup.CommittedRunID); err != nil {
|
|
||||||
return fmt.Errorf("post_publish_cleanup.committed_run_id is not a portable opaque identifier: %w", err)
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(cleanup.RemoteCommitKey) == "" || strings.TrimSpace(cleanup.CurrentCommitPointerKey) == "" {
|
|
||||||
return fmt.Errorf("post_publish_cleanup commit identity is required")
|
|
||||||
}
|
|
||||||
if len(cleanup.Targets) == 0 {
|
|
||||||
return fmt.Errorf("post_publish_cleanup.targets is required")
|
|
||||||
}
|
|
||||||
for index, target := range cleanup.Targets {
|
|
||||||
if strings.TrimSpace(target.Policy) == "" || strings.TrimSpace(target.Root) == "" || strings.TrimSpace(target.Path) == "" {
|
|
||||||
return fmt.Errorf("post_publish_cleanup.targets[%d] policy, root, and path are required", index)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateManifestIdentities(sessionID, campaign, runID string) error {
|
func validateManifestIdentities(sessionID, campaign, runID string) error {
|
||||||
identities := []struct {
|
identities := []struct {
|
||||||
field string
|
field string
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -256,6 +257,33 @@ func TestLoadRejectsInvalidManifest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLocalStoreRejectsRetiredCleanupFieldAndRoundTripsCanonicalV1(t *testing.T) {
|
||||||
|
store := &LocalStore{}
|
||||||
|
canonical := `{"schema_version":"narratio.session-manifest.v1","session_id":"2026-05-03","created_at":"2026-05-03T10:00:00Z","updated_at":"2026-05-03T10:01:00Z","stages":{}}`
|
||||||
|
|
||||||
|
loaded, err := store.LoadReader(context.Background(), strings.NewReader(canonical))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load canonical manifest: %v", err)
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(loaded)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal canonical manifest: %v", err)
|
||||||
|
}
|
||||||
|
roundTripped, err := store.LoadReader(context.Background(), strings.NewReader(string(encoded)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reload canonical manifest: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(roundTripped, loaded) {
|
||||||
|
t.Fatalf("round-tripped manifest = %#v, want %#v", roundTripped, loaded)
|
||||||
|
}
|
||||||
|
|
||||||
|
retired := strings.TrimSuffix(canonical, "}") + `,"post_publish_cleanup":{"committed_run_id":"20260905T010203Z-a1b2c3d4"}}`
|
||||||
|
_, err = store.LoadReader(context.Background(), strings.NewReader(retired))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), `unknown field "post_publish_cleanup"`) {
|
||||||
|
t.Fatalf("load retired cleanup field error = %v, want strict unknown-field rejection", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLocalStoreRejectsUnsupportedSchemaVersions(t *testing.T) {
|
func TestLocalStoreRejectsUnsupportedSchemaVersions(t *testing.T) {
|
||||||
store := &LocalStore{}
|
store := &LocalStore{}
|
||||||
now := time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC)
|
||||||
|
|||||||
Reference in New Issue
Block a user