Complete session-only manifest migration
This commit is contained in:
@@ -165,7 +165,7 @@ Behavior:
|
|||||||
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
- continues interrupted or partially completed sessions by running non-succeeded stages;
|
||||||
- writes the session manifest as the sole durable progress ledger. Each
|
- writes the session manifest as the sole durable progress ledger. Each
|
||||||
invocation also creates its run-local directory and returns an immediate
|
invocation also creates its run-local directory and returns an immediate
|
||||||
command summary; neither produces a run manifest.
|
command summary; neither creates another durable progress record.
|
||||||
- reports the resolved profile (or `none`) and effective configuration digest.
|
- reports the resolved profile (or `none`) and effective configuration digest.
|
||||||
|
|
||||||
When `--artifacts` is present, the selected range must contain `analyze` or
|
When `--artifacts` is present, the selected range must contain `analyze` or
|
||||||
|
|||||||
@@ -154,7 +154,7 @@ Supported source families:
|
|||||||
|
|
||||||
- `analyze` performs no remote storage calls for previous-session source resolution.
|
- `analyze` performs no remote storage calls for previous-session source resolution.
|
||||||
- input-identity resolution is read-only: it does not invoke adapters,
|
- input-identity resolution is read-only: it does not invoke adapters,
|
||||||
materialize outputs, update status, or create run records.
|
materialize outputs, update status, or create run-local files.
|
||||||
- fingerprints exclude timeouts, retries, timestamps, producer and Narratio run
|
- fingerprints exclude timeouts, retries, timestamps, producer and Narratio run
|
||||||
IDs, executable and config paths, workspace roots, diagnostic locations, and
|
IDs, executable and config paths, workspace roots, diagnostic locations, and
|
||||||
executable or private transitive configuration contents. A change that is
|
executable or private transitive configuration contents. A change that is
|
||||||
|
|||||||
@@ -101,8 +101,8 @@ has passed validation and installation.
|
|||||||
When a replacement attempt begins, the current session-stage record no longer
|
When a replacement attempt begins, the current session-stage record no longer
|
||||||
advertises payload from the previous success. A failed replacement therefore
|
advertises payload from the previous success. A failed replacement therefore
|
||||||
has no current outputs, logs, generated configuration references, or metadata,
|
has no current outputs, logs, generated configuration references, or metadata,
|
||||||
while the earlier invocation manifest and immutable promoted bundle remain
|
while the earlier immutable promoted bundle remains available for audit and
|
||||||
available for audit and recovery.
|
recovery.
|
||||||
|
|
||||||
## Implementation And Focused Tests
|
## Implementation And Focused Tests
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ Upload run/session outputs to object storage and atomically advance remote curre
|
|||||||
## Inputs
|
## Inputs
|
||||||
|
|
||||||
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
|
- successful preceding stages from the [canonical stage set](overview.md#pipeline-stage-set)
|
||||||
- invocation-scoped run files
|
- declared run-local diagnostic files
|
||||||
- resolved publish output rules
|
- resolved publish output rules
|
||||||
- effective publish locks (static + remote merged lock set), revalidated at the
|
- effective publish locks (static + remote merged lock set), revalidated at the
|
||||||
remote commit boundary
|
remote commit boundary
|
||||||
@@ -15,7 +15,8 @@ Upload run/session outputs to object storage and atomically advance remote curre
|
|||||||
|
|
||||||
## Outputs
|
## Outputs
|
||||||
|
|
||||||
- uploaded invocation record and selected publish outputs;
|
- uploaded session manifest, declared run-local diagnostics, and selected
|
||||||
|
publish outputs;
|
||||||
- uploaded durable previous-session cache files when present;
|
- uploaded durable previous-session cache files when present;
|
||||||
- immutable run-scoped commit manifest; and
|
- immutable run-scoped commit manifest; and
|
||||||
- current commit pointer, written last.
|
- current commit pointer, written last.
|
||||||
@@ -27,13 +28,15 @@ Exact remote placement and the operator workflow belong in
|
|||||||
|
|
||||||
- when publishing or run upload is disabled, completes successfully with no
|
- when publishing or run upload is disabled, completes successfully with no
|
||||||
outputs and records explanatory metadata. This is not an explicit self-skip:
|
outputs and records explanatory metadata. This is not an explicit self-skip:
|
||||||
both manifests record success. Enablement and upload policy are fingerprinted,
|
the session manifest records success. Enablement and upload policy are
|
||||||
so changing either automatically makes the prior result non-resumable.
|
fingerprinted, so changing either automatically makes the prior result
|
||||||
|
non-resumable.
|
||||||
- validates prerequisite stage success and object-store availability.
|
- validates prerequisite stage success and object-store availability.
|
||||||
- derives a deterministic run-archive allowlist from the session manifest's
|
- derives a deterministic run-archive allowlist from the session manifest's
|
||||||
declared current-run outputs, logs, and generated configurations. References
|
declared current-run outputs, logs, and generated configurations. References
|
||||||
outside the current run root, including canonical session files and prior-run
|
outside the current run root, including canonical session files and prior-run
|
||||||
diagnostics, are not archive candidates. No run manifest is read or uploaded.
|
diagnostics, are not archive candidates. No distinguished diagnostic file is
|
||||||
|
read or uploaded.
|
||||||
- opens each archive candidate beneath its archive root without following
|
- opens each archive candidate beneath its archive root without following
|
||||||
symlinked ancestors or leaf entries, verifies that it is a regular file and
|
symlinked ancestors or leaf entries, verifies that it is a regular file and
|
||||||
checks a declared checksum when present, then streams the opened descriptor.
|
checks a declared checksum when present, then streams the opened descriptor.
|
||||||
|
|||||||
@@ -168,11 +168,11 @@ selected artifact evidence is still semantically current.
|
|||||||
Before a bounded run or plan whose range starts after `prepare`, every excluded
|
Before a bounded run or plan whose range starts after `prepare`, every excluded
|
||||||
prefix stage must already have a session-manifest status of `succeeded` or
|
prefix stage must already have a session-manifest status of `succeeded` or
|
||||||
`skipped`. Narratio reports the first absent, pending, running, failed, stale,
|
`skipped`. Narratio reports the first absent, pending, running, failed, stale,
|
||||||
or interrupted prerequisite without creating a run record or changing session
|
or interrupted prerequisite without creating a run-local directory or changing
|
||||||
state. Widen `--from` to include that stage, or recover it explicitly before
|
session state. Widen `--from` to include that stage, or recover it explicitly
|
||||||
retrying. Excluded prefix stages are not resume-validated or repaired as part
|
before retrying. Excluded prefix stages are not resume-validated or repaired as
|
||||||
of the bounded invocation; selected stages still reject missing, unsafe, or
|
part of the bounded invocation; selected stages still reject missing, unsafe,
|
||||||
manifest-inconsistent inputs at their owning boundary.
|
or manifest-inconsistent inputs at their owning boundary.
|
||||||
|
|
||||||
Stages after `--through` are not prerequisites and are never scheduled by the
|
Stages after `--through` are not prerequisites and are never scheduled by the
|
||||||
bounded invocation. A selected forced stage can mark one of those succeeded
|
bounded invocation. A selected forced stage can mark one of those succeeded
|
||||||
@@ -279,9 +279,9 @@ narratio run-stage prepare 2026-04-04 --force
|
|||||||
|
|
||||||
Starting a replacement clears the previous extraction payload from the current
|
Starting a replacement clears the previous extraction payload from the current
|
||||||
session-stage record. If that replacement fails or self-skips, the current
|
session-stage record. If that replacement fails or self-skips, the current
|
||||||
record does not fall back to the earlier outputs. The earlier run manifest and
|
record does not fall back to the earlier outputs. The earlier immutable bundle
|
||||||
immutable bundle remain available for inspection, but downstream resolution
|
remains available for inspection, but downstream resolution requires a new
|
||||||
requires a new current successful extraction record.
|
current successful extraction record.
|
||||||
|
|
||||||
Notarius extraction validates its run-local bundle and moves it once to the
|
Notarius extraction validates its run-local bundle and moves it once to the
|
||||||
unique durable bundle path. A pre-existing durable destination is a collision;
|
unique durable bundle path. A pre-existing durable destination is a collision;
|
||||||
@@ -314,9 +314,9 @@ References to canonical session files and prior-run paths are excluded. Each
|
|||||||
declared source is opened without following symlinked ancestors or the leaf,
|
declared source is opened without following symlinked ancestors or the leaf,
|
||||||
verified as a regular file, and streamed from that verified descriptor.
|
verified as a regular file, and streamed from that verified descriptor.
|
||||||
Unlisted files and unsafe entries are never uploaded; a run with no declared
|
Unlisted files and unsafe entries are never uploaded; a run with no declared
|
||||||
current-run diagnostics uploads no run-local files. No run manifest is
|
current-run diagnostics uploads no run-local files. No distinguished diagnostic
|
||||||
uploaded. The durable bundle is never scanned for implicit publication; only
|
file is uploaded. The durable bundle is never scanned for implicit publication;
|
||||||
lanes named by explicit `pipeline.publish.outputs` rules are uploaded.
|
only lanes named by explicit `pipeline.publish.outputs` rules are uploaded.
|
||||||
|
|
||||||
To intentionally replace the current extraction result, run:
|
To intentionally replace the current extraction result, run:
|
||||||
|
|
||||||
@@ -482,10 +482,9 @@ Durable session paths:
|
|||||||
- `config/**`
|
- `config/**`
|
||||||
- `runs/**`
|
- `runs/**`
|
||||||
|
|
||||||
`manifest.json` is a versioned local-state document. Narratio accepts only
|
`manifest.json` is the versioned session-progress document. Narratio accepts
|
||||||
`narratio.session-manifest.v1`; each invocation record under `runs/**` accepts
|
only `narratio.session-manifest.v1`. Do not edit unsupported session state to
|
||||||
only `narratio.run-manifest.v1`. Do not edit an older manifest to add missing
|
add missing markers. Move it aside and regenerate it, restore a verified
|
||||||
markers. Move unsupported state aside and regenerate it, restore a verified
|
|
||||||
current committed snapshot, or republish healthy regenerated state as the
|
current committed snapshot, or republish healthy regenerated state as the
|
||||||
relevant workflow requires.
|
relevant workflow requires.
|
||||||
|
|
||||||
|
|||||||
@@ -7,9 +7,10 @@ This is the executable plan for [`refactoring.md`](refactoring.md), Workstream
|
|||||||
[`contract-audit.md`](contract-audit.md) and rows H14, H15, H17, C02, and C03
|
[`contract-audit.md`](contract-audit.md) and rows H14, H15, H17, C02, and C03
|
||||||
of [`audit-decisions.md`](audit-decisions.md) own the approved decisions.
|
of [`audit-decisions.md`](audit-decisions.md) own the approved decisions.
|
||||||
|
|
||||||
All five stages are pending and must be implemented in order. Each stage is
|
**Status: Completed**
|
||||||
bounded for one `gpt-5.6-terra` prompt. Workstreams 1 through 3 are complete.
|
|
||||||
Workstreams 5 and later are out of scope: do not decompose the runner into a
|
All five stages are completed. Workstreams 1 through 4 are complete.
|
||||||
|
Workstreams 5 and later remain out of scope: do not decompose the runner into a
|
||||||
new lifecycle type, redesign pre-execution decisions, add per-artifact analyze
|
new lifecycle type, redesign pre-execution decisions, add per-artifact analyze
|
||||||
checkpoints, split publish into separate transactions, add cleanup as a stage,
|
checkpoints, split publish into separate transactions, add cleanup as a stage,
|
||||||
or change remote lock policy while following this plan.
|
or change remote lock policy while following this plan.
|
||||||
@@ -333,7 +334,7 @@ path, and mechanism tests from their owning packages.
|
|||||||
|
|
||||||
## Stage 5 — Reconcile Current Documentation And Close Workstream 4
|
## Stage 5 — Reconcile Current Documentation And Close Workstream 4
|
||||||
|
|
||||||
**Status: Pending**
|
**Status: Completed**
|
||||||
|
|
||||||
### Goal
|
### Goal
|
||||||
|
|
||||||
|
|||||||
@@ -189,7 +189,7 @@ code and fail clearly instead of being inferred or normalized.
|
|||||||
|
|
||||||
### Workstream 4 — One durable local manifest authority
|
### Workstream 4 — One durable local manifest authority
|
||||||
|
|
||||||
**Status:** Pending
|
**Status:** Complete
|
||||||
|
|
||||||
**Audit sources:** H14, H15, H17, C02, and C03.
|
**Audit sources:** H14, H15, H17, C02, and C03.
|
||||||
|
|
||||||
|
|||||||
@@ -197,8 +197,9 @@ Safe fix:
|
|||||||
|
|
||||||
- widen the bounded range to include the first reported stage, or recover that
|
- widen the bounded range to include the first reported stage, or recover that
|
||||||
stage explicitly with `run-stage` before retrying. The failed check does not
|
stage explicitly with `run-stage` before retrying. The failed check does not
|
||||||
create a run record or modify the manifest. Narratio does not resume-validate
|
create a run-local directory or modify the manifest. Narratio does not
|
||||||
excluded prefix stages, and stages after `--through` are not prerequisites.
|
resume-validate excluded prefix stages, and stages after `--through` are not
|
||||||
|
prerequisites.
|
||||||
|
|
||||||
If prerequisite statuses are terminal but a selected stage reports a missing,
|
If prerequisite statuses are terminal but a selected stage reports a missing,
|
||||||
unsafe, or checksum-inconsistent artifact, repair the artifact at the stage
|
unsafe, or checksum-inconsistent artifact, repair the artifact at the stage
|
||||||
|
|||||||
@@ -142,7 +142,7 @@ func publishCleanupEligible(cfg *config.Config, m *manifest.Manifest, sr *manife
|
|||||||
return "", false, "publish stage was skipped"
|
return "", false, "publish stage was skipped"
|
||||||
}
|
}
|
||||||
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
|
if uploaded, _ := sr.Metadata["uploaded"].(bool); !uploaded {
|
||||||
return "", false, "publish did not upload run record"
|
return "", false, "publish did not complete an upload"
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(asString(sr.Metadata["remote_commit_key"])) == "" {
|
if strings.TrimSpace(asString(sr.Metadata["remote_commit_key"])) == "" {
|
||||||
return "", false, "publish remote commit key is missing"
|
return "", false, "publish remote commit key is missing"
|
||||||
|
|||||||
@@ -370,7 +370,6 @@ func TestPostPublishCleanupNotRunWhenOutputIsMissing(t *testing.T) {
|
|||||||
|
|
||||||
assertExists(t, seed.spoolAudioDir)
|
assertExists(t, seed.spoolAudioDir)
|
||||||
assertExists(t, seed.runWorkDir)
|
assertExists(t, seed.runWorkDir)
|
||||||
assertExists(t, filepath.Join(seed.runWorkDir, "manifest.json"))
|
|
||||||
assertExists(t, artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID))
|
assertExists(t, artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, runID))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -467,7 +466,6 @@ func cleanupFixtureConfig(t *testing.T) (*config.Config, cleanupSeed) {
|
|||||||
"session_recap.md",
|
"session_recap.md",
|
||||||
)
|
)
|
||||||
|
|
||||||
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
|
|
||||||
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "stage.log"), "log\n")
|
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "stage.log"), "log\n")
|
||||||
mustWriteFile(t, filepath.Join(otherRunDir, "logs", "stage.log"), "other\n")
|
mustWriteFile(t, filepath.Join(otherRunDir, "logs", "stage.log"), "other\n")
|
||||||
mustWriteFile(t, filepath.Join(spoolAudioDir, "speaker.flac"), "flac\n")
|
mustWriteFile(t, filepath.Join(spoolAudioDir, "speaker.flac"), "flac\n")
|
||||||
@@ -562,8 +560,6 @@ func writePublishFixtureRunFiles(t *testing.T, runWorkDir, sessionRoot string) {
|
|||||||
mustWriteFile(t, filepath.Join(runWorkDir, "polish", "reports", "audita.report.json"), "{}\n")
|
mustWriteFile(t, filepath.Join(runWorkDir, "polish", "reports", "audita.report.json"), "{}\n")
|
||||||
mustWriteFile(t, filepath.Join(runWorkDir, "merge", "config", "seriatim.generated.yml"), "key: value\n")
|
mustWriteFile(t, filepath.Join(runWorkDir, "merge", "config", "seriatim.generated.yml"), "key: value\n")
|
||||||
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "audita.stderr.log"), "stderr\n")
|
mustWriteFile(t, filepath.Join(runWorkDir, "logs", "audita.stderr.log"), "stderr\n")
|
||||||
mustWriteFile(t, filepath.Join(runWorkDir, "manifest.json"), "{}\n")
|
|
||||||
|
|
||||||
mustWriteFile(t, filepath.Join(sessionRoot, "transcripts", "final.trimmed.json"), "{\"segments\":[]}\n")
|
mustWriteFile(t, filepath.Join(sessionRoot, "transcripts", "final.trimmed.json"), "{\"segments\":[]}\n")
|
||||||
mustWriteFile(t, filepath.Join(sessionRoot, "artifacts", "session_recap.md"), "# recap\n")
|
mustWriteFile(t, filepath.Join(sessionRoot, "artifacts", "session_recap.md"), "# recap\n")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -135,9 +135,6 @@ func TestRestoreThenRunStageForceAnalyzeUsesRestoredDurableState(t *testing.T) {
|
|||||||
if info, statErr := os.Stat(runRoot); statErr != nil || !info.IsDir() {
|
if info, statErr := os.Stat(runRoot); statErr != nil || !info.IsDir() {
|
||||||
t.Fatalf("run directory %q = %#v, %v; want directory", runRoot, info, statErr)
|
t.Fatalf("run directory %q = %#v, %v; want directory", runRoot, info, statErr)
|
||||||
}
|
}
|
||||||
if _, statErr := os.Stat(filepath.Join(runRoot, "manifest.json")); !os.IsNotExist(statErr) {
|
|
||||||
t.Fatalf("run manifest stat = %v, want not exist", statErr)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRestoreThenAnalyzeUsesRestoredPreviousCacheWithoutObjectStore(t *testing.T) {
|
func TestRestoreThenAnalyzeUsesRestoredPreviousCacheWithoutObjectStore(t *testing.T) {
|
||||||
|
|||||||
@@ -447,10 +447,9 @@ func persistPostPublishCleanupFailure(
|
|||||||
if sessionManifest != nil {
|
if sessionManifest != nil {
|
||||||
sessionManifest.RecordFailure(nowUTC(), operationErr.Error())
|
sessionManifest.RecordFailure(nowUTC(), operationErr.Error())
|
||||||
}
|
}
|
||||||
// Do not rewrite the run manifest here: cleanup may have deliberately
|
// Cleanup may have deliberately removed the selected run directory. The
|
||||||
// removed the directory that contains it. The completed run record remains
|
// committed remote snapshot remains authoritative, while the session
|
||||||
// the authoritative evidence of the remote publish; the session manifest
|
// manifest records the outstanding local cleanup for the next invocation.
|
||||||
// records the outstanding local cleanup for the next invocation.
|
|
||||||
sessionErr := sessionStore.Save(ctx, sessionManifestPath, sessionManifest)
|
sessionErr := sessionStore.Save(ctx, sessionManifestPath, sessionManifest)
|
||||||
return errors.Join(operationErr, wrapTerminalPersistenceError("save terminal session manifest", sessionErr))
|
return errors.Join(operationErr, wrapTerminalPersistenceError("save terminal session manifest", sessionErr))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -854,9 +854,6 @@ func TestExecuteStagesCreatesRunDirectoryPerInvocation(t *testing.T) {
|
|||||||
if info, statErr := os.Stat(wantWorkDir); statErr != nil || !info.IsDir() {
|
if info, statErr := os.Stat(wantWorkDir); statErr != nil || !info.IsDir() {
|
||||||
t.Fatalf("run directory %q = %#v, %v; want directory", wantWorkDir, info, statErr)
|
t.Fatalf("run directory %q = %#v, %v; want directory", wantWorkDir, info, statErr)
|
||||||
}
|
}
|
||||||
if _, statErr := os.Stat(filepath.Join(wantWorkDir, "manifest.json")); !os.IsNotExist(statErr) {
|
|
||||||
t.Fatalf("run manifest stat = %v, want not exist", statErr)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if sessionManifest.LocalWorkDir == artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, run1.RunID) {
|
if sessionManifest.LocalWorkDir == artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, cfg.Session.Campaign, cfg.Session.SessionID, run1.RunID) {
|
||||||
t.Fatal("session manifest retained the prior invocation work directory")
|
t.Fatal("session manifest retained the prior invocation work directory")
|
||||||
|
|||||||
@@ -413,8 +413,8 @@ func analyzeProducerRunID(execution analyzeExecutionContext) string {
|
|||||||
return runID
|
return runID
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Direct stage callers predate invocation manifests. Application-owned
|
// Direct stage callers use a stable synthetic ID. Application-owned execution
|
||||||
// execution always supplies the actual run identity.
|
// always supplies the actual run identity.
|
||||||
return "direct-analyze"
|
return "direct-analyze"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ func TestPublishFailsWhenPrerequisiteNotSucceeded(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPublishUploadsRunRecordPublishedOutputsAndCurrentPointer(t *testing.T) {
|
func TestPublishUploadsRunFilesPublishedOutputsAndCurrentPointer(t *testing.T) {
|
||||||
env, m, runRoot := publishFixture(t)
|
env, m, runRoot := publishFixture(t)
|
||||||
fake := env.ObjectStore.(*storage.FakeBackend)
|
fake := env.ObjectStore.(*storage.FakeBackend)
|
||||||
lanePath := configurePublishExtractionFixture(t, env, m)
|
lanePath := configurePublishExtractionFixture(t, env, m)
|
||||||
|
|||||||
Reference in New Issue
Block a user