Harden pipeline state and plan release upgrades
This commit is contained in:
@@ -129,12 +129,15 @@ metadata does not participate in YAML decoding or alter the public
|
||||
configuration model.
|
||||
|
||||
The effective digest is SHA-256 over deterministic canonical YAML produced from
|
||||
the defaulted and path-resolved `PipelineConfig`. Because composition and
|
||||
resolution metadata are private, the digest excludes source layout, profile
|
||||
name, and ownership. Configuration stores environment variable names rather
|
||||
than resolving raw credentials, so raw secret values are neither loaded nor
|
||||
hashed. `recomputePipelineEffectiveDigest` is the single package-owned refresh
|
||||
point for later runtime expansion.
|
||||
the defaulted `PipelineConfig`. Runtime Notarius paths remain absolute for
|
||||
execution, but the digest substitutes their normalized logical values captured
|
||||
before root-relative resolution, so relocating an equivalent configuration
|
||||
bundle does not change provenance. Because composition and resolution metadata
|
||||
are private, the digest excludes source layout, profile name, and ownership.
|
||||
Configuration stores environment variable names rather than resolving raw
|
||||
credentials, so raw secret values are neither loaded nor hashed.
|
||||
`recomputePipelineEffectiveDigest` is the single package-owned refresh point
|
||||
for later runtime expansion.
|
||||
|
||||
## Test Surfaces
|
||||
|
||||
|
||||
@@ -73,8 +73,12 @@ evidence rerun once when selected.
|
||||
Workspace, spool, and cache placement and absolute source relocation are not
|
||||
semantic when logical selection, canonical names, and bytes are equivalent.
|
||||
The fingerprint deliberately does not read or rehash large audio. Prepared
|
||||
input checksums remain the content provenance; force prepare after changing
|
||||
source bytes that are not otherwise reflected by the semantic selection.
|
||||
input checksums remain the content provenance. Before reusing success, prepare
|
||||
validates every durable prepared copy and compares current stable-input bytes,
|
||||
canonical party and derived-player bytes, local audio membership/checksums, or
|
||||
S3 key/size/entity-tag identity with that provenance. Source relocation with
|
||||
equivalent names and bytes remains reusable; changed or unavailable evidence
|
||||
causes a normal prepare rerun.
|
||||
|
||||
## Related Contracts And Tests
|
||||
|
||||
@@ -86,6 +90,8 @@ source bytes that are not otherwise reflected by the semantic selection.
|
||||
the internal collaborators.
|
||||
- Implementation and tests: `internal/stage/prepare.go`,
|
||||
`internal/stage/prepare_test.go`,
|
||||
`internal/stage/prepare_resume.go`,
|
||||
`internal/stage/prepare_resume_test.go`,
|
||||
`internal/stage/semantic_contracts_initial.go`,
|
||||
`internal/stage/semantic_contracts_initial_test.go`,
|
||||
`internal/audio/s3_audio_test.go`,
|
||||
|
||||
@@ -115,7 +115,11 @@ Execution rules:
|
||||
Every aggregate stage except analyze currently provides semantic-configuration
|
||||
evidence; analyze retains its more precise per-artifact fingerprints and
|
||||
validator.
|
||||
Changing prepare selection semantics reruns all fixed descendants; changing
|
||||
Prepare additionally validates current stable/local/S3 source identity and the
|
||||
checksums of its durable prepared copies before reuse. Changed bytes, audio
|
||||
membership, S3 object identity, or missing/tampered copies rerun prepare and
|
||||
its fixed descendants without requiring `--force`. Changing prepare selection
|
||||
semantics likewise reruns all fixed descendants; changing
|
||||
WhisperX language/service identity reuses prepare; changing a Seriatim merge
|
||||
transformation reuses prepare and transcribe; and changing an Audita model
|
||||
reuses prepare, transcribe, and merge while rebuilding transcript refinement.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
303
docs/roadmap/release-procedure.md
Normal file
303
docs/roadmap/release-procedure.md
Normal file
@@ -0,0 +1,303 @@
|
||||
# Release Procedure Upgrade
|
||||
|
||||
## Status
|
||||
|
||||
Accepted target state. This document owns the intended release-process scope,
|
||||
maintainer policy, and target end state until the work is implemented.
|
||||
|
||||
## Goal
|
||||
|
||||
Give Narratio one reproducible, guarded minor/patch release procedure that
|
||||
validates a source candidate before publication, creates the existing binary
|
||||
release inputs consistently, and makes the exact upstream tag push the clear
|
||||
completion boundary.
|
||||
|
||||
The procedure should borrow WeatherReporter's binary-publication model and
|
||||
Notarius's centralized source-candidate validation and immutable-tag guards,
|
||||
while retaining Narratio's own repository-wide checks and six supported build
|
||||
targets.
|
||||
|
||||
## Responsibility Boundary
|
||||
|
||||
A Narratio release is complete when the selected tag has been successfully
|
||||
pushed to its exact tag ref in the upstream Git repository. The release command
|
||||
must then return successfully without polling Woodpecker, waiting for a
|
||||
pipeline, querying a Gitea release, downloading assets, or otherwise making its
|
||||
result depend on CI availability.
|
||||
|
||||
The tag event may asynchronously cause Woodpecker to validate the tagged tree,
|
||||
build binaries and checksums, and create a Gitea release. That work is outside
|
||||
the synchronous release procedure. CI failure or delay does not change whether
|
||||
the Git tag was successfully published, and release tooling must never wait
|
||||
indefinitely for infrastructure outside the upstream Git push.
|
||||
|
||||
An operator may inspect CI or published assets later. Such inspection is an
|
||||
optional asynchronous follow-up, not a release completion condition and not a
|
||||
step that an automated coding agent must perform after pushing the tag.
|
||||
|
||||
## Release Model
|
||||
|
||||
- Releases use unused stable semantic-version tags in the form
|
||||
`vMAJOR.MINOR.PATCH`; prereleases are outside this feature.
|
||||
- Narratio is past `v1.0.0`, so a minor release adds backward-compatible
|
||||
functionality, a patch release contains backward-compatible fixes, and an
|
||||
intentional breaking change to a documented public contract requires a new
|
||||
major version.
|
||||
- Release tags remain lightweight, matching Narratio's established tag
|
||||
history, and are bound explicitly to a guarded commit on `main`.
|
||||
- Every new release has a nonempty checked-in note at
|
||||
`docs/releases/<tag>.md` in the tagged tree.
|
||||
- Published tags are immutable. They are never moved, reused, overwritten, or
|
||||
deleted, even when asynchronous CI fails to create a complete binary
|
||||
release.
|
||||
- The Gitea release, when asynchronous publication succeeds, remains the
|
||||
canonical source for downloadable binaries and checksums. The Git tag and
|
||||
checked-in note remain valid source-release history independently of CI.
|
||||
- Existing historical tags and notes are not retroactively rewritten or
|
||||
backfilled to satisfy the new procedure.
|
||||
|
||||
## Canonical Documentation
|
||||
|
||||
Add `docs/release.md` as the canonical maintainer procedure. Update
|
||||
`docs/development.md` to route release preparation, tagging, and publication to
|
||||
it, and update the documentation policy so it assigns ownership as follows:
|
||||
|
||||
- `docs/release.md` owns version selection, candidate preparation, validation,
|
||||
tag guards, tag publication, the responsibility boundary, and failure
|
||||
recovery;
|
||||
- `docs/releases/<tag>.md` owns the immutable historical summary for one
|
||||
release;
|
||||
- `docs/releases/README.md` owns the release-note index and points to the Gitea
|
||||
release collection for downloadable assets; and
|
||||
- current CLI, configuration, operation, integration, and internal contracts
|
||||
remain in their existing canonical documents rather than being restated in
|
||||
release notes.
|
||||
|
||||
The maintainer procedure must be written as a complete safe workflow suitable
|
||||
for either a human maintainer or a coding agent. Commands must be explicit,
|
||||
non-interactive, and scoped to the selected version and tag ref.
|
||||
|
||||
## Release Notes
|
||||
|
||||
Require each future note to use this minimum structure:
|
||||
|
||||
```markdown
|
||||
# Narratio vMAJOR.MINOR.PATCH
|
||||
|
||||
This release ...
|
||||
|
||||
## Summary
|
||||
|
||||
## Compatibility
|
||||
|
||||
## Upgrade
|
||||
|
||||
## Changes
|
||||
```
|
||||
|
||||
The compatibility section must identify changes to documented CLI,
|
||||
configuration, durable artifact, integration, or operating contracts. The
|
||||
upgrade section must state required operator actions or explicitly say that no
|
||||
special action is required. When the Notarius consumer contract changes, the
|
||||
note should identify the supported/tested contract version and any required
|
||||
Notarius upgrade.
|
||||
|
||||
Release notes summarize a historical candidate and link to current canonical
|
||||
documentation for exact behavior. They must not substitute for updating that
|
||||
documentation, include credentials or private infrastructure details, or make
|
||||
claims that have not been validated against the candidate.
|
||||
|
||||
## Shared Release Scripts
|
||||
|
||||
### Source-candidate checker
|
||||
|
||||
Add the POSIX-shell checker:
|
||||
|
||||
```text
|
||||
scripts/check-release-candidate.sh vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
It must be deterministic, non-interactive, safe to run repeatedly, and usable
|
||||
both by maintainers before tagging and by tag CI in a detached checkout. It
|
||||
must not create or push Git tags, modify tracked source, contact CI, or require
|
||||
live WhisperX, LLM, Notarius, Scriptorium, object-storage, or notification
|
||||
services.
|
||||
|
||||
The checker must fail clearly unless all of the following hold:
|
||||
|
||||
- the argument is a stable semantic version;
|
||||
- `go.mod` declares Narratio's expected module path;
|
||||
- the matching release note exists, has the exact Narratio/version heading,
|
||||
and contains the required sections;
|
||||
- Go workspace use is disabled for candidate commands;
|
||||
- tracked `go.work`/`go.work.sum`, vendoring, and `go.mod` replacement
|
||||
directives are absent;
|
||||
- uncached repository tests, race tests, vet, build, documentation checks, and
|
||||
maintained-example validation succeed;
|
||||
- `go mod tidy -diff` reports no module-file change;
|
||||
- every tracked Go source file is `gofmt` clean;
|
||||
- working-tree and cached whitespace checks succeed; and
|
||||
- all supported release binaries cross-compile with the selected version
|
||||
embedded, with the host-platform binary reporting exactly
|
||||
`narratio <tag>` through `narratio version`.
|
||||
|
||||
The checker must preserve the distinction already documented by Narratio:
|
||||
cross-compilation is compilation evidence, not native macOS or Windows runtime
|
||||
evidence.
|
||||
|
||||
### Asset builder
|
||||
|
||||
Add `scripts/build-release-assets.sh`, used by both the candidate checker and
|
||||
release CI. It must build exactly these CGO-disabled targets:
|
||||
|
||||
- Linux `amd64` and `arm64`;
|
||||
- macOS `amd64` and `arm64`; and
|
||||
- Windows `amd64` and `arm64`.
|
||||
|
||||
Asset names must remain deterministic and include the Narratio version,
|
||||
operating system, architecture, and `.exe` suffix for Windows. Build output
|
||||
must use a caller-supplied, narrowly validated staging directory and must not
|
||||
perform broad or ambiguous deletion. Official assets use `-trimpath` and the
|
||||
established release version linker override.
|
||||
|
||||
### Tag-publication command
|
||||
|
||||
Add the narrowly scoped release command:
|
||||
|
||||
```text
|
||||
scripts/release.sh vMAJOR.MINOR.PATCH
|
||||
```
|
||||
|
||||
The command owns local validation, publication guards, lightweight tag
|
||||
creation, local tag verification, and pushing only the selected tag ref. It
|
||||
must not commit candidate changes or push `main`; the candidate must already be
|
||||
committed and present on upstream `main`.
|
||||
|
||||
Before creating a tag, the command must:
|
||||
|
||||
- run the shared candidate checker;
|
||||
- require the current branch to be `main`;
|
||||
- require a clean worktree and index;
|
||||
- fetch upstream `main` and tags;
|
||||
- record the exact candidate commit and require local `HEAD` to equal
|
||||
`origin/main`;
|
||||
- require the version-matched release note from that commit; and
|
||||
- reject an existing local or upstream tag of the selected name.
|
||||
|
||||
It must create an explicitly lightweight tag against the recorded commit,
|
||||
verify that the local tag resolves directly to that commit, and push only:
|
||||
|
||||
```text
|
||||
refs/tags/<version>:refs/tags/<version>
|
||||
```
|
||||
|
||||
It must never use `git push --tags`. If the tag push succeeds, the command must
|
||||
report the version and commit and exit successfully immediately. In
|
||||
particular, it must not invoke a CI client, poll a build status, wait for a
|
||||
Gitea release, or verify release assets.
|
||||
|
||||
If a failure occurs before the upstream tag is published, no release has
|
||||
completed. An unpublished local tag may be removed only after the maintainer
|
||||
has inspected the failure. The script should not silently delete it. A
|
||||
successful upstream push is final even if later asynchronous work fails.
|
||||
|
||||
## Woodpecker Release Workflow
|
||||
|
||||
Retain tag-triggered asynchronous release automation, but make it consume the
|
||||
same checked-in contracts as local preparation:
|
||||
|
||||
- validate the tag syntax and matching release note before asset work;
|
||||
- call the shared candidate checker rather than maintaining an independent
|
||||
drifting validation list;
|
||||
- use the shared asset builder for the six official binaries;
|
||||
- pin the Go builder image to `golang:1.25.5`, an explicit patch version
|
||||
compatible with the current `go.mod` language version;
|
||||
- pin the release plugin to the reviewed explicit version
|
||||
`woodpeckerci/plugin-release:0.3.1` rather than a floating image;
|
||||
- set the Gitea release title explicitly to `Narratio <tag>`;
|
||||
- use `docs/releases/<tag>.md` from the tagged tree as the release body;
|
||||
- generate a flattened `SHA256SUMS` file for all official binaries;
|
||||
- keep prerelease publication disabled and overwrite disabled; and
|
||||
- keep validation, asset building, and publication dependency-ordered so a
|
||||
failed CI validation cannot publish assets.
|
||||
|
||||
The workflow must not require a callback to, acknowledgement from, or status
|
||||
update in the release command. The two sides have intentionally separate
|
||||
responsibilities:
|
||||
|
||||
```text
|
||||
local release command asynchronous infrastructure
|
||||
--------------------- ---------------------------
|
||||
validate candidate
|
||||
guard main and exact commit
|
||||
create lightweight tag
|
||||
push exact upstream tag ────────► validate tagged tree
|
||||
return success immediately build assets and checksums
|
||||
create Gitea release
|
||||
```
|
||||
|
||||
## Failure And Correction Policy
|
||||
|
||||
Failures before tag publication are corrected on `main`, followed by complete
|
||||
candidate revalidation and fresh guards.
|
||||
|
||||
After a tag has been pushed, neither CI failure nor a defect permits changing
|
||||
that tag. Correct the cause on `main`, choose a new patch version, write a new
|
||||
release note, and repeat the complete release procedure. Do not move or delete
|
||||
the published tag, manually overwrite generated assets, or make the release
|
||||
command wait for repaired infrastructure.
|
||||
|
||||
A maintainer may later inspect the remote tag, Gitea note, assets, checksums,
|
||||
and host binary. Documentation may provide these as optional asynchronous
|
||||
inspection commands, clearly outside the release responsibility boundary. No
|
||||
automated release instruction may require them after a successful tag push.
|
||||
|
||||
## Security And Operational Constraints
|
||||
|
||||
- Local scripts never read or print the Gitea release token; the token remains
|
||||
a Woodpecker-owned secret supplied only to the release plugin.
|
||||
- Candidate validation remains offline with respect to paid APIs, mutable
|
||||
application services, and private user data.
|
||||
- Temporary build output is confined to a validated temporary or staging
|
||||
directory and cleaned safely.
|
||||
- Scripts avoid environment dumps and never record credentials in release
|
||||
notes, logs, repository files, or command output.
|
||||
- The release command does not edit tracked files, create commits, or push
|
||||
branches. Apart from fetching refs for its guards, its only release-state
|
||||
mutations are creating the selected local tag and pushing that exact tag.
|
||||
|
||||
## Target End State
|
||||
|
||||
When this feature is complete:
|
||||
|
||||
1. A maintainer or coding agent prepares and commits a standardized release
|
||||
note plus any final canonical-documentation updates.
|
||||
2. The candidate is pushed to `main` through the ordinary development
|
||||
workflow.
|
||||
3. One shared checker proves the exact tagged source is clean, self-contained,
|
||||
formatted, tested, race-tested, statically checked, documented, module
|
||||
tidy, cross-buildable, and version-correct.
|
||||
4. One guarded release command refuses ambiguous or unpublished candidates,
|
||||
creates the established lightweight tag, and pushes only that tag.
|
||||
5. The command completes successfully at the upstream tag push and performs no
|
||||
CI or Gitea status wait.
|
||||
6. Woodpecker independently reuses the shared checks and asset builder and may
|
||||
publish six versioned binaries, release notes, and SHA-256 checksums.
|
||||
7. Published tags remain immutable, and failures after tag publication are
|
||||
corrected only through a new version.
|
||||
8. The release procedure is documented in one canonical location, linked from
|
||||
contributor guidance, enforced by scripts and CI, and clear about the
|
||||
boundary between guaranteed Git publication and best-effort asynchronous
|
||||
binary publication.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
This feature does not:
|
||||
|
||||
- repair, replace, monitor, or improve Woodpecker reliability;
|
||||
- wait for or synchronously verify CI or Gitea release completion;
|
||||
- introduce prereleases, signed tags, containers, package-manager publication,
|
||||
installers, auto-updaters, or native macOS/Windows test runners;
|
||||
- publish `main` automatically or create release-note content automatically;
|
||||
- run live external adapters or paid model calls during release validation; or
|
||||
- rewrite historical tags or manufacture retrospective release notes.
|
||||
@@ -125,7 +125,17 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
if err := RunStage(
|
||||
context.Background(),
|
||||
[]string{"prepare", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
||||
&bytes.Buffer{},
|
||||
); err != nil {
|
||||
t.Fatalf("seed prepare stage: %v", err)
|
||||
}
|
||||
seed, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load prepared manifest: %v", err)
|
||||
}
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
@@ -136,7 +146,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Run(
|
||||
err = Run(
|
||||
context.Background(),
|
||||
[]string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&out,
|
||||
|
||||
@@ -549,6 +549,17 @@ inputs:
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
}
|
||||
|
||||
func materializePrepareResumeFixture(t *testing.T, pipelinePath, campaignPath, sessionPath string) {
|
||||
t.Helper()
|
||||
if err := RunStage(
|
||||
context.Background(),
|
||||
[]string{"prepare", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
||||
&bytes.Buffer{},
|
||||
); err != nil {
|
||||
t.Fatalf("materialize prepare resume fixture: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeAppTestCampaignConfig(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||
|
||||
@@ -61,8 +61,11 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
m.MarkStageSucceeded("prepare", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
materializePrepareResumeFixture(t, pipelinePath, campaignPath, sessionPath)
|
||||
m, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load prepared manifest: %v", err)
|
||||
}
|
||||
m.MarkStageSucceeded("transcribe", time.Date(2026, 5, 3, 10, 2, 0, 0, time.UTC), nil)
|
||||
seedCurrentSemanticEvidence(t, loadConfigForSemanticEvidence(t, pipelinePath, campaignPath, sessionPath), m, "prepare", "transcribe")
|
||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||
|
||||
@@ -19,8 +19,11 @@ func TestRunContinuesAfterCompletedStages(t *testing.T) {
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
m.MarkStageSucceeded("prepare", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
materializePrepareResumeFixture(t, pipelinePath, campaignPath, sessionPath)
|
||||
m, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load prepared manifest: %v", err)
|
||||
}
|
||||
m.MarkStageSucceeded("transcribe", time.Date(2026, 5, 3, 10, 2, 0, 0, time.UTC), nil)
|
||||
seedCurrentSemanticEvidence(t, loadConfigForSemanticEvidence(t, pipelinePath, campaignPath, sessionPath), m, "prepare", "transcribe")
|
||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||
@@ -28,12 +31,9 @@ func TestRunContinuesAfterCompletedStages(t *testing.T) {
|
||||
}
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "raw", "alice.json"), `{"segments":[]}`)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Run(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||
err = Run(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
@@ -56,7 +56,11 @@ func TestRunNoRemainingStagesRecordsSkippedStages(t *testing.T) {
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
materializePrepareResumeFixture(t, pipelinePath, campaignPath, sessionPath)
|
||||
m, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load prepared manifest: %v", err)
|
||||
}
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
@@ -67,7 +71,7 @@ func TestRunNoRemainingStagesRecordsSkippedStages(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Run(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||
err = Run(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
@@ -204,10 +208,13 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "base.json"), `{"segments":[]}`)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
materializePrepareResumeFixture(t, pipelinePath, campaignPath, sessionPath)
|
||||
seed, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load prepared manifest: %v", err)
|
||||
}
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||
seed.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
@@ -217,7 +224,7 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := RunStage(context.Background(), []string{"polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &out)
|
||||
err = RunStage(context.Background(), []string{"polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage(force) error = %v", err)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
@@ -77,7 +79,14 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return fmt.Errorf("pipeline resolution metadata is required")
|
||||
}
|
||||
data, err := yaml.Marshal(cfg)
|
||||
digestConfig := *cfg
|
||||
if cfg.Notarius != nil && cfg.resolution.logicalNotariusCaptured {
|
||||
notarius := *cfg.Notarius
|
||||
notarius.ConfigPath = cfg.resolution.logicalNotariusConfig
|
||||
notarius.WorkingDirectory = cfg.resolution.logicalNotariusWorking
|
||||
digestConfig.Notarius = ¬arius
|
||||
}
|
||||
data, err := yaml.Marshal(&digestConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize normalized effective pipeline: %w", err)
|
||||
}
|
||||
@@ -93,3 +102,28 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
|
||||
cfg.resolution.effectiveDigest = hex.EncodeToString(digest[:])
|
||||
return nil
|
||||
}
|
||||
|
||||
// captureLogicalNotariusPaths retains normalized user-facing path semantics
|
||||
// before runtime resolution makes relative paths depend on the checkout or
|
||||
// installation directory. Runtime paths remain absolute; provenance does not.
|
||||
func captureLogicalNotariusPaths(cfg *PipelineConfig) {
|
||||
if cfg == nil || cfg.resolution == nil || cfg.Notarius == nil {
|
||||
return
|
||||
}
|
||||
configPath := normalizeLogicalFilesystemPath(cfg.Notarius.ConfigPath)
|
||||
workingDirectory := normalizeLogicalFilesystemPath(cfg.Notarius.WorkingDirectory)
|
||||
if cfg.Notarius.Enabled && workingDirectory == "" && configPath != "" {
|
||||
workingDirectory = normalizeLogicalFilesystemPath(filepath.Dir(configPath))
|
||||
}
|
||||
cfg.resolution.logicalNotariusConfig = configPath
|
||||
cfg.resolution.logicalNotariusWorking = workingDirectory
|
||||
cfg.resolution.logicalNotariusCaptured = true
|
||||
}
|
||||
|
||||
func normalizeLogicalFilesystemPath(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.ToSlash(filepath.Clean(value))
|
||||
}
|
||||
|
||||
@@ -95,6 +95,7 @@ func LoadPipelineProfilePair(path, leftProfile, rightProfile string) (*PipelineC
|
||||
func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig, error) {
|
||||
cfg.resolution.publishDeclared = cfg.Publish != nil
|
||||
applyPipelineDefaults(cfg)
|
||||
captureLogicalNotariusPaths(cfg)
|
||||
if err := resolveNotariusPaths(cfg, path); err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
|
||||
@@ -100,6 +100,36 @@ notarius:
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotariusRelativePathsDoNotMakeEffectiveDigestLocationDependent(t *testing.T) {
|
||||
pipelineYAML := testPipelineBaseYAML + `
|
||||
notarius:
|
||||
enabled: true
|
||||
config_path: notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
`
|
||||
paths := make([]string, 2)
|
||||
configs := make([]*PipelineConfig, 2)
|
||||
for index := range paths {
|
||||
dir := t.TempDir()
|
||||
paths[index] = filepath.Join(dir, "pipeline.yml")
|
||||
if err := os.WriteFile(paths[index], []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline %d: %v", index, err)
|
||||
}
|
||||
loaded, err := LoadPipeline(paths[index])
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPipeline(%d) error = %v", index, err)
|
||||
}
|
||||
configs[index] = loaded
|
||||
}
|
||||
if configs[0].Notarius.ConfigPath == configs[1].Notarius.ConfigPath ||
|
||||
configs[0].Notarius.WorkingDirectory == configs[1].Notarius.WorkingDirectory {
|
||||
t.Fatalf("runtime Notarius paths should remain location-specific: %#v / %#v", configs[0].Notarius, configs[1].Notarius)
|
||||
}
|
||||
if first, second := EffectivePipelineDigest(configs[0]), EffectivePipelineDigest(configs[1]); first == "" || first != second {
|
||||
t.Fatalf("relocated logical configuration digests = %q / %q, want equal non-empty values", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotariusStrictYAML(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -18,6 +18,9 @@ type pipelineResolutionMetadata struct {
|
||||
sources []string
|
||||
selectedProfile *pipelineProfileSelection
|
||||
effectiveDigest string
|
||||
logicalNotariusConfig string
|
||||
logicalNotariusWorking string
|
||||
logicalNotariusCaptured bool
|
||||
ownership []pipelineFieldOwnership
|
||||
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
|
||||
artifactFamiliesExpanded bool
|
||||
|
||||
@@ -406,6 +406,10 @@ func currentAnalyzeArtifactRecord(
|
||||
Logs: dedupeAndSortPaths(result.Logs),
|
||||
GeneratedConfigs: dedupeAndSortPaths(result.GeneratedConfigs),
|
||||
}
|
||||
if origin, ok := analyzeArtifactOrigin(execution, plan.Name); ok {
|
||||
record.Family = origin.Family
|
||||
record.CharacterID = origin.CharacterID
|
||||
}
|
||||
if err := manifest.ValidateAnalyzeArtifactCollection(
|
||||
manifest.AnalyzeStateContractVersion,
|
||||
map[string]manifest.AnalyzeArtifactRecord{plan.Name: record},
|
||||
|
||||
56
internal/stage/analyze_family_origin_test.go
Normal file
56
internal/stage/analyze_family_origin_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func TestCurrentAnalyzeArtifactRecordPreservesResolvedFamilyOrigin(t *testing.T) {
|
||||
examples := filepath.Join("..", "..", "examples")
|
||||
cfg, err := config.LoadWithSessionOptions(
|
||||
filepath.Join(examples, "production-testing", "pipeline.yml"),
|
||||
filepath.Join(examples, "campaigns", "sample-campaign", "campaign.yml"),
|
||||
filepath.Join(examples, "session.local-audio.yml"),
|
||||
config.SessionLoadOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||
}
|
||||
const key = "character_meta_arannis"
|
||||
artifactConfig, ok := cfg.Pipeline.Scriptorium.Artifacts[key]
|
||||
if !ok {
|
||||
t.Fatalf("expanded artifact %q is unavailable", key)
|
||||
}
|
||||
execution := analyzeExecutionContext{
|
||||
Env: &Env{Config: cfg},
|
||||
Manifest: &manifest.Manifest{RunID: "run-family-origin"},
|
||||
}
|
||||
result := &analyzeArtifactExecutionResult{
|
||||
Output: artifacts.Ref{
|
||||
Checksum: strings.Repeat("b", 64),
|
||||
Contract: &artifactmodel.ContractMetadata{
|
||||
MediaType: "text/markdown", SchemaID: "narratio.character_meta", SchemaVersion: "1",
|
||||
},
|
||||
},
|
||||
OutputSize: 12,
|
||||
Scriptorium: manifest.AnalyzeArtifactProvenance{PromptID: artifactConfig.PromptID},
|
||||
}
|
||||
record, err := currentAnalyzeArtifactRecord(
|
||||
execution,
|
||||
analyzeArtifactExecutionPlan{Name: key, Cfg: artifactConfig},
|
||||
strings.Repeat("a", 64),
|
||||
result,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("currentAnalyzeArtifactRecord() error = %v", err)
|
||||
}
|
||||
if record.Family != "character_meta" || record.CharacterID != "arannis" {
|
||||
t.Fatalf("family origin = (%q, %q), want (character_meta, arannis)", record.Family, record.CharacterID)
|
||||
}
|
||||
}
|
||||
@@ -437,29 +437,9 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("run id is required for s3 audio input")
|
||||
}
|
||||
|
||||
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
|
||||
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
|
||||
objects, err := env.ObjectStore.List(ctx, audioPrefix)
|
||||
audioObjects, err := listS3AudioObjects(ctx, env, campaign, sessionID)
|
||||
if err != nil {
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||
}
|
||||
|
||||
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
|
||||
for _, obj := range objects {
|
||||
key := strings.TrimSpace(obj.Key)
|
||||
if key == "" || strings.HasSuffix(key, "/") {
|
||||
continue
|
||||
}
|
||||
if !isFlac(key) {
|
||||
continue
|
||||
}
|
||||
audioObjects = append(audioObjects, obj)
|
||||
}
|
||||
sort.Slice(audioObjects, func(i, j int) bool {
|
||||
return audioObjects[i].Key < audioObjects[j].Key
|
||||
})
|
||||
if len(audioObjects) == 0 {
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
|
||||
return s3AudioMaterializationStats{}, err
|
||||
}
|
||||
|
||||
spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir)
|
||||
@@ -475,16 +455,10 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
|
||||
}
|
||||
|
||||
seenBase := map[string]string{}
|
||||
stats := s3AudioMaterializationStats{}
|
||||
cacheEnabled := env.Config.Pipeline.Cache.S3Audio == nil || *env.Config.Pipeline.Cache.S3Audio
|
||||
for _, obj := range audioObjects {
|
||||
base := path.Base(obj.Key)
|
||||
if prev, exists := seenBase[base]; exists && prev != obj.Key {
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
|
||||
}
|
||||
seenBase[base] = obj.Key
|
||||
|
||||
spoolPath := filepath.Join(spoolAudioDir, base)
|
||||
workPath := filepath.Join(workAudioDir, base)
|
||||
result, err := audio.MaterializeS3Audio(ctx, audio.S3MaterializeRequest{
|
||||
@@ -525,6 +499,42 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func listS3AudioObjects(ctx context.Context, env *Env, campaign, sessionID string) ([]storage.ObjectInfo, error) {
|
||||
if env == nil || env.ObjectStore == nil || env.Config == nil || env.Config.Pipeline == nil ||
|
||||
env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil {
|
||||
return nil, fmt.Errorf("s3 audio input requires object store and resolved storage configuration")
|
||||
}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
|
||||
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
|
||||
objects, err := env.ObjectStore.List(ctx, audioPrefix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||
}
|
||||
|
||||
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
|
||||
seenBase := map[string]string{}
|
||||
for _, obj := range objects {
|
||||
key := strings.TrimSpace(obj.Key)
|
||||
if key == "" || strings.HasSuffix(key, "/") || !isFlac(key) {
|
||||
continue
|
||||
}
|
||||
base := path.Base(key)
|
||||
if previous, exists := seenBase[base]; exists && previous != key {
|
||||
return nil, fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, previous, key)
|
||||
}
|
||||
seenBase[base] = key
|
||||
obj.Key = key
|
||||
audioObjects = append(audioObjects, obj)
|
||||
}
|
||||
sort.Slice(audioObjects, func(i, j int) bool {
|
||||
return audioObjects[i].Key < audioObjects[j].Key
|
||||
})
|
||||
if len(audioObjects) == 0 {
|
||||
return nil, fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
|
||||
}
|
||||
return audioObjects, nil
|
||||
}
|
||||
|
||||
func countAudioInputs(inputs []manifest.InputRecord) int {
|
||||
count := 0
|
||||
for _, in := range inputs {
|
||||
|
||||
278
internal/stage/prepare_resume.go
Normal file
278
internal/stage/prepare_resume.go
Normal file
@@ -0,0 +1,278 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
const prepareResumeChangedReason = "prepared inputs or their current sources changed; rerun prepare"
|
||||
|
||||
type prepareExpectedInput struct {
|
||||
kind string
|
||||
destinationName string
|
||||
checksum string
|
||||
s3Bucket string
|
||||
s3Key string
|
||||
s3Size int64
|
||||
s3ETag string
|
||||
}
|
||||
|
||||
func (prepareStage) ValidateResume(ctx context.Context, env *Env, m *manifest.Manifest) (ResumeValidation, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ResumeValidation{}, err
|
||||
}
|
||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
||||
return ResumeValidation{}, fmt.Errorf("prepare resume: resolved stage environment config is required")
|
||||
}
|
||||
if env.ArtifactStore == nil {
|
||||
return ResumeValidation{}, fmt.Errorf("prepare resume: artifact store is required")
|
||||
}
|
||||
if m == nil || len(m.Inputs) == 0 {
|
||||
return NonResumable(prepareResumeChangedReason), nil
|
||||
}
|
||||
|
||||
current, err := currentPrepareSourceInputs(ctx, env, m)
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ResumeValidation{}, ctxErr
|
||||
}
|
||||
return NonResumable(prepareResumeChangedReason), nil
|
||||
}
|
||||
if !preparedInputRecordsCurrent(env, m.Inputs) || !prepareSourceInputsMatch(current, m.Inputs) {
|
||||
return NonResumable(prepareResumeChangedReason), nil
|
||||
}
|
||||
return Resumable(), nil
|
||||
}
|
||||
|
||||
func preparedInputRecordsCurrent(env *Env, records []manifest.InputRecord) bool {
|
||||
for _, record := range records {
|
||||
path := strings.TrimSpace(record.Path)
|
||||
expected := strings.TrimSpace(record.Checksum)
|
||||
if path == "" || expected == "" {
|
||||
return false
|
||||
}
|
||||
actual, err := env.ArtifactStore.Checksum(path)
|
||||
if err != nil || !strings.EqualFold(actual, expected) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func currentPrepareSourceInputs(ctx context.Context, env *Env, m *manifest.Manifest) ([]prepareExpectedInput, error) {
|
||||
cfg := env.Config
|
||||
expected := make([]prepareExpectedInput, 0, 8)
|
||||
appendFile := func(kind string, input config.ResolvedInputFile, fallback string) error {
|
||||
resolved := stableInputSource(input, fallback, cfg.SessionPath)
|
||||
sourcePath, err := resolveConfigRelativePath(resolved)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
checksum, err := env.ArtifactStore.Checksum(sourcePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
expected = append(expected, prepareExpectedInput{kind: kind, checksum: checksum})
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := appendFile("speakers", cfg.StableInputs.SpeakersFile, cfg.Session.Inputs.SpeakersFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := appendFile("autocorrect", cfg.StableInputs.AutocorrectFile, cfg.Session.Inputs.AutocorrectFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
glossary, ok := artifactpolicy.DescribePreparedInputSource(artifactpolicy.SourceInputGlossary)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("glossary prepared-input descriptor is unavailable")
|
||||
}
|
||||
if err := appendFile(glossary.ManifestKind, cfg.StableInputs.GlossaryFile, cfg.Session.Inputs.GlossaryFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
spellCatalog, ok := artifactpolicy.DescribePreparedInputSource(artifactpolicy.SourceInputSpellCatalog)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("spell catalog prepared-input descriptor is unavailable")
|
||||
}
|
||||
spellInput := stableInputSource(cfg.StableInputs.SpellCatalogFile, cfg.Session.Inputs.SpellCatalogFile, cfg.SessionPath)
|
||||
if strings.TrimSpace(spellInput.Path) != "" {
|
||||
if err := appendFile(spellCatalog.ManifestKind, cfg.StableInputs.SpellCatalogFile, cfg.Session.Inputs.SpellCatalogFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
partyInputs, err := currentPreparePartySourceInputs(env)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expected = append(expected, partyInputs...)
|
||||
|
||||
audioInputs, err := currentPrepareAudioSourceInputs(ctx, env, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expected = append(expected, audioInputs...)
|
||||
return expected, nil
|
||||
}
|
||||
|
||||
func currentPreparePartySourceInputs(env *Env) ([]prepareExpectedInput, error) {
|
||||
party, ok := artifactpolicy.DescribePreparedInputSource(artifactpolicy.SourceInputParty)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("party prepared-input descriptor is unavailable")
|
||||
}
|
||||
players, ok := artifactpolicy.DescribePreparedInputSource(artifactpolicy.SourceInputPlayers)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("players prepared-input descriptor is unavailable")
|
||||
}
|
||||
if env.Config.Party.Mode == config.PartyModeCanonical {
|
||||
if env.Config.Party.Canonical == nil || len(env.Config.Party.Canonical.Raw) == 0 {
|
||||
return nil, fmt.Errorf("canonical party data is unavailable")
|
||||
}
|
||||
playersBytes, err := env.Config.Party.Canonical.PlayersYAML()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []prepareExpectedInput{
|
||||
{kind: party.ManifestKind, checksum: checksumPrepareBytes(env.Config.Party.Canonical.Raw)},
|
||||
{kind: players.ManifestKind, checksum: checksumPrepareBytes(playersBytes)},
|
||||
}, nil
|
||||
}
|
||||
|
||||
cfg := env.Config
|
||||
inputs := []struct {
|
||||
kind string
|
||||
resolved config.ResolvedInputFile
|
||||
fallback string
|
||||
}{
|
||||
{kind: party.ManifestKind, resolved: cfg.StableInputs.PartyFile, fallback: cfg.Session.Inputs.PartyFile},
|
||||
{kind: players.ManifestKind, resolved: cfg.StableInputs.PlayersFile, fallback: cfg.Session.Inputs.PlayersFile},
|
||||
}
|
||||
expected := make([]prepareExpectedInput, 0, len(inputs))
|
||||
for _, input := range inputs {
|
||||
resolved := stableInputSource(input.resolved, input.fallback, cfg.SessionPath)
|
||||
path, err := resolveConfigRelativePath(resolved)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
checksum, err := env.ArtifactStore.Checksum(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expected = append(expected, prepareExpectedInput{kind: input.kind, checksum: checksum})
|
||||
}
|
||||
return expected, nil
|
||||
}
|
||||
|
||||
func currentPrepareAudioSourceInputs(ctx context.Context, env *Env, m *manifest.Manifest) ([]prepareExpectedInput, error) {
|
||||
sessionID := strings.TrimSpace(m.SessionID)
|
||||
if sessionID == "" {
|
||||
sessionID = strings.TrimSpace(env.Config.Session.SessionID)
|
||||
}
|
||||
local, useS3, err := resolveAudioInputs(filepath.Dir(env.Config.SessionPath), env.Config.Session.Inputs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if useS3 {
|
||||
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||
objects, err := listS3AudioObjects(ctx, env, campaign, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket := strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket)
|
||||
expected := make([]prepareExpectedInput, 0, len(objects))
|
||||
for _, object := range objects {
|
||||
if strings.TrimSpace(object.ETag) == "" {
|
||||
return nil, fmt.Errorf("s3 audio object %q has no stable entity tag", object.Key)
|
||||
}
|
||||
expected = append(expected, prepareExpectedInput{
|
||||
kind: "audio", destinationName: path.Base(object.Key),
|
||||
s3Bucket: bucket, s3Key: object.Key, s3Size: object.Size, s3ETag: strings.TrimSpace(object.ETag),
|
||||
})
|
||||
}
|
||||
return expected, nil
|
||||
}
|
||||
|
||||
destinations := localAudioDestinations(local)
|
||||
expected := make([]prepareExpectedInput, 0, len(local))
|
||||
for _, source := range local {
|
||||
checksum, err := env.ArtifactStore.Checksum(source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expected = append(expected, prepareExpectedInput{
|
||||
kind: "audio", destinationName: destinations[source], checksum: checksum,
|
||||
})
|
||||
}
|
||||
return expected, nil
|
||||
}
|
||||
|
||||
func prepareSourceInputsMatch(expected []prepareExpectedInput, records []manifest.InputRecord) bool {
|
||||
if len(expected) == 0 {
|
||||
return false
|
||||
}
|
||||
managedKinds := map[string]struct{}{
|
||||
"audio": {}, "speakers": {}, "autocorrect": {}, "glossary": {},
|
||||
"players": {}, "party": {}, "spell_catalog": {},
|
||||
}
|
||||
expectedCounts := make(map[string]int, len(managedKinds))
|
||||
recordCounts := make(map[string]int, len(managedKinds))
|
||||
for _, input := range expected {
|
||||
expectedCounts[input.kind]++
|
||||
}
|
||||
for _, record := range records {
|
||||
if _, managed := managedKinds[record.Kind]; managed {
|
||||
recordCounts[record.Kind]++
|
||||
}
|
||||
}
|
||||
for kind := range managedKinds {
|
||||
if expectedCounts[kind] != recordCounts[kind] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
matched := make(map[int]struct{}, len(expected))
|
||||
for _, input := range expected {
|
||||
found := -1
|
||||
for index, record := range records {
|
||||
if _, used := matched[index]; used || !prepareInputRecordMatches(input, record) {
|
||||
continue
|
||||
}
|
||||
if found != -1 {
|
||||
return false
|
||||
}
|
||||
found = index
|
||||
}
|
||||
if found == -1 {
|
||||
return false
|
||||
}
|
||||
matched[found] = struct{}{}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func prepareInputRecordMatches(expected prepareExpectedInput, record manifest.InputRecord) bool {
|
||||
if record.Kind != expected.kind {
|
||||
return false
|
||||
}
|
||||
if expected.destinationName != "" && filepath.Base(record.Path) != expected.destinationName {
|
||||
return false
|
||||
}
|
||||
if expected.s3Key != "" {
|
||||
return record.Source == "s3" && strings.TrimSpace(record.S3Bucket) == expected.s3Bucket &&
|
||||
strings.TrimSpace(record.S3Key) == expected.s3Key && record.S3Size == expected.s3Size &&
|
||||
strings.TrimSpace(record.S3ETag) == expected.s3ETag
|
||||
}
|
||||
return strings.EqualFold(strings.TrimSpace(record.Checksum), expected.checksum)
|
||||
}
|
||||
|
||||
func checksumPrepareBytes(data []byte) string {
|
||||
digest := sha256.Sum256(data)
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
160
internal/stage/prepare_resume_test.go
Normal file
160
internal/stage/prepare_resume_test.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func TestPrepareResumeValidatesCurrentLocalSourcesAndPreparedCopies(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*testing.T, *Env, string)
|
||||
}{
|
||||
{
|
||||
name: "stable input bytes changed",
|
||||
mutate: func(t *testing.T, env *Env, _ string) {
|
||||
writeFile(t, filepath.Join(filepath.Dir(env.Config.CampaignPath), "party.yml"), "changed: true\n")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "audio bytes changed",
|
||||
mutate: func(t *testing.T, _ *Env, audioPath string) {
|
||||
writeFile(t, audioPath, "changed audio")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "audio directory membership changed",
|
||||
mutate: func(t *testing.T, _ *Env, audioPath string) {
|
||||
writeFile(t, filepath.Join(filepath.Dir(audioPath), "second.flac"), "second")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "prepared copy changed",
|
||||
mutate: func(t *testing.T, env *Env, _ string) {
|
||||
paths := env.ArtifactStore.SessionPathsFor(env.Config.Session.Campaign, env.Config.Session.SessionID)
|
||||
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "tampered: true\n")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
env, m, audioPath := prepareResumeLocalFixture(t)
|
||||
assertPrepareResume(t, env, m, true)
|
||||
test.mutate(t, env, audioPath)
|
||||
assertPrepareResume(t, env, m, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareResumeAcceptsRelocatedEquivalentLocalSource(t *testing.T) {
|
||||
env, m, _ := prepareResumeLocalFixture(t)
|
||||
cfgDir := filepath.Dir(env.Config.CampaignPath)
|
||||
relocated := filepath.Join(cfgDir, "relocated-party.yml")
|
||||
writeFile(t, relocated, "[]\n")
|
||||
env.Config.StableInputs.PartyFile.Path = "./relocated-party.yml"
|
||||
assertPrepareResume(t, env, m, true)
|
||||
}
|
||||
|
||||
func TestPrepareResumeDetectsCanonicalPartyChange(t *testing.T) {
|
||||
env, m := setupPrepareEnv(t)
|
||||
audioPath := filepath.Join(filepath.Dir(env.Config.SessionPath), "audio", "one.flac")
|
||||
writeFile(t, audioPath, "audio")
|
||||
first := []byte(`schema_version: narratio.party.v1
|
||||
characters:
|
||||
arannis:
|
||||
player: {name: Eric}
|
||||
character:
|
||||
name: Arannis
|
||||
classes: [{name: wizard, level: 8}]
|
||||
`)
|
||||
setCanonicalPrepareParty(t, env, first)
|
||||
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("prepare.Run() error = %v", err)
|
||||
}
|
||||
assertPrepareResume(t, env, m, true)
|
||||
|
||||
second := []byte(`schema_version: narratio.party.v1
|
||||
characters:
|
||||
arannis:
|
||||
player: {name: Eric}
|
||||
character:
|
||||
name: Arannis
|
||||
alias: [The Red]
|
||||
classes: [{name: wizard, level: 9}]
|
||||
`)
|
||||
setCanonicalPrepareParty(t, env, second)
|
||||
assertPrepareResume(t, env, m, false)
|
||||
}
|
||||
|
||||
func TestPrepareResumeValidatesCurrentS3ObjectIdentity(t *testing.T) {
|
||||
env, m := setupPrepareEnv(t)
|
||||
env.Config.Session.Campaign = "forsaken"
|
||||
env.Config.Session.Inputs.AudioDir = ""
|
||||
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
|
||||
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
|
||||
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "archive", RootPrefix: "dnd"}
|
||||
m.RunID = "20260515T031522Z-a1b2c3d4"
|
||||
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
|
||||
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
|
||||
key := "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac"
|
||||
fake := &storage.FakeBackend{}
|
||||
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("alice")})
|
||||
env.ObjectStore = fake
|
||||
|
||||
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("prepare.Run() error = %v", err)
|
||||
}
|
||||
assertPrepareResume(t, env, m, true)
|
||||
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("changed")})
|
||||
assertPrepareResume(t, env, m, false)
|
||||
}
|
||||
|
||||
func prepareResumeLocalFixture(t *testing.T) (*Env, *manifest.Manifest, string) {
|
||||
t.Helper()
|
||||
env, m := setupPrepareEnv(t)
|
||||
audioPath := filepath.Join(filepath.Dir(env.Config.SessionPath), "audio", "one.flac")
|
||||
writeFile(t, audioPath, "audio")
|
||||
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("prepare.Run() error = %v", err)
|
||||
}
|
||||
return env, m, audioPath
|
||||
}
|
||||
|
||||
func setCanonicalPrepareParty(t *testing.T, env *Env, raw []byte) {
|
||||
t.Helper()
|
||||
document, err := config.ParseParty(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseParty() error = %v", err)
|
||||
}
|
||||
env.Config.Party = config.ResolvedParty{
|
||||
Mode: config.PartyModeCanonical, Source: config.PartySource{Source: "campaign_config"}, Canonical: document.Canonical,
|
||||
}
|
||||
}
|
||||
|
||||
func assertPrepareResume(t *testing.T, env *Env, m *manifest.Manifest, want bool) {
|
||||
t.Helper()
|
||||
validation, err := (prepareStage{}).ValidateResume(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateResume() error = %v", err)
|
||||
}
|
||||
if validation.Resumable != want {
|
||||
t.Fatalf("ValidateResume() = %#v, want resumable=%t", validation, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareResumeMissingPreparedFileReruns(t *testing.T) {
|
||||
env, m, _ := prepareResumeLocalFixture(t)
|
||||
paths := env.ArtifactStore.SessionPathsFor(env.Config.Session.Campaign, env.Config.Session.SessionID)
|
||||
if err := os.Remove(filepath.Join(paths.InputsDir, "players.yml")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertPrepareResume(t, env, m, false)
|
||||
}
|
||||
Reference in New Issue
Block a user