Harden pipeline state and plan release upgrades
This commit is contained in:
@@ -125,7 +125,17 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
if err := RunStage(
|
||||
context.Background(),
|
||||
[]string{"prepare", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
||||
&bytes.Buffer{},
|
||||
); err != nil {
|
||||
t.Fatalf("seed prepare stage: %v", err)
|
||||
}
|
||||
seed, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load prepared manifest: %v", err)
|
||||
}
|
||||
for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
seed.MarkStageSucceeded(stageName, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
@@ -136,7 +146,7 @@ func TestRunArtifactsWithSucceededAnalyzeSkipsUnlessForced(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Run(
|
||||
err = Run(
|
||||
context.Background(),
|
||||
[]string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--artifacts", "session_recap"},
|
||||
&out,
|
||||
|
||||
@@ -549,6 +549,17 @@ inputs:
|
||||
return pipelinePath, campaignPath, sessionPath
|
||||
}
|
||||
|
||||
func materializePrepareResumeFixture(t *testing.T, pipelinePath, campaignPath, sessionPath string) {
|
||||
t.Helper()
|
||||
if err := RunStage(
|
||||
context.Background(),
|
||||
[]string{"prepare", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath},
|
||||
&bytes.Buffer{},
|
||||
); err != nil {
|
||||
t.Fatalf("materialize prepare resume fixture: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeAppTestCampaignConfig(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
campaignPath := filepath.Join(dir, "campaign.yml")
|
||||
|
||||
@@ -61,8 +61,11 @@ func TestPlanShowsRunAndSkipFromManifest(t *testing.T) {
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
m.MarkStageSucceeded("prepare", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
materializePrepareResumeFixture(t, pipelinePath, campaignPath, sessionPath)
|
||||
m, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load prepared manifest: %v", err)
|
||||
}
|
||||
m.MarkStageSucceeded("transcribe", time.Date(2026, 5, 3, 10, 2, 0, 0, time.UTC), nil)
|
||||
seedCurrentSemanticEvidence(t, loadConfigForSemanticEvidence(t, pipelinePath, campaignPath, sessionPath), m, "prepare", "transcribe")
|
||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||
|
||||
@@ -19,8 +19,11 @@ func TestRunContinuesAfterCompletedStages(t *testing.T) {
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
m.MarkStageSucceeded("prepare", time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
materializePrepareResumeFixture(t, pipelinePath, campaignPath, sessionPath)
|
||||
m, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load prepared manifest: %v", err)
|
||||
}
|
||||
m.MarkStageSucceeded("transcribe", time.Date(2026, 5, 3, 10, 2, 0, 0, time.UTC), nil)
|
||||
seedCurrentSemanticEvidence(t, loadConfigForSemanticEvidence(t, pipelinePath, campaignPath, sessionPath), m, "prepare", "transcribe")
|
||||
if err := store.Save(context.Background(), manifestPath, m); err != nil {
|
||||
@@ -28,12 +31,9 @@ func TestRunContinuesAfterCompletedStages(t *testing.T) {
|
||||
}
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "raw", "alice.json"), `{"segments":[]}`)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "speakers.yml"), "match:\n - speaker: Alice\n match: [\"alice\"]\n")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "autocorrect.yml"), "[]\n")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Run(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||
err = Run(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
@@ -56,7 +56,11 @@ func TestRunNoRemainingStagesRecordsSkippedStages(t *testing.T) {
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
m := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
materializePrepareResumeFixture(t, pipelinePath, campaignPath, sessionPath)
|
||||
m, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load prepared manifest: %v", err)
|
||||
}
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze", "publish", "notify"} {
|
||||
m.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
@@ -67,7 +71,7 @@ func TestRunNoRemainingStagesRecordsSkippedStages(t *testing.T) {
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := Run(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||
err = Run(context.Background(), []string{"2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
@@ -204,10 +208,13 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
|
||||
manifestPath := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03", "manifest.json")
|
||||
workRoot := filepath.Join(workspaceRoot, "work", "sample-campaign", "2026-05-03")
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "base.json"), `{"segments":[]}`)
|
||||
mustWriteTestFile(t, filepath.Join(workRoot, "inputs", "glossary.yml"), "terms: []\n")
|
||||
|
||||
store := &manifest.LocalStore{}
|
||||
seed := manifest.New("2026-05-03", time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC))
|
||||
materializePrepareResumeFixture(t, pipelinePath, campaignPath, sessionPath)
|
||||
seed, err := store.Load(context.Background(), manifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("load prepared manifest: %v", err)
|
||||
}
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "extract", "render", "analyze", "publish", "notify"} {
|
||||
seed.MarkStageSucceeded(name, time.Date(2026, 5, 3, 10, 1, 0, 0, time.UTC), nil)
|
||||
}
|
||||
@@ -217,7 +224,7 @@ func TestRunStageForceMarksDownstreamStaleAndRunContinuesFromStale(t *testing.T)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := RunStage(context.Background(), []string{"polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &out)
|
||||
err = RunStage(context.Background(), []string{"polish", "2026-05-03", "--config", pipelinePath, "--campaign-file", campaignPath, "--session", sessionPath, "--force"}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("RunStage(force) error = %v", err)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
@@ -77,7 +79,14 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return fmt.Errorf("pipeline resolution metadata is required")
|
||||
}
|
||||
data, err := yaml.Marshal(cfg)
|
||||
digestConfig := *cfg
|
||||
if cfg.Notarius != nil && cfg.resolution.logicalNotariusCaptured {
|
||||
notarius := *cfg.Notarius
|
||||
notarius.ConfigPath = cfg.resolution.logicalNotariusConfig
|
||||
notarius.WorkingDirectory = cfg.resolution.logicalNotariusWorking
|
||||
digestConfig.Notarius = ¬arius
|
||||
}
|
||||
data, err := yaml.Marshal(&digestConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serialize normalized effective pipeline: %w", err)
|
||||
}
|
||||
@@ -93,3 +102,28 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
|
||||
cfg.resolution.effectiveDigest = hex.EncodeToString(digest[:])
|
||||
return nil
|
||||
}
|
||||
|
||||
// captureLogicalNotariusPaths retains normalized user-facing path semantics
|
||||
// before runtime resolution makes relative paths depend on the checkout or
|
||||
// installation directory. Runtime paths remain absolute; provenance does not.
|
||||
func captureLogicalNotariusPaths(cfg *PipelineConfig) {
|
||||
if cfg == nil || cfg.resolution == nil || cfg.Notarius == nil {
|
||||
return
|
||||
}
|
||||
configPath := normalizeLogicalFilesystemPath(cfg.Notarius.ConfigPath)
|
||||
workingDirectory := normalizeLogicalFilesystemPath(cfg.Notarius.WorkingDirectory)
|
||||
if cfg.Notarius.Enabled && workingDirectory == "" && configPath != "" {
|
||||
workingDirectory = normalizeLogicalFilesystemPath(filepath.Dir(configPath))
|
||||
}
|
||||
cfg.resolution.logicalNotariusConfig = configPath
|
||||
cfg.resolution.logicalNotariusWorking = workingDirectory
|
||||
cfg.resolution.logicalNotariusCaptured = true
|
||||
}
|
||||
|
||||
func normalizeLogicalFilesystemPath(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.ToSlash(filepath.Clean(value))
|
||||
}
|
||||
|
||||
@@ -95,6 +95,7 @@ func LoadPipelineProfilePair(path, leftProfile, rightProfile string) (*PipelineC
|
||||
func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig, error) {
|
||||
cfg.resolution.publishDeclared = cfg.Publish != nil
|
||||
applyPipelineDefaults(cfg)
|
||||
captureLogicalNotariusPaths(cfg)
|
||||
if err := resolveNotariusPaths(cfg, path); err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
|
||||
@@ -100,6 +100,36 @@ notarius:
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotariusRelativePathsDoNotMakeEffectiveDigestLocationDependent(t *testing.T) {
|
||||
pipelineYAML := testPipelineBaseYAML + `
|
||||
notarius:
|
||||
enabled: true
|
||||
config_path: notarius/config.yml
|
||||
pipeline_id: dnd-session
|
||||
`
|
||||
paths := make([]string, 2)
|
||||
configs := make([]*PipelineConfig, 2)
|
||||
for index := range paths {
|
||||
dir := t.TempDir()
|
||||
paths[index] = filepath.Join(dir, "pipeline.yml")
|
||||
if err := os.WriteFile(paths[index], []byte(pipelineYAML), 0o644); err != nil {
|
||||
t.Fatalf("write pipeline %d: %v", index, err)
|
||||
}
|
||||
loaded, err := LoadPipeline(paths[index])
|
||||
if err != nil {
|
||||
t.Fatalf("LoadPipeline(%d) error = %v", index, err)
|
||||
}
|
||||
configs[index] = loaded
|
||||
}
|
||||
if configs[0].Notarius.ConfigPath == configs[1].Notarius.ConfigPath ||
|
||||
configs[0].Notarius.WorkingDirectory == configs[1].Notarius.WorkingDirectory {
|
||||
t.Fatalf("runtime Notarius paths should remain location-specific: %#v / %#v", configs[0].Notarius, configs[1].Notarius)
|
||||
}
|
||||
if first, second := EffectivePipelineDigest(configs[0]), EffectivePipelineDigest(configs[1]); first == "" || first != second {
|
||||
t.Fatalf("relocated logical configuration digests = %q / %q, want equal non-empty values", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotariusStrictYAML(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -18,6 +18,9 @@ type pipelineResolutionMetadata struct {
|
||||
sources []string
|
||||
selectedProfile *pipelineProfileSelection
|
||||
effectiveDigest string
|
||||
logicalNotariusConfig string
|
||||
logicalNotariusWorking string
|
||||
logicalNotariusCaptured bool
|
||||
ownership []pipelineFieldOwnership
|
||||
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
|
||||
artifactFamiliesExpanded bool
|
||||
|
||||
@@ -406,6 +406,10 @@ func currentAnalyzeArtifactRecord(
|
||||
Logs: dedupeAndSortPaths(result.Logs),
|
||||
GeneratedConfigs: dedupeAndSortPaths(result.GeneratedConfigs),
|
||||
}
|
||||
if origin, ok := analyzeArtifactOrigin(execution, plan.Name); ok {
|
||||
record.Family = origin.Family
|
||||
record.CharacterID = origin.CharacterID
|
||||
}
|
||||
if err := manifest.ValidateAnalyzeArtifactCollection(
|
||||
manifest.AnalyzeStateContractVersion,
|
||||
map[string]manifest.AnalyzeArtifactRecord{plan.Name: record},
|
||||
|
||||
56
internal/stage/analyze_family_origin_test.go
Normal file
56
internal/stage/analyze_family_origin_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func TestCurrentAnalyzeArtifactRecordPreservesResolvedFamilyOrigin(t *testing.T) {
|
||||
examples := filepath.Join("..", "..", "examples")
|
||||
cfg, err := config.LoadWithSessionOptions(
|
||||
filepath.Join(examples, "production-testing", "pipeline.yml"),
|
||||
filepath.Join(examples, "campaigns", "sample-campaign", "campaign.yml"),
|
||||
filepath.Join(examples, "session.local-audio.yml"),
|
||||
config.SessionLoadOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadWithSessionOptions() error = %v", err)
|
||||
}
|
||||
const key = "character_meta_arannis"
|
||||
artifactConfig, ok := cfg.Pipeline.Scriptorium.Artifacts[key]
|
||||
if !ok {
|
||||
t.Fatalf("expanded artifact %q is unavailable", key)
|
||||
}
|
||||
execution := analyzeExecutionContext{
|
||||
Env: &Env{Config: cfg},
|
||||
Manifest: &manifest.Manifest{RunID: "run-family-origin"},
|
||||
}
|
||||
result := &analyzeArtifactExecutionResult{
|
||||
Output: artifacts.Ref{
|
||||
Checksum: strings.Repeat("b", 64),
|
||||
Contract: &artifactmodel.ContractMetadata{
|
||||
MediaType: "text/markdown", SchemaID: "narratio.character_meta", SchemaVersion: "1",
|
||||
},
|
||||
},
|
||||
OutputSize: 12,
|
||||
Scriptorium: manifest.AnalyzeArtifactProvenance{PromptID: artifactConfig.PromptID},
|
||||
}
|
||||
record, err := currentAnalyzeArtifactRecord(
|
||||
execution,
|
||||
analyzeArtifactExecutionPlan{Name: key, Cfg: artifactConfig},
|
||||
strings.Repeat("a", 64),
|
||||
result,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("currentAnalyzeArtifactRecord() error = %v", err)
|
||||
}
|
||||
if record.Family != "character_meta" || record.CharacterID != "arannis" {
|
||||
t.Fatalf("family origin = (%q, %q), want (character_meta, arannis)", record.Family, record.CharacterID)
|
||||
}
|
||||
}
|
||||
@@ -437,29 +437,9 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("run id is required for s3 audio input")
|
||||
}
|
||||
|
||||
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
|
||||
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
|
||||
objects, err := env.ObjectStore.List(ctx, audioPrefix)
|
||||
audioObjects, err := listS3AudioObjects(ctx, env, campaign, sessionID)
|
||||
if err != nil {
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||
}
|
||||
|
||||
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
|
||||
for _, obj := range objects {
|
||||
key := strings.TrimSpace(obj.Key)
|
||||
if key == "" || strings.HasSuffix(key, "/") {
|
||||
continue
|
||||
}
|
||||
if !isFlac(key) {
|
||||
continue
|
||||
}
|
||||
audioObjects = append(audioObjects, obj)
|
||||
}
|
||||
sort.Slice(audioObjects, func(i, j int) bool {
|
||||
return audioObjects[i].Key < audioObjects[j].Key
|
||||
})
|
||||
if len(audioObjects) == 0 {
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
|
||||
return s3AudioMaterializationStats{}, err
|
||||
}
|
||||
|
||||
spoolAudioDir := strings.TrimSpace(m.LocalSpoolDir)
|
||||
@@ -475,16 +455,10 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("create work audio directory %q: %w", workAudioDir, err)
|
||||
}
|
||||
|
||||
seenBase := map[string]string{}
|
||||
stats := s3AudioMaterializationStats{}
|
||||
cacheEnabled := env.Config.Pipeline.Cache.S3Audio == nil || *env.Config.Pipeline.Cache.S3Audio
|
||||
for _, obj := range audioObjects {
|
||||
base := path.Base(obj.Key)
|
||||
if prev, exists := seenBase[base]; exists && prev != obj.Key {
|
||||
return s3AudioMaterializationStats{}, fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, prev, obj.Key)
|
||||
}
|
||||
seenBase[base] = obj.Key
|
||||
|
||||
spoolPath := filepath.Join(spoolAudioDir, base)
|
||||
workPath := filepath.Join(workAudioDir, base)
|
||||
result, err := audio.MaterializeS3Audio(ctx, audio.S3MaterializeRequest{
|
||||
@@ -525,6 +499,42 @@ func materializeS3AudioInputs(ctx context.Context, env *Env, m *manifest.Manifes
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func listS3AudioObjects(ctx context.Context, env *Env, campaign, sessionID string) ([]storage.ObjectInfo, error) {
|
||||
if env == nil || env.ObjectStore == nil || env.Config == nil || env.Config.Pipeline == nil ||
|
||||
env.Config.Session == nil || env.Config.Pipeline.Storage.S3 == nil || env.Config.Session.Inputs.AudioS3 == nil {
|
||||
return nil, fmt.Errorf("s3 audio input requires object store and resolved storage configuration")
|
||||
}
|
||||
sessionPrefix := artifacts.S3SessionPrefix(env.Config.Pipeline.Storage.S3.RootPrefix, campaign, sessionID)
|
||||
audioPrefix := artifacts.S3AudioPrefix(sessionPrefix, env.Config.Session.Inputs.AudioS3.Prefix)
|
||||
objects, err := env.ObjectStore.List(ctx, audioPrefix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list s3 audio objects under %q: %w", audioPrefix, err)
|
||||
}
|
||||
|
||||
audioObjects := make([]storage.ObjectInfo, 0, len(objects))
|
||||
seenBase := map[string]string{}
|
||||
for _, obj := range objects {
|
||||
key := strings.TrimSpace(obj.Key)
|
||||
if key == "" || strings.HasSuffix(key, "/") || !isFlac(key) {
|
||||
continue
|
||||
}
|
||||
base := path.Base(key)
|
||||
if previous, exists := seenBase[base]; exists && previous != key {
|
||||
return nil, fmt.Errorf("duplicate s3 audio basename %q from %q and %q", base, previous, key)
|
||||
}
|
||||
seenBase[base] = key
|
||||
obj.Key = key
|
||||
audioObjects = append(audioObjects, obj)
|
||||
}
|
||||
sort.Slice(audioObjects, func(i, j int) bool {
|
||||
return audioObjects[i].Key < audioObjects[j].Key
|
||||
})
|
||||
if len(audioObjects) == 0 {
|
||||
return nil, fmt.Errorf("no .flac files found under s3 audio prefix %q", audioPrefix)
|
||||
}
|
||||
return audioObjects, nil
|
||||
}
|
||||
|
||||
func countAudioInputs(inputs []manifest.InputRecord) int {
|
||||
count := 0
|
||||
for _, in := range inputs {
|
||||
|
||||
278
internal/stage/prepare_resume.go
Normal file
278
internal/stage/prepare_resume.go
Normal file
@@ -0,0 +1,278 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
const prepareResumeChangedReason = "prepared inputs or their current sources changed; rerun prepare"
|
||||
|
||||
type prepareExpectedInput struct {
|
||||
kind string
|
||||
destinationName string
|
||||
checksum string
|
||||
s3Bucket string
|
||||
s3Key string
|
||||
s3Size int64
|
||||
s3ETag string
|
||||
}
|
||||
|
||||
func (prepareStage) ValidateResume(ctx context.Context, env *Env, m *manifest.Manifest) (ResumeValidation, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return ResumeValidation{}, err
|
||||
}
|
||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
||||
return ResumeValidation{}, fmt.Errorf("prepare resume: resolved stage environment config is required")
|
||||
}
|
||||
if env.ArtifactStore == nil {
|
||||
return ResumeValidation{}, fmt.Errorf("prepare resume: artifact store is required")
|
||||
}
|
||||
if m == nil || len(m.Inputs) == 0 {
|
||||
return NonResumable(prepareResumeChangedReason), nil
|
||||
}
|
||||
|
||||
current, err := currentPrepareSourceInputs(ctx, env, m)
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ResumeValidation{}, ctxErr
|
||||
}
|
||||
return NonResumable(prepareResumeChangedReason), nil
|
||||
}
|
||||
if !preparedInputRecordsCurrent(env, m.Inputs) || !prepareSourceInputsMatch(current, m.Inputs) {
|
||||
return NonResumable(prepareResumeChangedReason), nil
|
||||
}
|
||||
return Resumable(), nil
|
||||
}
|
||||
|
||||
func preparedInputRecordsCurrent(env *Env, records []manifest.InputRecord) bool {
|
||||
for _, record := range records {
|
||||
path := strings.TrimSpace(record.Path)
|
||||
expected := strings.TrimSpace(record.Checksum)
|
||||
if path == "" || expected == "" {
|
||||
return false
|
||||
}
|
||||
actual, err := env.ArtifactStore.Checksum(path)
|
||||
if err != nil || !strings.EqualFold(actual, expected) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func currentPrepareSourceInputs(ctx context.Context, env *Env, m *manifest.Manifest) ([]prepareExpectedInput, error) {
|
||||
cfg := env.Config
|
||||
expected := make([]prepareExpectedInput, 0, 8)
|
||||
appendFile := func(kind string, input config.ResolvedInputFile, fallback string) error {
|
||||
resolved := stableInputSource(input, fallback, cfg.SessionPath)
|
||||
sourcePath, err := resolveConfigRelativePath(resolved)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
checksum, err := env.ArtifactStore.Checksum(sourcePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
expected = append(expected, prepareExpectedInput{kind: kind, checksum: checksum})
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := appendFile("speakers", cfg.StableInputs.SpeakersFile, cfg.Session.Inputs.SpeakersFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := appendFile("autocorrect", cfg.StableInputs.AutocorrectFile, cfg.Session.Inputs.AutocorrectFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
glossary, ok := artifactpolicy.DescribePreparedInputSource(artifactpolicy.SourceInputGlossary)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("glossary prepared-input descriptor is unavailable")
|
||||
}
|
||||
if err := appendFile(glossary.ManifestKind, cfg.StableInputs.GlossaryFile, cfg.Session.Inputs.GlossaryFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
spellCatalog, ok := artifactpolicy.DescribePreparedInputSource(artifactpolicy.SourceInputSpellCatalog)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("spell catalog prepared-input descriptor is unavailable")
|
||||
}
|
||||
spellInput := stableInputSource(cfg.StableInputs.SpellCatalogFile, cfg.Session.Inputs.SpellCatalogFile, cfg.SessionPath)
|
||||
if strings.TrimSpace(spellInput.Path) != "" {
|
||||
if err := appendFile(spellCatalog.ManifestKind, cfg.StableInputs.SpellCatalogFile, cfg.Session.Inputs.SpellCatalogFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
partyInputs, err := currentPreparePartySourceInputs(env)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expected = append(expected, partyInputs...)
|
||||
|
||||
audioInputs, err := currentPrepareAudioSourceInputs(ctx, env, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expected = append(expected, audioInputs...)
|
||||
return expected, nil
|
||||
}
|
||||
|
||||
func currentPreparePartySourceInputs(env *Env) ([]prepareExpectedInput, error) {
|
||||
party, ok := artifactpolicy.DescribePreparedInputSource(artifactpolicy.SourceInputParty)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("party prepared-input descriptor is unavailable")
|
||||
}
|
||||
players, ok := artifactpolicy.DescribePreparedInputSource(artifactpolicy.SourceInputPlayers)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("players prepared-input descriptor is unavailable")
|
||||
}
|
||||
if env.Config.Party.Mode == config.PartyModeCanonical {
|
||||
if env.Config.Party.Canonical == nil || len(env.Config.Party.Canonical.Raw) == 0 {
|
||||
return nil, fmt.Errorf("canonical party data is unavailable")
|
||||
}
|
||||
playersBytes, err := env.Config.Party.Canonical.PlayersYAML()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []prepareExpectedInput{
|
||||
{kind: party.ManifestKind, checksum: checksumPrepareBytes(env.Config.Party.Canonical.Raw)},
|
||||
{kind: players.ManifestKind, checksum: checksumPrepareBytes(playersBytes)},
|
||||
}, nil
|
||||
}
|
||||
|
||||
cfg := env.Config
|
||||
inputs := []struct {
|
||||
kind string
|
||||
resolved config.ResolvedInputFile
|
||||
fallback string
|
||||
}{
|
||||
{kind: party.ManifestKind, resolved: cfg.StableInputs.PartyFile, fallback: cfg.Session.Inputs.PartyFile},
|
||||
{kind: players.ManifestKind, resolved: cfg.StableInputs.PlayersFile, fallback: cfg.Session.Inputs.PlayersFile},
|
||||
}
|
||||
expected := make([]prepareExpectedInput, 0, len(inputs))
|
||||
for _, input := range inputs {
|
||||
resolved := stableInputSource(input.resolved, input.fallback, cfg.SessionPath)
|
||||
path, err := resolveConfigRelativePath(resolved)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
checksum, err := env.ArtifactStore.Checksum(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expected = append(expected, prepareExpectedInput{kind: input.kind, checksum: checksum})
|
||||
}
|
||||
return expected, nil
|
||||
}
|
||||
|
||||
func currentPrepareAudioSourceInputs(ctx context.Context, env *Env, m *manifest.Manifest) ([]prepareExpectedInput, error) {
|
||||
sessionID := strings.TrimSpace(m.SessionID)
|
||||
if sessionID == "" {
|
||||
sessionID = strings.TrimSpace(env.Config.Session.SessionID)
|
||||
}
|
||||
local, useS3, err := resolveAudioInputs(filepath.Dir(env.Config.SessionPath), env.Config.Session.Inputs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if useS3 {
|
||||
campaign := strings.TrimSpace(env.Config.Session.Campaign)
|
||||
objects, err := listS3AudioObjects(ctx, env, campaign, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket := strings.TrimSpace(env.Config.Pipeline.Storage.S3.Bucket)
|
||||
expected := make([]prepareExpectedInput, 0, len(objects))
|
||||
for _, object := range objects {
|
||||
if strings.TrimSpace(object.ETag) == "" {
|
||||
return nil, fmt.Errorf("s3 audio object %q has no stable entity tag", object.Key)
|
||||
}
|
||||
expected = append(expected, prepareExpectedInput{
|
||||
kind: "audio", destinationName: path.Base(object.Key),
|
||||
s3Bucket: bucket, s3Key: object.Key, s3Size: object.Size, s3ETag: strings.TrimSpace(object.ETag),
|
||||
})
|
||||
}
|
||||
return expected, nil
|
||||
}
|
||||
|
||||
destinations := localAudioDestinations(local)
|
||||
expected := make([]prepareExpectedInput, 0, len(local))
|
||||
for _, source := range local {
|
||||
checksum, err := env.ArtifactStore.Checksum(source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expected = append(expected, prepareExpectedInput{
|
||||
kind: "audio", destinationName: destinations[source], checksum: checksum,
|
||||
})
|
||||
}
|
||||
return expected, nil
|
||||
}
|
||||
|
||||
func prepareSourceInputsMatch(expected []prepareExpectedInput, records []manifest.InputRecord) bool {
|
||||
if len(expected) == 0 {
|
||||
return false
|
||||
}
|
||||
managedKinds := map[string]struct{}{
|
||||
"audio": {}, "speakers": {}, "autocorrect": {}, "glossary": {},
|
||||
"players": {}, "party": {}, "spell_catalog": {},
|
||||
}
|
||||
expectedCounts := make(map[string]int, len(managedKinds))
|
||||
recordCounts := make(map[string]int, len(managedKinds))
|
||||
for _, input := range expected {
|
||||
expectedCounts[input.kind]++
|
||||
}
|
||||
for _, record := range records {
|
||||
if _, managed := managedKinds[record.Kind]; managed {
|
||||
recordCounts[record.Kind]++
|
||||
}
|
||||
}
|
||||
for kind := range managedKinds {
|
||||
if expectedCounts[kind] != recordCounts[kind] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
matched := make(map[int]struct{}, len(expected))
|
||||
for _, input := range expected {
|
||||
found := -1
|
||||
for index, record := range records {
|
||||
if _, used := matched[index]; used || !prepareInputRecordMatches(input, record) {
|
||||
continue
|
||||
}
|
||||
if found != -1 {
|
||||
return false
|
||||
}
|
||||
found = index
|
||||
}
|
||||
if found == -1 {
|
||||
return false
|
||||
}
|
||||
matched[found] = struct{}{}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func prepareInputRecordMatches(expected prepareExpectedInput, record manifest.InputRecord) bool {
|
||||
if record.Kind != expected.kind {
|
||||
return false
|
||||
}
|
||||
if expected.destinationName != "" && filepath.Base(record.Path) != expected.destinationName {
|
||||
return false
|
||||
}
|
||||
if expected.s3Key != "" {
|
||||
return record.Source == "s3" && strings.TrimSpace(record.S3Bucket) == expected.s3Bucket &&
|
||||
strings.TrimSpace(record.S3Key) == expected.s3Key && record.S3Size == expected.s3Size &&
|
||||
strings.TrimSpace(record.S3ETag) == expected.s3ETag
|
||||
}
|
||||
return strings.EqualFold(strings.TrimSpace(record.Checksum), expected.checksum)
|
||||
}
|
||||
|
||||
func checksumPrepareBytes(data []byte) string {
|
||||
digest := sha256.Sum256(data)
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
160
internal/stage/prepare_resume_test.go
Normal file
160
internal/stage/prepare_resume_test.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func TestPrepareResumeValidatesCurrentLocalSourcesAndPreparedCopies(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*testing.T, *Env, string)
|
||||
}{
|
||||
{
|
||||
name: "stable input bytes changed",
|
||||
mutate: func(t *testing.T, env *Env, _ string) {
|
||||
writeFile(t, filepath.Join(filepath.Dir(env.Config.CampaignPath), "party.yml"), "changed: true\n")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "audio bytes changed",
|
||||
mutate: func(t *testing.T, _ *Env, audioPath string) {
|
||||
writeFile(t, audioPath, "changed audio")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "audio directory membership changed",
|
||||
mutate: func(t *testing.T, _ *Env, audioPath string) {
|
||||
writeFile(t, filepath.Join(filepath.Dir(audioPath), "second.flac"), "second")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "prepared copy changed",
|
||||
mutate: func(t *testing.T, env *Env, _ string) {
|
||||
paths := env.ArtifactStore.SessionPathsFor(env.Config.Session.Campaign, env.Config.Session.SessionID)
|
||||
writeFile(t, filepath.Join(paths.InputsDir, "glossary.yml"), "tampered: true\n")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
env, m, audioPath := prepareResumeLocalFixture(t)
|
||||
assertPrepareResume(t, env, m, true)
|
||||
test.mutate(t, env, audioPath)
|
||||
assertPrepareResume(t, env, m, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareResumeAcceptsRelocatedEquivalentLocalSource(t *testing.T) {
|
||||
env, m, _ := prepareResumeLocalFixture(t)
|
||||
cfgDir := filepath.Dir(env.Config.CampaignPath)
|
||||
relocated := filepath.Join(cfgDir, "relocated-party.yml")
|
||||
writeFile(t, relocated, "[]\n")
|
||||
env.Config.StableInputs.PartyFile.Path = "./relocated-party.yml"
|
||||
assertPrepareResume(t, env, m, true)
|
||||
}
|
||||
|
||||
func TestPrepareResumeDetectsCanonicalPartyChange(t *testing.T) {
|
||||
env, m := setupPrepareEnv(t)
|
||||
audioPath := filepath.Join(filepath.Dir(env.Config.SessionPath), "audio", "one.flac")
|
||||
writeFile(t, audioPath, "audio")
|
||||
first := []byte(`schema_version: narratio.party.v1
|
||||
characters:
|
||||
arannis:
|
||||
player: {name: Eric}
|
||||
character:
|
||||
name: Arannis
|
||||
classes: [{name: wizard, level: 8}]
|
||||
`)
|
||||
setCanonicalPrepareParty(t, env, first)
|
||||
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("prepare.Run() error = %v", err)
|
||||
}
|
||||
assertPrepareResume(t, env, m, true)
|
||||
|
||||
second := []byte(`schema_version: narratio.party.v1
|
||||
characters:
|
||||
arannis:
|
||||
player: {name: Eric}
|
||||
character:
|
||||
name: Arannis
|
||||
alias: [The Red]
|
||||
classes: [{name: wizard, level: 9}]
|
||||
`)
|
||||
setCanonicalPrepareParty(t, env, second)
|
||||
assertPrepareResume(t, env, m, false)
|
||||
}
|
||||
|
||||
func TestPrepareResumeValidatesCurrentS3ObjectIdentity(t *testing.T) {
|
||||
env, m := setupPrepareEnv(t)
|
||||
env.Config.Session.Campaign = "forsaken"
|
||||
env.Config.Session.Inputs.AudioDir = ""
|
||||
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "audio/"}
|
||||
env.Config.Pipeline.Spool = config.SpoolConfig{Root: filepath.Join(t.TempDir(), "spool")}
|
||||
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "archive", RootPrefix: "dnd"}
|
||||
m.RunID = "20260515T031522Z-a1b2c3d4"
|
||||
m.LocalWorkDir = artifacts.SessionRunRootForCampaign(env.Config.Pipeline.Workspace.Root, "forsaken", m.SessionID, m.RunID)
|
||||
m.LocalSpoolDir = artifacts.SessionSpoolAudioDir(env.Config.Pipeline.Spool.Root, "forsaken", m.SessionID, m.RunID)
|
||||
key := "dnd/campaigns/forsaken/sessions/2026-05-03/audio/alice.flac"
|
||||
fake := &storage.FakeBackend{}
|
||||
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("alice")})
|
||||
env.ObjectStore = fake
|
||||
|
||||
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("prepare.Run() error = %v", err)
|
||||
}
|
||||
assertPrepareResume(t, env, m, true)
|
||||
fake.SeedObject(storage.FakeObject{Key: key, Data: []byte("changed")})
|
||||
assertPrepareResume(t, env, m, false)
|
||||
}
|
||||
|
||||
func prepareResumeLocalFixture(t *testing.T) (*Env, *manifest.Manifest, string) {
|
||||
t.Helper()
|
||||
env, m := setupPrepareEnv(t)
|
||||
audioPath := filepath.Join(filepath.Dir(env.Config.SessionPath), "audio", "one.flac")
|
||||
writeFile(t, audioPath, "audio")
|
||||
if _, err := (prepareStage{}).Run(context.Background(), env, m); err != nil {
|
||||
t.Fatalf("prepare.Run() error = %v", err)
|
||||
}
|
||||
return env, m, audioPath
|
||||
}
|
||||
|
||||
func setCanonicalPrepareParty(t *testing.T, env *Env, raw []byte) {
|
||||
t.Helper()
|
||||
document, err := config.ParseParty(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseParty() error = %v", err)
|
||||
}
|
||||
env.Config.Party = config.ResolvedParty{
|
||||
Mode: config.PartyModeCanonical, Source: config.PartySource{Source: "campaign_config"}, Canonical: document.Canonical,
|
||||
}
|
||||
}
|
||||
|
||||
func assertPrepareResume(t *testing.T, env *Env, m *manifest.Manifest, want bool) {
|
||||
t.Helper()
|
||||
validation, err := (prepareStage{}).ValidateResume(context.Background(), env, m)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateResume() error = %v", err)
|
||||
}
|
||||
if validation.Resumable != want {
|
||||
t.Fatalf("ValidateResume() = %#v, want resumable=%t", validation, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareResumeMissingPreparedFileReruns(t *testing.T) {
|
||||
env, m, _ := prepareResumeLocalFixture(t)
|
||||
paths := env.ArtifactStore.SessionPathsFor(env.Config.Session.Campaign, env.Config.Session.SessionID)
|
||||
if err := os.Remove(filepath.Join(paths.InputsDir, "players.yml")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertPrepareResume(t, env, m, false)
|
||||
}
|
||||
Reference in New Issue
Block a user