Protect initial pipeline stage resume semantics
This commit is contained in:
180
internal/stage/semantic_contracts_initial.go
Normal file
180
internal/stage/semantic_contracts_initial.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
const (
|
||||
prepareSemanticConfigVersion = 1
|
||||
transcribeSemanticConfigVersion = 1
|
||||
mergeSemanticConfigVersion = 1
|
||||
)
|
||||
|
||||
type prepareSemanticConfig struct {
|
||||
CampaignID string `json:"campaign_id"`
|
||||
SessionID string `json:"session_id"`
|
||||
PreviousSession string `json:"previous_session_id"`
|
||||
Audio prepareAudioSelection `json:"audio"`
|
||||
StableInputs []prepareStableInputSelection `json:"stable_inputs"`
|
||||
PreviousArtifacts []preparePreviousRequirement `json:"previous_artifacts"`
|
||||
}
|
||||
|
||||
type prepareAudioSelection struct {
|
||||
Mode string `json:"mode"`
|
||||
LocalNames []string `json:"local_names,omitempty"`
|
||||
S3Bucket string `json:"s3_bucket,omitempty"`
|
||||
S3RootPrefix string `json:"s3_root_prefix,omitempty"`
|
||||
S3AudioPrefix string `json:"s3_audio_prefix,omitempty"`
|
||||
}
|
||||
|
||||
type prepareStableInputSelection struct {
|
||||
Kind string `json:"kind"`
|
||||
Source string `json:"source"`
|
||||
Present bool `json:"present"`
|
||||
}
|
||||
|
||||
type preparePreviousRequirement struct {
|
||||
Name string `json:"name"`
|
||||
Required bool `json:"required"`
|
||||
OutputPath string `json:"output_path,omitempty"`
|
||||
}
|
||||
|
||||
type transcribeSemanticConfig struct {
|
||||
ServiceURL string `json:"service_url"`
|
||||
Language string `json:"language"`
|
||||
}
|
||||
|
||||
type mergeSemanticConfig struct {
|
||||
Operation string `json:"operation"`
|
||||
OutputSchema string `json:"output_schema"`
|
||||
CoalesceGap *float64 `json:"coalesce_gap,omitempty"`
|
||||
OverlapWordRunGap *float64 `json:"overlap_word_run_gap,omitempty"`
|
||||
OverlapWordRunReorderWindow *float64 `json:"overlap_word_run_reorder_window,omitempty"`
|
||||
BackchannelMaxDuration *float64 `json:"backchannel_max_duration,omitempty"`
|
||||
FillerMaxDuration *float64 `json:"filler_max_duration,omitempty"`
|
||||
}
|
||||
|
||||
func (prepareStage) SemanticConfigFingerprint(env *Env) (manifest.SemanticConfigFingerprint, error) {
|
||||
payload, err := buildPrepareSemanticConfig(env)
|
||||
if err != nil {
|
||||
return manifest.SemanticConfigFingerprint{}, err
|
||||
}
|
||||
return FingerprintSemanticConfig(prepareSemanticConfigVersion, payload)
|
||||
}
|
||||
|
||||
func (transcribeStage) SemanticConfigFingerprint(env *Env) (manifest.SemanticConfigFingerprint, error) {
|
||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil {
|
||||
return manifest.SemanticConfigFingerprint{}, fmt.Errorf("transcribe semantic configuration requires resolved pipeline config")
|
||||
}
|
||||
whisper := env.Config.Pipeline.WhisperX
|
||||
return FingerprintSemanticConfig(transcribeSemanticConfigVersion, transcribeSemanticConfig{
|
||||
ServiceURL: strings.TrimSpace(whisper.TranscribeURL),
|
||||
Language: strings.TrimSpace(whisper.Language),
|
||||
})
|
||||
}
|
||||
|
||||
func (mergeStage) SemanticConfigFingerprint(env *Env) (manifest.SemanticConfigFingerprint, error) {
|
||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil {
|
||||
return manifest.SemanticConfigFingerprint{}, fmt.Errorf("merge semantic configuration requires resolved pipeline config")
|
||||
}
|
||||
seriatim := env.Config.Pipeline.Seriatim
|
||||
return FingerprintSemanticConfig(mergeSemanticConfigVersion, mergeSemanticConfig{
|
||||
Operation: "merge",
|
||||
OutputSchema: strings.TrimSpace(seriatim.OutputSchema),
|
||||
CoalesceGap: cloneFloat64(seriatim.CoalesceGap),
|
||||
OverlapWordRunGap: cloneFloat64(seriatim.Env.OverlapWordRunGap),
|
||||
OverlapWordRunReorderWindow: cloneFloat64(seriatim.Env.OverlapWordRunReorderWindow),
|
||||
BackchannelMaxDuration: cloneFloat64(seriatim.Env.BackchannelMaxDuration),
|
||||
FillerMaxDuration: cloneFloat64(seriatim.Env.FillerMaxDuration),
|
||||
})
|
||||
}
|
||||
|
||||
func buildPrepareSemanticConfig(env *Env) (prepareSemanticConfig, error) {
|
||||
if env == nil || env.Config == nil || env.Config.Pipeline == nil || env.Config.Session == nil {
|
||||
return prepareSemanticConfig{}, fmt.Errorf("prepare semantic configuration requires resolved pipeline and session config")
|
||||
}
|
||||
cfg := env.Config
|
||||
requirements, err := collectPreparePreviousRequirements(cfg, env.EffectiveArtifacts)
|
||||
if err != nil {
|
||||
return prepareSemanticConfig{}, fmt.Errorf("resolve previous artifact requirements: %w", err)
|
||||
}
|
||||
payload := prepareSemanticConfig{
|
||||
CampaignID: strings.TrimSpace(cfg.Session.Campaign),
|
||||
SessionID: strings.TrimSpace(cfg.Session.SessionID),
|
||||
PreviousSession: strings.TrimSpace(cfg.Session.PreviousSessionID),
|
||||
Audio: prepareAudioSemantics(cfg),
|
||||
StableInputs: prepareStableInputSemantics(cfg),
|
||||
}
|
||||
for _, requirement := range requirements {
|
||||
entry := preparePreviousRequirement{Name: strings.TrimSpace(requirement.Name), Required: requirement.Required}
|
||||
if cfg.Pipeline.Scriptorium != nil {
|
||||
entry.OutputPath = strings.TrimSpace(cfg.Pipeline.Scriptorium.Artifacts[requirement.Name].OutputPath)
|
||||
}
|
||||
payload.PreviousArtifacts = append(payload.PreviousArtifacts, entry)
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func prepareAudioSemantics(cfg *config.Config) prepareAudioSelection {
|
||||
inputs := cfg.Session.Inputs
|
||||
if inputs.AudioS3 != nil {
|
||||
selection := prepareAudioSelection{Mode: "s3", S3AudioPrefix: strings.TrimSpace(inputs.AudioS3.Prefix)}
|
||||
if cfg.Pipeline.Storage.S3 != nil {
|
||||
selection.S3Bucket = strings.TrimSpace(cfg.Pipeline.Storage.S3.Bucket)
|
||||
selection.S3RootPrefix = strings.TrimSpace(cfg.Pipeline.Storage.S3.RootPrefix)
|
||||
}
|
||||
return selection
|
||||
}
|
||||
selection := prepareAudioSelection{Mode: "local_directory"}
|
||||
if len(inputs.AudioFiles) > 0 {
|
||||
selection.Mode = "local_files"
|
||||
destinations := localAudioDestinations(inputs.AudioFiles)
|
||||
for _, name := range destinations {
|
||||
selection.LocalNames = append(selection.LocalNames, filepath.ToSlash(name))
|
||||
}
|
||||
sort.Strings(selection.LocalNames)
|
||||
}
|
||||
return selection
|
||||
}
|
||||
|
||||
func prepareStableInputSemantics(cfg *config.Config) []prepareStableInputSelection {
|
||||
stable := cfg.StableInputs
|
||||
return []prepareStableInputSelection{
|
||||
prepareStableInput("speakers", stable.SpeakersFile, cfg.Session.Inputs.SpeakersFile),
|
||||
prepareStableInput("autocorrect", stable.AutocorrectFile, cfg.Session.Inputs.AutocorrectFile),
|
||||
prepareStableInput("glossary", stable.GlossaryFile, cfg.Session.Inputs.GlossaryFile),
|
||||
prepareStableInput("players", stable.PlayersFile, cfg.Session.Inputs.PlayersFile),
|
||||
prepareStableInput("party", stable.PartyFile, cfg.Session.Inputs.PartyFile),
|
||||
prepareStableInput("spell_catalog", stable.SpellCatalogFile, cfg.Session.Inputs.SpellCatalogFile),
|
||||
}
|
||||
}
|
||||
|
||||
func prepareStableInput(kind string, resolved config.ResolvedInputFile, fallback string) prepareStableInputSelection {
|
||||
source := strings.TrimSpace(resolved.Source)
|
||||
if source == "" {
|
||||
source = "session_config"
|
||||
}
|
||||
path := resolved.Path
|
||||
if strings.TrimSpace(path) == "" {
|
||||
path = fallback
|
||||
}
|
||||
return prepareStableInputSelection{Kind: kind, Source: source, Present: strings.TrimSpace(path) != ""}
|
||||
}
|
||||
|
||||
func cloneFloat64(value *float64) *float64 {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
copy := *value
|
||||
return ©
|
||||
}
|
||||
|
||||
var _ SemanticConfigFingerprinter = prepareStage{}
|
||||
var _ SemanticConfigFingerprinter = transcribeStage{}
|
||||
var _ SemanticConfigFingerprinter = mergeStage{}
|
||||
174
internal/stage/semantic_contracts_initial_test.go
Normal file
174
internal/stage/semantic_contracts_initial_test.go
Normal file
@@ -0,0 +1,174 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func TestPrepareSemanticConfigSensitivity(t *testing.T) {
|
||||
assertSemanticFingerprintChanges(t, prepareStage{}, initialSemanticEnv(), []struct {
|
||||
name string
|
||||
mutate func(*Env)
|
||||
}{
|
||||
{name: "audio mode", mutate: func(env *Env) {
|
||||
env.Config.Session.Inputs.AudioFiles = nil
|
||||
env.Config.Session.Inputs.AudioS3 = &config.SessionAudioS3Input{Prefix: "sessions/audio"}
|
||||
env.Config.Pipeline.Storage.S3 = &config.StorageS3Config{Bucket: "archive", RootPrefix: "dnd"}
|
||||
}},
|
||||
{name: "audio canonical name", mutate: func(env *Env) {
|
||||
env.Config.Session.Inputs.AudioFiles = []string{"/audio/other.flac"}
|
||||
}},
|
||||
{name: "previous session", mutate: func(env *Env) {
|
||||
env.Config.Session.PreviousSessionID = "2026-04-26"
|
||||
}},
|
||||
{name: "stable input owner", mutate: func(env *Env) {
|
||||
env.Config.StableInputs.PartyFile.Source = "campaign_config"
|
||||
}},
|
||||
})
|
||||
|
||||
assertSemanticFingerprintUnchanged(t, prepareStage{}, initialSemanticEnv(), []struct {
|
||||
name string
|
||||
mutate func(*Env)
|
||||
}{
|
||||
{name: "workspace and spool", mutate: func(env *Env) {
|
||||
env.Config.Pipeline.Workspace.Root = "/different/work"
|
||||
env.Config.Pipeline.Spool.Root = "/different/spool"
|
||||
env.Config.Pipeline.Cache.Root = "/different/cache"
|
||||
}},
|
||||
{name: "absolute stable source path", mutate: func(env *Env) {
|
||||
env.Config.StableInputs.PartyFile.Path = "/relocated/party.yml"
|
||||
env.Config.StableInputs.PartyFile.ConfigPath = "/relocated/campaign.yml"
|
||||
}},
|
||||
{name: "absolute unique audio source", mutate: func(env *Env) {
|
||||
env.Config.Session.Inputs.AudioFiles = []string{"/relocated/speaker.flac"}
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
func TestTranscribeSemanticConfigSensitivity(t *testing.T) {
|
||||
assertSemanticFingerprintChanges(t, transcribeStage{}, initialSemanticEnv(), []struct {
|
||||
name string
|
||||
mutate func(*Env)
|
||||
}{
|
||||
{name: "language", mutate: func(env *Env) { env.Config.Pipeline.WhisperX.Language = "fr" }},
|
||||
{name: "service", mutate: func(env *Env) { env.Config.Pipeline.WhisperX.TranscribeURL = "https://other.example/transcribe" }},
|
||||
})
|
||||
assertSemanticFingerprintUnchanged(t, transcribeStage{}, initialSemanticEnv(), []struct {
|
||||
name string
|
||||
mutate func(*Env)
|
||||
}{
|
||||
{name: "transport tuning", mutate: func(env *Env) {
|
||||
env.Config.Pipeline.WhisperX.Timeout = "30m"
|
||||
env.Config.Pipeline.WhisperX.Retries = intPointer(9)
|
||||
env.Config.Pipeline.WhisperX.Concurrency = intPointer(12)
|
||||
env.Config.Pipeline.WhisperX.RetryDelay = "10s"
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
func TestMergeSemanticConfigSensitivity(t *testing.T) {
|
||||
assertSemanticFingerprintChanges(t, mergeStage{}, initialSemanticEnv(), []struct {
|
||||
name string
|
||||
mutate func(*Env)
|
||||
}{
|
||||
{name: "output schema", mutate: func(env *Env) { env.Config.Pipeline.Seriatim.OutputSchema = "seriatim.transcript.v2" }},
|
||||
{name: "coalesce gap", mutate: func(env *Env) { env.Config.Pipeline.Seriatim.CoalesceGap = floatPointer(1.25) }},
|
||||
{name: "overlap transformation", mutate: func(env *Env) { env.Config.Pipeline.Seriatim.Env.OverlapWordRunGap = floatPointer(0.75) }},
|
||||
})
|
||||
assertSemanticFingerprintUnchanged(t, mergeStage{}, initialSemanticEnv(), []struct {
|
||||
name string
|
||||
mutate func(*Env)
|
||||
}{
|
||||
{name: "process and diagnostics", mutate: func(env *Env) {
|
||||
env.Config.Pipeline.Seriatim.Binary = "/opt/seriatim"
|
||||
env.Config.Pipeline.Seriatim.Timeout = "45m"
|
||||
env.Config.Pipeline.Seriatim.Report = boolPointer(false)
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
func initialSemanticEnv() *Env {
|
||||
return &Env{Config: &config.Config{
|
||||
Pipeline: &config.PipelineConfig{
|
||||
Workspace: config.WorkspaceConfig{Root: "/work"},
|
||||
Spool: config.SpoolConfig{Root: "/spool"},
|
||||
Cache: config.CacheConfig{Root: "/cache"},
|
||||
WhisperX: config.WhisperXConfig{
|
||||
TranscribeURL: "https://whisper.example/transcribe", Language: "en",
|
||||
Timeout: "10m", Retries: intPointer(3), RetryDelay: "1s", Concurrency: intPointer(2),
|
||||
},
|
||||
Seriatim: config.SeriatimConfig{
|
||||
Binary: "seriatim", Timeout: "10m", OutputSchema: "seriatim.transcript.v1",
|
||||
CoalesceGap: floatPointer(0.5), Report: boolPointer(true),
|
||||
Env: config.SeriatimEnvConfig{OverlapWordRunGap: floatPointer(0.25)},
|
||||
},
|
||||
},
|
||||
Session: &config.SessionConfig{
|
||||
SessionID: "2026-05-03", Campaign: "campaign",
|
||||
Inputs: config.SessionInputsConfig{
|
||||
AudioFiles: []string{"/audio/speaker.flac"}, PartyFile: "party.yml",
|
||||
},
|
||||
},
|
||||
StableInputs: config.ResolvedStableInputs{
|
||||
PartyFile: config.ResolvedInputFile{Path: "/campaign/party.yml", ConfigPath: "/campaign/campaign.yml", Source: "session_config"},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
type semanticFingerprintProvider interface {
|
||||
SemanticConfigFingerprint(*Env) (manifest.SemanticConfigFingerprint, error)
|
||||
}
|
||||
|
||||
func assertSemanticFingerprintChanges(t *testing.T, provider semanticFingerprintProvider, base *Env, tests []struct {
|
||||
name string
|
||||
mutate func(*Env)
|
||||
}) {
|
||||
t.Helper()
|
||||
want, err := provider.SemanticConfigFingerprint(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run("semantic "+test.name, func(t *testing.T) {
|
||||
candidate := initialSemanticEnv()
|
||||
test.mutate(candidate)
|
||||
got, err := provider.SemanticConfigFingerprint(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want.Equal(got) {
|
||||
t.Fatalf("semantic change %q retained fingerprint %q", test.name, got.Digest)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertSemanticFingerprintUnchanged(t *testing.T, provider semanticFingerprintProvider, base *Env, tests []struct {
|
||||
name string
|
||||
mutate func(*Env)
|
||||
}) {
|
||||
t.Helper()
|
||||
want, err := provider.SemanticConfigFingerprint(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run("operational "+test.name, func(t *testing.T) {
|
||||
candidate := initialSemanticEnv()
|
||||
test.mutate(candidate)
|
||||
got, err := provider.SemanticConfigFingerprint(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !want.Equal(got) {
|
||||
t.Fatalf("operational change %q changed fingerprint: want %#v got %#v", test.name, want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func intPointer(value int) *int { return &value }
|
||||
func floatPointer(value float64) *float64 { return &value }
|
||||
func boolPointer(value bool) *bool { return &value }
|
||||
Reference in New Issue
Block a user