Files
narratio/internal/app/semantic_resume_test.go

578 lines
22 KiB
Go

package app
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
type semanticStage struct {
name string
fingerprint manifest.SemanticConfigFingerprint
fingerprintErr error
result *stage.StageResult
runErr error
runs *int
}
func (s semanticStage) Name() string { return s.name }
func (s semanticStage) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
if s.runs != nil {
*s.runs++
}
return s.result, s.runErr
}
func (s semanticStage) SemanticConfigFingerprint(_ *stage.Env) (manifest.SemanticConfigFingerprint, error) {
return s.fingerprint, s.fingerprintErr
}
type semanticResumeCheckingStage struct {
semanticStage
validation stage.ResumeValidation
validationCalls *int
}
type semanticContractRunStub struct {
name string
provider stage.SemanticConfigFingerprinter
runs *int
}
func (s semanticContractRunStub) Name() string { return s.name }
func (s semanticContractRunStub) Run(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (*stage.StageResult, error) {
*s.runs++
return &stage.StageResult{}, nil
}
func (s semanticContractRunStub) SemanticConfigFingerprint(env *stage.Env) (manifest.SemanticConfigFingerprint, error) {
return s.provider.SemanticConfigFingerprint(env)
}
func (s semanticResumeCheckingStage) ValidateResume(_ context.Context, _ *stage.Env, _ *manifest.Manifest) (stage.ResumeValidation, error) {
if s.validationCalls != nil {
*s.validationCalls++
}
return s.validation, nil
}
func TestSemanticResumeComparison(t *testing.T) {
current := semanticFingerprint(1, "a")
selected := semanticStage{name: "render", fingerprint: current}
for _, test := range []struct {
name string
persisted *manifest.SemanticConfigFingerprint
resumable bool
want string
}{
{name: "matching", persisted: &current, resumable: true},
{name: "missing", want: "missing"},
{name: "version", persisted: fingerprintPointer(semanticFingerprint(2, "a")), want: "version"},
{name: "digest", persisted: fingerprintPointer(semanticFingerprint(1, "b")), want: "configuration changed"},
{name: "malformed", persisted: &manifest.SemanticConfigFingerprint{Version: 1, Digest: "bad"}, want: "malformed"},
} {
t.Run(test.name, func(t *testing.T) {
model := manifest.New("session", time.Now().UTC())
model.MarkStageSucceeded("render", time.Now().UTC(), nil)
model.Stages["render"].SemanticConfig = test.persisted
got := validateStageSemanticResume(selected, model, &current)
if got.Resumable != test.resumable || (test.want != "" && !strings.Contains(got.Reason, test.want)) {
t.Fatalf("validation = %#v, want resumable=%v reason %q", got, test.resumable, test.want)
}
})
}
}
func TestExecuteStagesSemanticEvidenceLifecycle(t *testing.T) {
for _, test := range []struct {
name string
persisted *manifest.SemanticConfigFingerprint
force bool
wantRuns int
wantSkipped int
}{
{name: "matching skips", persisted: fingerprintPointer(semanticFingerprint(1, "current")), wantSkipped: 1},
{name: "legacy missing reruns", wantRuns: 1},
{name: "version reruns", persisted: fingerprintPointer(semanticFingerprint(2, "current")), wantRuns: 1},
{name: "digest reruns", persisted: fingerprintPointer(semanticFingerprint(1, "old")), wantRuns: 1},
{name: "force reruns matching", persisted: fingerprintPointer(semanticFingerprint(1, "current")), force: true, wantRuns: 1},
} {
t.Run(test.name, func(t *testing.T) {
cfg := testConfig(t)
store := &manifest.LocalStore{}
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
seed.MarkStageSucceeded("render", time.Now().UTC(), nil)
seed.Stages["render"].SemanticConfig = test.persisted
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatal(err)
}
runs := 0
current := semanticFingerprint(1, "current")
candidate := semanticStage{name: "render", fingerprint: current, result: &stage.StageResult{}, runs: &runs}
summary, err := executeStages(context.Background(), cfg, []stage.Stage{candidate}, RunOptions{Force: test.force})
if err != nil {
t.Fatal(err)
}
if runs != test.wantRuns || len(summary.Skipped) != test.wantSkipped {
t.Fatalf("runs=%d summary=%#v", runs, summary)
}
loaded, err := store.Load(context.Background(), summary.ManifestPath)
if err != nil {
t.Fatal(err)
}
if loaded.Stages["render"].SemanticConfig == nil || !loaded.Stages["render"].SemanticConfig.Equal(current) {
t.Fatalf("session evidence = %#v", loaded.Stages["render"].SemanticConfig)
}
run, err := store.LoadRun(context.Background(), summary.RunManifestPath)
if err != nil {
t.Fatal(err)
}
if run.Stages["render"].SemanticConfig == nil || !run.Stages["render"].SemanticConfig.Equal(current) {
t.Fatalf("run evidence = %#v", run.Stages["render"].SemanticConfig)
}
})
}
}
func TestSemanticEvidenceRequiresBothChecksAndNeverPromotesFailure(t *testing.T) {
t.Run("semantic mismatch precedes resume validator", func(t *testing.T) {
cfg := testConfig(t)
store := &manifest.LocalStore{}
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
seed.MarkStageSucceeded("render", time.Now().UTC(), nil)
seed.Stages["render"].SemanticConfig = fingerprintPointer(semanticFingerprint(1, "old"))
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatal(err)
}
runs, validations := 0, 0
candidate := semanticResumeCheckingStage{
semanticStage: semanticStage{name: "render", fingerprint: semanticFingerprint(1, "new"), result: &stage.StageResult{}, runs: &runs},
validation: stage.Resumable(), validationCalls: &validations,
}
if _, err := executeStages(context.Background(), cfg, []stage.Stage{candidate}, RunOptions{}); err != nil {
t.Fatal(err)
}
if runs != 1 || validations != 0 {
t.Fatalf("runs=%d validations=%d", runs, validations)
}
})
t.Run("matching semantic evidence still requires validator", func(t *testing.T) {
cfg := testConfig(t)
store := &manifest.LocalStore{}
current := semanticFingerprint(1, "same")
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
seed.MarkStageSucceeded("render", time.Now().UTC(), nil)
seed.Stages["render"].SemanticConfig = &current
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatal(err)
}
runs, validations := 0, 0
candidate := semanticResumeCheckingStage{
semanticStage: semanticStage{name: "render", fingerprint: current, result: &stage.StageResult{}, runs: &runs},
validation: stage.NonResumable("output changed"), validationCalls: &validations,
}
if _, err := executeStages(context.Background(), cfg, []stage.Stage{candidate}, RunOptions{}); err != nil {
t.Fatal(err)
}
if runs != 1 || validations != 1 {
t.Fatalf("runs=%d validations=%d", runs, validations)
}
})
t.Run("failed execution has no promoted evidence", func(t *testing.T) {
cfg := testConfig(t)
runs := 0
candidate := semanticStage{
name: "render", fingerprint: semanticFingerprint(1, "new"), runErr: errors.New("render failed"), runs: &runs,
}
_, err := executeStages(context.Background(), cfg, []stage.Stage{candidate}, RunOptions{})
if err == nil {
t.Fatal("executeStages() error = nil")
}
loaded, loadErr := (&manifest.LocalStore{}).Load(context.Background(), manifestPathFor(cfg))
if loadErr != nil {
t.Fatal(loadErr)
}
if loaded.Stages["render"].SemanticConfig != nil || runs != 1 {
t.Fatalf("failed evidence=%#v runs=%d", loaded.Stages["render"].SemanticConfig, runs)
}
})
}
func TestIntentionalStageSkipPersistsSemanticEvidence(t *testing.T) {
cfg := testConfig(t)
current := semanticFingerprint(1, "disabled")
runs := 0
candidate := semanticStage{
name: "render", fingerprint: current, runs: &runs,
result: &stage.StageResult{Disposition: stage.StageDispositionSkipped, SkipReason: "render disabled"},
}
summary, err := executeStages(context.Background(), cfg, []stage.Stage{candidate}, RunOptions{})
if err != nil {
t.Fatal(err)
}
store := &manifest.LocalStore{}
session, err := store.Load(context.Background(), summary.ManifestPath)
if err != nil {
t.Fatal(err)
}
run, err := store.LoadRun(context.Background(), summary.RunManifestPath)
if err != nil {
t.Fatal(err)
}
if runs != 1 || session.Stages["render"].SemanticConfig == nil || run.Stages["render"].SemanticConfig == nil ||
!session.Stages["render"].SemanticConfig.Equal(current) || !run.Stages["render"].SemanticConfig.Equal(current) {
t.Fatalf("runs=%d session=%#v run=%#v", runs, session.Stages["render"], run.Stages["render"])
}
}
func TestInvalidCurrentSemanticEvidenceStopsBeforeExecution(t *testing.T) {
cfg := testConfig(t)
runs := 0
candidate := semanticStage{
name: "render", fingerprint: manifest.SemanticConfigFingerprint{Version: 1, Digest: "invalid"}, runs: &runs,
}
_, err := executeStages(context.Background(), cfg, []stage.Stage{candidate}, RunOptions{Force: true})
if err == nil || !strings.Contains(err.Error(), "invalid current semantic configuration fingerprint") || runs != 0 {
t.Fatalf("error=%v runs=%d", err, runs)
}
}
func TestReadOnlySemanticResumeDecisionMutatesOnlyPlanModel(t *testing.T) {
cfg := testConfig(t)
original := manifest.New(cfg.Session.SessionID, time.Now().UTC())
original.MarkStageSucceeded("render", time.Now().UTC(), nil)
original.MarkStageSucceeded("extract", time.Now().UTC(), nil)
original.MarkStageSucceeded("analyze", time.Now().UTC(), nil)
original.Stages["render"].SemanticConfig = fingerprintPointer(semanticFingerprint(1, "old"))
model, err := cloneManifestForPlan(original, cfg)
if err != nil {
t.Fatal(err)
}
selected := semanticStage{name: "render", fingerprint: semanticFingerprint(1, "new")}
current, err := currentStageSemanticConfig(selected, &stage.Env{Config: cfg})
if err != nil {
t.Fatal(err)
}
validation, err := evaluateStageResume(context.Background(), selected, &stage.Env{Config: cfg}, model, current)
if err != nil {
t.Fatal(err)
}
if validation == nil || validation.Resumable {
t.Fatalf("validation = %#v, want planned rerun", validation)
}
at := time.Now().UTC()
model.MarkStageStale("render", at, validation.Reason)
if _, err := invalidateDependentSucceededStagesWithReason(model, "render", at, staleReasonNotResumable); err != nil {
t.Fatal(err)
}
if model.Stages["render"].Status != manifest.StatusStale || model.Stages["analyze"].Status != manifest.StatusStale {
t.Fatalf("model render=%q analyze=%q", model.Stages["render"].Status, model.Stages["analyze"].Status)
}
if original.Stages["render"].Status != manifest.StatusSucceeded || original.Stages["analyze"].Status != manifest.StatusSucceeded {
t.Fatalf("authoritative manifest mutated: render=%q analyze=%q", original.Stages["render"].Status, original.Stages["analyze"].Status)
}
}
func TestSemanticMismatchInvalidatesOnlyFixedDependents(t *testing.T) {
cfg := testConfig(t)
store := &manifest.LocalStore{}
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
for _, name := range []string{"render", "extract", "analyze", "publish", "notify"} {
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
seed.Stages["render"].SemanticConfig = fingerprintPointer(semanticFingerprint(1, "old"))
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatal(err)
}
runs := 0
candidate := semanticStage{name: "render", fingerprint: semanticFingerprint(1, "new"), result: &stage.StageResult{}, runs: &runs}
if _, err := executeStages(context.Background(), cfg, []stage.Stage{candidate}, RunOptions{}); err != nil {
t.Fatal(err)
}
loaded, err := store.Load(context.Background(), manifestPathFor(cfg))
if err != nil {
t.Fatal(err)
}
if loaded.Stages["render"].Status != manifest.StatusSucceeded || loaded.Stages["extract"].Status != manifest.StatusSucceeded {
t.Fatalf("render=%q extract=%q", loaded.Stages["render"].Status, loaded.Stages["extract"].Status)
}
for _, name := range []string{"analyze", "publish", "notify"} {
if loaded.Stages[name].Status != manifest.StatusStale {
t.Fatalf("%s status = %q, want stale", name, loaded.Stages[name].Status)
}
}
}
func TestInitialPipelineSemanticChangesRerunOnlyAffectedLineage(t *testing.T) {
for _, test := range []struct {
name string
mutate func(*config.Config)
wantRuns [3]int
}{
{name: "prepare selection", mutate: func(cfg *config.Config) {
cfg.Session.PreviousSessionID = "2026-04-26"
}, wantRuns: [3]int{1, 1, 1}},
{name: "transcribe language", mutate: func(cfg *config.Config) {
cfg.Pipeline.WhisperX.Language = "fr"
}, wantRuns: [3]int{0, 1, 1}},
{name: "merge transformation", mutate: func(cfg *config.Config) {
value := 1.75
cfg.Pipeline.Seriatim.CoalesceGap = &value
}, wantRuns: [3]int{0, 0, 1}},
} {
t.Run(test.name, func(t *testing.T) {
cfg := testConfig(t)
names := []string{"prepare", "transcribe", "merge"}
providers := make([]stage.SemanticConfigFingerprinter, len(names))
for index, name := range names {
providers[index] = canonicalSemanticProvider(t, name)
}
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
for _, candidate := range stage.All() {
seed.MarkStageSucceeded(candidate.Name(), time.Now().UTC(), nil)
}
for index, name := range names {
fingerprint, err := providers[index].SemanticConfigFingerprint(&stage.Env{Config: cfg})
if err != nil {
t.Fatal(err)
}
seed.Stages[name].SemanticConfig = &fingerprint
}
store := &manifest.LocalStore{}
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatal(err)
}
test.mutate(cfg)
runs := [3]int{}
selected := make([]stage.Stage, 0, len(names))
for index, name := range names {
selected = append(selected, semanticContractRunStub{name: name, provider: providers[index], runs: &runs[index]})
}
if _, err := executeStages(context.Background(), cfg, selected, RunOptions{}); err != nil {
t.Fatal(err)
}
if runs != test.wantRuns {
t.Fatalf("runs = %v, want %v", runs, test.wantRuns)
}
})
}
}
func TestRefinementSemanticChangesRespectDependencyBranches(t *testing.T) {
names := []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render"}
for _, test := range []struct {
name string
mutate func(*config.Config)
wantRuns [7]int
wantExtractStatus manifest.StageStatus
}{
{name: "polish model", mutate: func(cfg *config.Config) {
cfg.Pipeline.Audita.Model = "production"
}, wantRuns: [7]int{0, 0, 0, 1, 1, 1, 1}, wantExtractStatus: manifest.StatusStale},
{name: "normalize schema", mutate: func(cfg *config.Config) {
cfg.Pipeline.Normalize = &config.NormalizeConfig{OutputSchema: "seriatim.transcript.v2"}
}, wantRuns: [7]int{0, 0, 0, 0, 1, 1, 1}, wantExtractStatus: manifest.StatusStale},
{name: "trim prompt", mutate: func(cfg *config.Config) {
enabled := true
cfg.Pipeline.Trim = &config.TrimConfig{
Enabled: &enabled,
Bounds: config.TrimBoundsConfig{PromptID: "session-bounds-v2"},
}
}, wantRuns: [7]int{0, 0, 0, 0, 0, 1, 1}, wantExtractStatus: manifest.StatusStale},
{name: "render format", mutate: func(cfg *config.Config) {
cfg.Pipeline.Render = &config.RenderConfig{Format: "html"}
}, wantRuns: [7]int{0, 0, 0, 0, 0, 0, 1}, wantExtractStatus: manifest.StatusSucceeded},
} {
t.Run(test.name, func(t *testing.T) {
cfg := testConfig(t)
providers := make([]stage.SemanticConfigFingerprinter, len(names))
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
for _, candidate := range stage.All() {
seed.MarkStageSucceeded(candidate.Name(), time.Now().UTC(), nil)
}
for index, name := range names {
providers[index] = canonicalSemanticProvider(t, name)
fingerprint, err := providers[index].SemanticConfigFingerprint(&stage.Env{Config: cfg})
if err != nil {
t.Fatal(err)
}
seed.Stages[name].SemanticConfig = &fingerprint
}
store := &manifest.LocalStore{}
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatal(err)
}
test.mutate(cfg)
runs := [7]int{}
selected := make([]stage.Stage, 0, len(names))
for index, name := range names {
selected = append(selected, semanticContractRunStub{name: name, provider: providers[index], runs: &runs[index]})
}
if _, err := executeStages(context.Background(), cfg, selected, RunOptions{}); err != nil {
t.Fatal(err)
}
if runs != test.wantRuns {
t.Fatalf("runs = %v, want %v", runs, test.wantRuns)
}
loaded, err := store.Load(context.Background(), manifestPathFor(cfg))
if err != nil {
t.Fatal(err)
}
if got := loaded.Stages["extract"].Status; got != test.wantExtractStatus {
t.Fatalf("extract status = %q, want %q", got, test.wantExtractStatus)
}
})
}
}
func TestDeliverySemanticChangesInvalidateOnlyTheirFixedDependents(t *testing.T) {
t.Run("extract contract", func(t *testing.T) {
cfg := testConfig(t)
configureDeliverySemanticConfig(cfg)
provider := canonicalSemanticProvider(t, "extract")
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
for _, name := range []string{"render", "extract", "analyze", "publish", "notify"} {
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
fingerprint, err := provider.SemanticConfigFingerprint(&stage.Env{Config: cfg})
if err != nil {
t.Fatal(err)
}
seed.Stages["extract"].SemanticConfig = &fingerprint
store := &manifest.LocalStore{}
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatal(err)
}
cfg.Pipeline.Notarius.PipelineID = "session-v2"
runs := 0
selected := semanticContractRunStub{name: "extract", provider: provider, runs: &runs}
if _, err := executeStages(context.Background(), cfg, []stage.Stage{selected}, RunOptions{}); err != nil {
t.Fatal(err)
}
loaded, err := store.Load(context.Background(), manifestPathFor(cfg))
if err != nil {
t.Fatal(err)
}
if runs != 1 || loaded.Stages["render"].Status != manifest.StatusSucceeded {
t.Fatalf("runs=%d render=%q", runs, loaded.Stages["render"].Status)
}
for _, name := range []string{"analyze", "publish", "notify"} {
if loaded.Stages[name].Status != manifest.StatusStale {
t.Fatalf("%s status = %q, want stale", name, loaded.Stages[name].Status)
}
}
})
for _, test := range []struct {
name string
mutate func(*config.Config)
wantRuns [2]int
}{
{name: "publish destination", mutate: func(cfg *config.Config) {
cfg.Pipeline.Publish.Outputs[0].Dest = "published/alternate.json"
}, wantRuns: [2]int{1, 1}},
{name: "publish credentials", mutate: func(cfg *config.Config) {
cfg.Pipeline.Storage.S3.AccessKeyIDEnv = "OTHER_ACCESS_KEY"
cfg.Pipeline.Storage.S3.SecretKeyEnv = "OTHER_SECRET_KEY"
}, wantRuns: [2]int{0, 0}},
{name: "notify mode", mutate: func(cfg *config.Config) {
cfg.Pipeline.Notification.Mode = "webhook"
}, wantRuns: [2]int{0, 1}},
} {
t.Run(test.name, func(t *testing.T) {
cfg := testConfig(t)
configureDeliverySemanticConfig(cfg)
names := []string{"publish", "notify"}
providers := []stage.SemanticConfigFingerprinter{
canonicalSemanticProvider(t, names[0]), canonicalSemanticProvider(t, names[1]),
}
seed := manifest.New(cfg.Session.SessionID, time.Now().UTC())
for index, name := range names {
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
fingerprint, err := providers[index].SemanticConfigFingerprint(&stage.Env{Config: cfg})
if err != nil {
t.Fatal(err)
}
seed.Stages[name].SemanticConfig = &fingerprint
}
store := &manifest.LocalStore{}
if err := store.Save(context.Background(), manifestPathFor(cfg), seed); err != nil {
t.Fatal(err)
}
test.mutate(cfg)
runs := [2]int{}
selected := []stage.Stage{
semanticContractRunStub{name: names[0], provider: providers[0], runs: &runs[0]},
semanticContractRunStub{name: names[1], provider: providers[1], runs: &runs[1]},
}
if _, err := executeStages(context.Background(), cfg, selected, RunOptions{}); err != nil {
t.Fatal(err)
}
if runs != test.wantRuns {
t.Fatalf("runs = %v, want %v", runs, test.wantRuns)
}
})
}
}
func configureDeliverySemanticConfig(cfg *config.Config) {
enabled := true
disabled := false
cfg.Pipeline.Notarius = &config.NotariusConfig{
Enabled: true, PipelineID: "session",
References: map[string]string{"party": "narratio.input.party"},
Outputs: map[string]config.NotariusOutputConfig{
"encounters": {LaneID: "encounters", MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1"},
},
}
cfg.Pipeline.Publish = &config.PublishConfig{
Enabled: &disabled, UploadRun: &enabled,
Outputs: []config.PublishOutputRule{{
Source: "narratio.transcript.final_trimmed", Dest: "transcripts/final.trimmed.json", Required: &enabled,
}},
}
cfg.Pipeline.Storage = config.StorageConfig{
Backend: config.StorageBackendLocal,
S3: &config.StorageS3Config{
Bucket: "campaign", RootPrefix: "narratio", Region: "us-east-1",
Endpoint: "https://objects.example", AccessKeyIDEnv: "ACCESS_KEY", SecretKeyEnv: "SECRET_KEY",
},
}
cfg.Pipeline.Notification.Mode = config.DefaultNotificationMode
}
func canonicalSemanticProvider(t *testing.T, name string) stage.SemanticConfigFingerprinter {
t.Helper()
for _, candidate := range stage.All() {
if candidate.Name() != name {
continue
}
provider, ok := candidate.(stage.SemanticConfigFingerprinter)
if !ok {
t.Fatalf("canonical stage %q does not implement semantic fingerprinting", name)
}
return provider
}
t.Fatalf("canonical stage %q not found", name)
return nil
}
func semanticFingerprint(version int, seed string) manifest.SemanticConfigFingerprint {
digest := sha256.Sum256([]byte(seed))
return manifest.SemanticConfigFingerprint{Version: version, Digest: hex.EncodeToString(digest[:])}
}
func fingerprintPointer(value manifest.SemanticConfigFingerprint) *manifest.SemanticConfigFingerprint {
return &value
}