notarius/internal/cli/compatibility_test.go

package cli

import (
	"bytes"
	"context"
	"encoding/json"
	"io/fs"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"sync"
	"testing"

	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
	frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
	spellshape "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/shape"
	spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_refs"
	spellrelatedness "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_relatedness"
	validjson "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json"
	validjsonschema "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json_schema"
)

func TestProductionCompatibilitySnapshot(t *testing.T) {
	registries, err := productionRegistries()
	if err != nil {
		t.Fatalf("productionRegistries() error = %v, want nil", err)
	}

	keySnapshots := []struct {
		name string
		got  []string
		want []string
	}{
		{name: "inputs", got: registries.Inputs.RegisteredKeys(), want: []string{"seriatim"}},
		{name: "chunkers", got: registries.Chunkers.RegisteredKeys(), want: []string{"dnd/scenes", "generic"}},
		{name: "extractors", got: registries.Extractors.RegisteredKeys(), want: []string{"dnd/spells"}},
		{name: "mergers", got: registries.Mergers.RegisteredKeys(), want: []string{"appendorder"}},
		{name: "normalizers", got: registries.Normalizers.RegisteredKeys(), want: []string{"noop"}},
		{name: "validators", got: registries.Validators.RegisteredKeys(), want: []string{
			"extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
			"generic/always_accept", "generic/always_reject", "generic/valid_json", "generic/valid_json_schema",
		}},
		{name: "outputs", got: registries.Outputs.RegisteredKeys(), want: []string{"json"}},
	}
	for _, snapshot := range keySnapshots {
		t.Run(snapshot.name, func(t *testing.T) {
			if !reflect.DeepEqual(snapshot.got, snapshot.want) {
				t.Fatalf("registered keys = %#v, want compatibility snapshot %#v", snapshot.got, snapshot.want)
			}
		})
	}

	wantChain := []pipeline.ModuleBinding{
		pipeline.Binding(validjson.Key),
		pipeline.Binding(validjsonschema.Key),
		pipeline.Binding(spellshape.Key),
		pipeline.Binding(spellsourcerefs.Key),
		pipeline.Binding(spellrelatedness.Key),
	}
	if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
		t.Fatalf("spell validator chain = %#v, want compatibility snapshot %#v", got, wantChain)
	}

	assets, err := productionPromptAssets()
	if err != nil {
		t.Fatalf("productionPromptAssets() error = %v, want nil", err)
	}
	assertAssetNames(t, assets.PromptFS, []string{
		"dnd.scenes/dnd.scenes.yaml",
		"dnd.scenes/instructions.md",
		"dnd.scenes/sharedassets/common-dnd-references.md",
		"dnd.scenes/sharedassets/common-dnd-system.md",
		"dnd.scenes/sharedassets/common-dnd-transcript.md",
		"dnd.scenes/task.md",
		"dnd.spells/dnd.spells.yaml",
		"dnd.spells/instructions.md",
		"dnd.spells/sharedassets/common-dnd-references.md",
		"dnd.spells/sharedassets/common-dnd-system.md",
		"dnd.spells/sharedassets/common-dnd-transcript.md",
		"dnd.spells/task.md",
	})
	assertAssetNames(t, assets.SchemaFS, []string{
		"dnd_scenes.v1.json",
		"dnd_spells.v1.json",
		"dnd_spells_llm.v1.json",
	})

	identitySnapshot := map[string]map[string]any{
		"scenes": scenes.New().ManifestMetadata(),
		"spells": spells.New().ManifestMetadata(),
	}
	for name, metadata := range identitySnapshot {
		for _, key := range []string{"prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_id", "response_schema_name", "response_schema_version", "response_schema_sha256"} {
			if value, ok := metadata[key].(string); !ok || value == "" {
				t.Fatalf("%s metadata[%q] = %#v, want non-empty identity", name, key, metadata[key])
			}
		}
	}
	if got := []any{
		identitySnapshot["scenes"]["prompt_id"], identitySnapshot["scenes"]["prompt_version"],
		identitySnapshot["scenes"]["response_schema_key"], identitySnapshot["scenes"]["response_schema_id"], identitySnapshot["scenes"]["response_schema_name"], identitySnapshot["scenes"]["response_schema_version"],
	}; !reflect.DeepEqual(got, []any{"dnd.scenes", "v1", "dnd_scenes", "notarius.dnd.scenes", "notarius_dnd_scenes_v1", "v1"}) {
		t.Fatalf("scene identities = %#v, want compatibility snapshot", got)
	}
	if got := []any{
		identitySnapshot["spells"]["prompt_id"], identitySnapshot["spells"]["prompt_version"],
		identitySnapshot["spells"]["response_schema_key"], identitySnapshot["spells"]["response_schema_id"], identitySnapshot["spells"]["response_schema_name"], identitySnapshot["spells"]["response_schema_version"],
	}; !reflect.DeepEqual(got, []any{"dnd.spells", "v1", "dnd_spells", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"}) {
		t.Fatalf("spell identities = %#v, want compatibility snapshot", got)
	}

	fileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells.config.yml"))
	if err != nil {
		t.Fatalf("LoadFileConfig() error = %v, want nil", err)
	}
	cfg := config.Default()
	if err := cfg.ApplyFileConfig(fileConfig); err != nil {
		t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
	}
	effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)})
	if err != nil {
		t.Fatalf("Resolve() error = %v, want nil", err)
	}
	resolved := effective.ResolvedPipeline
	if resolved.Input.Module != "seriatim" || resolved.Chunk.Module != "generic" || resolved.Output.Module != "json" || len(resolved.ArtifactLanes) != 1 {
		t.Fatalf("resolved example = %#v, want maintained production topology", resolved)
	}
	lane := resolved.ArtifactLanes[0]
	if lane.ID != "spells" || lane.Extract.Module != "dnd/spells" || lane.Merge.Module != "appendorder" || lane.Normalize.Module != "noop" {
		t.Fatalf("resolved lane = %#v, want maintained spell lane", lane)
	}
	if got := resolvedValidatorKeys(resolved.ValidatorChains, pipeline.StageExtract, "spells", spells.Key); !reflect.DeepEqual(got, []string{
		"generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
	}) {
		t.Fatalf("resolved validator keys = %#v, want compatibility snapshot", got)
	}

	productionFileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells-production.config.yml"))
	if err != nil {
		t.Fatalf("LoadFileConfig(production example) error = %v, want nil", err)
	}
	productionConfig := config.Default()
	if err := productionConfig.ApplyFileConfig(productionFileConfig); err != nil {
		t.Fatalf("ApplyFileConfig(production example) error = %v, want nil", err)
	}
	productionEffective, err := productionConfig.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)})
	if err != nil {
		t.Fatalf("Resolve(production example) error = %v, want nil", err)
	}
	productionResolved := productionEffective.ResolvedPipeline
	if productionConfig.Concurrency.TotalLLM != 1 || !reflect.DeepEqual(productionResolved.Chunk.Options, map[string]any{"max_units": 50}) {
		t.Fatalf("production example concurrency/options = %d/%#v, want compatibility snapshot", productionConfig.Concurrency.TotalLLM, productionResolved.Chunk.Options)
	}
	bindings := productionResolved.ArtifactLanes[0].ExtractReferences.Bindings
	if len(bindings) != 2 || bindings[0].SlotName != "glossary" || bindings[0].Source != "./dnd-spells-glossary.txt" || bindings[1].SlotName != "party" || bindings[1].Source != "./dnd-spells-roster.txt" {
		t.Fatalf("production example reference bindings = %#v, want maintained glossary and party bindings", bindings)
	}
}

func TestMaintainedSeriatimToDNDCompatibilityBundle(t *testing.T) {
	tests := []struct {
		name              string
		client            contracts.StructuredLLMClient
		wantStatus        string
		wantLaneFile      bool
		wantRejectedCount int
	}{
		{name: "approved", client: newFakeRunLLMClient(false), wantStatus: "approved", wantLaneFile: true},
		{name: "validator rejection is nonfatal", client: newFakeRunLLMClient(true), wantStatus: "rejected", wantRejectedCount: 1},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			outputDir := t.TempDir()
			var stdout bytes.Buffer
			var stderr bytes.Buffer
			code := RunWithOptions([]string{
				"run", "dnd-session",
				"--config", fixturePath(t, "examples/dnd-spells.config.yml"),
				"--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"),
				"--output-dir", outputDir,
				"--diagnostics-dir", t.TempDir(),
			}, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(test.client, nil)})
			if code != 0 {
				t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
			}

			runDir := onlyChildDir(t, outputDir)
			wantFiles := []string{"index.json", "manifest.json", "rejected.json", "warnings.json"}
			if test.wantLaneFile {
				wantFiles = append(wantFiles, "lanes/spells.json")
			}
			sort.Strings(wantFiles)
			if got := relativeFileNames(t, runDir); !reflect.DeepEqual(got, wantFiles) {
				t.Fatalf("durable files = %#v, want compatibility snapshot %#v", got, wantFiles)
			}

			var manifest artifacts.RunManifest
			readJSONFile(t, filepath.Join(runDir, "manifest.json"), &manifest)
			if manifest.PipelineID != "dnd-session" || manifest.InputModule != "seriatim" || manifest.Chunker != "generic" || manifest.OutputEncoder != "json" {
				t.Fatalf("manifest module provenance = %#v, want maintained production modules", manifest)
			}
			if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].ID != "spells" {
				t.Fatalf("artifact lanes = %#v, want one spells lane", manifest.ArtifactLanes)
			}
			laneManifest := manifest.ArtifactLanes[0]
			if laneManifest.Extractor != "dnd/spells" || laneManifest.Merger != "appendorder" || laneManifest.Normalizer != "noop" {
				t.Fatalf("manifest lane module provenance = %#v, want maintained production modules", laneManifest)
			}
			if len(manifest.Extractors) != 0 || manifest.Merger != "" || manifest.Normalizer != "" {
				t.Fatalf("legacy top-level lane summaries = %#v/%q/%q, want empty compatibility snapshot", manifest.Extractors, manifest.Merger, manifest.Normalizer)
			}
			if manifest.ValidationStatus != test.wantStatus || len(manifest.RejectedOutputs) != test.wantRejectedCount {
				t.Fatalf("manifest outcome = status %q rejected %#v, want %q/%d", manifest.ValidationStatus, manifest.RejectedOutputs, test.wantStatus, test.wantRejectedCount)
			}
			if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:346f0b0b0cf2d8081ca222dbea47ff231ae9d81c8120ac51a4fc9fa6dfb6cc07"}) {
				t.Fatalf("source digests = %#v, want maintained fixture provenance", manifest.SourceDigests)
			}
			if got := manifestValidatorKeys(manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)); !reflect.DeepEqual(got, []string{
				"generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
			}) {
				t.Fatalf("manifest validator chain = %#v, want compatibility snapshot", got)
			}

			var index struct {
				ManifestFile string `json:"manifest_file"`
				OutputFiles  []struct {
					LaneID        string `json:"lane_id"`
					MediaType     string `json:"media_type"`
					File          string `json:"file"`
					ModuleKey     string `json:"module_key"`
					SchemaID      string `json:"schema_id"`
					SchemaName    string `json:"schema_name"`
					SchemaVersion string `json:"schema_version"`
				} `json:"output_files"`
				RejectedFile string `json:"rejected_file"`
				WarningsFile string `json:"warnings_file"`
			}
			readJSONFile(t, filepath.Join(runDir, "index.json"), &index)
			if index.ManifestFile != "manifest.json" || index.RejectedFile != "rejected.json" || index.WarningsFile != "warnings.json" {
				t.Fatalf("output index fixed files = %#v, want compatibility snapshot", index)
			}
			if test.wantLaneFile {
				if len(index.OutputFiles) != 1 {
					t.Fatalf("output index entries = %#v, want one", index.OutputFiles)
				}
				wantOutput := struct {
					LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string
				}{"spells", "application/json", "lanes/spells.json", "noop", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"}
				gotOutput := index.OutputFiles[0]
				got := struct {
					LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string
				}{gotOutput.LaneID, gotOutput.MediaType, gotOutput.File, gotOutput.ModuleKey, gotOutput.SchemaID, gotOutput.SchemaName, gotOutput.SchemaVersion}
				if got != wantOutput {
					t.Fatalf("output index entries = %#v, want compatibility snapshot %#v", index.OutputFiles, wantOutput)
				}
				assertJSONEqual(t, readFile(t, filepath.Join(runDir, "lanes/spells.json")), []byte(`{
					"spell_casts": [{
						"caster": "Aria",
						"spell": "Cure Wounds",
						"effect": "Heals a wounded ally.",
						"narrative_description": "Aria casts Cure Wounds.",
						"source_refs": [{"source_id": "session-alpha", "start_unit_id": 1, "end_unit_id": 1}]
					}]
				}`))
			} else if len(index.OutputFiles) != 0 {
				t.Fatalf("output index entries = %#v, want none for rejected lane", index.OutputFiles)
			}

			var warnings struct {
				Warnings []contracts.Warning `json:"warnings"`
			}
			readJSONFile(t, filepath.Join(runDir, "warnings.json"), &warnings)
			if len(warnings.Warnings) != 0 {
				t.Fatalf("warnings = %#v, want empty compatibility snapshot", warnings.Warnings)
			}
			var rejected struct {
				Rejected []contracts.RejectedOutput `json:"rejected"`
			}
			readJSONFile(t, filepath.Join(runDir, "rejected.json"), &rejected)
			if len(rejected.Rejected) != test.wantRejectedCount {
				t.Fatalf("rejected outputs = %#v, want %d", rejected.Rejected, test.wantRejectedCount)
			}
			if test.wantRejectedCount == 1 {
				got := rejected.Rejected[0]
				if got.Stage != "extract" || got.LaneID != "spells" || got.ModuleKey != "dnd/spells" || got.ChunkID != "chunk-000001" || got.ChunkIndex != 0 || got.ValidatorName != "extract/dnd/spells/source_refs" || got.ReasonCode != "invalid_source_refs" || got.AttemptCount != 1 {
					t.Fatalf("rejection = %#v, want maintained nonfatal validator outcome", got)
				}
			}
		})
	}
}

func TestProductionLLMCallersShareScheduledClient(t *testing.T) {
	underlying := newBlockingProductionLLMClient()
	scheduler, err := frameworkllm.NewScheduler(1)
	if err != nil {
		t.Fatalf("NewScheduler() error = %v, want nil", err)
	}
	client := frameworkllm.NewScheduledClient(underlying, scheduler)
	doc := &source.SourceDocument{
		ID: "session-alpha", Kind: "transcript", Format: "application/json", Digest: "sha256:source",
		Units: []source.SourceUnit{{ID: 1, Kind: "segment", Text: "Aria casts Cure Wounds."}, {ID: 2, Kind: "segment", Text: "The spell takes effect."}},
	}
	chunk := contracts.SourceChunk{
		ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, StartUnitID: 1, EndUnitID: 2,
		Content: []byte(`{"scene":"Aria casts Cure Wounds."}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), doc.Units...),
	}

	var started sync.WaitGroup
	started.Add(2)
	errs := make(chan error, 2)
	go func() {
		started.Done()
		_, err := scenes.New().Chunk(context.Background(), contracts.ChunkRequest{Source: doc, LLMClient: client})
		errs <- err
	}()
	go func() {
		started.Done()
		_, err := spells.New().Extract(context.Background(), contracts.ExtractionRequest{Source: doc, Chunk: &chunk, LLMClient: client})
		errs <- err
	}()
	started.Wait()

	for i := 0; i < 2; i++ {
		<-underlying.entered
		underlying.release <- struct{}{}
	}
	for i := 0; i < 2; i++ {
		if err := <-errs; err != nil {
			t.Fatalf("production LLM caller error = %v, want nil", err)
		}
	}
	if underlying.maxActive != 1 {
		t.Fatalf("maximum concurrent provider calls = %d, want total_llm limit 1", underlying.maxActive)
	}
	sort.Strings(underlying.stageNames)
	if !reflect.DeepEqual(underlying.stageNames, []string{"dnd/scenes", "dnd/spells"}) {
		t.Fatalf("scheduled stage names = %#v, want both production LLM callers", underlying.stageNames)
	}
}

func TestProductionBundlePreservesLaneAndChunkOrder(t *testing.T) {
	configPath := writeTestConfig(t, `version: 2
pipelines:
  dnd-session:
    input: seriatim
    chunk:
      module: generic
      options:
        max_units: 1
    artifacts:
      zeta:
        extract: dnd/spells
      alpha:
        extract: dnd/spells
`)
	outputDir := t.TempDir()
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	code := RunWithOptions([]string{
		"run", "dnd-session",
		"--config", configPath,
		"--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"),
		"--output-dir", outputDir,
		"--diagnostics-dir", t.TempDir(),
	}, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(orderingProductionLLMClient{}, nil)})
	if code != 0 {
		t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
	}

	runDir := onlyChildDir(t, outputDir)
	var index struct {
		OutputFiles []struct {
			LaneID string `json:"lane_id"`
		} `json:"output_files"`
	}
	readJSONFile(t, filepath.Join(runDir, "index.json"), &index)
	if len(index.OutputFiles) != 2 {
		t.Fatalf("output index entries = %#v, want two lanes", index.OutputFiles)
	}
	if got := []string{index.OutputFiles[0].LaneID, index.OutputFiles[1].LaneID}; !reflect.DeepEqual(got, []string{"alpha", "zeta"}) {
		t.Fatalf("output lane order = %#v, want resolved lane order", got)
	}
	for _, laneID := range []string{"alpha", "zeta"} {
		var payload struct {
			SpellCasts []struct {
				Spell      string             `json:"spell"`
				SourceRefs []source.SourceRef `json:"source_refs"`
			} `json:"spell_casts"`
		}
		readJSONFile(t, filepath.Join(runDir, "lanes", laneID+".json"), &payload)
		if len(payload.SpellCasts) != 2 {
			t.Fatalf("lane %q spell casts = %#v, want one per source chunk", laneID, payload.SpellCasts)
		}
		got := []any{
			payload.SpellCasts[0].Spell, payload.SpellCasts[0].SourceRefs[0].StartUnitID,
			payload.SpellCasts[1].Spell, payload.SpellCasts[1].SourceRefs[0].StartUnitID,
		}
		if !reflect.DeepEqual(got, []any{"Cure Wounds", 1, "Shield", 2}) {
			t.Fatalf("lane %q chunk handoff order = %#v, want source chunk order", laneID, got)
		}
	}
}

type blockingProductionLLMClient struct {
	mu         sync.Mutex
	active     int
	maxActive  int
	stageNames []string
	entered    chan struct{}
	release    chan struct{}
}

type orderingProductionLLMClient struct{}

func (orderingProductionLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
	material := req.Inputs["transcript"]
	unitID := 1
	spellName := "Cure Wounds"
	if strings.Contains(string(material.Content), "Shield") {
		unitID = 2
		spellName = "Shield"
	}
	payload, err := json.Marshal(map[string]any{
		"spell_casts": []map[string]any{{
			"caster":                "Aria",
			"spell":                 spellName,
			"effect":                "Fixture effect.",
			"narrative_description": "Fixture spell cast.",
			"source_refs": []map[string]any{{
				"start_unit_id": unitID,
				"end_unit_id":   unitID,
			}},
		}},
	})
	if err != nil {
		return contracts.StructuredCompletionResponse{}, err
	}
	if err := json.Unmarshal(payload, out); err != nil {
		return contracts.StructuredCompletionResponse{}, err
	}
	return contracts.StructuredCompletionResponse{Content: payload}, nil
}

func newBlockingProductionLLMClient() *blockingProductionLLMClient {
	return &blockingProductionLLMClient{entered: make(chan struct{}, 2), release: make(chan struct{}, 2)}
}

func (client *blockingProductionLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
	client.mu.Lock()
	client.active++
	if client.active > client.maxActive {
		client.maxActive = client.active
	}
	client.stageNames = append(client.stageNames, req.StageName)
	client.mu.Unlock()
	client.entered <- struct{}{}

	select {
	case <-ctx.Done():
		return contracts.StructuredCompletionResponse{}, ctx.Err()
	case <-client.release:
	}

	client.mu.Lock()
	client.active--
	client.mu.Unlock()

	var payload []byte
	switch req.StageName {
	case scenes.Key:
		payload = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Spell","primary_mode":"Narrative","main_participants":["Aria"],"summary":"Aria casts a spell.","boundary_note":"Complete source.","boundary_confidence":"High"}],"boundary_caveats":[]}`)
	case spells.Key:
		payload = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Healing","narrative_description":"Aria casts Cure Wounds.","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`)
	}
	if err := json.Unmarshal(payload, out); err != nil {
		return contracts.StructuredCompletionResponse{}, err
	}
	return contracts.StructuredCompletionResponse{Content: payload}, nil
}

func assertAssetNames(t *testing.T, getFS func() (fs.FS, error), want []string) {
	t.Helper()
	fSys, err := getFS()
	if err != nil {
		t.Fatalf("asset filesystem error = %v, want nil", err)
	}
	var got []string
	if err := fs.WalkDir(fSys, ".", func(path string, entry fs.DirEntry, err error) error {
		if err == nil && !entry.IsDir() {
			got = append(got, path)
		}
		return err
	}); err != nil {
		t.Fatalf("walk assets: %v", err)
	}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("asset names = %#v, want compatibility snapshot %#v", got, want)
	}
}

func resolvedValidatorKeys(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID, moduleKey string) []string {
	for _, chain := range chains {
		if chain.Stage == stage && chain.LaneID == laneID && chain.ModuleKey == moduleKey {
			keys := make([]string, 0, len(chain.Validators))
			for _, validator := range chain.Validators {
				keys = append(keys, validator.Binding.Module)
			}
			return keys
		}
	}
	return nil
}

func relativeFileNames(t *testing.T, root string) []string {
	t.Helper()
	var names []string
	if err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
		if err != nil || entry.IsDir() {
			return err
		}
		rel, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		names = append(names, filepath.ToSlash(rel))
		return nil
	}); err != nil {
		t.Fatalf("walk durable output: %v", err)
	}
	sort.Strings(names)
	return names
}

func assertJSONEqual(t *testing.T, got, want []byte) {
	t.Helper()
	var gotValue any
	var wantValue any
	if err := json.Unmarshal(got, &gotValue); err != nil {
		t.Fatalf("unmarshal actual JSON: %v", err)
	}
	if err := json.Unmarshal(want, &wantValue); err != nil {
		t.Fatalf("unmarshal expected JSON: %v", err)
	}
	if !reflect.DeepEqual(gotValue, wantValue) {
		t.Fatalf("JSON = %#v, want compatibility snapshot %#v", gotValue, wantValue)
	}
}