notarius/internal/cli/production_contract_test.go

package cli

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
	combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
	spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
	combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
	combatnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/combatturns"
	spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
)

func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
	components := productionTestComponents(t)
	registries := components.registries

	assertProductionContains(t, "inputs", registries.Inputs.RegisteredKeys(), []string{"seriatim"})
	assertProductionContains(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes", "generic"})
	assertProductionContains(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells", "dnd/npcs", combatextract.Key})
	assertProductionContains(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"})
	assertProductionContains(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop", spellnormalize.Key, "dnd/npcs", combatnormalize.Key})
	assertProductionContains(t, "outputs", registries.Outputs.RegisteredKeys(), []string{"json"})
	assertProductionContains(t, "validators", registries.Validators.RegisteredKeys(), []string{
		"extract/dnd/spells/catalog",
		"extract/dnd/spells/shape",
		"extract/dnd/spells/source_refs",
		"extract/dnd/spells/source_relatedness",
		"extract/dnd/combat-turns/shape",
		"extract/dnd/combat-turns/source_refs",
		"extract/dnd/combat-turns/source_relatedness",
		"normalize/dnd/combat-turns/invariants",
		"generic/always_accept",
		"generic/always_reject",
		"generic/valid_json",
		"generic/valid_json_schema",
	})
	assertProductionContains(t, "artifact codec kinds", registries.ArtifactCodecs.RegisteredKinds(), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind})
	assertProductionContains(t, "merger variants", registries.Mergers.RegisteredArtifactKinds(pipeline.DefaultMergeModule), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind})
	assertProductionContains(t, "normalizer variants", registries.Normalizers.RegisteredArtifactKinds(pipeline.DefaultNormalizeModule), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind})
	assertProductionContains(t, "spell normalizer variants", registries.Normalizers.RegisteredArtifactKinds(spellnormalize.Key), []contracts.ArtifactKind{dnd.SpellListKind})
	assertProductionContains(t, "combat normalizer variants", registries.Normalizers.RegisteredArtifactKinds(combatnormalize.Key), []contracts.ArtifactKind{dnd.CombatTurnListKind})

	wantChain := []pipeline.ModuleBinding{
		pipeline.Binding("generic/valid_json"),
		pipeline.Binding("extract/dnd/spells/shape"),
		pipeline.Binding("extract/dnd/spells/catalog"),
		pipeline.Binding("extract/dnd/spells/source_refs"),
		pipeline.Binding("generic/valid_json_schema"),
		pipeline.Binding("extract/dnd/spells/source_relatedness"),
	}
	if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
		t.Fatalf("spell validator chain = %#v, want %#v", got, wantChain)
	}
	if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, spellnormalize.Key); !reflect.DeepEqual(got, wantChain) {
		t.Fatalf("spell normalize validator chain = %#v, want %#v", got, wantChain)
	}
	combatExtractChain := []pipeline.ModuleBinding{
		pipeline.Binding("generic/valid_json"),
		pipeline.Binding("extract/dnd/combat-turns/shape"),
		pipeline.Binding("extract/dnd/combat-turns/source_refs"),
		pipeline.Binding("generic/valid_json_schema"),
		pipeline.Binding("extract/dnd/combat-turns/source_relatedness"),
	}
	combatNormalizeChain := []pipeline.ModuleBinding{
		pipeline.Binding("generic/valid_json"),
		pipeline.Binding("extract/dnd/combat-turns/shape"),
		pipeline.Binding("normalize/dnd/combat-turns/invariants"),
		pipeline.Binding("extract/dnd/combat-turns/source_refs"),
		pipeline.Binding("generic/valid_json_schema"),
		pipeline.Binding("extract/dnd/combat-turns/source_relatedness"),
	}
	if got := registries.ValidatorChains.Validators(pipeline.StageExtract, combatextract.Key); !reflect.DeepEqual(got, combatExtractChain) {
		t.Fatalf("combat extract validator chain = %#v, want %#v", got, combatExtractChain)
	}
	if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, combatnormalize.Key); !reflect.DeepEqual(got, combatNormalizeChain) {
		t.Fatalf("combat normalize validator chain = %#v, want %#v", got, combatNormalizeChain)
	}

	assetNames := productionAssetNames(t, components.assets.PromptFS)
	requiredAssets := []string{
		"dnd.scenes/dnd.scenes.yaml",
		"dnd.scenes/instructions.md",
		"dnd.scenes/sharedassets/common-dnd-references.md",
		"dnd.scenes/sharedassets/common-dnd-system.md",
		"dnd.scenes/sharedassets/common-dnd-transcript.md",
		"dnd.scenes/task.md",
		"dnd.spells/dnd.spells.yaml",
		"dnd.spells/catalog.md",
		"dnd.spells/instructions.md",
		"dnd.spells/sharedassets/common-dnd-references.md",
		"dnd.spells/sharedassets/common-dnd-system.md",
		"dnd.spells/sharedassets/common-dnd-transcript.md",
		"dnd.spells/task.md",
		"dnd.combat_turns/dnd.combat_turns.yaml",
		"dnd.combat_turns/instructions.md",
		"dnd.combat_turns/sharedassets/common-dnd-references.md",
		"dnd.combat_turns/sharedassets/common-dnd-system.md",
		"dnd.combat_turns/sharedassets/common-dnd-transcript.md",
		"dnd.combat_turns/task.md",
	}
	assertProductionContains(t, "production prompt assets", assetNames, requiredAssets)

	catalog := catalogFromRegistries(registries)
	converted := registriesFromCatalog(catalog)
	if converted.ArtifactCodecs != registries.ArtifactCodecs || converted.ValidatorChains != registries.ValidatorChains {
		t.Fatal("catalog/registry conversion did not preserve codec and validator-chain registries")
	}
	codecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.SpellListKind)
	if !ok || codecSpec.Kind != dnd.SpellListKind || codecSpec.Schema.ID != spellcodec.SchemaID {
		t.Fatalf("catalog codec spec = %#v, ok=%t, want typed D&D spell codec", codecSpec, ok)
	}
	combatCodecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.CombatTurnListKind)
	if !ok || combatCodecSpec.Kind != dnd.CombatTurnListKind || combatCodecSpec.Schema.ID != combatcodec.SchemaID {
		t.Fatalf("combat codec spec = %#v, ok=%t, want typed D&D combat codec", combatCodecSpec, ok)
	}
	if got := catalog.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
		t.Fatalf("catalog validator chain = %#v, want %#v", got, wantChain)
	}
	if got := catalog.ValidatorChains.Validators(pipeline.StageNormalize, spellnormalize.Key); !reflect.DeepEqual(got, wantChain) {
		t.Fatalf("catalog spell normalize validator chain = %#v, want %#v", got, wantChain)
	}

}

func TestDefaultCLICompositionValidatesRepresentativeConfiguration(t *testing.T) {
	var stdout, stderr strings.Builder
	code := RunWithOptions([]string{
		"config", "validate", "--config", repositoryPath("examples", "dnd-spells.config.yml"), "--pipeline", "dnd-session",
	}, &stdout, &stderr, Options{})
	if code != 0 || stderr.Len() != 0 {
		t.Fatalf("validate representative config with default composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
	}
}

func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
	components := productionTestComponents(t)
	cfg := config.Default()
	cfg.Pipelines["dnd-scenes"] = pipeline.PipelineProfile{
		ID:    "dnd-scenes",
		Input: pipeline.Binding("seriatim"),
		Chunk: pipeline.Binding("dnd/scenes"),
		Artifacts: map[string]pipeline.ArtifactLaneProfile{
			"spells": {Extract: pipeline.Binding("dnd/spells")},
		},
	}
	effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-scenes", Catalog: catalogFromRegistries(components.registries)})
	if err != nil {
		t.Fatalf("resolve production scene pipeline: %v", err)
	}
	if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
		t.Fatalf("prepare production scene and spell modules: %v", err)
	}
}

func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
	components := productionTestComponents(t)
	configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
	effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
	if err != nil {
		t.Fatalf("resolve production spell configuration: %v", err)
	}
	materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
		ConfigPath: configPath,
		WorkingDir: filepath.Dir(configPath),
	})
	if err != nil {
		t.Fatalf("materialize production spell references: %v", err)
	}
	extractItems := materialized.Steps[0].ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items
	normalizeItems := materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items
	if len(extractItems) != 1 || extractItems[0].MediaType != "application/json" || len(extractItems[0].Content) == 0 {
		t.Fatalf("materialized extract spell catalog items = %#v, want one JSON item", extractItems)
	}
	if len(normalizeItems) != 1 || normalizeItems[0].MediaType != "application/json" || !reflect.DeepEqual(normalizeItems[0].Content, extractItems[0].Content) {
		t.Fatalf("materialized normalize spell catalog items = %#v, want an independent binding of the extract catalog", normalizeItems)
	}
	if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
		t.Fatalf("prepare production spell pipeline from materialized catalog: %v", err)
	}
}

func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution(t *testing.T) {
	components := productionTestComponents(t)
	configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
	resolve := func(t *testing.T) pipeline.ResolvedPipeline {
		t.Helper()
		effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
		if err != nil {
			t.Fatalf("resolve production spell configuration: %v", err)
		}
		return effective.ResolvedPipeline
	}
	materialize := func(resolved pipeline.ResolvedPipeline) (pipeline.ResolvedPipeline, error) {
		materialized, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
			ConfigPath: configPath,
			WorkingDir: filepath.Dir(configPath),
		})
		return materialized, err
	}

	t.Run("malformed catalog fails preparation", func(t *testing.T) {
		catalogPath := filepath.Join(t.TempDir(), "malformed.json")
		if err := os.WriteFile(catalogPath, []byte(`{"schema_version":`), 0o600); err != nil {
			t.Fatal(err)
		}
		resolved := resolve(t)
		setNormalizeSpellCatalogSource(t, &resolved, catalogPath)
		materialized, err := materialize(resolved)
		if err != nil {
			t.Fatalf("MaterializeReferences() error = %v, want malformed JSON to reach preparation", err)
		}
		_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
		for _, fragment := range []string{`pipeline "dnd-session"`, `lane "spells"`, "normalize", `module "dnd/spells"`, "decode spell catalog overlay"} {
			if err == nil || !strings.Contains(err.Error(), fragment) {
				t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment)
			}
		}
	})

	t.Run("multiple catalog items fail preparation", func(t *testing.T) {
		materialized, err := materialize(resolve(t))
		if err != nil {
			t.Fatal(err)
		}
		slot := materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"]
		slot.Items = append(slot.Items, slot.Items[0])
		materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"] = slot
		_, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
		for _, fragment := range []string{"normalize", `module "dnd/spells"`, "zero or one item"} {
			if err == nil || !strings.Contains(err.Error(), fragment) {
				t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment)
			}
		}
	})

	t.Run("oversized catalog fails materialization", func(t *testing.T) {
		catalogPath := filepath.Join(t.TempDir(), "oversized.json")
		if err := os.WriteFile(catalogPath, []byte(strings.Repeat("x", 1048577)), 0o600); err != nil {
			t.Fatal(err)
		}
		checkpointRoot := filepath.Join(t.TempDir(), "checkpoints")
		content := string(readRepositoryFile(t, "examples", "dnd-spells-production.config.yml"))
		content = replaceRequiredOnce(t, content, "./dnd-spells-roster.txt", repositoryPath("examples", "dnd-spells-roster.txt"))
		content = replaceRequiredOnce(t, content, "./dnd-spells-glossary.txt", repositoryPath("examples", "dnd-spells-glossary.txt"))
		content = strings.Replace(content, "./dnd-spells-catalog.json", repositoryPath("examples", "dnd-spells-catalog.json"), 1)
		content = replaceRequiredOnce(t, content, "./dnd-spells-catalog.json", catalogPath)
		content = replaceRequiredOnce(t, content, "    enabled: false\n    directory: /var/cache/notarius/checkpoints", "    enabled: true\n    directory: "+checkpointRoot)
		configFile := filepath.Join(t.TempDir(), "config.yml")
		if err := os.WriteFile(configFile, []byte(content), 0o600); err != nil {
			t.Fatal(err)
		}

		llmConstructed := false
		chunkStoreConstructed := false
		options := Options{
			Catalog:    catalogFromRegistries(components.registries),
			Registries: components.registries,
			LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
				llmConstructed = true
				return nil, nil, errors.New("LLM client must not be constructed")
			},
			ChunkPlanStoreFactory: func(string) (pipeline.ChunkPlanStore, error) {
				chunkStoreConstructed = true
				return nil, errors.New("chunk-plan store must not be constructed")
			},
		}
		var stdout, stderr strings.Builder
		code := RunWithOptions([]string{
			"run", "dnd-session", "--config", configFile,
			"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
		}, &stdout, &stderr, options)
		errText := stderr.String()
		for _, fragment := range []string{"normalize", `lane "spells"`, `reference slot "spell_catalog"`, "1048577 bytes", "limit 1048576"} {
			if code == 0 || !strings.Contains(errText, fragment) {
				t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, errText, fragment)
			}
		}
		if llmConstructed || chunkStoreConstructed {
			t.Fatalf("runtime construction = LLM %t, chunk store %t; want materialization failure first", llmConstructed, chunkStoreConstructed)
		}
		if _, err := os.Stat(checkpointRoot); !errors.Is(err, fs.ErrNotExist) {
			t.Fatalf("checkpoint root stat error = %v, want no checkpoint allocation", err)
		}
	})
}

func setNormalizeSpellCatalogSource(t *testing.T, resolved *pipeline.ResolvedPipeline, sourcePath string) {
	t.Helper()
	if resolved == nil || len(resolved.Steps[0].ArtifactLanes) != 1 {
		t.Fatalf("resolved pipeline = %#v, want one artifact lane", resolved)
	}
	bindings := resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings
	matches := 0
	for index := range bindings {
		if bindings[index].SlotName == "spell_catalog" {
			bindings[index].Source = sourcePath
			matches++
		}
	}
	if matches != 1 {
		t.Fatalf("normalize reference bindings = %#v, want exactly one spell_catalog binding", bindings)
	}
	resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings = bindings
}

func TestProductionLLMClientFactoriesBuildOfflineRuntime(t *testing.T) {
	components := productionTestComponents(t)
	factories := []struct {
		name    string
		factory LLMClientFactory
	}{
		{name: "default production assets", factory: productionLLMClientFactory},
		{name: "provided production assets", factory: productionLLMClientFactoryWithAssets(components.assets)},
	}
	for _, tt := range factories {
		t.Run(tt.name, func(t *testing.T) {
			client, manifests, err := tt.factory(context.Background(), config.Default(), "test-profile")
			if err != nil {
				t.Fatalf("build production LLM runtime: %v", err)
			}
			if client == nil {
				t.Fatal("production LLM runtime returned a nil client")
			}
			if len(manifests) != 0 {
				t.Fatalf("eager profile manifests = %#v, want none", manifests)
			}
			if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
				t.Fatalf("production LLM client %T does not provide profile manifests", client)
			}
		})
	}
}

func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) {
	t.Run("canceled context", func(t *testing.T) {
		ctx, cancel := context.WithCancel(context.Background())
		cancel()
		client, manifests, err := productionLLMClientFactory(ctx, config.Default(), "test-profile")
		if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 {
			t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err)
		}
	})

	t.Run("nil assets", func(t *testing.T) {
		client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile")
		if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 {
			t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err)
		}
	})

	t.Run("invalid scheduler concurrency", func(t *testing.T) {
		components := productionTestComponents(t)
		cfg := config.Default()
		cfg.Concurrency.TotalLLM = 0
		client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile")
		if err == nil || !strings.Contains(err.Error(), "create LLM scheduler") || !strings.Contains(err.Error(), "greater than zero") || client != nil || len(manifests) != 0 {
			t.Fatalf("client=%T manifests=%#v error=%v, want scheduler-construction failure", client, manifests, err)
		}
	})
}

func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) {
	base := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml"))
	validPath := writeProductionContractConfig(t, base)
	options := productionCLIOptions(t)
	var stdout, stderr strings.Builder
	if code := RunWithOptions([]string{"config", "validate", "--config", validPath, "--pipeline", "dnd-session"}, &stdout, &stderr, options); code != 0 {
		t.Fatalf("valid production config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
	}
	tests := []struct {
		name      string
		content   string
		options   Options
		fragments []string
	}{
		{
			name:      "unknown module",
			content:   replaceRequiredOnce(t, base, "    input: seriatim\n", "    input: missing/input\n"),
			options:   productionCLIOptions(t),
			fragments: []string{"pipeline \"dnd-session\"", "input", "missing/input"},
		},
		{
			name:      "unknown validator",
			content:   replaceRequiredOnce(t, base, "        extract: dnd/spells\n", "        extract:\n          module: dnd/spells\n          validators:\n            - module: missing/validator\n"),
			options:   productionCLIOptions(t),
			fragments: []string{"validator", "missing/validator"},
		},
		{
			name:      "invalid artifact variant",
			content:   base,
			options:   productionCLIOptionsWithoutSpellNormalizer(t),
			fragments: []string{"normalizer", spellnormalize.Key, string(dnd.SpellListKind), "variant"},
		},
		{
			name:      "deterministic validator with profile",
			content:   replaceRequiredOnce(t, base, "        extract: dnd/spells\n", "        extract:\n          module: dnd/spells\n          validators:\n            - module: generic/valid_json\n              llm_profile: forbidden-profile\n"),
			options:   productionCLIOptions(t),
			fragments: []string{"deterministic validator", "llm_profile"},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			path := writeProductionContractConfig(t, tt.content)
			var stdout, stderr strings.Builder
			code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "dnd-session"}, &stdout, &stderr, tt.options)
			if code != 1 {
				t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
			}
			for _, fragment := range tt.fragments {
				if !strings.Contains(stderr.String(), fragment) {
					t.Fatalf("stderr=%q, want %q", stderr.String(), fragment)
				}
			}
		})
	}
}

func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) {
	base := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml"))
	content := replaceRequiredOnce(t, base, "        normalize: dnd/spells\n", "        normalize:\n          module: dnd/spells\n          validators:\n            - module: generic/always_accept\n            - module: generic/valid_json\n")
	path := writeProductionContractConfig(t, content)
	components := productionTestComponents(t)
	effective, err := loadMaintainedExample(t, path).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
	if err != nil {
		t.Fatalf("resolve normalize override: %v", err)
	}
	for _, chain := range effective.ResolvedPipeline.ValidatorChains {
		if chain.Stage != pipeline.StageNormalize || chain.ModuleKey != spellnormalize.Key {
			continue
		}
		if len(chain.Validators) != 2 || chain.Validators[0].Binding.Module != "generic/always_accept" || chain.Validators[1].Binding.Module != "generic/valid_json" {
			t.Fatalf("normalize validator chain = %#v, want explicit validator order", chain)
		}
		return
	}
	t.Fatalf("resolved validator chains = %#v, want normalize chain for %q", effective.ResolvedPipeline.ValidatorChains, spellnormalize.Key)
}

func TestProductionSceneRunRecordsChunkerWarningsAndProvenance(t *testing.T) {
	outputRoot := filepath.Join(t.TempDir(), "output")
	configPath := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes"))
	fake := &productionFakeLLMClient{}
	options := productionRunOptions(t, fake)
	var stdout, stderr strings.Builder
	code := RunWithOptions([]string{
		"run", "dnd-session", "--config", configPath,
		"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
		"--chunk_cache", "bypass", "--session-id", "offline-session",
	}, &stdout, &stderr, options)
	if code != 0 {
		t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
	}
	manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(outputRoot, productionRunID, "manifest.json"))
	if manifest.Chunker != scenes.Key || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" || manifest.ChunkPlan.ProducerModule != scenes.Key {
		t.Fatalf("chunk manifest = %#v, want dnd scene producer", manifest.ChunkPlan)
	}
	if got := manifest.ModuleMetadata["chunker"]["prompt_id"]; got != scenes.PromptID {
		t.Fatalf("chunker prompt metadata = %#v, want %q", got, scenes.PromptID)
	}
	if got := manifest.ChunkPlan.ProducerMetadata["response_schema_id"]; got != scenes.ResponseSchemaID {
		t.Fatalf("chunk producer schema metadata = %#v, want %q", got, scenes.ResponseSchemaID)
	}
	warnings := readProductionJSON[struct {
		Warnings []contracts.Warning `json:"warnings"`
	}](t, filepath.Join(outputRoot, productionRunID, "warnings.json"))
	if len(warnings.Warnings) != 1 || warnings.Warnings[0].ReasonCode != "scene_boundary_caveat" {
		t.Fatalf("warnings = %#v, want one scene boundary warning", warnings.Warnings)
	}
	if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 {
		t.Fatalf("fake prompt requests = %#v, want one scene and one spell request", fake.requestPrompts())
	}
}

type maintainedExample struct {
	name        string
	path        string
	pipelineIDs []string
}

func maintainedExampleFiles(t *testing.T) []maintainedExample {
	t.Helper()
	return []maintainedExample{
		{name: "minimal", path: repositoryPath("examples", "dnd-spells.config.yml"), pipelineIDs: []string{"dnd-session"}},
		{name: "production", path: repositoryPath("examples", "dnd-spells-production.config.yml"), pipelineIDs: []string{"dnd-session"}},
		{name: "npcs", path: repositoryPath("examples", "dnd-npcs.config.yml"), pipelineIDs: []string{"dnd-session"}},
		{name: "combat", path: repositoryPath("examples", "dnd-combat-turns.config.yml"), pipelineIDs: []string{"dnd-combat"}},
		{name: "npc-grounded", path: repositoryPath("examples", "dnd-npc-grounded.config.yml"), pipelineIDs: []string{"dnd-npc-grounded"}},
	}
}

func loadMaintainedExample(t *testing.T, path string) config.Config {
	t.Helper()
	fileConfig, err := config.LoadFileConfig(path)
	if err != nil {
		t.Fatalf("load maintained config %q: %v", path, err)
	}
	cfg := config.Default()
	if err := cfg.ApplyFileConfig(fileConfig); err != nil {
		t.Fatalf("apply maintained config %q: %v", path, err)
	}
	if err := cfg.Validate(); err != nil {
		t.Fatalf("validate maintained config %q: %v", path, err)
	}
	return cfg
}

func productionTestComponents(t *testing.T) productionComponents {
	t.Helper()
	components, err := newProductionComponents()
	if err != nil {
		t.Fatalf("new production components: %v", err)
	}
	return components
}

func productionCLIOptions(t *testing.T) Options {
	t.Helper()
	components := productionTestComponents(t)
	return productionOptionsFromComponents(components)
}

func productionOptionsFromComponents(components productionComponents) Options {
	return Options{
		Catalog:    catalogFromRegistries(components.registries),
		Registries: components.registries,
		LookupEnv:  emptyLookup,
	}
}

func productionCLIOptionsWithoutSpellNormalizer(t *testing.T) Options {
	t.Helper()
	components := productionTestComponents(t)
	registries := components.registries
	registries.Normalizers = pipeline.NewNormalizerRegistry()
	if err := noop.RegisterTyped[dnd.SpellList](registries.Normalizers, contracts.ArtifactKind("test/other")); err != nil {
		t.Fatalf("register mismatched normalizer: %v", err)
	}
	return productionOptionsFromComponents(productionComponents{registries: registries, assets: components.assets})
}

const productionRunID = "run-1700000000000000000-0123456789abcdef0123456789abcdef"

func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options {
	t.Helper()
	options := productionCLIOptions(t)
	options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() }
	options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil }
	options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") }
	options.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
		return fake, nil, nil
	}
	return options
}

func productionRunConfig(outputRoot, chunkModule string) string {
	return fmt.Sprintf(`version: 3
output:
  directory: %q
cache:
  chunk_plans:
    mode: bypass
  checkpoints: {}
debug:
  directory: %q
pipelines:
  dnd-session:
    input: seriatim
    chunk: %s
    artifacts:
      spells:
        extract: dnd/spells
`, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule)
}

func writeProductionContractConfig(t *testing.T, content string) string {
	t.Helper()
	path := filepath.Join(t.TempDir(), "config.yml")
	if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
		t.Fatal(err)
	}
	return path
}

func productionAssetNames(t *testing.T, getFS func() (fs.FS, error)) []string {
	t.Helper()
	fileSystem, err := getFS()
	if err != nil {
		t.Fatalf("load production prompt assets: %v", err)
	}
	var names []string
	if err := fs.WalkDir(fileSystem, ".", func(path string, entry fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if !entry.IsDir() {
			names = append(names, path)
		}
		return nil
	}); err != nil {
		t.Fatalf("walk production prompt assets: %v", err)
	}
	sort.Strings(names)
	return names
}

func assertProductionContains[T comparable](t *testing.T, name string, got, required []T) {
	t.Helper()
	available := make(map[T]struct{}, len(got))
	for _, entry := range got {
		available[entry] = struct{}{}
	}
	var missing []T
	for _, entry := range required {
		if _, ok := available[entry]; !ok {
			missing = append(missing, entry)
		}
	}
	if len(missing) > 0 {
		t.Fatalf("%s missing required entries %#v; registered entries are %#v", name, missing, got)
	}
}

func readProductionJSON[T any](t *testing.T, path string) T {
	t.Helper()
	data, err := os.ReadFile(path)
	if err != nil {
		t.Fatalf("read %s: %v", path, err)
	}
	var value T
	if err := json.Unmarshal(data, &value); err != nil {
		t.Fatalf("decode %s: %v", path, err)
	}
	return value
}

type productionFakeLLMClient struct {
	mu            sync.Mutex
	requests      []contracts.StructuredCompletionRequest
	spellResponse string
}

func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
	if err := ctx.Err(); err != nil {
		return contracts.StructuredCompletionResponse{}, err
	}
	var content []byte
	switch req.PromptID {
	case scenes.PromptID:
		content = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Opening scene","primary_mode":"Narrative","main_participants":["Aria"],"summary":"The session opens.","boundary_note":"The opening covers the available transcript.","boundary_confidence":"High"}],"boundary_caveats":["The opening boundary is inferred from the short transcript."]}`)
	case spells.PromptID:
		if client.spellResponse != "" {
			content = []byte(client.spellResponse)
		} else {
			content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Heals an injured ally.","narrative_description":"Aria restores the fighter after the fight.","source_refs":[{"source_id":"session-alpha","start_unit_id":1,"end_unit_id":1}]}]}`)
		}
	default:
		return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
	}
	if err := json.Unmarshal(content, out); err != nil {
		return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
	}
	client.mu.Lock()
	client.requests = append(client.requests, req)
	client.mu.Unlock()
	return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
}

func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.StructuredCompletionRequest {
	client.mu.Lock()
	defer client.mu.Unlock()
	var requests []contracts.StructuredCompletionRequest
	for _, req := range client.requests {
		if req.PromptID == promptID {
			requests = append(requests, req)
		}
	}
	return requests
}

func (client *productionFakeLLMClient) requestPrompts() []string {
	client.mu.Lock()
	defer client.mu.Unlock()
	prompts := make([]string, 0, len(client.requests))
	for _, req := range client.requests {
		prompts = append(prompts, req.PromptID)
	}
	return prompts
}

func repositoryPath(parts ...string) string {
	_, file, _, _ := runtime.Caller(0)
	return filepath.Join(append([]string{filepath.Dir(file), "..", ".."}, parts...)...)
}

func readRepositoryFile(t *testing.T, parts ...string) []byte {
	t.Helper()
	data, err := os.ReadFile(repositoryPath(parts...))
	if err != nil {
		t.Fatal(err)
	}
	return data
}