package cli import ( "context" "encoding/json" "errors" "fmt" "io/fs" "net/http" "net/http/httptest" "os" "path/filepath" "reflect" "runtime" "sort" "strings" "sync" "sync/atomic" "testing" "testing/fstest" "time" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes" combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns" enemyeventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/enemyevents" itemeventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemevents" spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells" combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns" enemyeventextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/enemyevents" itemeventextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/itemevents" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" combatnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/combatturns" enemyeventnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/enemyevents" itemeventnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/itemevents" spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop" "gitea.maximumdirect.net/eric/promptkit" ) func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) { components := productionTestComponents(t) registries := components.registries assertProductionContains(t, "inputs", registries.Inputs.RegisteredKeys(), []string{"seriatim"}) assertProductionContains(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes", "generic"}) assertProductionContains(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells", "dnd/npcs", combatextract.Key, itemeventextract.Key, enemyeventextract.Key}) assertProductionContains(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"}) assertProductionContains(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop", spellnormalize.Key, "dnd/npcs", combatnormalize.Key, itemeventnormalize.Key, enemyeventnormalize.Key}) assertProductionContains(t, "outputs", registries.Outputs.RegisteredKeys(), []string{"json"}) assertProductionContains(t, "validators", registries.Validators.RegisteredKeys(), []string{ "extract/dnd/spells/catalog", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness", "extract/dnd/combat-turns/shape", "extract/dnd/combat-turns/source_refs", "extract/dnd/combat-turns/source_relatedness", "normalize/dnd/combat-turns/invariants", "extract/dnd/item-events/shape", "extract/dnd/item-events/source_refs", "extract/dnd/item-events/source_relatedness", "normalize/dnd/item-events/invariants", "extract/dnd/enemy-events/shape", "extract/dnd/enemy-events/engagements", "extract/dnd/enemy-events/source_refs", "extract/dnd/enemy-events/source_relatedness", "normalize/dnd/enemy-events/invariants", "generic/always_accept", "generic/always_reject", "generic/valid_json", "generic/valid_json_schema", }) assertProductionContains(t, "artifact codec kinds", registries.ArtifactCodecs.RegisteredKinds(), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind, dnd.ItemEventListKind, dnd.EnemyEventListKind}) assertProductionContains(t, "merger variants", registries.Mergers.RegisteredArtifactKinds(pipeline.DefaultMergeModule), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind, dnd.ItemEventListKind, dnd.EnemyEventListKind}) assertProductionContains(t, "normalizer variants", registries.Normalizers.RegisteredArtifactKinds(pipeline.DefaultNormalizeModule), []contracts.ArtifactKind{dnd.SpellListKind, dnd.NPCListKind, dnd.CombatTurnListKind, dnd.ItemEventListKind, dnd.EnemyEventListKind}) assertProductionContains(t, "spell normalizer variants", registries.Normalizers.RegisteredArtifactKinds(spellnormalize.Key), []contracts.ArtifactKind{dnd.SpellListKind}) assertProductionContains(t, "combat normalizer variants", registries.Normalizers.RegisteredArtifactKinds(combatnormalize.Key), []contracts.ArtifactKind{dnd.CombatTurnListKind}) assertProductionContains(t, "item event normalizer variants", registries.Normalizers.RegisteredArtifactKinds(itemeventnormalize.Key), []contracts.ArtifactKind{dnd.ItemEventListKind}) assertProductionContains(t, "enemy event normalizer variants", registries.Normalizers.RegisteredArtifactKinds(enemyeventnormalize.Key), []contracts.ArtifactKind{dnd.EnemyEventListKind}) wantChain := []pipeline.ModuleBinding{ pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/spells/shape"), pipeline.Binding("extract/dnd/spells/catalog"), pipeline.Binding("extract/dnd/spells/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/spells/source_relatedness"), } if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) { t.Fatalf("spell validator chain = %#v, want %#v", got, wantChain) } if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, spellnormalize.Key); !reflect.DeepEqual(got, wantChain) { t.Fatalf("spell normalize validator chain = %#v, want %#v", got, wantChain) } combatExtractChain := []pipeline.ModuleBinding{ pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/combat-turns/shape"), pipeline.Binding("extract/dnd/combat-turns/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/combat-turns/source_relatedness"), } combatNormalizeChain := []pipeline.ModuleBinding{ pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/combat-turns/shape"), pipeline.Binding("normalize/dnd/combat-turns/invariants"), pipeline.Binding("extract/dnd/combat-turns/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/combat-turns/source_relatedness"), } if got := registries.ValidatorChains.Validators(pipeline.StageExtract, combatextract.Key); !reflect.DeepEqual(got, combatExtractChain) { t.Fatalf("combat extract validator chain = %#v, want %#v", got, combatExtractChain) } if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, combatnormalize.Key); !reflect.DeepEqual(got, combatNormalizeChain) { t.Fatalf("combat normalize validator chain = %#v, want %#v", got, combatNormalizeChain) } itemEventExtractChain := []pipeline.ModuleBinding{ pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/item-events/shape"), pipeline.Binding("extract/dnd/item-events/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/item-events/source_relatedness"), } itemEventNormalizeChain := []pipeline.ModuleBinding{ pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/item-events/shape"), pipeline.Binding("normalize/dnd/item-events/invariants"), pipeline.Binding("extract/dnd/item-events/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/item-events/source_relatedness"), } if got := registries.ValidatorChains.Validators(pipeline.StageExtract, itemeventextract.Key); !reflect.DeepEqual(got, itemEventExtractChain) { t.Fatalf("item event extract validator chain = %#v, want %#v", got, itemEventExtractChain) } if got := registries.ValidatorChains.Validators(pipeline.StageNormalize, itemeventnormalize.Key); !reflect.DeepEqual(got, itemEventNormalizeChain) { t.Fatalf("item event normalize validator chain = %#v, want %#v", got, itemEventNormalizeChain) } enemyEventExtractChain := []pipeline.ModuleBinding{ pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/enemy-events/shape"), pipeline.Binding("extract/dnd/enemy-events/engagements"), pipeline.Binding("extract/dnd/enemy-events/source_refs"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/enemy-events/source_relatedness"), } if got := registries.ValidatorChains.Validators(pipeline.StageExtract, enemyeventextract.Key); !reflect.DeepEqual(got, enemyEventExtractChain) { t.Fatalf("enemy event extract validator chain = %#v, want %#v", got, enemyEventExtractChain) } assetNames := productionAssetNames(t, components.assets.PromptFS) requiredAssets := []string{ "dnd.scenes/dnd.scenes.yaml", "dnd.scenes/instructions.md", "dnd.scenes/sharedassets/common-dnd-references.md", "dnd.scenes/sharedassets/common-dnd-system.md", "dnd.scenes/sharedassets/common-dnd-transcript-full.md", "dnd.scenes/task.md", "dnd.spells/dnd.spells.yaml", "dnd.spells/catalog.md", "dnd.spells/instructions.md", "dnd.spells/sharedassets/common-dnd-references.md", "dnd.spells/sharedassets/common-dnd-system.md", "dnd.spells/sharedassets/common-dnd-transcript-chunk.md", "dnd.spells/task.md", "dnd.combat_turns/dnd.combat_turns.yaml", "dnd.combat_turns/instructions.md", "dnd.combat_turns/sharedassets/common-dnd-references.md", "dnd.combat_turns/sharedassets/common-dnd-system.md", "dnd.combat_turns/sharedassets/common-dnd-transcript-chunk.md", "dnd.combat_turns/task.md", "dnd.item_events/dnd.item_events.yaml", "dnd.item_events/instructions.md", "dnd.item_events/sharedassets/common-dnd-extraction-evidence.md", "dnd.item_events/sharedassets/common-dnd-identity.md", "dnd.item_events/sharedassets/common-dnd-references.md", "dnd.item_events/sharedassets/common-dnd-system.md", "dnd.item_events/sharedassets/common-dnd-transcript-chunk.md", "dnd.item_events/task.md", "dnd.enemy_events/dnd.enemy_events.yaml", "dnd.enemy_events/grounding.md", "dnd.enemy_events/instructions.md", "dnd.enemy_events/task.md", } assertProductionContains(t, "production prompt assets", assetNames, requiredAssets) catalog := catalogFromRegistries(registries) for _, test := range []struct { stage pipeline.ModuleStage key string want contracts.ExecutionClass }{ {stage: pipeline.StageInput, key: "seriatim", want: contracts.ExecutionClassDeterministic}, {stage: pipeline.StageChunk, key: "generic", want: contracts.ExecutionClassDeterministic}, {stage: pipeline.StageChunk, key: "dnd/scenes", want: contracts.ExecutionClassLLMBacked}, {stage: pipeline.StageExtract, key: "dnd/spells", want: contracts.ExecutionClassLLMBacked}, {stage: pipeline.StageExtract, key: "dnd/npcs", want: contracts.ExecutionClassLLMBacked}, {stage: pipeline.StageExtract, key: "dnd/combat-turns", want: contracts.ExecutionClassLLMBacked}, {stage: pipeline.StageExtract, key: "dnd/item-events", want: contracts.ExecutionClassLLMBacked}, {stage: pipeline.StageExtract, key: "dnd/npc-interactions", want: contracts.ExecutionClassLLMBacked}, {stage: pipeline.StageExtract, key: "dnd/scene-descriptions", want: contracts.ExecutionClassLLMBacked}, {stage: pipeline.StageExtract, key: enemyeventextract.Key, want: contracts.ExecutionClassLLMBacked}, {stage: pipeline.StageMerge, key: "appendorder", want: contracts.ExecutionClassDeterministic}, {stage: pipeline.StageNormalize, key: "noop", want: contracts.ExecutionClassDeterministic}, {stage: pipeline.StageNormalize, key: "dnd/spells", want: contracts.ExecutionClassDeterministic}, {stage: pipeline.StageNormalize, key: "dnd/npcs", want: contracts.ExecutionClassLLMBacked}, {stage: pipeline.StageNormalize, key: "dnd/combat-turns", want: contracts.ExecutionClassDeterministic}, {stage: pipeline.StageNormalize, key: "dnd/item-events", want: contracts.ExecutionClassDeterministic}, {stage: pipeline.StageNormalize, key: "dnd/npc-interactions", want: contracts.ExecutionClassDeterministic}, {stage: pipeline.StageNormalize, key: "dnd/scene-descriptions", want: contracts.ExecutionClassDeterministic}, {stage: pipeline.StageNormalize, key: enemyeventnormalize.Key, want: contracts.ExecutionClassDeterministic}, {stage: pipeline.StageOutput, key: "json", want: contracts.ExecutionClassDeterministic}, } { got, ok := catalog.ExecutionClass(test.stage, test.key) if !ok || got != test.want { t.Fatalf("production execution class for %s/%s = %q, %t; want %q, true", test.stage, test.key, got, ok, test.want) } } converted := registriesFromCatalog(catalog) if converted.ArtifactCodecs != registries.ArtifactCodecs || converted.ArtifactEvidence != registries.ArtifactEvidence || converted.ValidatorChains != registries.ValidatorChains { t.Fatal("catalog/registry conversion did not preserve artifact and validator registries") } codecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.SpellListKind) if !ok || codecSpec.Kind != dnd.SpellListKind || codecSpec.Schema.ID != spellcodec.SchemaID { t.Fatalf("catalog codec spec = %#v, ok=%t, want typed D&D spell codec", codecSpec, ok) } combatCodecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.CombatTurnListKind) if !ok || combatCodecSpec.Kind != dnd.CombatTurnListKind || combatCodecSpec.Schema.ID != combatcodec.SchemaID { t.Fatalf("combat codec spec = %#v, ok=%t, want typed D&D combat codec", combatCodecSpec, ok) } itemEventCodecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.ItemEventListKind) if !ok || itemEventCodecSpec.Kind != dnd.ItemEventListKind || itemEventCodecSpec.Schema.ID != itemeventcodec.SchemaID { t.Fatalf("item event codec spec = %#v, ok=%t, want typed D&D item-event codec", itemEventCodecSpec, ok) } enemyEventCodecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.EnemyEventListKind) if !ok || enemyEventCodecSpec.Kind != dnd.EnemyEventListKind || enemyEventCodecSpec.Schema.ID != enemyeventcodec.SchemaID { t.Fatalf("enemy event codec spec = %#v, ok=%t, want typed D&D enemy-event codec", enemyEventCodecSpec, ok) } if got := catalog.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) { t.Fatalf("catalog validator chain = %#v, want %#v", got, wantChain) } if got := catalog.ValidatorChains.Validators(pipeline.StageNormalize, spellnormalize.Key); !reflect.DeepEqual(got, wantChain) { t.Fatalf("catalog spell normalize validator chain = %#v, want %#v", got, wantChain) } } func TestDefaultCLICompositionValidatesRepresentativeConfiguration(t *testing.T) { var stdout, stderr strings.Builder code := RunWithOptions([]string{ "config", "validate", "--config", repositoryPath("examples", "dnd-minimal.config.yml"), "--pipeline", "dnd-session", }, &stdout, &stderr, Options{}) if code != 0 || stderr.Len() != 0 { t.Fatalf("validate representative config with default composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) { components := productionTestComponents(t) newEngine := func(profileFile string) (*promptkit.Engine, error) { t.Helper() options, err := components.assets.PromptKitOptions() if err != nil { return nil, err } if profileFile != "" { options = append(options, promptkit.WithProfileFile(profileFile)) } return promptkit.NewEngine(promptkit.Config{}, options...) } t.Run("fallback", func(t *testing.T) { engine, err := newEngine("") if err != nil { t.Fatal(err) } inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction") if err != nil { t.Fatalf("InspectProfile() error = %v, want fallback profile", err) } params := inspection.EffectiveModelParams if params.BackendID != "openrouter" || params.Model != "openai/gpt-5.6-luna" || params.TimeoutSeconds != 240 || params.ServiceTier != "flex" { t.Fatalf("fallback profile parameters = %#v", params) } if params.ReasoningEffort != "" || params.Temperature != 0 || params.MaxTokens != 0 || params.TopP != 0 { t.Fatalf("fallback profile selected optional provider controls: %#v", params) } }) t.Run("valid operator profile wins", func(t *testing.T) { profilePath := filepath.Join(t.TempDir(), "profiles.yaml") if err := os.WriteFile(profilePath, []byte(`id: dnd-extraction endpoint: http://operator.example.test/v1 model: operator-model timeout_seconds: 75 `), 0o600); err != nil { t.Fatal(err) } engine, err := newEngine(profilePath) if err != nil { t.Fatal(err) } inspection, err := engine.InspectProfile(context.Background(), "dnd-extraction") if err != nil { t.Fatalf("InspectProfile() error = %v, want operator profile", err) } params := inspection.EffectiveModelParams if params.BackendID != "" || params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "" { t.Fatalf("operator profile parameters = %#v, want complete replacement", params) } }) t.Run("invalid operator profile does not fall through", func(t *testing.T) { profilePath := filepath.Join(t.TempDir(), "profiles.yaml") if err := os.WriteFile(profilePath, []byte("id: dnd-extraction\nendpoint: http://operator.example.test/v1\nmodel: operator-model\nunknown: value\n"), 0o600); err != nil { t.Fatal(err) } engine, err := newEngine(profilePath) if err == nil { _, err = engine.InspectProfile(context.Background(), "dnd-extraction") } if err == nil { t.Fatal("operator profile error = nil, want failure instead of fallback") } }) } func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) { components := productionTestComponents(t) cfg := config.Default() cfg.Pipelines["dnd-scenes"] = pipeline.PipelineProfile{ ID: "dnd-scenes", Input: pipeline.Binding("seriatim"), Chunk: pipeline.Binding("dnd/scenes"), Artifacts: map[string]pipeline.ArtifactLaneProfile{ "spells": {Extract: pipeline.Binding("dnd/spells")}, }, } effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-scenes", Catalog: catalogFromRegistries(components.registries)}) if err != nil { t.Fatalf("resolve production scene pipeline: %v", err) } if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil { t.Fatalf("prepare production scene and spell modules: %v", err) } } func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) { components := productionTestComponents(t) configPath := writeProductionSpellCatalogContractConfig(t) effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatalf("resolve production spell configuration: %v", err) } materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ ConfigPath: configPath, WorkingDir: filepath.Dir(configPath), }) if err != nil { t.Fatalf("materialize production spell references: %v", err) } extractItems := materialized.Steps[0].ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items normalizeItems := materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items if len(extractItems) != 1 || extractItems[0].MediaType != "application/json" || len(extractItems[0].Content) == 0 { t.Fatalf("materialized extract spell catalog items = %#v, want one JSON item", extractItems) } if len(normalizeItems) != 1 || normalizeItems[0].MediaType != "application/json" || !reflect.DeepEqual(normalizeItems[0].Content, extractItems[0].Content) { t.Fatalf("materialized normalize spell catalog items = %#v, want an independent binding of the extract catalog", normalizeItems) } if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil { t.Fatalf("prepare production spell pipeline from materialized catalog: %v", err) } } func TestProductionSpellNormalizerRejectsInvalidCatalogReferencesBeforeExecution(t *testing.T) { components := productionTestComponents(t) configPath := writeProductionSpellCatalogContractConfig(t) resolve := func(t *testing.T) pipeline.ResolvedPipeline { t.Helper() effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatalf("resolve production spell configuration: %v", err) } return effective.ResolvedPipeline } materialize := func(resolved pipeline.ResolvedPipeline) (pipeline.ResolvedPipeline, error) { materialized, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ ConfigPath: configPath, WorkingDir: filepath.Dir(configPath), }) return materialized, err } t.Run("malformed catalog fails preparation", func(t *testing.T) { catalogPath := filepath.Join(t.TempDir(), "malformed.json") if err := os.WriteFile(catalogPath, []byte(`{"schema_version":`), 0o600); err != nil { t.Fatal(err) } resolved := resolve(t) setNormalizeSpellCatalogSource(t, &resolved, catalogPath) materialized, err := materialize(resolved) if err != nil { t.Fatalf("MaterializeReferences() error = %v, want malformed JSON to reach preparation", err) } _, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) for _, fragment := range []string{`pipeline "dnd-session"`, `lane "spells"`, "normalize", `module "dnd/spells"`, "decode spell catalog overlay"} { if err == nil || !strings.Contains(err.Error(), fragment) { t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment) } } }) t.Run("multiple catalog items fail preparation", func(t *testing.T) { materialized, err := materialize(resolve(t)) if err != nil { t.Fatal(err) } slot := materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"] slot.Items = append(slot.Items, slot.Items[0]) materialized.Steps[0].ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["spell_catalog"] = slot _, err = pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}) for _, fragment := range []string{"normalize", `module "dnd/spells"`, "zero or one item"} { if err == nil || !strings.Contains(err.Error(), fragment) { t.Fatalf("Prepare() error = %v, want context fragment %q", err, fragment) } } }) t.Run("oversized catalog fails materialization", func(t *testing.T) { catalogPath := filepath.Join(t.TempDir(), "oversized.json") if err := os.WriteFile(catalogPath, []byte(strings.Repeat("x", 1048577)), 0o600); err != nil { t.Fatal(err) } checkpointRoot := filepath.Join(t.TempDir(), "checkpoints") content := productionSpellCatalogContractConfig(t) catalogSource := repositoryPath("examples", "dnd-spell-catalog.json") if count := strings.Count(content, catalogSource); count != 2 { t.Fatalf("spell catalog source occurs %d times, want extract and normalize bindings", count) } content = strings.Replace(content, catalogSource, "__extract_catalog__", 1) content = replaceRequiredOnce(t, content, catalogSource, catalogPath) content = replaceRequiredOnce(t, content, "__extract_catalog__", catalogSource) content = replaceRequiredOnce(t, content, " enabled: false\n directory: \"\"", " enabled: true\n directory: "+checkpointRoot) configFile := filepath.Join(t.TempDir(), "config.yml") if err := os.WriteFile(configFile, []byte(content), 0o600); err != nil { t.Fatal(err) } llmConstructed := false chunkStoreConstructed := false options := Options{ Catalog: catalogFromRegistries(components.registries), Registries: components.registries, LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { llmConstructed = true return nil, nil, errors.New("LLM client must not be constructed") }, ChunkPlanStoreFactory: func(string) (pipeline.ChunkPlanStore, error) { chunkStoreConstructed = true return nil, errors.New("chunk-plan store must not be constructed") }, } var stdout, stderr strings.Builder code := RunWithOptions([]string{ "run", "dnd-session", "--config", configFile, "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), }, &stdout, &stderr, options) errText := stderr.String() for _, fragment := range []string{"normalize", `lane "spells"`, `reference slot "spell_catalog"`, "1048577 bytes", "limit 1048576"} { if code == 0 || !strings.Contains(errText, fragment) { t.Fatalf("RunWithOptions() code = %d stderr = %q, want context fragment %q", code, errText, fragment) } } if llmConstructed || chunkStoreConstructed { t.Fatalf("runtime construction = LLM %t, chunk store %t; want materialization failure first", llmConstructed, chunkStoreConstructed) } if _, err := os.Stat(checkpointRoot); !errors.Is(err, fs.ErrNotExist) { t.Fatalf("checkpoint root stat error = %v, want no checkpoint allocation", err) } }) } func setNormalizeSpellCatalogSource(t *testing.T, resolved *pipeline.ResolvedPipeline, sourcePath string) { t.Helper() if resolved == nil || len(resolved.Steps[0].ArtifactLanes) != 1 { t.Fatalf("resolved pipeline = %#v, want one artifact lane", resolved) } bindings := resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings matches := 0 for index := range bindings { if bindings[index].SlotName == "spell_catalog" { bindings[index].Source = sourcePath matches++ } } if matches != 1 { t.Fatalf("normalize reference bindings = %#v, want exactly one spell_catalog binding", bindings) } resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings = bindings } func TestProductionLLMClientFactoryBuildsOfflineRuntime(t *testing.T) { components := productionTestComponents(t) client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{}) if err != nil { t.Fatalf("build production LLM runtime: %v", err) } if client == nil { t.Fatal("production LLM runtime returned a nil client") } if len(manifests) != 0 { t.Fatalf("eager profile manifests = %#v, want none", manifests) } fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider) if !ok { t.Fatalf("production LLM client %T does not provide one profile-source checkpoint fingerprint", client) } fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints() if err != nil || len(fingerprints) != 1 { t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one profile-source identity", fingerprints, err) } if _, ok := client.(contracts.LLMProfileManifestProvider); !ok { t.Fatalf("production LLM client %T does not provide profile manifests", client) } } func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *testing.T) { opts, err := normalizeOptions(Options{ Catalog: pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry()}, }) if err != nil { t.Fatal(err) } if opts.promptKitAssets == nil || opts.LLMClientFactory == nil { t.Fatalf("normalized options = %#v, want shared profile assets and default runtime factory", opts) } if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"dnd-extraction"}, opts.promptKitAssets); err != nil { t.Fatalf("inspect application fallback profile: %v", err) } client, _, err := opts.LLMClientFactory(context.Background(), config.Default(), "dnd-extraction", LLMRuntimeOverrides{}) if err != nil { t.Fatalf("build default runtime: %v", err) } fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider) if !ok { t.Fatalf("default runtime client %T does not provide checkpoint fingerprints", client) } runtimeFingerprints, err := fingerprintProvider.LLMCheckpointFingerprints() if err != nil { t.Fatal(err) } directClient, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{Assets: opts.promptKitAssets}) if err != nil { t.Fatal(err) } inspectionFingerprints, err := directClient.LLMCheckpointFingerprints() if err != nil { t.Fatal(err) } if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) { t.Fatalf("runtime profile fingerprints = %#v, inspection profile fingerprints = %#v", runtimeFingerprints, inspectionFingerprints) } } func TestProductionLLMClientFactoryUsesConfiguredLocalBackend(t *testing.T) { var providerCalls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { providerCalls.Add(1) if r.URL.Path != "/v1/chat/completions" { t.Errorf("provider path = %q, want /v1/chat/completions", r.URL.Path) } w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{ "choices": [{"message": {"role": "assistant", "content": "{\"ok\":true}"}}], "usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7} }`)) })) defer server.Close() profilePath := filepath.Join(t.TempDir(), "profiles.yml") if err := os.WriteFile(profilePath, []byte(`id: local-profile backend: local model: local-model `), 0o600); err != nil { t.Fatal(err) } assets := llm.NewAssetRegistry() if err := assets.RegisterPromptFS(fstest.MapFS{ "production.local.yaml": {Data: []byte(`id: production.local version: "v1" inputs: - name: transcript required: true messages: - role: user content: '{{ input "transcript" }}' output: format: json validation_mode: json `)}, }, "."); err != nil { t.Fatalf("register prompt assets: %v", err) } cfg := config.Default() cfg.PromptKit.ProfileFile = profilePath cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{ Endpoint: server.URL + "/v1", ConcurrencyLimit: 2, } client, manifests, err := productionLLMClientFactoryWithAssets(assets)( context.Background(), cfg, "local-profile", LLMRuntimeOverrides{}, ) if err != nil { t.Fatalf("build production LLM runtime: %v", err) } if len(manifests) != 0 { t.Fatalf("eager profile manifests = %#v, want none", manifests) } var out map[string]any _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "production.local", ProfileID: "local-profile", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "text/plain", []byte("local request"), "", ""), }, }, &out) if err != nil { t.Fatalf("CompleteStructured() error = %v, want nil", err) } if providerCalls.Load() != 1 { t.Fatalf("provider calls = %d, want 1", providerCalls.Load()) } provider, ok := client.(contracts.LLMProfileManifestProvider) if !ok { t.Fatalf("production client %T does not provide profile manifests", client) } recorded := provider.LLMProfileManifests() if len(recorded) != 1 || recorded[0].BackendID != promptkit.BackendLocal { t.Fatalf("production profile manifests = %#v, want local backend", recorded) } } func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) { t.Run("canceled context", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() components := productionTestComponents(t) client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(ctx, config.Default(), "test-profile", LLMRuntimeOverrides{}) if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 { t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err) } }) t.Run("nil assets", func(t *testing.T) { client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile", LLMRuntimeOverrides{}) if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 { t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err) } }) t.Run("invalid scheduler concurrency", func(t *testing.T) { components := productionTestComponents(t) cfg := config.Default() cfg.Concurrency.TotalLLM = 0 client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile", LLMRuntimeOverrides{}) if err == nil || !strings.Contains(err.Error(), "create LLM scheduler") || !strings.Contains(err.Error(), "greater than zero") || client != nil || len(manifests) != 0 { t.Fatalf("client=%T manifests=%#v error=%v, want scheduler-construction failure", client, manifests, err) } }) } func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) { base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml")) validPath := writeProductionContractConfig(t, base) options := productionCLIOptions(t) var stdout, stderr strings.Builder if code := RunWithOptions([]string{"config", "validate", "--config", validPath, "--pipeline", "dnd-session"}, &stdout, &stderr, options); code != 0 { t.Fatalf("valid production config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } tests := []struct { name string content string options Options fragments []string }{ { name: "unknown module", content: replaceRequiredOnce(t, base, " input: seriatim\n", " input: missing/input\n"), options: productionCLIOptions(t), fragments: []string{"pipeline \"dnd-session\"", "input", "missing/input"}, }, { name: "unknown validator", content: replaceRequiredOnce(t, base, " extract: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: missing/validator\n"), options: productionCLIOptions(t), fragments: []string{"validator", "missing/validator"}, }, { name: "invalid artifact variant", content: base, options: productionCLIOptionsWithoutSpellNormalizer(t), fragments: []string{"normalizer", spellnormalize.Key, string(dnd.SpellListKind), "variant"}, }, { name: "deterministic validator with profile", content: replaceRequiredOnce(t, base, " extract: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: generic/valid_json\n llm_profile: forbidden-profile\n"), options: productionCLIOptions(t), fragments: []string{"deterministic validator", "llm_profile"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { path := writeProductionContractConfig(t, tt.content) var stdout, stderr strings.Builder code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "dnd-session"}, &stdout, &stderr, tt.options) if code != 1 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } for _, fragment := range tt.fragments { if !strings.Contains(stderr.String(), fragment) { t.Fatalf("stderr=%q, want %q", stderr.String(), fragment) } } }) } } func TestProductionNormalizeValidatorOverrideRemainsAuthoritative(t *testing.T) { base := string(readRepositoryFile(t, "examples", "dnd-minimal.config.yml")) content := replaceRequiredOnce(t, base, " normalize: dnd/spells\n", " normalize:\n module: dnd/spells\n validators:\n - module: generic/always_accept\n - module: generic/valid_json\n") path := writeProductionContractConfig(t, content) components := productionTestComponents(t) effective, err := loadMaintainedExample(t, path).Resolve(resolveInputForMaintainedExample(components, "dnd-session")) if err != nil { t.Fatalf("resolve normalize override: %v", err) } for _, chain := range effective.ResolvedPipeline.ValidatorChains { if chain.Stage != pipeline.StageNormalize || chain.ModuleKey != spellnormalize.Key { continue } if len(chain.Validators) != 2 || chain.Validators[0].Binding.Module != "generic/always_accept" || chain.Validators[1].Binding.Module != "generic/valid_json" { t.Fatalf("normalize validator chain = %#v, want explicit validator order", chain) } return } t.Fatalf("resolved validator chains = %#v, want normalize chain for %q", effective.ResolvedPipeline.ValidatorChains, spellnormalize.Key) } func TestProductionSceneRunRecordsAnnotationFreeChunkPlanAndProvenance(t *testing.T) { outputRoot := filepath.Join(t.TempDir(), "output") configPath := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes")) fake := &productionFakeLLMClient{} options := productionRunOptions(t, fake) var stdout, stderr strings.Builder code := RunWithOptions([]string{ "run", "dnd-session", "--config", configPath, "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--chunk_cache", "bypass", "--session-id", "offline-session", }, &stdout, &stderr, options) if code != 0 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(outputRoot, productionRunID, "manifest.json")) if manifest.Chunker != scenes.Key || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" || manifest.ChunkPlan.ProducerModule != scenes.Key { t.Fatalf("chunk manifest = %#v, want dnd scene producer", manifest.ChunkPlan) } if got := manifest.ModuleMetadata["chunker"]["prompt_id"]; got != scenes.PromptID { t.Fatalf("chunker prompt metadata = %#v, want %q", got, scenes.PromptID) } if got := manifest.ChunkPlan.ProducerMetadata["response_schema_id"]; got != scenes.ResponseSchemaID { t.Fatalf("chunk producer schema metadata = %#v, want %q", got, scenes.ResponseSchemaID) } index := readProductionJSON[productionChunkMapIndex](t, filepath.Join(outputRoot, productionRunID, "index.json")) if index.ChunkMap == nil || index.ChunkMap.ArtifactKind != chunkmap.ArtifactKind || index.ChunkMap.File != "chunk-map.json" || index.ChunkMap.MediaType != chunkmap.MediaType || index.ChunkMap.SchemaID != chunkmap.SchemaID || index.ChunkMap.SchemaName != chunkmap.SchemaName || index.ChunkMap.SchemaVersion != chunkmap.SchemaVersion { t.Fatalf("chunk map index = %#v, want fixed chunk map descriptor", index.ChunkMap) } for _, output := range index.OutputFiles { if output.File == index.ChunkMap.File { t.Fatalf("lane output files = %#v, want no chunk map", index.OutputFiles) } } content, err := os.ReadFile(filepath.Join(outputRoot, productionRunID, index.ChunkMap.File)) if err != nil { t.Fatal(err) } chunkMap, err := chunkmap.New().Decode(content) if err != nil { t.Fatalf("Decode(chunk map) error = %v", err) } if chunkMap.SourceID != "session-alpha" || chunkMap.SourceDigest != manifest.ChunkPlan.SourceDigest || chunkMap.PlanDigest != manifest.ChunkPlan.PlanDigest || chunkMap.RequestedChunker != scenes.Key || chunkMap.Producer.InputModule != "seriatim" || chunkMap.Producer.ChunkModule != scenes.Key || chunkMap.Producer.LLMProfile != manifest.ChunkPlan.ProducerLLMProfile { t.Fatalf("chunk map identity and producer = %#v, want accepted scene plan provenance", chunkMap) } if len(chunkMap.Chunks) != 1 || chunkMap.Chunks[0].ID != "chunk-000001" || chunkMap.Chunks[0].Index != 0 || chunkMap.Chunks[0].SourceRef.SourceID != "session-alpha" || chunkMap.Chunks[0].SourceRef.StartUnitID != 1 || chunkMap.Chunks[0].SourceRef.EndUnitID != 2 || chunkMap.Chunks[0].UnitCount != 2 { t.Fatalf("chunk map chunks = %#v, want one stable accepted scene range", chunkMap.Chunks) } if len(chunkMap.PlanAnnotations) != 0 { t.Fatalf("chunk map plan annotations = %#v, want none", chunkMap.PlanAnnotations) } if len(chunkMap.Chunks[0].Annotations) != 0 { t.Fatalf("chunk map range annotations = %#v, want none", chunkMap.Chunks[0].Annotations) } warnings := readProductionJSON[struct { Warnings []contracts.Warning `json:"warnings"` }](t, filepath.Join(outputRoot, productionRunID, "warnings.json")) if len(warnings.Warnings) != 0 { t.Fatalf("warnings = %#v, want none", warnings.Warnings) } if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemeventextract.PromptID)) != 1 { t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-event request", fake.requestPrompts()) } } type maintainedExample struct { name string path string transcriptPath string pipelineIDs []string } func maintainedExampleFiles(t *testing.T) []maintainedExample { t.Helper() return []maintainedExample{ {name: "minimal", path: repositoryPath("examples", "dnd-minimal.config.yml"), transcriptPath: repositoryPath("examples", "seriatim-minimal-transcript.json"), pipelineIDs: []string{"dnd-session"}}, {name: "complete", path: repositoryPath("examples", "dnd-complete.config.yml"), transcriptPath: repositoryPath("examples", "dnd-complete-transcript.json"), pipelineIDs: []string{"dnd-session"}}, } } func productionSpellCatalogContractConfig(t *testing.T) string { t.Helper() return fmt.Sprintf(`version: 4 cache: chunk_plans: mode: bypass checkpoints: enabled: false directory: "" pipelines: dnd-session: input: seriatim references: party: %q glossary: %q artifacts: spells: extract: module: dnd/spells retries: 2 references: spell_catalog: %q normalize: module: dnd/spells references: spell_catalog: %q `, repositoryPath("examples", "dnd-party.txt"), repositoryPath("examples", "dnd-glossary.txt"), repositoryPath("examples", "dnd-spell-catalog.json"), repositoryPath("examples", "dnd-spell-catalog.json")) } func writeProductionSpellCatalogContractConfig(t *testing.T) string { t.Helper() return writeProductionContractConfig(t, productionSpellCatalogContractConfig(t)) } func loadMaintainedExample(t *testing.T, path string) config.Config { t.Helper() fileConfig, err := config.LoadFileConfig(path) if err != nil { t.Fatalf("load maintained config %q: %v", path, err) } cfg := config.Default() if err := cfg.ApplyFileConfig(fileConfig); err != nil { t.Fatalf("apply maintained config %q: %v", path, err) } if err := cfg.Validate(); err != nil { t.Fatalf("validate maintained config %q: %v", path, err) } return cfg } func productionTestComponents(t *testing.T) productionComponents { t.Helper() components, err := newProductionComponents() if err != nil { t.Fatalf("new production components: %v", err) } return components } func productionCLIOptions(t *testing.T) Options { t.Helper() components := productionTestComponents(t) return productionOptionsFromComponents(components) } func productionOptionsFromComponents(components productionComponents) Options { return Options{ Catalog: catalogFromRegistries(components.registries), Registries: components.registries, LookupEnv: emptyLookup, promptKitAssets: components.assets, } } func productionCLIOptionsWithoutSpellNormalizer(t *testing.T) Options { t.Helper() components := productionTestComponents(t) registries := components.registries registries.Normalizers = pipeline.NewNormalizerRegistry() if err := noop.RegisterTyped[dnd.SpellList](registries.Normalizers, contracts.ArtifactKind("test/other")); err != nil { t.Fatalf("register mismatched normalizer: %v", err) } return productionOptionsFromComponents(productionComponents{registries: registries, assets: components.assets}) } const productionRunID = "run-1700000000000000000-0123456789abcdef0123456789abcdef" func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options { t.Helper() options := productionCLIOptions(t) options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() } options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil } options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") } options.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { return fake, nil, nil } return options } func productionRunConfig(outputRoot, chunkModule string) string { return fmt.Sprintf(`version: 4 output: directory: %q cache: chunk_plans: mode: bypass checkpoints: {} debug: directory: %q pipelines: dnd-session: input: seriatim chunk: %s output: module: json options: include_chunk_map: true artifacts: spells: extract: dnd/spells item-events: extract: dnd/item-events `, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule) } type productionChunkMapIndex struct { OutputFiles []struct { File string `json:"file"` } `json:"output_files"` ChunkMap *struct { ArtifactKind contracts.ArtifactKind `json:"artifact_kind"` File string `json:"file"` MediaType string `json:"media_type"` SchemaID string `json:"schema_id"` SchemaName string `json:"schema_name"` SchemaVersion string `json:"schema_version"` } `json:"chunk_map"` } func writeProductionContractConfig(t *testing.T, content string) string { t.Helper() path := filepath.Join(t.TempDir(), "config.yml") if err := os.WriteFile(path, []byte(content), 0o600); err != nil { t.Fatal(err) } return path } func productionAssetNames(t *testing.T, getFS func() (fs.FS, error)) []string { t.Helper() fileSystem, err := getFS() if err != nil { t.Fatalf("load production prompt assets: %v", err) } var names []string if err := fs.WalkDir(fileSystem, ".", func(path string, entry fs.DirEntry, err error) error { if err != nil { return err } if !entry.IsDir() { names = append(names, path) } return nil }); err != nil { t.Fatalf("walk production prompt assets: %v", err) } sort.Strings(names) return names } func assertProductionContains[T comparable](t *testing.T, name string, got, required []T) { t.Helper() available := make(map[T]struct{}, len(got)) for _, entry := range got { available[entry] = struct{}{} } var missing []T for _, entry := range required { if _, ok := available[entry]; !ok { missing = append(missing, entry) } } if len(missing) > 0 { t.Fatalf("%s missing required entries %#v; registered entries are %#v", name, missing, got) } } func readProductionJSON[T any](t *testing.T, path string) T { t.Helper() data, err := os.ReadFile(path) if err != nil { t.Fatalf("read %s: %v", path, err) } var value T if err := json.Unmarshal(data, &value); err != nil { t.Fatalf("decode %s: %v", path, err) } return value } type productionFakeLLMClient struct { mu sync.Mutex requests []contracts.StructuredCompletionRequest spellResponse string } func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { if err := ctx.Err(); err != nil { return contracts.StructuredCompletionResponse{}, err } var content []byte switch req.PromptID { case scenes.PromptID: content = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2}]}`) case spells.PromptID: if client.spellResponse != "" { content = []byte(client.spellResponse) } else { content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`) } case itemeventextract.PromptID: content = []byte(`{"events":[{"name":"Cure Wounds","kind":"acquired","to":"party","source_refs":[{"start_segment":1,"end_segment":1}]}]}`) default: return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID) } if err := json.Unmarshal(content, out); err != nil { return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err) } client.mu.Lock() client.requests = append(client.requests, req) client.mu.Unlock() return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil } func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.StructuredCompletionRequest { client.mu.Lock() defer client.mu.Unlock() var requests []contracts.StructuredCompletionRequest for _, req := range client.requests { if req.PromptID == promptID { requests = append(requests, req) } } return requests } func (client *productionFakeLLMClient) requestPrompts() []string { client.mu.Lock() defer client.mu.Unlock() prompts := make([]string, 0, len(client.requests)) for _, req := range client.requests { prompts = append(prompts, req.PromptID) } return prompts } func repositoryPath(parts ...string) string { _, file, _, _ := runtime.Caller(0) return filepath.Join(append([]string{filepath.Dir(file), "..", ".."}, parts...)...) } func readRepositoryFile(t *testing.T, parts ...string) []byte { t.Helper() data, err := os.ReadFile(repositoryPath(parts...)) if err != nil { t.Fatal(err) } return data }