package cli import ( "context" "encoding/json" "errors" "fmt" "io/fs" "os" "path/filepath" "reflect" "runtime" "sort" "strings" "sync" "testing" "time" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes" spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop" ) func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) { components := productionTestComponents(t) registries := components.registries assertProductionContains(t, "inputs", registries.Inputs.RegisteredKeys(), []string{"seriatim"}) assertProductionContains(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes", "generic"}) assertProductionContains(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells"}) assertProductionContains(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"}) assertProductionContains(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop"}) assertProductionContains(t, "outputs", registries.Outputs.RegisteredKeys(), []string{"json"}) assertProductionContains(t, "validators", registries.Validators.RegisteredKeys(), []string{ "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness", "generic/always_accept", "generic/always_reject", "generic/valid_json", "generic/valid_json_schema", }) assertProductionContains(t, "artifact codec kinds", registries.ArtifactCodecs.RegisteredKinds(), []contracts.ArtifactKind{dnd.SpellListKind}) assertProductionContains(t, "merger variants", registries.Mergers.RegisteredArtifactKinds(pipeline.DefaultMergeModule), []contracts.ArtifactKind{dnd.SpellListKind}) assertProductionContains(t, "normalizer variants", registries.Normalizers.RegisteredArtifactKinds(pipeline.DefaultNormalizeModule), []contracts.ArtifactKind{dnd.SpellListKind}) wantChain := []pipeline.ModuleBinding{ pipeline.Binding("generic/valid_json"), pipeline.Binding("generic/valid_json_schema"), pipeline.Binding("extract/dnd/spells/shape"), pipeline.Binding("extract/dnd/spells/source_refs"), pipeline.Binding("extract/dnd/spells/source_relatedness"), } if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) { t.Fatalf("spell validator chain = %#v, want %#v", got, wantChain) } assetNames := productionAssetNames(t, components.assets.PromptFS) requiredAssets := []string{ "dnd.scenes/dnd.scenes.yaml", "dnd.scenes/instructions.md", "dnd.scenes/sharedassets/common-dnd-references.md", "dnd.scenes/sharedassets/common-dnd-system.md", "dnd.scenes/sharedassets/common-dnd-transcript.md", "dnd.scenes/task.md", "dnd.spells/dnd.spells.yaml", "dnd.spells/catalog.md", "dnd.spells/instructions.md", "dnd.spells/sharedassets/common-dnd-references.md", "dnd.spells/sharedassets/common-dnd-system.md", "dnd.spells/sharedassets/common-dnd-transcript.md", "dnd.spells/task.md", } assertProductionContains(t, "production prompt assets", assetNames, requiredAssets) catalog := catalogFromRegistries(registries) converted := registriesFromCatalog(catalog) if converted.ArtifactCodecs != registries.ArtifactCodecs || converted.ValidatorChains != registries.ValidatorChains { t.Fatal("catalog/registry conversion did not preserve codec and validator-chain registries") } codecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.SpellListKind) if !ok || codecSpec.Kind != dnd.SpellListKind || codecSpec.Schema.ID != spellcodec.SchemaID { t.Fatalf("catalog codec spec = %#v, ok=%t, want typed D&D spell codec", codecSpec, ok) } if got := catalog.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) { t.Fatalf("catalog validator chain = %#v, want %#v", got, wantChain) } } func TestDefaultCLICompositionValidatesRepresentativeConfiguration(t *testing.T) { var stdout, stderr strings.Builder code := RunWithOptions([]string{ "config", "validate", "--config", repositoryPath("examples", "dnd-spells.config.yml"), "--pipeline", "dnd-session", }, &stdout, &stderr, Options{}) if code != 0 || stderr.Len() != 0 { t.Fatalf("validate representative config with default composition: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) { components := productionTestComponents(t) cfg := config.Default() cfg.Pipelines["dnd-scenes"] = pipeline.PipelineProfile{ ID: "dnd-scenes", Input: pipeline.Binding("seriatim"), Chunk: pipeline.Binding("dnd/scenes"), Artifacts: map[string]pipeline.ArtifactLaneProfile{ "spells": {Extract: pipeline.Binding("dnd/spells")}, }, } effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-scenes", Catalog: catalogFromRegistries(components.registries)}) if err != nil { t.Fatalf("resolve production scene pipeline: %v", err) } if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil { t.Fatalf("prepare production scene and spell modules: %v", err) } } func TestProductionLLMClientFactoriesBuildOfflineRuntime(t *testing.T) { components := productionTestComponents(t) factories := []struct { name string factory LLMClientFactory }{ {name: "default production assets", factory: productionLLMClientFactory}, {name: "provided production assets", factory: productionLLMClientFactoryWithAssets(components.assets)}, } for _, tt := range factories { t.Run(tt.name, func(t *testing.T) { client, manifests, err := tt.factory(context.Background(), config.Default(), "test-profile") if err != nil { t.Fatalf("build production LLM runtime: %v", err) } if client == nil { t.Fatal("production LLM runtime returned a nil client") } if len(manifests) != 0 { t.Fatalf("eager profile manifests = %#v, want none", manifests) } if _, ok := client.(contracts.LLMProfileManifestProvider); !ok { t.Fatalf("production LLM client %T does not provide profile manifests", client) } }) } } func TestProductionLLMClientFactoriesRejectInvalidConstruction(t *testing.T) { t.Run("canceled context", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() client, manifests, err := productionLLMClientFactory(ctx, config.Default(), "test-profile") if !errors.Is(err, context.Canceled) || client != nil || len(manifests) != 0 { t.Fatalf("client=%T manifests=%#v error=%v, want canceled construction", client, manifests, err) } }) t.Run("nil assets", func(t *testing.T) { client, manifests, err := productionLLMClientFactoryWithAssets(nil)(context.Background(), config.Default(), "test-profile") if err == nil || !strings.Contains(err.Error(), "asset registry must not be nil") || client != nil || len(manifests) != 0 { t.Fatalf("client=%T manifests=%#v error=%v, want nil-assets failure", client, manifests, err) } }) t.Run("invalid scheduler concurrency", func(t *testing.T) { components := productionTestComponents(t) cfg := config.Default() cfg.Concurrency.TotalLLM = 0 client, manifests, err := productionLLMClientFactoryWithAssets(components.assets)(context.Background(), cfg, "test-profile") if err == nil || !strings.Contains(err.Error(), "create LLM scheduler") || !strings.Contains(err.Error(), "greater than zero") || client != nil || len(manifests) != 0 { t.Fatalf("client=%T manifests=%#v error=%v, want scheduler-construction failure", client, manifests, err) } }) } func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) { base := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml")) validPath := writeProductionContractConfig(t, base) options := productionCLIOptions(t) var stdout, stderr strings.Builder if code := RunWithOptions([]string{"config", "validate", "--config", validPath, "--pipeline", "dnd-session"}, &stdout, &stderr, options); code != 0 { t.Fatalf("valid production config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } tests := []struct { name string content string options Options fragments []string }{ { name: "unknown module", content: replaceRequiredOnce(t, base, " input: seriatim\n", " input: missing/input\n"), options: productionCLIOptions(t), fragments: []string{"pipeline \"dnd-session\"", "input", "missing/input"}, }, { name: "unknown validator", content: replaceRequiredOnce(t, base, " extract: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: missing/validator\n"), options: productionCLIOptions(t), fragments: []string{"validator", "missing/validator"}, }, { name: "invalid artifact variant", content: base, options: productionCLIOptionsWithoutSpellNormalizer(t), fragments: []string{"normalizer", "noop", string(dnd.SpellListKind), "variant"}, }, { name: "deterministic validator with profile", content: replaceRequiredOnce(t, base, " extract: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: generic/valid_json\n llm_profile: forbidden-profile\n"), options: productionCLIOptions(t), fragments: []string{"deterministic validator", "llm_profile"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { path := writeProductionContractConfig(t, tt.content) var stdout, stderr strings.Builder code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "dnd-session"}, &stdout, &stderr, tt.options) if code != 1 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } for _, fragment := range tt.fragments { if !strings.Contains(stderr.String(), fragment) { t.Fatalf("stderr=%q, want %q", stderr.String(), fragment) } } }) } } func TestProductionSceneRunRecordsChunkerWarningsAndProvenance(t *testing.T) { outputRoot := filepath.Join(t.TempDir(), "output") configPath := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes")) fake := &productionFakeLLMClient{} options := productionRunOptions(t, fake) var stdout, stderr strings.Builder code := RunWithOptions([]string{ "run", "dnd-session", "--config", configPath, "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--chunk_cache", "bypass", "--session-id", "offline-session", }, &stdout, &stderr, options) if code != 0 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(outputRoot, productionRunID, "manifest.json")) if manifest.Chunker != scenes.Key || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" || manifest.ChunkPlan.ProducerModule != scenes.Key { t.Fatalf("chunk manifest = %#v, want dnd scene producer", manifest.ChunkPlan) } if got := manifest.ModuleMetadata["chunker"]["prompt_id"]; got != scenes.PromptID { t.Fatalf("chunker prompt metadata = %#v, want %q", got, scenes.PromptID) } if got := manifest.ChunkPlan.ProducerMetadata["response_schema_id"]; got != scenes.ResponseSchemaID { t.Fatalf("chunk producer schema metadata = %#v, want %q", got, scenes.ResponseSchemaID) } warnings := readProductionJSON[struct { Warnings []contracts.Warning `json:"warnings"` }](t, filepath.Join(outputRoot, productionRunID, "warnings.json")) if len(warnings.Warnings) != 1 || warnings.Warnings[0].ReasonCode != "scene_boundary_caveat" { t.Fatalf("warnings = %#v, want one scene boundary warning", warnings.Warnings) } if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 { t.Fatalf("fake prompt requests = %#v, want one scene and one spell request", fake.requestPrompts()) } } type maintainedExample struct { name string path string } func maintainedExampleFiles(t *testing.T) []maintainedExample { t.Helper() return []maintainedExample{ {name: "minimal", path: repositoryPath("examples", "dnd-spells.config.yml")}, {name: "production", path: repositoryPath("examples", "dnd-spells-production.config.yml")}, } } func loadMaintainedExample(t *testing.T, path string) config.Config { t.Helper() fileConfig, err := config.LoadFileConfig(path) if err != nil { t.Fatalf("load maintained config %q: %v", path, err) } cfg := config.Default() if err := cfg.ApplyFileConfig(fileConfig); err != nil { t.Fatalf("apply maintained config %q: %v", path, err) } if err := cfg.Validate(); err != nil { t.Fatalf("validate maintained config %q: %v", path, err) } return cfg } func productionTestComponents(t *testing.T) productionComponents { t.Helper() components, err := newProductionComponents() if err != nil { t.Fatalf("new production components: %v", err) } return components } func productionCLIOptions(t *testing.T) Options { t.Helper() components := productionTestComponents(t) return productionOptionsFromComponents(components) } func productionOptionsFromComponents(components productionComponents) Options { return Options{ Catalog: catalogFromRegistries(components.registries), Registries: components.registries, LookupEnv: emptyLookup, } } func productionCLIOptionsWithoutSpellNormalizer(t *testing.T) Options { t.Helper() components := productionTestComponents(t) registries := components.registries registries.Normalizers = pipeline.NewNormalizerRegistry() if err := noop.RegisterTyped[dnd.SpellList](registries.Normalizers, contracts.ArtifactKind("test/other")); err != nil { t.Fatalf("register mismatched normalizer: %v", err) } return productionOptionsFromComponents(productionComponents{registries: registries, assets: components.assets}) } const productionRunID = "run-1700000000000000000-0123456789abcdef0123456789abcdef" func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options { t.Helper() options := productionCLIOptions(t) options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() } options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil } options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") } options.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { return fake, nil, nil } return options } func productionRunConfig(outputRoot, chunkModule string) string { return fmt.Sprintf(`version: 3 output: directory: %q cache: chunk_plans: mode: bypass checkpoints: {} debug: directory: %q pipelines: dnd-session: input: seriatim chunk: %s artifacts: spells: extract: dnd/spells `, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule) } func writeProductionContractConfig(t *testing.T, content string) string { t.Helper() path := filepath.Join(t.TempDir(), "config.yml") if err := os.WriteFile(path, []byte(content), 0o600); err != nil { t.Fatal(err) } return path } func productionAssetNames(t *testing.T, getFS func() (fs.FS, error)) []string { t.Helper() fileSystem, err := getFS() if err != nil { t.Fatalf("load production prompt assets: %v", err) } var names []string if err := fs.WalkDir(fileSystem, ".", func(path string, entry fs.DirEntry, err error) error { if err != nil { return err } if !entry.IsDir() { names = append(names, path) } return nil }); err != nil { t.Fatalf("walk production prompt assets: %v", err) } sort.Strings(names) return names } func assertProductionContains[T comparable](t *testing.T, name string, got, required []T) { t.Helper() available := make(map[T]struct{}, len(got)) for _, entry := range got { available[entry] = struct{}{} } var missing []T for _, entry := range required { if _, ok := available[entry]; !ok { missing = append(missing, entry) } } if len(missing) > 0 { t.Fatalf("%s missing required entries %#v; registered entries are %#v", name, missing, got) } } func readProductionJSON[T any](t *testing.T, path string) T { t.Helper() data, err := os.ReadFile(path) if err != nil { t.Fatalf("read %s: %v", path, err) } var value T if err := json.Unmarshal(data, &value); err != nil { t.Fatalf("decode %s: %v", path, err) } return value } type productionFakeLLMClient struct { mu sync.Mutex requests []contracts.StructuredCompletionRequest } func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { if err := ctx.Err(); err != nil { return contracts.StructuredCompletionResponse{}, err } var content []byte switch req.PromptID { case scenes.PromptID: content = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Opening scene","primary_mode":"Narrative","main_participants":["Aria"],"summary":"The session opens.","boundary_note":"The opening covers the available transcript.","boundary_confidence":"High"}],"boundary_caveats":["The opening boundary is inferred from the short transcript."]}`) case spells.PromptID: content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Heals an injured ally.","narrative_description":"Aria restores the fighter after the fight.","source_refs":[{"source_id":"session-alpha","start_unit_id":1,"end_unit_id":1}]}]}`) default: return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID) } if err := json.Unmarshal(content, out); err != nil { return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err) } client.mu.Lock() client.requests = append(client.requests, req) client.mu.Unlock() return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil } func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.StructuredCompletionRequest { client.mu.Lock() defer client.mu.Unlock() var requests []contracts.StructuredCompletionRequest for _, req := range client.requests { if req.PromptID == promptID { requests = append(requests, req) } } return requests } func (client *productionFakeLLMClient) requestPrompts() []string { client.mu.Lock() defer client.mu.Unlock() prompts := make([]string, 0, len(client.requests)) for _, req := range client.requests { prompts = append(prompts, req.PromptID) } return prompts } func repositoryPath(parts ...string) string { _, file, _, _ := runtime.Caller(0) return filepath.Join(append([]string{filepath.Dir(file), "..", ".."}, parts...)...) } func readRepositoryFile(t *testing.T, parts ...string) []byte { t.Helper() data, err := os.ReadFile(repositoryPath(parts...)) if err != nil { t.Fatal(err) } return data }