package cli import ( "encoding/json" "os" "path/filepath" "sort" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells" ) func TestMaintainedExamplesLoadResolveAndList(t *testing.T) { components := productionTestComponents(t) for _, example := range maintainedExampleFiles(t) { t.Run(example.name, func(t *testing.T) { cfg := loadMaintainedExample(t, example.path) for _, pipelineID := range example.pipelineIDs { effective, err := cfg.Resolve(resolveInputForMaintainedExample(components, pipelineID)) if err != nil { t.Fatalf("resolve maintained example %q: %v", pipelineID, err) } materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ ConfigPath: example.path, WorkingDir: filepath.Dir(example.path), }) if err != nil { t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err) } if example.name == "complete" { if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:npcs,scene-descriptions|extract-events:combat-turns,npc-interactions,spells" { t.Fatalf("complete example steps and lanes = %v, want every D&D extractor in the documented two-step composition", got) } spellLane := referenceContractLane(t, materialized, "spells") if len(spellLane.ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 || len(spellLane.NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 { t.Fatalf("complete example spell catalog reference was not materialized: %#v", spellLane) } } } var stdout, stderr strings.Builder code := RunWithOptions([]string{"pipelines", "list", "--config", example.path}, &stdout, &stderr, productionOptionsFromComponents(components)) if code != 0 || stdout.String() != strings.Join(example.pipelineIDs, "\n")+"\n" || stderr.Len() != 0 { t.Fatalf("pipelines list: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) } } func TestMaintainedConfigurationExampleSet(t *testing.T) { entries, err := os.ReadDir(repositoryPath("examples")) if err != nil { t.Fatal(err) } var names []string for _, entry := range entries { if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".config.yml") { names = append(names, entry.Name()) } } sort.Strings(names) if got := strings.Join(names, ","); got != "dnd-complete.config.yml,dnd-minimal.config.yml" { t.Fatalf("maintained configuration examples = %q, want only the minimal and complete D&D examples", got) } } func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string { result := make([]string, 0, len(resolved.Steps)) for _, step := range resolved.Steps { laneIDs := make([]string, 0, len(step.ArtifactLanes)) for _, lane := range step.ArtifactLanes { laneIDs = append(laneIDs, lane.ID) } sort.Strings(laneIDs) result = append(result, step.ID+":"+strings.Join(laneIDs, ",")) } return result } func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) { outputRoot := filepath.Join(t.TempDir(), "output") fake := &productionFakeLLMClient{} options := productionRunOptions(t, fake) var stdout, stderr strings.Builder code := RunWithOptions([]string{ "run", "dnd-session", "--config", repositoryPath("examples", "dnd-minimal.config.yml"), "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, }, &stdout, &stderr, options) if code != 0 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } if !strings.Contains(stdout.String(), `pipeline "dnd-session"`) || !strings.Contains(stdout.String(), "outputs=1 rejected=0") { t.Fatalf("stdout=%q, want completed pipeline and counts", stdout.String()) } runRoot := filepath.Join(outputRoot, productionRunID) index := readProductionJSON[exampleOutputIndex](t, filepath.Join(runRoot, "index.json")) if index.ManifestFile != "manifest.json" || index.RejectedFile != "rejected.json" || index.WarningsFile != "warnings.json" || len(index.OutputFiles) != 1 { t.Fatalf("index = %#v, want one spells output and fixed companion files", index) } entry := index.OutputFiles[0] if entry.LaneID != "spells" || entry.File != "lanes/spells.json" || entry.MediaType != "application/json" || entry.SchemaID != "notarius.dnd.spells" || entry.SchemaVersion != "v1" { t.Fatalf("index output entry = %#v, want spells JSON contract", entry) } manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json")) if manifest.PipelineID != "dnd-session" || manifest.InputModule != "seriatim" || manifest.Chunker != "generic" || manifest.OutputEncoder != "json" || manifest.ValidationStatus != "approved" || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" { t.Fatalf("manifest = %#v, want approved minimal run", manifest) } if len(manifest.ArtifactLanes) != 1 { t.Fatalf("manifest lanes = %#v, want exactly spells", manifest.ArtifactLanes) } lane := manifest.ArtifactLanes[0] if lane.ID != "spells" || lane.Extractor != "dnd/spells" || lane.Merger != "appendorder" || lane.Normalizer != spellnormalize.Key { t.Fatalf("manifest lane = %#v, want production spells composition", lane) } if len(manifest.References) != 0 { t.Fatalf("base-only manifest references = %#v, want no overlay provenance", manifest.References) } extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any) if !ok || len(stringValues(extractorMetadata["catalog_overlay_ids"])) != 0 { t.Fatalf("base-only extractor metadata = %#v, want no overlay IDs", lane.Metadata) } artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, entry.File)) if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Cure Wounds" || artifact.SpellCasts[0].SourceRefs[0].SourceID != "session-alpha" { t.Fatalf("artifact = %#v, want one source-linked Cure Wounds cast", artifact) } rejected := readProductionJSON[struct { Rejected []json.RawMessage `json:"rejected"` }](t, filepath.Join(runRoot, "rejected.json")) if len(rejected.Rejected) != 0 { t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected) } warnings := readProductionJSON[struct { Warnings []json.RawMessage `json:"warnings"` }](t, filepath.Join(runRoot, "warnings.json")) if len(warnings.Warnings) != 0 { t.Fatalf("warnings = %#v, want empty warning list", warnings.Warnings) } } func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing.T) { malformed := filepath.Join(t.TempDir(), "malformed.json") if err := os.WriteFile(malformed, []byte("{not valid json"), 0o600); err != nil { t.Fatal(err) } for _, debug := range []bool{false, true} { name := "without debug" if debug { name = "with debug" } t.Run(name, func(t *testing.T) { outputRoot := filepath.Join(t.TempDir(), "output") debugRoot := filepath.Join(t.TempDir(), "debug") options := productionRunOptions(t, &productionFakeLLMClient{}) args := []string{ "run", "dnd-session", "--config", repositoryPath("examples", "dnd-minimal.config.yml"), "--input", malformed, "--chunk_cache", "bypass", "--output-dir", outputRoot, } if debug { args = append(args, "--debug", "--debug-dir", debugRoot) } var stdout, stderr strings.Builder code := RunWithOptions(args, &stdout, &stderr, options) if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "parse input") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } assertAbsent(t, outputRoot) if !debug { assertAbsent(t, debugRoot) return } bundle := onlyChildDir(t, debugRoot) report := readProductionJSON[debugbundle.RunReport](t, filepath.Join(bundle, "summary", "run-report.json")) if report.Succeeded || report.PipelineID != "dnd-session" { t.Fatalf("failure report = %#v, want failed dnd-session report", report) } }) } } type exampleOutputIndex struct { ManifestFile string `json:"manifest_file"` OutputFiles []exampleOutputIndexEntry `json:"output_files"` RejectedFile string `json:"rejected_file"` WarningsFile string `json:"warnings_file"` } type exampleOutputIndexEntry struct { LaneID string `json:"lane_id"` MediaType string `json:"media_type"` File string `json:"file"` SchemaID string `json:"schema_id"` SchemaVersion string `json:"schema_version"` } func resolveInputForMaintainedExample(components productionComponents, pipelineID string) config.ResolveInput { return config.ResolveInput{PipelineID: pipelineID, Catalog: catalogFromRegistries(components.registries)} }