210 lines
9.0 KiB
Go
210 lines
9.0 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
|
)
|
|
|
|
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
for _, example := range maintainedExampleFiles(t) {
|
|
t.Run(example.name, func(t *testing.T) {
|
|
cfg := loadMaintainedExample(t, example.path)
|
|
for _, pipelineID := range example.pipelineIDs {
|
|
effective, err := cfg.Resolve(resolveInputForMaintainedExample(components, pipelineID))
|
|
if err != nil {
|
|
t.Fatalf("resolve maintained example %q: %v", pipelineID, err)
|
|
}
|
|
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
|
|
ConfigPath: example.path,
|
|
WorkingDir: filepath.Dir(example.path),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err)
|
|
}
|
|
if example.name == "complete" {
|
|
if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:npcs,scene-descriptions|extract-events:combat-turns,npc-interactions,spells" {
|
|
t.Fatalf("complete example steps and lanes = %v, want every D&D extractor in the documented two-step composition", got)
|
|
}
|
|
spellLane := referenceContractLane(t, materialized, "spells")
|
|
if len(spellLane.ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 ||
|
|
len(spellLane.NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 {
|
|
t.Fatalf("complete example spell catalog reference was not materialized: %#v", spellLane)
|
|
}
|
|
}
|
|
}
|
|
var stdout, stderr strings.Builder
|
|
code := RunWithOptions([]string{"pipelines", "list", "--config", example.path}, &stdout, &stderr, productionOptionsFromComponents(components))
|
|
if code != 0 || stdout.String() != strings.Join(example.pipelineIDs, "\n")+"\n" || stderr.Len() != 0 {
|
|
t.Fatalf("pipelines list: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMaintainedConfigurationExampleSet(t *testing.T) {
|
|
entries, err := os.ReadDir(repositoryPath("examples"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var names []string
|
|
for _, entry := range entries {
|
|
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".config.yml") {
|
|
names = append(names, entry.Name())
|
|
}
|
|
}
|
|
sort.Strings(names)
|
|
if got := strings.Join(names, ","); got != "dnd-complete.config.yml,dnd-minimal.config.yml" {
|
|
t.Fatalf("maintained configuration examples = %q, want only the minimal and complete D&D examples", got)
|
|
}
|
|
}
|
|
|
|
func exampleStepLaneIDs(resolved pipeline.ResolvedPipeline) []string {
|
|
result := make([]string, 0, len(resolved.Steps))
|
|
for _, step := range resolved.Steps {
|
|
laneIDs := make([]string, 0, len(step.ArtifactLanes))
|
|
for _, lane := range step.ArtifactLanes {
|
|
laneIDs = append(laneIDs, lane.ID)
|
|
}
|
|
sort.Strings(laneIDs)
|
|
result = append(result, step.ID+":"+strings.Join(laneIDs, ","))
|
|
}
|
|
return result
|
|
}
|
|
|
|
func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
|
fake := &productionFakeLLMClient{}
|
|
options := productionRunOptions(t, fake)
|
|
var stdout, stderr strings.Builder
|
|
code := RunWithOptions([]string{
|
|
"run", "dnd-session",
|
|
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
|
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
|
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
|
}, &stdout, &stderr, options)
|
|
if code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
if !strings.Contains(stdout.String(), `pipeline "dnd-session"`) || !strings.Contains(stdout.String(), "outputs=1 rejected=0") {
|
|
t.Fatalf("stdout=%q, want completed pipeline and counts", stdout.String())
|
|
}
|
|
|
|
runRoot := filepath.Join(outputRoot, productionRunID)
|
|
index := readProductionJSON[exampleOutputIndex](t, filepath.Join(runRoot, "index.json"))
|
|
if index.ManifestFile != "manifest.json" || index.RejectedFile != "rejected.json" || index.WarningsFile != "warnings.json" || len(index.OutputFiles) != 1 {
|
|
t.Fatalf("index = %#v, want one spells output and fixed companion files", index)
|
|
}
|
|
entry := index.OutputFiles[0]
|
|
if entry.LaneID != "spells" || entry.File != "lanes/spells.json" || entry.MediaType != "application/json" || entry.SchemaID != "notarius.dnd.spells" || entry.SchemaVersion != "v1" {
|
|
t.Fatalf("index output entry = %#v, want spells JSON contract", entry)
|
|
}
|
|
|
|
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
|
|
if manifest.PipelineID != "dnd-session" || manifest.InputModule != "seriatim" || manifest.Chunker != "generic" || manifest.OutputEncoder != "json" || manifest.ValidationStatus != "approved" || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" {
|
|
t.Fatalf("manifest = %#v, want approved minimal run", manifest)
|
|
}
|
|
if len(manifest.ArtifactLanes) != 1 {
|
|
t.Fatalf("manifest lanes = %#v, want exactly spells", manifest.ArtifactLanes)
|
|
}
|
|
lane := manifest.ArtifactLanes[0]
|
|
if lane.ID != "spells" || lane.Extractor != "dnd/spells" || lane.Merger != "appendorder" || lane.Normalizer != spellnormalize.Key {
|
|
t.Fatalf("manifest lane = %#v, want production spells composition", lane)
|
|
}
|
|
if len(manifest.References) != 0 {
|
|
t.Fatalf("base-only manifest references = %#v, want no overlay provenance", manifest.References)
|
|
}
|
|
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
|
if !ok || len(stringValues(extractorMetadata["catalog_overlay_ids"])) != 0 {
|
|
t.Fatalf("base-only extractor metadata = %#v, want no overlay IDs", lane.Metadata)
|
|
}
|
|
|
|
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, entry.File))
|
|
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Cure Wounds" || artifact.SpellCasts[0].SourceRefs[0].SourceID != "session-alpha" {
|
|
t.Fatalf("artifact = %#v, want one source-linked Cure Wounds cast", artifact)
|
|
}
|
|
rejected := readProductionJSON[struct {
|
|
Rejected []json.RawMessage `json:"rejected"`
|
|
}](t, filepath.Join(runRoot, "rejected.json"))
|
|
if len(rejected.Rejected) != 0 {
|
|
t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected)
|
|
}
|
|
warnings := readProductionJSON[struct {
|
|
Warnings []json.RawMessage `json:"warnings"`
|
|
}](t, filepath.Join(runRoot, "warnings.json"))
|
|
if len(warnings.Warnings) != 0 {
|
|
t.Fatalf("warnings = %#v, want empty warning list", warnings.Warnings)
|
|
}
|
|
}
|
|
|
|
func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing.T) {
|
|
malformed := filepath.Join(t.TempDir(), "malformed.json")
|
|
if err := os.WriteFile(malformed, []byte("{not valid json"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, debug := range []bool{false, true} {
|
|
name := "without debug"
|
|
if debug {
|
|
name = "with debug"
|
|
}
|
|
t.Run(name, func(t *testing.T) {
|
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
|
debugRoot := filepath.Join(t.TempDir(), "debug")
|
|
options := productionRunOptions(t, &productionFakeLLMClient{})
|
|
args := []string{
|
|
"run", "dnd-session",
|
|
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
|
"--input", malformed, "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
|
}
|
|
if debug {
|
|
args = append(args, "--debug", "--debug-dir", debugRoot)
|
|
}
|
|
var stdout, stderr strings.Builder
|
|
code := RunWithOptions(args, &stdout, &stderr, options)
|
|
if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "parse input") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
assertAbsent(t, outputRoot)
|
|
if !debug {
|
|
assertAbsent(t, debugRoot)
|
|
return
|
|
}
|
|
bundle := onlyChildDir(t, debugRoot)
|
|
report := readProductionJSON[debugbundle.RunReport](t, filepath.Join(bundle, "summary", "run-report.json"))
|
|
if report.Succeeded || report.PipelineID != "dnd-session" {
|
|
t.Fatalf("failure report = %#v, want failed dnd-session report", report)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type exampleOutputIndex struct {
|
|
ManifestFile string `json:"manifest_file"`
|
|
OutputFiles []exampleOutputIndexEntry `json:"output_files"`
|
|
RejectedFile string `json:"rejected_file"`
|
|
WarningsFile string `json:"warnings_file"`
|
|
}
|
|
|
|
type exampleOutputIndexEntry struct {
|
|
LaneID string `json:"lane_id"`
|
|
MediaType string `json:"media_type"`
|
|
File string `json:"file"`
|
|
SchemaID string `json:"schema_id"`
|
|
SchemaVersion string `json:"schema_version"`
|
|
}
|
|
|
|
func resolveInputForMaintainedExample(components productionComponents, pipelineID string) config.ResolveInput {
|
|
return config.ResolveInput{PipelineID: pipelineID, Catalog: catalogFromRegistries(components.registries)}
|
|
}
|