diff --git a/docs/adr/0002-linear-pipes-and-filters-pipeline.md b/docs/adr/0002-linear-pipes-and-filters-pipeline.md index 4b39563..9d6d730 100644 --- a/docs/adr/0002-linear-pipes-and-filters-pipeline.md +++ b/docs/adr/0002-linear-pipes-and-filters-pipeline.md @@ -1,6 +1,6 @@ # ADR-0002: Linear pipes-and-filters pipeline, not a general DAG -**Status:** Proposed +**Status:** Accepted **Date:** 2026-07-13 ## Context diff --git a/docs/adr/0003-typed-interfaces-with-two-zone-data-model.md b/docs/adr/0003-typed-interfaces-with-two-zone-data-model.md index 273b789..2d56478 100644 --- a/docs/adr/0003-typed-interfaces-with-two-zone-data-model.md +++ b/docs/adr/0003-typed-interfaces-with-two-zone-data-model.md @@ -1,6 +1,6 @@ # ADR-0003: Strongly typed stage interfaces with a two-zone data model -**Status:** Proposed +**Status:** Accepted **Date:** 2026-07-13 ## Context diff --git a/docs/adr/0004-package-modules-by-domain.md b/docs/adr/0004-package-modules-by-domain.md index b5f157a..3ddbd8e 100644 --- a/docs/adr/0004-package-modules-by-domain.md +++ b/docs/adr/0004-package-modules-by-domain.md @@ -1,6 +1,6 @@ # ADR-0004: Package modules by domain, not by stage -**Status:** Proposed +**Status:** Accepted **Date:** 2026-07-13 ## Context diff --git a/internal/cli/compatibility_test.go b/internal/cli/compatibility_test.go new file mode 100644 index 0000000..4e2ff10 --- /dev/null +++ b/internal/cli/compatibility_test.go @@ -0,0 +1,556 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "io/fs" + "path/filepath" + "reflect" + "sort" + "strings" + "sync" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" + "gitea.maximumdirect.net/eric/notarius/internal/core/config" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes" + "gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells" + spellshape "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/shape" + spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_refs" + spellrelatedness "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_relatedness" + validjson "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json" + validjsonschema "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json_schema" +) + +func TestProductionCompatibilitySnapshot(t *testing.T) { + registries, err := productionRegistries() + if err != nil { + t.Fatalf("productionRegistries() error = %v, want nil", err) + } + + keySnapshots := []struct { + name string + got []string + want []string + }{ + {name: "inputs", got: registries.Inputs.RegisteredKeys(), want: []string{"seriatim"}}, + {name: "chunkers", got: registries.Chunkers.RegisteredKeys(), want: []string{"dnd/scenes", "generic"}}, + {name: "extractors", got: registries.Extractors.RegisteredKeys(), want: []string{"dnd/spells"}}, + {name: "mergers", got: registries.Mergers.RegisteredKeys(), want: []string{"appendorder"}}, + {name: "normalizers", got: registries.Normalizers.RegisteredKeys(), want: []string{"noop"}}, + {name: "validators", got: registries.Validators.RegisteredKeys(), want: []string{ + "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness", + "generic/always_accept", "generic/always_reject", "generic/valid_json", "generic/valid_json_schema", + }}, + {name: "outputs", got: registries.Outputs.RegisteredKeys(), want: []string{"json"}}, + } + for _, snapshot := range keySnapshots { + t.Run(snapshot.name, func(t *testing.T) { + if !reflect.DeepEqual(snapshot.got, snapshot.want) { + t.Fatalf("registered keys = %#v, want compatibility snapshot %#v", snapshot.got, snapshot.want) + } + }) + } + + wantChain := []pipeline.ModuleBinding{ + pipeline.Binding(validjson.Key), + pipeline.Binding(validjsonschema.Key), + pipeline.Binding(spellshape.Key), + pipeline.Binding(spellsourcerefs.Key), + pipeline.Binding(spellrelatedness.Key), + } + if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) { + t.Fatalf("spell validator chain = %#v, want compatibility snapshot %#v", got, wantChain) + } + + assets, err := productionPromptAssets() + if err != nil { + t.Fatalf("productionPromptAssets() error = %v, want nil", err) + } + assertAssetNames(t, assets.PromptFS, []string{ + "dnd.scenes/dnd.scenes.yaml", + "dnd.scenes/instructions.md", + "dnd.scenes/sharedassets/common-dnd-references.md", + "dnd.scenes/sharedassets/common-dnd-system.md", + "dnd.scenes/sharedassets/common-dnd-transcript.md", + "dnd.scenes/task.md", + "dnd.spells/dnd.spells.yaml", + "dnd.spells/instructions.md", + "dnd.spells/sharedassets/common-dnd-references.md", + "dnd.spells/sharedassets/common-dnd-system.md", + "dnd.spells/sharedassets/common-dnd-transcript.md", + "dnd.spells/task.md", + }) + assertAssetNames(t, assets.SchemaFS, []string{ + "dnd_scenes.v1.json", + "dnd_spells.v1.json", + "dnd_spells_llm.v1.json", + }) + + identitySnapshot := map[string]map[string]any{ + "scenes": scenes.New().ManifestMetadata(), + "spells": spells.New().ManifestMetadata(), + } + for name, metadata := range identitySnapshot { + for _, key := range []string{"prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_id", "response_schema_name", "response_schema_version", "response_schema_sha256"} { + if value, ok := metadata[key].(string); !ok || value == "" { + t.Fatalf("%s metadata[%q] = %#v, want non-empty identity", name, key, metadata[key]) + } + } + } + if got := []any{ + identitySnapshot["scenes"]["prompt_id"], identitySnapshot["scenes"]["prompt_version"], + identitySnapshot["scenes"]["response_schema_key"], identitySnapshot["scenes"]["response_schema_id"], identitySnapshot["scenes"]["response_schema_name"], identitySnapshot["scenes"]["response_schema_version"], + }; !reflect.DeepEqual(got, []any{"dnd.scenes", "v1", "dnd_scenes", "notarius.dnd.scenes", "notarius_dnd_scenes_v1", "v1"}) { + t.Fatalf("scene identities = %#v, want compatibility snapshot", got) + } + if got := []any{ + identitySnapshot["spells"]["prompt_id"], identitySnapshot["spells"]["prompt_version"], + identitySnapshot["spells"]["response_schema_key"], identitySnapshot["spells"]["response_schema_id"], identitySnapshot["spells"]["response_schema_name"], identitySnapshot["spells"]["response_schema_version"], + }; !reflect.DeepEqual(got, []any{"dnd.spells", "v1", "dnd_spells", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"}) { + t.Fatalf("spell identities = %#v, want compatibility snapshot", got) + } + + fileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells.config.yml")) + if err != nil { + t.Fatalf("LoadFileConfig() error = %v, want nil", err) + } + cfg := config.Default() + if err := cfg.ApplyFileConfig(fileConfig); err != nil { + t.Fatalf("ApplyFileConfig() error = %v, want nil", err) + } + effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)}) + if err != nil { + t.Fatalf("Resolve() error = %v, want nil", err) + } + resolved := effective.ResolvedPipeline + if resolved.Input.Module != "seriatim" || resolved.Chunk.Module != "generic" || resolved.Output.Module != "json" || len(resolved.ArtifactLanes) != 1 { + t.Fatalf("resolved example = %#v, want maintained production topology", resolved) + } + lane := resolved.ArtifactLanes[0] + if lane.ID != "spells" || lane.Extract.Module != "dnd/spells" || lane.Merge.Module != "appendorder" || lane.Normalize.Module != "noop" { + t.Fatalf("resolved lane = %#v, want maintained spell lane", lane) + } + if got := resolvedValidatorKeys(resolved.ValidatorChains, pipeline.StageExtract, "spells", spells.Key); !reflect.DeepEqual(got, []string{ + "generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness", + }) { + t.Fatalf("resolved validator keys = %#v, want compatibility snapshot", got) + } + + productionFileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells-production.config.yml")) + if err != nil { + t.Fatalf("LoadFileConfig(production example) error = %v, want nil", err) + } + productionConfig := config.Default() + if err := productionConfig.ApplyFileConfig(productionFileConfig); err != nil { + t.Fatalf("ApplyFileConfig(production example) error = %v, want nil", err) + } + productionEffective, err := productionConfig.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)}) + if err != nil { + t.Fatalf("Resolve(production example) error = %v, want nil", err) + } + productionResolved := productionEffective.ResolvedPipeline + if productionConfig.Concurrency.TotalLLM != 1 || !reflect.DeepEqual(productionResolved.Chunk.Options, map[string]any{"max_units": 50}) { + t.Fatalf("production example concurrency/options = %d/%#v, want compatibility snapshot", productionConfig.Concurrency.TotalLLM, productionResolved.Chunk.Options) + } + bindings := productionResolved.ArtifactLanes[0].ExtractReferences.Bindings + if len(bindings) != 2 || bindings[0].SlotName != "glossary" || bindings[0].Source != "./dnd-spells-glossary.txt" || bindings[1].SlotName != "party" || bindings[1].Source != "./dnd-spells-roster.txt" { + t.Fatalf("production example reference bindings = %#v, want maintained glossary and party bindings", bindings) + } +} + +func TestMaintainedSeriatimToDNDCompatibilityBundle(t *testing.T) { + tests := []struct { + name string + client contracts.StructuredLLMClient + wantStatus string + wantLaneFile bool + wantRejectedCount int + }{ + {name: "approved", client: newFakeRunLLMClient(false), wantStatus: "approved", wantLaneFile: true}, + {name: "validator rejection is nonfatal", client: newFakeRunLLMClient(true), wantStatus: "rejected", wantRejectedCount: 1}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + outputDir := t.TempDir() + var stdout bytes.Buffer + var stderr bytes.Buffer + code := RunWithOptions([]string{ + "run", "dnd-session", + "--config", fixturePath(t, "examples/dnd-spells.config.yml"), + "--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"), + "--output-dir", outputDir, + "--diagnostics-dir", t.TempDir(), + }, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(test.client, nil)}) + if code != 0 { + t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) + } + + runDir := onlyChildDir(t, outputDir) + wantFiles := []string{"index.json", "manifest.json", "rejected.json", "warnings.json"} + if test.wantLaneFile { + wantFiles = append(wantFiles, "lanes/spells.json") + } + sort.Strings(wantFiles) + if got := relativeFileNames(t, runDir); !reflect.DeepEqual(got, wantFiles) { + t.Fatalf("durable files = %#v, want compatibility snapshot %#v", got, wantFiles) + } + + var manifest artifacts.RunManifest + readJSONFile(t, filepath.Join(runDir, "manifest.json"), &manifest) + if manifest.PipelineID != "dnd-session" || manifest.InputModule != "seriatim" || manifest.Chunker != "generic" || manifest.OutputEncoder != "json" { + t.Fatalf("manifest module provenance = %#v, want maintained production modules", manifest) + } + if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].ID != "spells" { + t.Fatalf("artifact lanes = %#v, want one spells lane", manifest.ArtifactLanes) + } + laneManifest := manifest.ArtifactLanes[0] + if laneManifest.Extractor != "dnd/spells" || laneManifest.Merger != "appendorder" || laneManifest.Normalizer != "noop" { + t.Fatalf("manifest lane module provenance = %#v, want maintained production modules", laneManifest) + } + if len(manifest.Extractors) != 0 || manifest.Merger != "" || manifest.Normalizer != "" { + t.Fatalf("legacy top-level lane summaries = %#v/%q/%q, want empty compatibility snapshot", manifest.Extractors, manifest.Merger, manifest.Normalizer) + } + if manifest.ValidationStatus != test.wantStatus || len(manifest.RejectedOutputs) != test.wantRejectedCount { + t.Fatalf("manifest outcome = status %q rejected %#v, want %q/%d", manifest.ValidationStatus, manifest.RejectedOutputs, test.wantStatus, test.wantRejectedCount) + } + if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:346f0b0b0cf2d8081ca222dbea47ff231ae9d81c8120ac51a4fc9fa6dfb6cc07"}) { + t.Fatalf("source digests = %#v, want maintained fixture provenance", manifest.SourceDigests) + } + if got := manifestValidatorKeys(manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)); !reflect.DeepEqual(got, []string{ + "generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness", + }) { + t.Fatalf("manifest validator chain = %#v, want compatibility snapshot", got) + } + + var index struct { + ManifestFile string `json:"manifest_file"` + OutputFiles []struct { + LaneID string `json:"lane_id"` + MediaType string `json:"media_type"` + File string `json:"file"` + ModuleKey string `json:"module_key"` + SchemaID string `json:"schema_id"` + SchemaName string `json:"schema_name"` + SchemaVersion string `json:"schema_version"` + } `json:"output_files"` + RejectedFile string `json:"rejected_file"` + WarningsFile string `json:"warnings_file"` + } + readJSONFile(t, filepath.Join(runDir, "index.json"), &index) + if index.ManifestFile != "manifest.json" || index.RejectedFile != "rejected.json" || index.WarningsFile != "warnings.json" { + t.Fatalf("output index fixed files = %#v, want compatibility snapshot", index) + } + if test.wantLaneFile { + if len(index.OutputFiles) != 1 { + t.Fatalf("output index entries = %#v, want one", index.OutputFiles) + } + wantOutput := struct { + LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string + }{"spells", "application/json", "lanes/spells.json", "noop", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"} + gotOutput := index.OutputFiles[0] + got := struct { + LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string + }{gotOutput.LaneID, gotOutput.MediaType, gotOutput.File, gotOutput.ModuleKey, gotOutput.SchemaID, gotOutput.SchemaName, gotOutput.SchemaVersion} + if got != wantOutput { + t.Fatalf("output index entries = %#v, want compatibility snapshot %#v", index.OutputFiles, wantOutput) + } + assertJSONEqual(t, readFile(t, filepath.Join(runDir, "lanes/spells.json")), []byte(`{ + "spell_casts": [{ + "caster": "Aria", + "spell": "Cure Wounds", + "effect": "Heals a wounded ally.", + "narrative_description": "Aria casts Cure Wounds.", + "source_refs": [{"source_id": "session-alpha", "start_unit_id": 1, "end_unit_id": 1}] + }] + }`)) + } else if len(index.OutputFiles) != 0 { + t.Fatalf("output index entries = %#v, want none for rejected lane", index.OutputFiles) + } + + var warnings struct { + Warnings []contracts.Warning `json:"warnings"` + } + readJSONFile(t, filepath.Join(runDir, "warnings.json"), &warnings) + if len(warnings.Warnings) != 0 { + t.Fatalf("warnings = %#v, want empty compatibility snapshot", warnings.Warnings) + } + var rejected struct { + Rejected []contracts.RejectedOutput `json:"rejected"` + } + readJSONFile(t, filepath.Join(runDir, "rejected.json"), &rejected) + if len(rejected.Rejected) != test.wantRejectedCount { + t.Fatalf("rejected outputs = %#v, want %d", rejected.Rejected, test.wantRejectedCount) + } + if test.wantRejectedCount == 1 { + got := rejected.Rejected[0] + if got.Stage != "extract" || got.LaneID != "spells" || got.ModuleKey != "dnd/spells" || got.ChunkID != "chunk-000001" || got.ChunkIndex != 0 || got.ValidatorName != "extract/dnd/spells/source_refs" || got.ReasonCode != "invalid_source_refs" || got.AttemptCount != 1 { + t.Fatalf("rejection = %#v, want maintained nonfatal validator outcome", got) + } + } + }) + } +} + +func TestProductionLLMCallersShareScheduledClient(t *testing.T) { + underlying := newBlockingProductionLLMClient() + scheduler, err := frameworkllm.NewScheduler(1) + if err != nil { + t.Fatalf("NewScheduler() error = %v, want nil", err) + } + client := frameworkllm.NewScheduledClient(underlying, scheduler) + doc := &source.SourceDocument{ + ID: "session-alpha", Kind: "transcript", Format: "application/json", Digest: "sha256:source", + Units: []source.SourceUnit{{ID: 1, Kind: "segment", Text: "Aria casts Cure Wounds."}, {ID: 2, Kind: "segment", Text: "The spell takes effect."}}, + } + chunk := contracts.SourceChunk{ + ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, StartUnitID: 1, EndUnitID: 2, + Content: []byte(`{"scene":"Aria casts Cure Wounds."}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), doc.Units...), + } + + var started sync.WaitGroup + started.Add(2) + errs := make(chan error, 2) + go func() { + started.Done() + _, err := scenes.New().Chunk(context.Background(), contracts.ChunkRequest{Source: doc, LLMClient: client}) + errs <- err + }() + go func() { + started.Done() + _, err := spells.New().Extract(context.Background(), contracts.ExtractionRequest{Source: doc, Chunk: &chunk, LLMClient: client}) + errs <- err + }() + started.Wait() + + for i := 0; i < 2; i++ { + <-underlying.entered + underlying.release <- struct{}{} + } + for i := 0; i < 2; i++ { + if err := <-errs; err != nil { + t.Fatalf("production LLM caller error = %v, want nil", err) + } + } + if underlying.maxActive != 1 { + t.Fatalf("maximum concurrent provider calls = %d, want total_llm limit 1", underlying.maxActive) + } + sort.Strings(underlying.stageNames) + if !reflect.DeepEqual(underlying.stageNames, []string{"dnd/scenes", "dnd/spells"}) { + t.Fatalf("scheduled stage names = %#v, want both production LLM callers", underlying.stageNames) + } +} + +func TestProductionBundlePreservesLaneAndChunkOrder(t *testing.T) { + configPath := writeTestConfig(t, `version: 2 +pipelines: + dnd-session: + input: seriatim + chunk: + module: generic + options: + max_units: 1 + artifacts: + zeta: + extract: dnd/spells + alpha: + extract: dnd/spells +`) + outputDir := t.TempDir() + var stdout bytes.Buffer + var stderr bytes.Buffer + code := RunWithOptions([]string{ + "run", "dnd-session", + "--config", configPath, + "--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"), + "--output-dir", outputDir, + "--diagnostics-dir", t.TempDir(), + }, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(orderingProductionLLMClient{}, nil)}) + if code != 0 { + t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) + } + + runDir := onlyChildDir(t, outputDir) + var index struct { + OutputFiles []struct { + LaneID string `json:"lane_id"` + } `json:"output_files"` + } + readJSONFile(t, filepath.Join(runDir, "index.json"), &index) + if len(index.OutputFiles) != 2 { + t.Fatalf("output index entries = %#v, want two lanes", index.OutputFiles) + } + if got := []string{index.OutputFiles[0].LaneID, index.OutputFiles[1].LaneID}; !reflect.DeepEqual(got, []string{"alpha", "zeta"}) { + t.Fatalf("output lane order = %#v, want resolved lane order", got) + } + for _, laneID := range []string{"alpha", "zeta"} { + var payload struct { + SpellCasts []struct { + Spell string `json:"spell"` + SourceRefs []source.SourceRef `json:"source_refs"` + } `json:"spell_casts"` + } + readJSONFile(t, filepath.Join(runDir, "lanes", laneID+".json"), &payload) + if len(payload.SpellCasts) != 2 { + t.Fatalf("lane %q spell casts = %#v, want one per source chunk", laneID, payload.SpellCasts) + } + got := []any{ + payload.SpellCasts[0].Spell, payload.SpellCasts[0].SourceRefs[0].StartUnitID, + payload.SpellCasts[1].Spell, payload.SpellCasts[1].SourceRefs[0].StartUnitID, + } + if !reflect.DeepEqual(got, []any{"Cure Wounds", 1, "Shield", 2}) { + t.Fatalf("lane %q chunk handoff order = %#v, want source chunk order", laneID, got) + } + } +} + +type blockingProductionLLMClient struct { + mu sync.Mutex + active int + maxActive int + stageNames []string + entered chan struct{} + release chan struct{} +} + +type orderingProductionLLMClient struct{} + +func (orderingProductionLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + material := req.Inputs["transcript"] + unitID := 1 + spellName := "Cure Wounds" + if strings.Contains(string(material.Content), "Shield") { + unitID = 2 + spellName = "Shield" + } + payload, err := json.Marshal(map[string]any{ + "spell_casts": []map[string]any{{ + "caster": "Aria", + "spell": spellName, + "effect": "Fixture effect.", + "narrative_description": "Fixture spell cast.", + "source_refs": []map[string]any{{ + "start_unit_id": unitID, + "end_unit_id": unitID, + }}, + }}, + }) + if err != nil { + return contracts.StructuredCompletionResponse{}, err + } + if err := json.Unmarshal(payload, out); err != nil { + return contracts.StructuredCompletionResponse{}, err + } + return contracts.StructuredCompletionResponse{Content: payload}, nil +} + +func newBlockingProductionLLMClient() *blockingProductionLLMClient { + return &blockingProductionLLMClient{entered: make(chan struct{}, 2), release: make(chan struct{}, 2)} +} + +func (client *blockingProductionLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + client.mu.Lock() + client.active++ + if client.active > client.maxActive { + client.maxActive = client.active + } + client.stageNames = append(client.stageNames, req.StageName) + client.mu.Unlock() + client.entered <- struct{}{} + + select { + case <-ctx.Done(): + return contracts.StructuredCompletionResponse{}, ctx.Err() + case <-client.release: + } + + client.mu.Lock() + client.active-- + client.mu.Unlock() + + var payload []byte + switch req.StageName { + case scenes.Key: + payload = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Spell","primary_mode":"Narrative","main_participants":["Aria"],"summary":"Aria casts a spell.","boundary_note":"Complete source.","boundary_confidence":"High"}],"boundary_caveats":[]}`) + case spells.Key: + payload = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Healing","narrative_description":"Aria casts Cure Wounds.","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`) + } + if err := json.Unmarshal(payload, out); err != nil { + return contracts.StructuredCompletionResponse{}, err + } + return contracts.StructuredCompletionResponse{Content: payload}, nil +} + +func assertAssetNames(t *testing.T, getFS func() (fs.FS, error), want []string) { + t.Helper() + fSys, err := getFS() + if err != nil { + t.Fatalf("asset filesystem error = %v, want nil", err) + } + var got []string + if err := fs.WalkDir(fSys, ".", func(path string, entry fs.DirEntry, err error) error { + if err == nil && !entry.IsDir() { + got = append(got, path) + } + return err + }); err != nil { + t.Fatalf("walk assets: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("asset names = %#v, want compatibility snapshot %#v", got, want) + } +} + +func resolvedValidatorKeys(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID, moduleKey string) []string { + for _, chain := range chains { + if chain.Stage == stage && chain.LaneID == laneID && chain.ModuleKey == moduleKey { + keys := make([]string, 0, len(chain.Validators)) + for _, validator := range chain.Validators { + keys = append(keys, validator.Binding.Module) + } + return keys + } + } + return nil +} + +func relativeFileNames(t *testing.T, root string) []string { + t.Helper() + var names []string + if err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + names = append(names, filepath.ToSlash(rel)) + return nil + }); err != nil { + t.Fatalf("walk durable output: %v", err) + } + sort.Strings(names) + return names +} + +func assertJSONEqual(t *testing.T, got, want []byte) { + t.Helper() + var gotValue any + var wantValue any + if err := json.Unmarshal(got, &gotValue); err != nil { + t.Fatalf("unmarshal actual JSON: %v", err) + } + if err := json.Unmarshal(want, &wantValue); err != nil { + t.Fatalf("unmarshal expected JSON: %v", err) + } + if !reflect.DeepEqual(gotValue, wantValue) { + t.Fatalf("JSON = %#v, want compatibility snapshot %#v", gotValue, wantValue) + } +} diff --git a/internal/framework/pipeline/runner_test.go b/internal/framework/pipeline/runner_test.go index 83e6c61..c385774 100644 --- a/internal/framework/pipeline/runner_test.go +++ b/internal/framework/pipeline/runner_test.go @@ -1823,6 +1823,9 @@ func TestRunReturnsPartialOutputWhenLaterLaneFails(t *testing.T) { if len(output.NormalizeOutputs) != 1 { t.Fatalf("len(NormalizeOutputs) = %d, want first lane output", len(output.NormalizeOutputs)) } + if len(modules.output.requests) != 0 { + t.Fatalf("output requests = %d, want framework error to abort before output", len(modules.output.requests)) + } } func resolvedPipeline() ResolvedPipeline {