package integration_test import ( "context" "encoding/json" "fmt" "os" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry" npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register" npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/shape" npcregistrysourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_refs" genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register" "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" seriatimregister "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/register" ) func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T) { raw := readNPCFixture(t) doc, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw}) if err != nil { t.Fatalf("Parse() error = %v, want nil", err) } registries := productionNPCRegistries(t) configValue := loadNPCPipelineConfig(t) effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-registry-fixture", Catalog: moduleCatalog(registries)}) if err != nil { t.Fatalf("Resolve() error = %v, want nil", err) } client := &fakeNPCProductionLLMClient{response: npcProductionResponse{ NPCs: []npcProductionRecord{ { Name: "Mira Thorn", SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 1}}, }, { Name: " mira thorn ", SourceRefs: []npcProductionSourceRef{{StartUnitID: 2, EndUnitID: 2}}, }, { Name: "Hooded Guard", SourceRefs: []npcProductionSourceRef{{StartUnitID: 3, EndUnitID: 3}}, }, }}, } output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: raw}) if err != nil { t.Fatalf("Run() error = %v, want nil", err) } if len(output.NormalizeOutputs) != 1 { t.Fatalf("normalize outputs = %d, want one NPC output; rejected=%#v", len(output.NormalizeOutputs), output.Rejected) } serialized := output.NormalizeOutputs[0] if serialized.LaneID != "npc_registry" || serialized.NormalizerKey != npcregistry.Key || serialized.Artifact.Schema.ID != npccodec.SchemaID || serialized.Artifact.Schema.Version != npccodec.SchemaVersion { t.Fatalf("serialized output = %#v, want durable NPC lane schema", serialized) } value, err := npccodec.New().Decode(serialized.Artifact.Content) if err != nil { t.Fatalf("Decode(output) error = %v, want durable NPC payload", err) } if len(value.NPCs) != 2 { t.Fatalf("NPC output = %#v, want repeated name consolidated and group/PC omitted", value.NPCs) } first, second := value.NPCs[0], value.NPCs[1] if first.Name != "Mira Thorn" || len(first.SourceRefs) != 2 { t.Fatalf("first NPC = %#v, want consolidated Mira identity", first) } if first.ID != identity.DeriveID(first.Name) || second.Name != "Hooded Guard" || second.ID != identity.DeriveID(second.Name) { t.Fatalf("NPC IDs = %q/%q, want derived IDs", first.ID, second.ID) } for _, npc := range value.NPCs { for _, ref := range npc.SourceRefs { if ref.SourceID != doc.ID { t.Fatalf("NPC source ref = %#v, want source document %q", ref, doc.ID) } } } if !hasNPCWarning(output.Warnings, "duplicate_npc_collapsed") { t.Fatalf("warnings = %#v, want name-only consolidation warning", output.Warnings) } if output.Manifest.ValidationStatus != "approved" || len(output.Manifest.ArtifactLanes) != 1 { t.Fatalf("manifest = %#v, want approved NPC lane", output.Manifest) } lane := output.Manifest.ArtifactLanes[0] if lane.ID != "npc_registry" || lane.Extractor != npcregistry.Key || lane.Merger != pipeline.DefaultMergeModule || lane.Normalizer != npcregistry.Key { t.Fatalf("manifest lane = %#v, want NPC production composition", lane) } normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any) if !ok || normalizerMetadata["identity_policy"] != identity.Policy || normalizerMetadata["normalization_policy"] != npcnormalize.NormalizationPolicy || normalizerMetadata["prompt_id"] != npcnormalize.PromptID || normalizerMetadata["response_schema_id"] != semanticreconcile.ResponseSchemaID { t.Fatalf("normalizer metadata = %#v, want identity and normalization policies", lane.Metadata) } var npcOutputFile *contracts.OutputFile for index := range output.OutputFiles { if output.OutputFiles[index].Name == "lanes/npc_registry.json" { npcOutputFile = &output.OutputFiles[index] break } } if npcOutputFile == nil || npcOutputFile.ContentType != npccodec.MediaType { t.Fatalf("output files = %#v, want JSON NPC lane file", output.OutputFiles) } if len(client.requests) != 2 || client.requests[0].PromptID != npcregistry.PromptID || client.requests[1].PromptID != npcnormalize.PromptID { t.Fatalf("LLM requests = %#v, want extraction and normalization prompt requests", client.requests) } } func TestProductionNPCPipelineRoutesSemanticCandidatesToDeterministicValidators(t *testing.T) { registries := productionNPCRegistries(t) effective, err := loadNPCPipelineConfig(t).Resolve(config.ResolveInput{ PipelineID: "dnd-npc-registry-fixture", Catalog: moduleCatalog(registries), }) if err != nil { t.Fatalf("Resolve() error = %v, want nil", err) } for _, test := range []struct { name string response []byte reasonCode string validatorName string }{ { name: "blank string", response: []byte(`{"npcs":[{"name":"","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), reasonCode: npcshape.ReasonCode, validatorName: npcshape.Key, }, { name: "empty evidence", response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[]}]}`), reasonCode: npcshape.ReasonCode, validatorName: npcshape.Key, }, { name: "nonpositive unit candidate", response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":0,"end_unit_id":1}]}]}`), reasonCode: npcregistrysourcerefs.ReasonCode, validatorName: npcregistrysourcerefs.Key, }, { name: "unknown unit candidate", response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":99,"end_unit_id":99}]}]}`), reasonCode: npcregistrysourcerefs.ReasonCode, validatorName: npcregistrysourcerefs.Key, }, } { t.Run(test.name, func(t *testing.T) { client := &fakeNPCProductionLLMClient{rawResponses: [][]byte{test.response, test.response, test.response}} output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: readNPCFixture(t)}) if err != nil { t.Fatalf("Run() error = %v, want non-fatal rejected output", err) } if len(client.requests) != 3 || len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 { t.Fatalf("LLM requests = %d rejected = %#v normalized = %#v, want exhausted rejection", len(client.requests), output.Rejected, output.NormalizeOutputs) } rejection := output.Rejected[0] if rejection.ReasonCode != test.reasonCode || rejection.ValidatorName != test.validatorName || rejection.AttemptCount != 3 { t.Fatalf("rejection = %#v, want exhausted %s rejection", rejection, test.validatorName) } }) } } func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) { registries := productionNPCRegistries(t) baseResponse := npcProductionResponse{NPCs: []npcProductionRecord{ {Name: "Mira", SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 1}}}, {Name: "Mira Thorn", SourceRefs: []npcProductionSourceRef{{StartUnitID: 2, EndUnitID: 2}}}, {Name: "Hooded Guard", SourceRefs: []npcProductionSourceRef{{StartUnitID: 3, EndUnitID: 3}}}, }} partial := []byte(`{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2},{"candidate_ids":[3,99],"canonical_candidate_id":3}]}`) safe := []byte(`{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`) for _, test := range []struct { name string retries int responses [][]byte wantCalls int wantExhaustion bool }{ {name: "default retry budget", responses: [][]byte{partial}, wantCalls: 1, wantExhaustion: true}, {name: "later complete proposal", retries: 1, responses: [][]byte{partial, safe}, wantCalls: 2}, } { t.Run(test.name, func(t *testing.T) { cfg := loadNPCPipelineConfig(t) profile := cfg.Pipelines["dnd-npc-registry-fixture"] lane := profile.Artifacts["npc_registry"] lane.Normalize.Retries = test.retries profile.Artifacts["npc_registry"] = lane cfg.Pipelines["dnd-npc-registry-fixture"] = profile effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-npc-registry-fixture", Catalog: moduleCatalog(registries)}) if err != nil { t.Fatal(err) } client := &fakeNPCProductionLLMClient{response: baseResponse, normalizeResponses: test.responses} output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: readNPCFixture(t)}) if err != nil { t.Fatalf("Run() error = %v", err) } if client.requestCount(npcnormalize.PromptID) != test.wantCalls || len(output.NormalizeOutputs) != 1 { t.Fatalf("normalization calls = %d output = %#v, want %d accepted result", client.requestCount(npcnormalize.PromptID), output.NormalizeOutputs, test.wantCalls) } if got := hasWarningReason(output.Warnings, npcnormalize.ReasonCodeNPCSemanticReconciliationExhausted); got != test.wantExhaustion { t.Fatalf("warnings = %#v, exhaustion = %t, want %t", output.Warnings, got, test.wantExhaustion) } }) } } type npcProductionResponse struct { NPCs []npcProductionRecord `json:"npcs"` } type npcProductionRecord struct { Name string `json:"name"` SourceRefs []npcProductionSourceRef `json:"source_refs"` } type npcProductionSourceRef struct { StartUnitID int `json:"start_unit_id"` EndUnitID int `json:"end_unit_id"` } type fakeNPCProductionLLMClient struct { response npcProductionResponse rawResponses [][]byte normalizeResponses [][]byte requests []contracts.StructuredCompletionRequest } func (client *fakeNPCProductionLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { client.requests = append(client.requests, req) var content []byte switch req.PromptID { case npcregistry.PromptID: if client.rawResponses != nil { index := client.requestCount(npcregistry.PromptID) - 1 if index >= len(client.rawResponses) { return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake NPC response %d", index) } content = append([]byte(nil), client.rawResponses[index]...) } else { var err error content, err = json.Marshal(client.response) if err != nil { return contracts.StructuredCompletionResponse{}, err } } case npcnormalize.PromptID: content = []byte(`{"duplicate_groups":[]}`) if client.normalizeResponses != nil { index := client.requestCount(npcnormalize.PromptID) - 1 if index >= len(client.normalizeResponses) { return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake NPC normalization response %d", index) } content = append([]byte(nil), client.normalizeResponses[index]...) } default: return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected fake NPC prompt %q", req.PromptID) } if err := json.Unmarshal(content, out); err != nil { return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate NPC structured target: %w", err) } return contracts.StructuredCompletionResponse{Content: content}, nil } func (client *fakeNPCProductionLLMClient) requestCount(promptID string) int { count := 0 for _, request := range client.requests { if request.PromptID == promptID { count++ } } return count } func productionNPCRegistries(t *testing.T) pipeline.Registries { t.Helper() registries := pipeline.Registries{ Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), ArtifactEvidence: pipeline.NewArtifactEvidenceRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry(), } assets := llm.NewAssetRegistry() for _, registration := range []struct { name string fn func(pipeline.Registries, *llm.AssetRegistry) error }{ {name: "generic", fn: genericregister.Register}, {name: "seriatim", fn: seriatimregister.Register}, {name: "dnd", fn: dndregister.Register}, } { if err := registration.fn(registries, assets); err != nil { t.Fatalf("register %s modules: %v", registration.name, err) } } return registries } func moduleCatalog(registries pipeline.Registries) pipeline.ModuleCatalog { return pipeline.ModuleCatalog{ Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs, ArtifactEvidence: registries.ArtifactEvidence, Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers, Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs, } } func loadNPCPipelineConfig(t *testing.T) config.Config { t.Helper() data, err := os.ReadFile("testdata/dnd_npc_registry_pipeline.yml") if err != nil { t.Fatalf("ReadFile(dnd_npc_registry_pipeline.yml) error = %v", err) } fileConfig, err := config.ParseFileConfigYAML(data) if err != nil { t.Fatalf("ParseFileConfigYAML() error = %v", err) } cfg := config.Default() if err := cfg.ApplyFileConfig(fileConfig); err != nil { t.Fatalf("ApplyFileConfig() error = %v", err) } return cfg } func readNPCFixture(t *testing.T) []byte { t.Helper() raw, err := os.ReadFile("testdata/seriatim_npc_session.json") if err != nil { t.Fatalf("ReadFile(seriatim_npc_session.json) error = %v", err) } return raw } func hasNPCWarning(warnings []contracts.Warning, reason string) bool { for _, warning := range warnings { if warning.ReasonCode == reason && strings.HasPrefix(warning.Scope, "npcs[") { return true } } return false } func hasWarningReason(warnings []contracts.Warning, reason string) bool { for _, warning := range warnings { if warning.ReasonCode == reason { return true } } return false }