eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Integrate NPC semantic normalization

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-26 01:41:35 +00:00
parent d1c48db4bc
commit 8d9a496935
4 changed files with 234 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
internal/modules/integration/dnd_npcs_runner_test.go
View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs"
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register"
npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcs/shape"
@@ -95,7 +96,7 @@ func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T)
t.Fatalf("manifest lane = %#v, want NPC production composition", lane)
}
normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
if !ok || normalizerMetadata["identity_policy"] != identity.Policy || normalizerMetadata["normalization_policy"] != "dnd.npcs.normalize.v2" {
if !ok || normalizerMetadata["identity_policy"] != identity.Policy || normalizerMetadata["normalization_policy"] != npcnormalize.NormalizationPolicy || normalizerMetadata["prompt_id"] != npcnormalize.PromptID || normalizerMetadata["response_schema_id"] != npcnormalize.ResponseSchemaID {
t.Fatalf("normalizer metadata = %#v, want identity and normalization policies", lane.Metadata)
}
var npcOutputFile *contracts.OutputFile
@@ -108,8 +109,8 @@ func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T)
if npcOutputFile == nil || npcOutputFile.ContentType != npccodec.MediaType {
t.Fatalf("output files = %#v, want JSON NPC lane file", output.OutputFiles)
}
if len(client.requests) != 1 || client.requests[0].PromptID != npcs.PromptID {
t.Fatalf("LLM requests = %#v, want one NPC prompt request", client.requests)
if len(client.requests) != 2 || client.requests[0].PromptID != npcs.PromptID || client.requests[1].PromptID != npcnormalize.PromptID {
t.Fatalf("LLM requests = %#v, want extraction and normalization prompt requests", client.requests)
}
}
@@ -170,6 +171,52 @@ func TestProductionNPCPipelineRoutesSemanticCandidatesToDeterministicValidators(
}
}
func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) {
registries := productionNPCRegistries(t)
baseResponse := npcProductionResponse{NPCs: []npcProductionRecord{
{Name: "Mira", SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 1}}},
{Name: "Mira Thorn", SourceRefs: []npcProductionSourceRef{{StartUnitID: 2, EndUnitID: 2}}},
{Name: "Hooded Guard", SourceRefs: []npcProductionSourceRef{{StartUnitID: 3, EndUnitID: 3}}},
}}
partial := []byte(`{"duplicate_groups":[{"members":["Mira","Mira Thorn"],"canonical_name":"Mira Thorn"},{"members":["Hooded Guard","Unknown"],"canonical_name":"Hooded Guard"}]}`)
safe := []byte(`{"duplicate_groups":[{"members":["Mira","Mira Thorn"],"canonical_name":"Mira Thorn"}]}`)
for _, test := range []struct {
name string
retries int
responses [][]byte
wantCalls int
wantExhaustion bool
}{
{name: "default retry budget", responses: [][]byte{partial}, wantCalls: 1, wantExhaustion: true},
{name: "later complete proposal", retries: 1, responses: [][]byte{partial, safe}, wantCalls: 2},
} {
t.Run(test.name, func(t *testing.T) {
cfg := loadNPCPipelineConfig(t)
profile := cfg.Pipelines["dnd-npcs-fixture"]
lane := profile.Artifacts["npcs"]
lane.Normalize.Retries = test.retries
profile.Artifacts["npcs"] = lane
cfg.Pipelines["dnd-npcs-fixture"] = profile
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-npcs-fixture", Catalog: moduleCatalog(registries)})
if err != nil {
t.Fatal(err)
}
client := &fakeNPCProductionLLMClient{response: baseResponse, normalizeResponses: test.responses}
output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if client.requestCount(npcnormalize.PromptID) != test.wantCalls || len(output.NormalizeOutputs) != 1 {
t.Fatalf("normalization calls = %d output = %#v, want %d accepted result", client.requestCount(npcnormalize.PromptID), output.NormalizeOutputs, test.wantCalls)
}
if got := hasWarningReason(output.Warnings, npcnormalize.ReasonCodeNPCSemanticReconciliationExhausted); got != test.wantExhaustion {
t.Fatalf("warnings = %#v, exhaustion = %t, want %t", output.Warnings, got, test.wantExhaustion)
}
})
}
}
type npcProductionResponse struct {
NPCs []npcProductionRecord `json:"npcs"`
}
@@ -185,26 +232,41 @@ type npcProductionSourceRef struct {
}
type fakeNPCProductionLLMClient struct {
response npcProductionResponse
rawResponses [][]byte
requests []contracts.StructuredCompletionRequest
response npcProductionResponse
rawResponses [][]byte
normalizeResponses [][]byte
requests []contracts.StructuredCompletionRequest
}
func (client *fakeNPCProductionLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, req)
var content []byte
if client.rawResponses != nil {
index := len(client.requests) - 1
if index >= len(client.rawResponses) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake NPC response %d", index)
switch req.PromptID {
case npcs.PromptID:
if client.rawResponses != nil {
index := client.requestCount(npcs.PromptID) - 1
if index >= len(client.rawResponses) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake NPC response %d", index)
}
content = append([]byte(nil), client.rawResponses[index]...)
} else {
var err error
content, err = json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
}
content = append([]byte(nil), client.rawResponses[index]...)
} else {
var err error
content, err = json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
case npcnormalize.PromptID:
content = []byte(`{"duplicate_groups":[]}`)
if client.normalizeResponses != nil {
index := client.requestCount(npcnormalize.PromptID) - 1
if index >= len(client.normalizeResponses) {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake NPC normalization response %d", index)
}
content = append([]byte(nil), client.normalizeResponses[index]...)
}
default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected fake NPC prompt %q", req.PromptID)
}
if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate NPC structured target: %w", err)
@@ -212,6 +274,16 @@ func (client *fakeNPCProductionLLMClient) CompleteStructured(_ context.Context,
return contracts.StructuredCompletionResponse{Content: content}, nil
}
func (client *fakeNPCProductionLLMClient) requestCount(promptID string) int {
count := 0
for _, request := range client.requests {
if request.PromptID == promptID {
count++
}
}
return count
}
func productionNPCRegistries(t *testing.T) pipeline.Registries {
t.Helper()
registries := pipeline.Registries{
@@ -283,3 +355,12 @@ func hasNPCWarning(warnings []contracts.Warning, reason string) bool {
}
return false
}
func hasWarningReason(warnings []contracts.Warning, reason string) bool {
for _, warning := range warnings {
if warning.ReasonCode == reason {
return true
}
}
return false
}