600 lines
26 KiB
Go
600 lines
26 KiB
Go
package integration_test
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
|
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
|
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
|
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
|
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry"
|
|
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
|
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
|
)
|
|
|
|
func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
catalog := moduleCatalog(registries)
|
|
configValue := loadGroundedPipelineConfig(t)
|
|
campaignInputs := configureGroundedCampaignReferences(t, &configValue)
|
|
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog})
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v", err)
|
|
}
|
|
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
|
|
if err != nil || len(warnings) != 0 {
|
|
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
|
|
}
|
|
|
|
client := &groundedDNDLLMClient{}
|
|
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
|
|
if err != nil {
|
|
t.Fatalf("Prepare() error = %v", err)
|
|
}
|
|
for name, value := range map[string]string{
|
|
"extract:npc_registry:dnd/npc-registry:mapping_policy": "dnd.npc_registry.extract_mapping.v2",
|
|
"normalize:npc_registry:dnd/npc-registry:identity_policy": "dnd.npc_registry.identity.v1",
|
|
"normalize:npc_registry:dnd/npc-registry:normalization_policy": npcnormalize.NormalizationPolicy,
|
|
"normalize:npc_registry:dnd/npc-registry:semantic_reconciliation_policy": semanticreconcile.Policy,
|
|
"extract:spells:dnd/spells:mapping_policy": "dnd.spells.extract_mapping.v2",
|
|
"extract:combat:dnd/combat-turns:scene_gate_policy": "dnd.combat_turns.scene_gate.v1",
|
|
} {
|
|
assertFingerprintValue(t, prepared.CheckpointFingerprints(), name, value)
|
|
}
|
|
for _, name := range []string{
|
|
"extract:npc_registry:dnd/npc-registry:prompt",
|
|
"extract:npc_registry:dnd/npc-registry:response_schema",
|
|
"normalize:npc_registry:dnd/npc-registry:prompt",
|
|
"normalize:npc_registry:dnd/npc-registry:response_schema",
|
|
"normalize:npc_registry:dnd/npc-registry:semantic_reconciliation_limits",
|
|
"extract:spells:dnd/spells:prompt",
|
|
"extract:spells:dnd/spells:response_schema",
|
|
"extract:spells:dnd/spells:npc_registry",
|
|
"extract:combat:dnd/combat-turns:prompt",
|
|
"extract:combat:dnd/combat-turns:response_schema",
|
|
"extract:combat:dnd/combat-turns:scene_eligibility",
|
|
"extract:combat:dnd/combat-turns:npc_registry",
|
|
"normalize:combat:dnd/combat-turns:npc_registry",
|
|
} {
|
|
assertCombatFingerprint(t, prepared.CheckpointFingerprints(), name)
|
|
}
|
|
checkpoint := newGeneratedReferenceCheckpointLoader()
|
|
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
|
Prepared: prepared,
|
|
RawInput: readNPCFixture(t),
|
|
ExtractWorkers: 1,
|
|
Checkpoint: checkpoint,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v", err)
|
|
}
|
|
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 4 {
|
|
t.Fatalf("run outputs = %#v rejected = %#v, want NPC, scene, spell, and combat outputs", output.NormalizeOutputs, output.Rejected)
|
|
}
|
|
wantSchemas := map[string]string{"npc_registry": npccodec.SchemaID, "scene-descriptions": scenecodec.SchemaID, "spells": spellcodec.SchemaID, "combat": combatcodec.SchemaID}
|
|
for _, serialized := range output.NormalizeOutputs {
|
|
if serialized.Artifact.Schema.ID != wantSchemas[serialized.LaneID] || serialized.Artifact.Schema.Version != "v1" {
|
|
t.Fatalf("%s artifact schema = %#v, want minimal v1 identity", serialized.LaneID, serialized.Artifact.Schema)
|
|
}
|
|
}
|
|
wantExtractorIdentity := map[string]struct{ promptID, schemaID string }{
|
|
"npc_registry": {npcregistry.PromptID, npcregistry.ResponseSchemaID},
|
|
"scene-descriptions": {sceneextract.PromptID, sceneextract.ResponseSchemaID},
|
|
"spells": {spells.PromptID, spells.ResponseSchemaID},
|
|
"combat": {combatextract.PromptID, combatextract.ResponseSchemaID},
|
|
}
|
|
for _, lane := range output.Manifest.ArtifactLanes {
|
|
want, ok := wantExtractorIdentity[lane.ID]
|
|
metadata, metadataOK := lane.Metadata["extractor"].(map[string]any)
|
|
if !ok || !metadataOK || metadata["prompt_id"] != want.promptID || metadata["prompt_version"] != "v1" || metadata["response_schema_id"] != want.schemaID || metadata["response_schema_version"] != "v1" {
|
|
t.Fatalf("%s extractor metadata = %#v, want v1 prompt/schema identity", lane.ID, metadata)
|
|
}
|
|
}
|
|
|
|
var npcPayload, scenePayload []byte
|
|
seenSteps := map[string]string{}
|
|
for _, serialized := range output.NormalizeOutputs {
|
|
seenSteps[serialized.LaneID] = serialized.StepID
|
|
switch serialized.LaneID {
|
|
case "npc_registry":
|
|
npcPayload = append([]byte(nil), serialized.Artifact.Content...)
|
|
case "scene-descriptions":
|
|
scenePayload = append([]byte(nil), serialized.Artifact.Content...)
|
|
}
|
|
}
|
|
if seenSteps["npc_registry"] != "identify-npcs" || seenSteps["scene-descriptions"] != "identify-npcs" || seenSteps["spells"] != "grounded-events" || seenSteps["combat"] != "grounded-events" {
|
|
t.Fatalf("normalized output steps = %#v, want ordered producer and consumer steps", seenSteps)
|
|
}
|
|
if len(npcPayload) == 0 {
|
|
t.Fatal("NPC producer did not publish a canonical payload")
|
|
}
|
|
if len(scenePayload) == 0 {
|
|
t.Fatal("scene producer did not publish a canonical payload")
|
|
}
|
|
npcValue, err := npccodec.New().Decode(npcPayload)
|
|
if err != nil {
|
|
t.Fatalf("decode NPC producer payload: %v", err)
|
|
}
|
|
npcPayload, err = npccodec.New().Encode(npcValue)
|
|
if err != nil {
|
|
t.Fatalf("encode canonical NPC producer payload: %v", err)
|
|
}
|
|
canonicalDigest := digestBytes(npcPayload)
|
|
sceneValue, err := scenecodec.New().Decode(scenePayload)
|
|
if err != nil {
|
|
t.Fatalf("decode scene producer payload: %v", err)
|
|
}
|
|
scenePayload, err = scenecodec.New().Encode(sceneValue)
|
|
if err != nil {
|
|
t.Fatalf("encode canonical scene producer payload: %v", err)
|
|
}
|
|
sceneDigest := digestBytes(scenePayload)
|
|
projection := []byte(`{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}`)
|
|
projectionDigest := digestBytes(projection)
|
|
seenConsumers := map[string]bool{}
|
|
for _, request := range client.requestsSnapshot() {
|
|
if request.PromptID != spells.PromptID && request.PromptID != combatextract.PromptID {
|
|
continue
|
|
}
|
|
input := request.Inputs["npc_registry"]
|
|
if input.MediaType != npccodec.MediaType || input.Digest != projectionDigest || string(input.Content) != string(projection) || input.OriginURI != "" {
|
|
t.Fatalf("%s NPC prompt input = %#v, want names-only generated registry", request.PromptID, input)
|
|
}
|
|
if request.PromptID == combatextract.PromptID {
|
|
transcript, ok := request.Inputs["transcript"]
|
|
if !ok || !strings.Contains(string(transcript.Content), "Mira Thorn asks the party") {
|
|
t.Fatalf("combat transcript input = %#v, want configured transcript content", transcript)
|
|
}
|
|
for name, want := range campaignInputs {
|
|
input, ok := request.Inputs[name]
|
|
if !ok || string(input.Content) != want {
|
|
t.Fatalf("combat %s input = %#v, want configured campaign reference", name, input)
|
|
}
|
|
}
|
|
if _, ok := request.Inputs["scene_descriptions"]; ok {
|
|
t.Fatalf("combat prompt inputs = %#v, must not contain scene descriptions", request.Inputs)
|
|
}
|
|
}
|
|
seenConsumers[request.PromptID] = true
|
|
}
|
|
if !seenConsumers[spells.PromptID] || !seenConsumers[combatextract.PromptID] {
|
|
t.Fatalf("consumer prompt IDs = %#v, want spell and combat requests", seenConsumers)
|
|
}
|
|
if combatCompletionCount(client.requestsSnapshot()) != 1 {
|
|
t.Fatalf("combat completion requests = %#v, want exactly one for the exact combat scene", client.requestsSnapshot())
|
|
}
|
|
|
|
provenanceCount := 0
|
|
for _, reference := range output.Manifest.References {
|
|
if reference.SlotName != "npc_registry" {
|
|
continue
|
|
}
|
|
provenanceCount++
|
|
if reference.Digest != canonicalDigest || reference.OriginType != "generated" {
|
|
t.Fatalf("NPC generated provenance = %#v, want canonical digest and generated origin", reference)
|
|
}
|
|
}
|
|
if provenanceCount != 3 {
|
|
t.Fatalf("NPC generated provenance count = %d, want spell extract plus combat extract/normalize", provenanceCount)
|
|
}
|
|
sceneProvenanceCount := 0
|
|
for _, reference := range output.Manifest.References {
|
|
if reference.SlotName == "scene_descriptions" {
|
|
sceneProvenanceCount++
|
|
if reference.Stage != string(pipeline.StageExtract) || reference.StepID != "grounded-events" || reference.LaneID != "combat" || reference.OriginType != "generated" || reference.Digest != sceneDigest || reference.ArtifactKind != string(dnd.SceneDescriptionListKind) || reference.SchemaID != scenecodec.SchemaID || reference.SchemaName != scenecodec.SchemaName || reference.SchemaVersion != scenecodec.SchemaVersion || reference.MediaType != scenecodec.MediaType || reference.ProducerPipeline != "dnd-npc-grounded" || reference.ProducerStep != "identify-npcs" || reference.ProducerLane != "scene-descriptions" || reference.ProducerModule != "dnd/scene-descriptions" {
|
|
t.Fatalf("scene generated provenance = %#v, want canonical combat handoff identity", reference)
|
|
}
|
|
}
|
|
}
|
|
if sceneProvenanceCount != 1 {
|
|
t.Fatalf("scene generated provenance count = %d, want combat extractor handoff", sceneProvenanceCount)
|
|
}
|
|
assertGeneratedSceneDependency(t, checkpoint.extractDependencies("combat"))
|
|
manifestContent, err := json.Marshal(output.Manifest)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, forbidden := range []string{"Mira Thorn", "Hooded Guard", "For every NPC record", "Extract Dungeons & Dragons", "A combat encounter", "The party faces an active encounter.", "Mira Thorn asks the party", `"scenes":`} {
|
|
if strings.Contains(string(manifestContent), forbidden) {
|
|
t.Fatalf("manifest exposes prompt or reference payload content %q", forbidden)
|
|
}
|
|
}
|
|
for _, lane := range output.Manifest.ArtifactLanes {
|
|
componentContent, err := json.Marshal(lane.Metadata)
|
|
if err != nil {
|
|
t.Fatalf("marshal %s component metadata: %v", lane.ID, err)
|
|
}
|
|
for _, forbidden := range []string{"A combat encounter", "The party faces an active encounter.", "Mira Thorn asks the party", `"scenes":`} {
|
|
if strings.Contains(string(componentContent), forbidden) {
|
|
t.Fatalf("%s component metadata exposes scene or transcript content %q", lane.ID, forbidden)
|
|
}
|
|
}
|
|
for _, component := range []string{"extractor", "normalizer"} {
|
|
metadata, ok := lane.Metadata[component].(map[string]any)
|
|
if ok && (metadata["npc_registry_digest"] != nil || metadata["scene_eligibility_digest"] != nil) {
|
|
t.Fatalf("%s %s metadata = %#v, want generated identity only in framework provenance", lane.ID, component, metadata)
|
|
}
|
|
}
|
|
}
|
|
|
|
var combatValue dnd.CombatTurnList
|
|
for _, serialized := range output.NormalizeOutputs {
|
|
switch serialized.LaneID {
|
|
case "spells":
|
|
spellValue := decodeRunnerSpellResponse(t, serialized.Artifact.Content)
|
|
if len(spellValue.SpellCasts) != 1 || spellValue.SpellCasts[0].Caster != "Mira Thorn" {
|
|
t.Fatalf("spell output = %#v, want one registry-grounded-context spell", spellValue)
|
|
}
|
|
assertCurrentEvidence(t, spellValue.SpellCasts[0].SourceRefs)
|
|
case "combat":
|
|
decoded, decodeErr := combatcodec.New().Decode(serialized.Artifact.Content)
|
|
if decodeErr != nil {
|
|
t.Fatalf("decode combat output: %v", decodeErr)
|
|
}
|
|
combatValue = decoded
|
|
}
|
|
}
|
|
if len(combatValue.CombatTurns) != 1 || combatValue.CombatTurns[0].Actor != "Mira Thorn" {
|
|
t.Fatalf("combat output = %#v, want registry-normalized actor", combatValue)
|
|
}
|
|
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
|
|
}
|
|
|
|
func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
configValue := loadGroundedPipelineConfig(t)
|
|
profile := configValue.Pipelines["dnd-npc-grounded"]
|
|
profile.Output.Options = map[string]any{"evidence_context": map[string]any{
|
|
"enabled": true, "window_units": 0, "lanes": []any{"npc_registry", "spells", "combat"},
|
|
}}
|
|
configValue.Pipelines["dnd-npc-grounded"] = profile
|
|
raw := strings.NewReplacer(
|
|
`"id": 1`, `"id": 10`,
|
|
`"id": 2`, `"id": 30`,
|
|
`"id": 3`, `"id": 20`,
|
|
`"id": 4`, `"id": 50`,
|
|
`"id": 5`, `"id": 40`,
|
|
).Replace(string(readNPCFixture(t)))
|
|
output := runGroundedPipelineWithRaw(t, configValue, registries, &groundedDNDLLMClient{firstUnitID: 10, thirdUnitID: 20}, nil, []byte(raw))
|
|
|
|
value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json"))
|
|
if err != nil {
|
|
t.Fatalf("Decode(evidence context) error = %v", err)
|
|
}
|
|
if !reflect.DeepEqual(value.SelectedLanes, []string{"combat", "npc_registry", "spells"}) {
|
|
t.Fatalf("selected lanes = %#v, want configured production lanes without scene descriptions", value.SelectedLanes)
|
|
}
|
|
if len(value.Contexts) != 2 || len(value.Contexts[0].Units) != 1 || len(value.Contexts[1].Units) != 1 || value.Contexts[0].Units[0].ID != 10 || value.Contexts[1].Units[0].ID != 20 {
|
|
t.Fatalf("evidence contexts = %#v, want source-position union with non-monotonic unit IDs", value.Contexts)
|
|
}
|
|
firstRefs := value.Contexts[0].EvidenceRefs
|
|
if len(firstRefs) != 3 || firstRefs[0].LaneID != "combat" || firstRefs[1].LaneID != "npc_registry" || firstRefs[2].LaneID != "spells" {
|
|
t.Fatalf("first context evidence = %#v, want overlapping selected lane references", firstRefs)
|
|
}
|
|
for _, context := range value.Contexts {
|
|
for _, reference := range context.EvidenceRefs {
|
|
if reference.LaneID == "scene-descriptions" {
|
|
t.Fatalf("evidence refs = %#v, want scene descriptions excluded by allowlist", value.Contexts)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProductionDNDOutputCanExplicitlySelectSceneDescriptionEvidence(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
configValue := loadGroundedPipelineConfig(t)
|
|
profile := configValue.Pipelines["dnd-npc-grounded"]
|
|
profile.Output.Options = map[string]any{"evidence_context": map[string]any{
|
|
"enabled": true, "lanes": []any{"scene-descriptions"},
|
|
}}
|
|
configValue.Pipelines["dnd-npc-grounded"] = profile
|
|
output := runGroundedPipeline(t, configValue, registries, &groundedDNDLLMClient{}, nil)
|
|
|
|
value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json"))
|
|
if err != nil {
|
|
t.Fatalf("Decode(evidence context) error = %v", err)
|
|
}
|
|
if !reflect.DeepEqual(value.SelectedLanes, []string{"scene-descriptions"}) || len(value.Contexts) == 0 || len(value.Contexts[0].EvidenceRefs) == 0 || value.Contexts[0].EvidenceRefs[0].LaneID != "scene-descriptions" {
|
|
t.Fatalf("evidence context = %#v, want explicitly selected scene-description evidence", value)
|
|
}
|
|
}
|
|
|
|
func TestGroundedPipelineSkipsCombatForExactNarrativeScene(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
configValue := loadGroundedPipelineConfig(t)
|
|
profile := configValue.Pipelines["dnd-npc-grounded"]
|
|
combat := profile.Steps[1].Artifacts["combat"]
|
|
combat.Extract.Retries = 2
|
|
profile.Steps[1].Artifacts["combat"] = combat
|
|
configValue.Pipelines["dnd-npc-grounded"] = profile
|
|
|
|
client := &groundedDNDLLMClient{sceneKind: dnd.SceneKindNarrative}
|
|
output := runGroundedPipeline(t, configValue, registries, client, nil)
|
|
if combatCompletionCount(client.requestsSnapshot()) != 0 {
|
|
t.Fatalf("combat completion requests = %#v, want none for an exact narrative scene", client.requestsSnapshot())
|
|
}
|
|
if len(output.Rejected) != 0 || hasCombatWarning(output.Warnings, "scene_classification_unavailable") {
|
|
t.Fatalf("narrative output rejected/warnings = %#v / %#v, want accepted skip without unavailable warning", output.Rejected, output.Warnings)
|
|
}
|
|
combatValue := normalizedCombatOutput(t, output)
|
|
if combatValue.CombatTurns == nil || len(combatValue.CombatTurns) != 0 {
|
|
t.Fatalf("narrative combat output = %#v, want an accepted empty combat list", combatValue)
|
|
}
|
|
}
|
|
|
|
func TestGeneratedSceneReferenceChangesCombatCheckpointDependency(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
first := generatedSceneDependency(t, registries, "A combat encounter")
|
|
second := generatedSceneDependency(t, registries, "A revised combat encounter")
|
|
if first == second {
|
|
t.Fatalf("generated combat dependencies = %q and %q, want canonical scene artifact change to invalidate reuse", first, second)
|
|
}
|
|
}
|
|
|
|
func assertFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, want string) {
|
|
t.Helper()
|
|
for _, fingerprint := range fingerprints {
|
|
if fingerprint.Name == name && fingerprint.Value == want {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("fingerprints = %#v, want %q = %q", fingerprints, name, want)
|
|
}
|
|
|
|
func assertCurrentEvidence(t *testing.T, references []source.SourceRef) {
|
|
t.Helper()
|
|
for _, reference := range references {
|
|
if reference.SourceID != "npc-session" {
|
|
t.Fatalf("evidence reference = %#v, want current source only", reference)
|
|
}
|
|
}
|
|
}
|
|
|
|
func loadGroundedPipelineConfig(t *testing.T) config.Config {
|
|
t.Helper()
|
|
fileConfig, err := config.LoadFileConfig(repositoryPathForIntegration("internal", "modules", "integration", "testdata", "dnd_npc_grounded_pipeline.yml"))
|
|
if err != nil {
|
|
t.Fatalf("LoadFileConfig() error = %v", err)
|
|
}
|
|
cfg := config.Default()
|
|
if err := cfg.ApplyFileConfig(fileConfig); err != nil {
|
|
t.Fatalf("ApplyFileConfig() error = %v", err)
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
func configureGroundedCampaignReferences(t *testing.T, cfg *config.Config) map[string]string {
|
|
t.Helper()
|
|
values := map[string]string{
|
|
"party": "Mira Thorn: ally\n",
|
|
"glossary": "Greencloak: Mira Thorn's title\n",
|
|
}
|
|
profile := cfg.Pipelines["dnd-npc-grounded"]
|
|
profile.References = make(map[string]pipeline.ReferenceSource, len(values))
|
|
for name, content := range values {
|
|
path := filepath.Join(t.TempDir(), name+".txt")
|
|
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
|
t.Fatalf("write %s reference: %v", name, err)
|
|
}
|
|
profile.References[name] = pipeline.ExternalReference(path)
|
|
}
|
|
cfg.Pipelines["dnd-npc-grounded"] = profile
|
|
return values
|
|
}
|
|
|
|
func runGroundedPipeline(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, checkpoint pipeline.CheckpointLoader) pipeline.RunOutput {
|
|
return runGroundedPipelineWithRaw(t, configValue, registries, client, checkpoint, readNPCFixture(t))
|
|
}
|
|
|
|
func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, checkpoint pipeline.CheckpointLoader, raw []byte) pipeline.RunOutput {
|
|
t.Helper()
|
|
catalog := moduleCatalog(registries)
|
|
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog})
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v", err)
|
|
}
|
|
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
|
|
if err != nil || len(warnings) != 0 {
|
|
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
|
|
}
|
|
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
|
|
if err != nil {
|
|
t.Fatalf("Prepare() error = %v", err)
|
|
}
|
|
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
|
Prepared: prepared,
|
|
RawInput: append([]byte(nil), raw...),
|
|
ExtractWorkers: 1,
|
|
Checkpoint: checkpoint,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v", err)
|
|
}
|
|
return output
|
|
}
|
|
|
|
func outputFileContent(t *testing.T, files []contracts.OutputFile, name string) []byte {
|
|
t.Helper()
|
|
for _, file := range files {
|
|
if file.Name == name {
|
|
return append([]byte(nil), file.Bytes...)
|
|
}
|
|
}
|
|
t.Fatalf("output files = %#v, missing %q", files, name)
|
|
return nil
|
|
}
|
|
|
|
func normalizedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList {
|
|
t.Helper()
|
|
for _, serialized := range output.NormalizeOutputs {
|
|
if serialized.LaneID != "combat" {
|
|
continue
|
|
}
|
|
value, err := combatcodec.New().Decode(serialized.Artifact.Content)
|
|
if err != nil {
|
|
t.Fatalf("decode combat output: %v", err)
|
|
}
|
|
return value
|
|
}
|
|
t.Fatalf("normalized outputs = %#v, want combat lane", output.NormalizeOutputs)
|
|
return dnd.CombatTurnList{}
|
|
}
|
|
|
|
func combatCompletionCount(requests []contracts.StructuredCompletionRequest) int {
|
|
count := 0
|
|
for _, request := range requests {
|
|
if request.PromptID == combatextract.PromptID {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func generatedSceneDependency(t *testing.T, registries pipeline.Registries, sceneTitle string) string {
|
|
t.Helper()
|
|
client := &groundedDNDLLMClient{sceneTitle: sceneTitle}
|
|
checkpoint := newGeneratedReferenceCheckpointLoader()
|
|
runGroundedPipeline(t, loadGroundedPipelineConfig(t), registries, client, checkpoint)
|
|
return assertGeneratedSceneDependency(t, checkpoint.extractDependencies("combat"))
|
|
}
|
|
|
|
func assertGeneratedSceneDependency(t *testing.T, dependencies []pipeline.CheckpointFingerprint) string {
|
|
t.Helper()
|
|
for _, dependency := range dependencies {
|
|
if dependency.Name == "generated-reference:scene_descriptions:0" && strings.HasPrefix(dependency.Value, "sha256:") {
|
|
return dependency.Value
|
|
}
|
|
}
|
|
t.Fatalf("combat extract checkpoint dependencies = %#v, want generated scene reference fingerprint", dependencies)
|
|
return ""
|
|
}
|
|
|
|
type generatedReferenceCheckpointLoader struct {
|
|
pipeline.CheckpointLoader
|
|
extract map[string][][]pipeline.CheckpointFingerprint
|
|
}
|
|
|
|
func newGeneratedReferenceCheckpointLoader() *generatedReferenceCheckpointLoader {
|
|
return &generatedReferenceCheckpointLoader{
|
|
CheckpointLoader: pipeline.NoopCheckpointLoader(),
|
|
extract: make(map[string][][]pipeline.CheckpointFingerprint),
|
|
}
|
|
}
|
|
|
|
func (loader *generatedReferenceCheckpointLoader) Enabled() bool { return true }
|
|
|
|
func (loader *generatedReferenceCheckpointLoader) Extract(laneID, _ string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
|
|
loader.extract[laneID] = append(loader.extract[laneID], append([]pipeline.CheckpointFingerprint(nil), dependencies...))
|
|
return pipeline.ExtractCheckpoint{}, pipeline.NewCheckpointDecision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonMissing)
|
|
}
|
|
|
|
func (loader *generatedReferenceCheckpointLoader) extractDependencies(laneID string) []pipeline.CheckpointFingerprint {
|
|
if len(loader.extract[laneID]) != 1 {
|
|
return nil
|
|
}
|
|
return loader.extract[laneID][0]
|
|
}
|
|
|
|
type groundedDNDLLMClient struct {
|
|
mu sync.Mutex
|
|
requests []contracts.StructuredCompletionRequest
|
|
sceneKind dnd.SceneKind
|
|
sceneTitle string
|
|
firstUnitID int
|
|
thirdUnitID int
|
|
}
|
|
|
|
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
client.mu.Lock()
|
|
client.requests = append(client.requests, cloneStructuredCompletionRequest(request))
|
|
client.mu.Unlock()
|
|
|
|
var payload any
|
|
firstUnitID := client.firstUnitID
|
|
if firstUnitID == 0 {
|
|
firstUnitID = 1
|
|
}
|
|
thirdUnitID := client.thirdUnitID
|
|
if thirdUnitID == 0 {
|
|
thirdUnitID = 3
|
|
}
|
|
switch request.PromptID {
|
|
case npcregistry.PromptID:
|
|
payload = map[string]any{"npcs": []any{
|
|
map[string]any{
|
|
"name": "Mira Thorn", "source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
|
|
},
|
|
map[string]any{
|
|
"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": thirdUnitID, "end_unit_id": thirdUnitID}},
|
|
},
|
|
}}
|
|
case npcnormalize.PromptID:
|
|
payload = map[string]any{"duplicate_groups": []any{}}
|
|
case sceneextract.PromptID:
|
|
kind := client.sceneKind
|
|
if kind == "" {
|
|
kind = dnd.SceneKindCombat
|
|
}
|
|
title := client.sceneTitle
|
|
if title == "" {
|
|
title = "A combat encounter"
|
|
}
|
|
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
|
|
case spells.PromptID:
|
|
payload = map[string]any{"spell_casts": []any{map[string]any{
|
|
"caster": "Mira Thorn", "spell": "Cure Wounds",
|
|
"source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
|
|
}}}
|
|
case combatextract.PromptID:
|
|
payload = map[string]any{"combat_turns": []any{map[string]any{
|
|
"actor": "Mira Thorn",
|
|
"turn_kind": "turn",
|
|
"source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
|
|
}}}
|
|
default:
|
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected grounded prompt %q", request.PromptID)
|
|
}
|
|
content, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
if err := json.Unmarshal(content, out); err != nil {
|
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate grounded response: %w", err)
|
|
}
|
|
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "grounded-fake"}, nil
|
|
}
|
|
|
|
func (client *groundedDNDLLMClient) requestsSnapshot() []contracts.StructuredCompletionRequest {
|
|
client.mu.Lock()
|
|
defer client.mu.Unlock()
|
|
return append([]contracts.StructuredCompletionRequest(nil), client.requests...)
|
|
}
|
|
|
|
func digestBytes(content []byte) string {
|
|
sum := sha256.Sum256(content)
|
|
return fmt.Sprintf("sha256:%x", sum[:])
|
|
}
|
|
|
|
var _ contracts.StructuredLLMClient = (*groundedDNDLLMClient)(nil)
|