606 lines
26 KiB
Go
606 lines
26 KiB
Go
package combatturns
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
|
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
|
)
|
|
|
|
func TestExtractMapsAndOrdersCombatTurnsBySourcePosition(t *testing.T) {
|
|
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{
|
|
{
|
|
Actor: "Borin", TurnKind: "turn",
|
|
SourceRefs: []combatSourceRefResponse{{StartUnitID: 2, EndUnitID: 2}},
|
|
},
|
|
{
|
|
Actor: "Aria", TurnKind: "reaction", SourceRefs: []combatSourceRefResponse{
|
|
{StartUnitID: 10, EndUnitID: 10},
|
|
{StartUnitID: 10, EndUnitID: 10},
|
|
},
|
|
},
|
|
{
|
|
Actor: "Unknown", TurnKind: "other",
|
|
SourceRefs: []combatSourceRefResponse{{StartUnitID: 0, EndUnitID: 0}},
|
|
},
|
|
}}}
|
|
|
|
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest(t))
|
|
if err != nil {
|
|
t.Fatalf("Extract() error = %v, want nil", err)
|
|
}
|
|
if got := []string{result.Value.CombatTurns[0].Actor, result.Value.CombatTurns[1].Actor, result.Value.CombatTurns[2].Actor}; !reflect.DeepEqual(got, []string{"Aria", "Borin", "Unknown"}) {
|
|
t.Fatalf("actor order = %#v, want source-position order with invalid evidence last", got)
|
|
}
|
|
wantRefs := []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 10, EndUnitID: 10}}
|
|
if !reflect.DeepEqual(result.Value.CombatTurns[0].SourceRefs, wantRefs) {
|
|
t.Fatalf("canonical refs = %#v, want %#v", result.Value.CombatTurns[0].SourceRefs, wantRefs)
|
|
}
|
|
if ref := result.Value.CombatTurns[2].SourceRefs[0]; ref != (source.SourceRef{SourceID: "session-alpha"}) {
|
|
t.Fatalf("invalid evidence = %#v, want source identity and invalid range preserved", ref)
|
|
}
|
|
|
|
if len(client.requests) != 1 {
|
|
t.Fatalf("LLM calls = %d, want one call per chunk", len(client.requests))
|
|
}
|
|
request := client.requests[0]
|
|
if request.StageName != Key || request.PromptID != PromptID || request.PromptVersion != SchemaVersion || request.SessionID != "session-123" || request.ProfileID != "profile-combat" {
|
|
t.Fatalf("LLM request identity = %#v", request)
|
|
}
|
|
transcript := request.Inputs["transcript"]
|
|
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || !reflect.DeepEqual(transcript.Content, extractionRequest(t).Chunk.Content) {
|
|
t.Fatalf("transcript input = %#v, want chunk-scoped source input", transcript)
|
|
}
|
|
}
|
|
|
|
func TestExtractPreservesInvalidCandidatesForValidators(t *testing.T) {
|
|
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{
|
|
{
|
|
Actor: " ", TurnKind: "unsupported",
|
|
SourceRefs: []combatSourceRefResponse{{StartUnitID: 99, EndUnitID: 0}},
|
|
},
|
|
}}}
|
|
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest(t))
|
|
if err != nil {
|
|
t.Fatalf("Extract() error = %v, want nil for candidate values", err)
|
|
}
|
|
turn := result.Value.CombatTurns[0]
|
|
if turn.Actor != " " || turn.TurnKind != "unsupported" {
|
|
t.Fatalf("invalid turn fields = %#v, want preserved candidate values", turn)
|
|
}
|
|
if turn.SourceRefs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 99}) {
|
|
t.Fatalf("invalid source ref = %#v, want invalid range preserved", turn.SourceRefs[0])
|
|
}
|
|
}
|
|
|
|
func TestExtractUsesDocumentOrderForReferencesAndTurns(t *testing.T) {
|
|
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{
|
|
{Actor: "Later", TurnKind: "turn", SourceRefs: []combatSourceRefResponse{{StartUnitID: 10, EndUnitID: 10}}},
|
|
{Actor: "First", TurnKind: "reaction", SourceRefs: []combatSourceRefResponse{
|
|
{StartUnitID: 10, EndUnitID: 10},
|
|
{StartUnitID: 30, EndUnitID: 30},
|
|
{StartUnitID: 30, EndUnitID: 30},
|
|
{StartUnitID: 999, EndUnitID: 0},
|
|
}},
|
|
{Actor: "Second", TurnKind: "other", SourceRefs: []combatSourceRefResponse{{StartUnitID: 30, EndUnitID: 30}}},
|
|
}}}
|
|
req := extractionRequest(t)
|
|
req.Source.Units = []source.SourceUnit{{ID: 30}, {ID: 10}}
|
|
req.Chunk.Units = append([]source.SourceUnit(nil), req.Source.Units...)
|
|
req.Chunk.Ref = source.SourceRef{SourceID: req.Source.ID, StartUnitID: 10, EndUnitID: 30}
|
|
req.References = sceneReferences(t, req.Chunk, dnd.SceneKindCombat)
|
|
|
|
result, err := newExtractor(t, client).Extract(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("Extract() error = %v", err)
|
|
}
|
|
if got := []string{result.Value.CombatTurns[0].Actor, result.Value.CombatTurns[1].Actor, result.Value.CombatTurns[2].Actor}; !reflect.DeepEqual(got, []string{"First", "Second", "Later"}) {
|
|
t.Fatalf("turn order = %#v, want document chronology with stable equal-evidence ties", got)
|
|
}
|
|
refs := result.Value.CombatTurns[0].SourceRefs
|
|
if got := []int{refs[0].StartUnitID, refs[1].StartUnitID, refs[2].StartUnitID}; !reflect.DeepEqual(got, []int{30, 10, 999}) {
|
|
t.Fatalf("source refs = %#v, want document order with exact duplicate removed", refs)
|
|
}
|
|
refs[0].StartUnitID = 777
|
|
for _, turn := range client.response.CombatTurns {
|
|
for _, ref := range turn.SourceRefs {
|
|
if ref.StartUnitID == 777 {
|
|
t.Fatal("result source references alias the model response")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestExtractPassesReferencesAndNPCGroundingWithoutUsingItAsEvidence(t *testing.T) {
|
|
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}}
|
|
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
|
"players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Alice: Aria")}}},
|
|
"party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Aria: cleric")}}},
|
|
"glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("ogre: a large foe")}}},
|
|
NPCRegistryReferenceSlot: {Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: npcRegistryJSON(t)}}},
|
|
}}
|
|
req := extractionRequest(t)
|
|
for name, slot := range req.References.Slots {
|
|
references.Slots[name] = slot
|
|
}
|
|
req.References = references
|
|
if _, err := newExtractor(t, client, references).Extract(context.Background(), req); err != nil {
|
|
t.Fatalf("Extract() error = %v, want nil", err)
|
|
}
|
|
inputs := client.requests[0].Inputs
|
|
if string(inputs["players"].Content) != "Alice: Aria" || string(inputs["party"].Content) != "Aria: cleric" || string(inputs["glossary"].Content) != "ogre: a large foe" {
|
|
t.Fatalf("reference inputs = %#v, want configured references", inputs)
|
|
}
|
|
registryInput := inputs[NPCRegistryReferenceSlot]
|
|
if registryInput.Name != NPCRegistryReferenceSlot || registryInput.MediaType != "application/json" || !strings.Contains(string(registryInput.Content), "Mira Thorn") {
|
|
t.Fatalf("NPC registry input = %#v, want canonical registry grounding", registryInput)
|
|
}
|
|
if strings.Contains(string(inputs["transcript"].Content), "Aria: cleric") {
|
|
t.Fatal("transcript input contains reference content")
|
|
}
|
|
if _, ok := inputs[SceneDescriptionReferenceSlot]; ok {
|
|
t.Fatalf("prompt inputs include scene descriptions: %#v", inputs)
|
|
}
|
|
|
|
metadata := newExtractor(t, &fakeCombatTurnsLLMClient{}, references).ManifestMetadata()
|
|
if metadata["npc_count"] != 1 || !strings.HasPrefix(metadata["npc_registry_digest"].(string), "sha256:") {
|
|
t.Fatalf("bound registry metadata = %#v, want digest and count", metadata)
|
|
}
|
|
if metadata["scene_description_count"] != 1 || !strings.HasPrefix(metadata["scene_eligibility_digest"].(string), "sha256:") {
|
|
t.Fatalf("bound scene metadata = %#v, want digest and count", metadata)
|
|
}
|
|
fingerprints := newExtractor(t, &fakeCombatTurnsLLMClient{}, references).CheckpointFingerprints()
|
|
if len(fingerprints) != 6 || fingerprints[3].Name != "scene_gate_policy" || fingerprints[4].Name != "npc_registry" || fingerprints[5].Name != "scene_eligibility" {
|
|
t.Fatalf("bound fingerprints = %#v, want local identities plus scene gate and NPC registry", fingerprints)
|
|
}
|
|
encoded, err := json.Marshal(metadata)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(encoded), "Mira Thorn") || strings.Contains(string(encoded), "session-alpha") {
|
|
t.Fatalf("metadata leaked content: %s", encoded)
|
|
}
|
|
}
|
|
|
|
func TestExtractUnboundRegistryUsesExactEmptyPromptAndOmitsIdentity(t *testing.T) {
|
|
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}}
|
|
extractor := newExtractor(t, client)
|
|
if _, err := extractor.Extract(context.Background(), extractionRequest(t)); err != nil {
|
|
t.Fatalf("Extract() error = %v, want nil", err)
|
|
}
|
|
input := client.requests[0].Inputs[NPCRegistryReferenceSlot]
|
|
if string(input.Content) != `{"npcs":[]}` || input.Digest == "" || input.OriginURI != "" {
|
|
t.Fatalf("unbound registry input = %#v, want exact empty prompt without identity", input)
|
|
}
|
|
metadata := extractor.ManifestMetadata()
|
|
if _, ok := metadata["npc_registry_digest"]; ok {
|
|
t.Fatalf("unbound metadata has registry digest: %#v", metadata)
|
|
}
|
|
if fingerprints := extractor.CheckpointFingerprints(); len(fingerprints) != 6 || fingerprints[4].Name != "npc_registry" || fingerprints[4].Value != input.Digest || fingerprints[5].Name != "scene_eligibility" || fingerprints[5].Value == "" {
|
|
t.Fatalf("unbound fingerprints = %#v, want empty-projection identity", fingerprints)
|
|
}
|
|
}
|
|
|
|
func TestExtractorResolvesOperationNPCOverrideWithoutSingletonMetadata(t *testing.T) {
|
|
content := npcRegistryJSON(t)
|
|
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}}
|
|
extractor := newExtractor(t, client)
|
|
request := extractionRequest(t)
|
|
request.References.Slots[NPCRegistryReferenceSlot] = contracts.ResolvedReferenceSlot{
|
|
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
|
|
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: append([]byte(nil), content...), Origin: contracts.ReferenceOrigin{Type: "generated"}}},
|
|
}
|
|
if _, err := extractor.Extract(context.Background(), request); err != nil {
|
|
t.Fatalf("Extract() error = %v", err)
|
|
}
|
|
input := client.requests[0].Inputs[NPCRegistryReferenceSlot]
|
|
if input.Digest == "" || string(input.Content) != `{"npcs":[{"name":"Mira Thorn"}]}` || input.OriginURI != "" {
|
|
t.Fatalf("operation NPC input = %#v, want generated canonical grounding without provenance", input)
|
|
}
|
|
if metadata := extractor.ManifestMetadata(); metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil || metadata["scene_eligibility_digest"] != nil || metadata["scene_description_count"] != nil {
|
|
t.Fatalf("singleton metadata = %#v, want no operation-varying NPC identity", metadata)
|
|
}
|
|
}
|
|
|
|
func TestExtractAppliesSceneEligibilityBeforePromptConstruction(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
scenes []dnd.SceneDescription
|
|
wantWarning string
|
|
wantError string
|
|
wantCalls int
|
|
wantEmptyList bool
|
|
}{
|
|
{
|
|
name: "exact combat",
|
|
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)},
|
|
wantCalls: 1,
|
|
},
|
|
{
|
|
name: "exact narrative",
|
|
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindNarrative)},
|
|
wantEmptyList: true,
|
|
},
|
|
{
|
|
name: "exact recap",
|
|
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindRecap)},
|
|
wantEmptyList: true,
|
|
},
|
|
{
|
|
name: "exact meta",
|
|
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindMeta)},
|
|
wantEmptyList: true,
|
|
},
|
|
{
|
|
name: "missing ID",
|
|
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
|
|
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
|
|
value.ID = "other-chunk"
|
|
return value
|
|
}()},
|
|
wantWarning: "scene_classification_unavailable",
|
|
wantEmptyList: true,
|
|
},
|
|
{
|
|
name: "source mismatch",
|
|
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
|
|
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
|
|
value.SourceRef.SourceID = "other-source"
|
|
return value
|
|
}()},
|
|
wantWarning: "scene_classification_unavailable",
|
|
wantEmptyList: true,
|
|
},
|
|
{
|
|
name: "start mismatch",
|
|
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
|
|
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
|
|
value.SourceRef.StartUnitID = 3
|
|
return value
|
|
}()},
|
|
wantWarning: "scene_classification_unavailable",
|
|
wantEmptyList: true,
|
|
},
|
|
{
|
|
name: "end mismatch",
|
|
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
|
|
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
|
|
value.SourceRef.EndUnitID = 9
|
|
return value
|
|
}()},
|
|
wantWarning: "scene_classification_unavailable",
|
|
wantEmptyList: true,
|
|
},
|
|
{
|
|
name: "missing required reference",
|
|
wantError: "required scene descriptions reference is missing",
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}}
|
|
req := extractionRequest(t)
|
|
if test.wantError != "" {
|
|
req.References = contracts.ReferenceSet{}
|
|
} else {
|
|
req.References = sceneListReferences(t, test.scenes...)
|
|
}
|
|
|
|
result, err := newExtractor(t, client).Extract(context.Background(), req)
|
|
if test.wantError != "" {
|
|
if err == nil || !strings.Contains(err.Error(), test.wantError) {
|
|
t.Fatalf("Extract() error = %v, want %q", err, test.wantError)
|
|
}
|
|
} else if err != nil {
|
|
t.Fatalf("Extract() error = %v", err)
|
|
}
|
|
if len(client.requests) != test.wantCalls {
|
|
t.Fatalf("LLM calls = %d, want %d", len(client.requests), test.wantCalls)
|
|
}
|
|
if test.wantEmptyList && result.Value.CombatTurns == nil {
|
|
t.Fatal("CombatTurns = nil, want accepted non-nil empty list")
|
|
}
|
|
if test.wantWarning != "" {
|
|
if len(result.Warnings) != 1 || result.Warnings[0].Scope != SceneDescriptionReferenceSlot || result.Warnings[0].ReasonCode != test.wantWarning {
|
|
t.Fatalf("warnings = %#v", result.Warnings)
|
|
}
|
|
} else if len(result.Warnings) != 0 {
|
|
t.Fatalf("warnings = %#v, want none", result.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSceneEligibilityMetadataAndFingerprintsTrackGatingValues(t *testing.T) {
|
|
chunk := extractionRequest(t).Chunk
|
|
base := sceneDescription(chunk, dnd.SceneKindCombat)
|
|
baseExtractor := newExtractor(t, &fakeCombatTurnsLLMClient{}, sceneListReferences(t, base))
|
|
metadata := baseExtractor.ManifestMetadata()
|
|
if metadata["scene_gate_policy"] != sceneGatePolicy || metadata["scene_description_count"] != 1 {
|
|
t.Fatalf("metadata = %#v, want scene gate policy and count", metadata)
|
|
}
|
|
baseDigest := fingerprintValue(t, baseExtractor.CheckpointFingerprints(), "scene_eligibility")
|
|
proseChanged := base
|
|
proseChanged.Title = "Different title"
|
|
proseChanged.Summary = "Different summary."
|
|
if got := fingerprintValue(t, newExtractor(t, &fakeCombatTurnsLLMClient{}, sceneListReferences(t, proseChanged)).CheckpointFingerprints(), "scene_eligibility"); got != baseDigest {
|
|
t.Fatalf("prose-only eligibility digest = %q, want %q", got, baseDigest)
|
|
}
|
|
for _, test := range []struct {
|
|
name string
|
|
change func(*dnd.SceneDescription)
|
|
}{
|
|
{"kind", func(value *dnd.SceneDescription) { value.Kind = dnd.SceneKindNarrative }},
|
|
{"range", func(value *dnd.SceneDescription) { value.SourceRef.StartUnitID = 3 }},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
changed := base
|
|
test.change(&changed)
|
|
if got := fingerprintValue(t, newExtractor(t, &fakeCombatTurnsLLMClient{}, sceneListReferences(t, changed)).CheckpointFingerprints(), "scene_eligibility"); got == baseDigest {
|
|
t.Fatal("eligibility digest did not change")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNewRejectsMalformedNPCRegistryBeforeLLMCallWithoutContent(t *testing.T) {
|
|
client := &fakeCombatTurnsLLMClient{}
|
|
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
|
NPCRegistryReferenceSlot: {
|
|
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
|
|
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"private transcript detail"}`)}},
|
|
},
|
|
}}
|
|
_, err := New(client, Options{}, references)
|
|
if err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private transcript detail") {
|
|
t.Fatalf("New() error = %v, want bounded content-free registry failure", err)
|
|
}
|
|
if len(client.requests) != 0 {
|
|
t.Fatalf("LLM calls = %d, want none during failed construction", len(client.requests))
|
|
}
|
|
}
|
|
|
|
func TestNewRejectsInvalidSceneReferenceBeforeLLMCallWithoutContent(t *testing.T) {
|
|
client := &fakeCombatTurnsLLMClient{}
|
|
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
|
SceneDescriptionReferenceSlot: {
|
|
Slot: contracts.ReferenceSlot{Name: SceneDescriptionReferenceSlot},
|
|
Items: []contracts.ReferenceItem{{SlotName: SceneDescriptionReferenceSlot, MediaType: scenecodec.MediaType, Content: []byte(`{"scenes":[]}`)}},
|
|
},
|
|
}}
|
|
_, err := New(client, Options{}, references)
|
|
if err == nil || !strings.Contains(err.Error(), "prepare scene eligibility") || strings.Contains(err.Error(), `{"scenes":[]}`) {
|
|
t.Fatalf("New() error = %v, want bounded content-free scene failure", err)
|
|
}
|
|
if len(client.requests) != 0 {
|
|
t.Fatalf("LLM calls = %d, want none during failed construction", len(client.requests))
|
|
}
|
|
}
|
|
|
|
func TestExtractRejectsInvalidRequestsAndWrapsProviderFailures(t *testing.T) {
|
|
validReq := extractionRequest(t)
|
|
validExtractor := newExtractor(t, &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}})
|
|
var nilExtractor *Extractor
|
|
tests := []struct {
|
|
name string
|
|
extractor *Extractor
|
|
ctx context.Context
|
|
req contracts.TypedExtractionRequest
|
|
want string
|
|
}{
|
|
{name: "nil extractor", extractor: nilExtractor, ctx: context.Background(), req: validReq, want: "extractor"},
|
|
{name: "nil LLM client", extractor: &Extractor{}, ctx: context.Background(), req: validReq, want: "LLM client"},
|
|
{name: "wrapped preflight failure", extractor: validExtractor, ctx: context.Background(), req: mismatchedSourceInputRequest(validReq), want: "must match chunk"},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
_, err := test.extractor.Extract(test.ctx, test.req)
|
|
if err == nil || !strings.Contains(err.Error(), "dnd combat turns") || !strings.Contains(err.Error(), test.want) {
|
|
t.Fatalf("Extract() error = %v, want %q context", err, test.want)
|
|
}
|
|
})
|
|
}
|
|
_, err := newExtractor(t, &fakeCombatTurnsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), extractionRequest(t))
|
|
if err == nil || !strings.Contains(err.Error(), "dnd combat turns") || !strings.Contains(err.Error(), "provider unavailable") {
|
|
t.Fatalf("provider error = %v, want contextual wrapped error", err)
|
|
}
|
|
}
|
|
|
|
func TestExtractorManifestMetadataAndFingerprints(t *testing.T) {
|
|
extractor := newExtractor(t, &fakeCombatTurnsLLMClient{})
|
|
metadata := extractor.ManifestMetadata()
|
|
for key, want := range map[string]string{
|
|
"prompt_id": PromptID, "prompt_version": SchemaVersion, "mapping_policy": mappingPolicy,
|
|
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
|
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
|
|
} {
|
|
if metadata[key] != want {
|
|
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
|
}
|
|
}
|
|
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
|
|
value, ok := metadata[key].(string)
|
|
if !ok || !strings.HasPrefix(value, "sha256:") {
|
|
t.Fatalf("metadata[%q] = %#v, want digest", key, metadata[key])
|
|
}
|
|
}
|
|
wantNames := map[string]struct{}{"prompt": {}, "response_schema": {}, "mapping_policy": {}, "scene_gate_policy": {}, "npc_registry": {}, "scene_eligibility": {}}
|
|
for _, fingerprint := range extractor.CheckpointFingerprints() {
|
|
if _, ok := wantNames[fingerprint.Name]; !ok {
|
|
t.Fatalf("unexpected fingerprint = %#v", fingerprint)
|
|
}
|
|
delete(wantNames, fingerprint.Name)
|
|
}
|
|
if len(wantNames) != 0 {
|
|
t.Fatalf("missing fingerprints = %#v", wantNames)
|
|
}
|
|
}
|
|
|
|
func TestModuleSpecAndRegistration(t *testing.T) {
|
|
wantSlots := referenceSlots()
|
|
got := ModuleSpec()
|
|
if got.Key != Key || got.Stage != pipeline.StageExtract || got.ArtifactKind != dnd.CombatTurnListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.combat_turns"}) || !reflect.DeepEqual(got.ReferenceSlots, wantSlots) {
|
|
t.Fatalf("ModuleSpec() = %#v, want combat extractor contract", got)
|
|
}
|
|
got.Requires[0] = "changed"
|
|
got.Provides[0] = "changed"
|
|
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
|
|
again := ModuleSpec()
|
|
if again.Requires[0] == "changed" || again.Provides[0] == "changed" || again.ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" {
|
|
t.Fatalf("ModuleSpec() returned shared mutable values: %#v", again)
|
|
}
|
|
registry := pipeline.NewExtractorRegistry()
|
|
if err := Register(registry); err != nil {
|
|
t.Fatalf("Register() error = %v", err)
|
|
}
|
|
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
|
t.Fatal("DecodeOptions() accepted unknown option")
|
|
}
|
|
if _, ok := registry.Spec(Key); !ok {
|
|
t.Fatalf("registry missing spec for %q", Key)
|
|
}
|
|
}
|
|
|
|
func extractionRequest(t *testing.T) contracts.TypedExtractionRequest {
|
|
t.Helper()
|
|
doc := combatSourceDocument()
|
|
chunk := &source.Chunk{
|
|
ID: "session-alpha:chunk:0",
|
|
SourceID: doc.ID,
|
|
Index: 0,
|
|
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 2, EndUnitID: 10},
|
|
Content: []byte(`{"units":[10,2]}`),
|
|
MediaType: "application/json",
|
|
Units: append([]source.SourceUnit(nil), doc.Units...),
|
|
}
|
|
request := contracts.TypedExtractionRequest{
|
|
Source: doc,
|
|
Chunk: chunk,
|
|
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
|
|
SessionID: "session-123",
|
|
LLMProfile: "profile-combat",
|
|
}
|
|
request.References = sceneReferences(t, chunk, dnd.SceneKindCombat)
|
|
return request
|
|
}
|
|
|
|
func combatSourceDocument() *source.SourceDocument {
|
|
return &source.SourceDocument{
|
|
ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source",
|
|
Units: []source.SourceUnit{
|
|
{ID: 10, Kind: "transcript_segment", Text: "Aria reacts and strikes the ogre.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 10, EndUnitID: 10}},
|
|
{ID: 2, Kind: "transcript_segment", Text: "Borin retreats from the ogre.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}},
|
|
},
|
|
}
|
|
}
|
|
|
|
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
|
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"other":true}`), "sha256:other", "")
|
|
return req
|
|
}
|
|
|
|
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
|
t.Helper()
|
|
extractor, err := New(client, Options{}, references...)
|
|
if err != nil {
|
|
t.Fatalf("New() error = %v", err)
|
|
}
|
|
return extractor
|
|
}
|
|
|
|
func npcRegistryJSON(t *testing.T) []byte {
|
|
t.Helper()
|
|
value := dnd.NPCRegistry{NPCs: []dnd.NPC{{ID: identity.DeriveID("Mira Thorn"), Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: 1, EndUnitID: 1}}}}}
|
|
content, err := npccodec.New().Encode(value)
|
|
if err != nil {
|
|
t.Fatalf("encode NPC registry: %v", err)
|
|
}
|
|
return content
|
|
}
|
|
|
|
func sceneReferences(t *testing.T, chunk *source.Chunk, kind dnd.SceneKind) contracts.ReferenceSet {
|
|
t.Helper()
|
|
return sceneListReferences(t, sceneDescription(chunk, kind))
|
|
}
|
|
|
|
func sceneListReferences(t *testing.T, scenes ...dnd.SceneDescription) contracts.ReferenceSet {
|
|
t.Helper()
|
|
content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: scenes})
|
|
if err != nil {
|
|
t.Fatalf("encode scene descriptions: %v", err)
|
|
}
|
|
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
|
SceneDescriptionReferenceSlot: {
|
|
Slot: contracts.ReferenceSlot{Name: SceneDescriptionReferenceSlot},
|
|
Items: []contracts.ReferenceItem{{SlotName: SceneDescriptionReferenceSlot, MediaType: scenecodec.MediaType, Content: content}},
|
|
},
|
|
}}
|
|
}
|
|
|
|
func sceneDescription(chunk *source.Chunk, kind dnd.SceneKind) dnd.SceneDescription {
|
|
return dnd.SceneDescription{
|
|
ID: chunk.ID,
|
|
SourceRef: chunk.Ref,
|
|
Kind: kind,
|
|
Title: "Combat eligibility",
|
|
Summary: "A valid scene description for combat extraction.",
|
|
}
|
|
}
|
|
|
|
func fingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name string) string {
|
|
t.Helper()
|
|
for _, fingerprint := range fingerprints {
|
|
if fingerprint.Name == name {
|
|
return fingerprint.Value
|
|
}
|
|
}
|
|
t.Fatalf("fingerprints = %#v, missing %q", fingerprints, name)
|
|
return ""
|
|
}
|
|
|
|
type fakeCombatTurnsLLMClient struct {
|
|
response extractionResponse
|
|
err error
|
|
requests []contracts.StructuredCompletionRequest
|
|
}
|
|
|
|
func (client *fakeCombatTurnsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
|
if client.err != nil {
|
|
return contracts.StructuredCompletionResponse{}, client.err
|
|
}
|
|
target, ok := out.(*extractionResponse)
|
|
if !ok {
|
|
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
|
}
|
|
*target = client.response
|
|
content, err := json.Marshal(client.response)
|
|
if err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
return contracts.StructuredCompletionResponse{Content: content}, nil
|
|
}
|
|
|
|
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
|
req.Inputs = req.Inputs.Clone()
|
|
if len(req.Vars) == 0 {
|
|
req.Vars = nil
|
|
return req
|
|
}
|
|
vars := make(map[string]any, len(req.Vars))
|
|
for key, value := range req.Vars {
|
|
vars[key] = value
|
|
}
|
|
req.Vars = vars
|
|
return req
|
|
}
|