280 lines
13 KiB
Go
280 lines
13 KiB
Go
package spells
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"unicode/utf8"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/diagnostics"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
|
)
|
|
|
|
func TestResolveNPCRegistryUsesExactEmptyPromptWhenUnbound(t *testing.T) {
|
|
resolved, err := resolveNPCRegistry(contracts.ReferenceSet{})
|
|
if err != nil {
|
|
t.Fatalf("resolveNPCRegistry() error = %v, want nil", err)
|
|
}
|
|
if resolved.bound || resolved.digest != "" || resolved.count != 0 {
|
|
t.Fatalf("resolved unbound registry = %#v, want no semantic metadata", resolved)
|
|
}
|
|
if resolved.input.Name != NPCRegistryReferenceSlot || resolved.input.MediaType != "application/json" || resolved.input.Digest != "" || resolved.input.OriginURI != "" {
|
|
t.Fatalf("unbound prompt input metadata = %#v, want name/media type only", resolved.input)
|
|
}
|
|
if got := string(resolved.input.Content); got != `{"npcs":[]}` {
|
|
t.Fatalf("unbound prompt input = %q, want exact empty registry", got)
|
|
}
|
|
if resolved.input.SizeBytes != int64(len(`{"npcs":[]}`)) {
|
|
t.Fatalf("unbound prompt input size = %d, want %d", resolved.input.SizeBytes, len(`{"npcs":[]}`))
|
|
}
|
|
if metadata := newExtractor(t, &fakeSpellsLLMClient{}).ManifestMetadata(); metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil {
|
|
t.Fatalf("unbound extractor metadata = %#v, want no NPC registry fields", metadata)
|
|
}
|
|
fingerprints := newExtractor(t, &fakeSpellsLLMClient{}).CheckpointFingerprints()
|
|
for _, fingerprint := range fingerprints {
|
|
if fingerprint.Name == "npc_registry" {
|
|
t.Fatalf("unbound checkpoint fingerprints = %#v, want no NPC registry fingerprint", fingerprints)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResolveNPCRegistryCanonicalizesContentAndUsesSemanticDigest(t *testing.T) {
|
|
value := validNPCRegistryList()
|
|
canonical := encodeNPCRegistry(t, value)
|
|
raw := append([]byte(" \n"), canonical...)
|
|
raw = append(raw, []byte("\n ")...)
|
|
|
|
resolved, err := resolveNPCRegistry(npcRegistryReference(raw, "file:///another-session/npcs.json"))
|
|
if err != nil {
|
|
t.Fatalf("resolveNPCRegistry() error = %v, want nil", err)
|
|
}
|
|
if !resolved.bound || resolved.count != len(value.NPCs) {
|
|
t.Fatalf("resolved registry = %#v, want bound registry with %d NPC", resolved, len(value.NPCs))
|
|
}
|
|
if !bytes.Equal(resolved.input.Content, canonical) {
|
|
t.Fatalf("canonical prompt input = %s, want %s", resolved.input.Content, canonical)
|
|
}
|
|
if resolved.input.Digest != semanticNPCRegistryDigest(canonical) || resolved.digest != resolved.input.Digest {
|
|
t.Fatalf("semantic digest = %q/%q, want %q", resolved.input.Digest, resolved.digest, semanticNPCRegistryDigest(canonical))
|
|
}
|
|
if resolved.input.OriginURI != "" {
|
|
t.Fatalf("prompt input origin = %q, want no provenance path", resolved.input.OriginURI)
|
|
}
|
|
|
|
resolved.input.Content[0] = 'X'
|
|
again, err := resolveNPCRegistry(npcRegistryReference(raw, "file:///another-session/npcs.json"))
|
|
if err != nil {
|
|
t.Fatalf("second resolveNPCRegistry() error = %v, want nil", err)
|
|
}
|
|
if !bytes.Equal(again.input.Content, canonical) {
|
|
t.Fatalf("canonical content changed after caller mutation = %s, want %s", again.input.Content, canonical)
|
|
}
|
|
}
|
|
|
|
func TestResolveNPCRegistryRejectsInvalidBoundaryValues(t *testing.T) {
|
|
valid := validNPCRegistryList()
|
|
second := validNPCRegistryList().NPCs[0]
|
|
second.ID = identity.DeriveID("Captain Vale")
|
|
second.Name = "Captain Vale"
|
|
second.Aliases = []string{"The Greencloak"}
|
|
valueWithAliasCollision := dnd.NPCList{NPCs: []dnd.NPC{valid.NPCs[0], second}}
|
|
invalidID := valid
|
|
invalidID.NPCs[0].ID = "not-an-npc-id"
|
|
|
|
tests := []struct {
|
|
name string
|
|
reference contracts.ReferenceSet
|
|
wantError string
|
|
forbidden []string
|
|
}{
|
|
{name: "zero items", reference: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{NPCRegistryReferenceSlot: {Items: []contracts.ReferenceItem{}}}}, wantError: "exactly one"},
|
|
{name: "multiple", reference: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{NPCRegistryReferenceSlot: {Items: []contracts.ReferenceItem{{Content: []byte(`{"npcs":[]}`)}, {Content: []byte(`{"npcs":[]}`)}}}}}, wantError: "exactly one"},
|
|
{name: "wrong media type", reference: npcRegistryReferenceWithMedia([]byte(`{"npcs":[]}`), "text/plain"), wantError: "must be application/json"},
|
|
{name: "malformed JSON", reference: npcRegistryReference([]byte(`{"npcs":[],"MALFORMED_REGISTRY_SECRET":`), "file:///private.json"), wantError: "invalid approved NPC JSON", forbidden: []string{"MALFORMED_REGISTRY_SECRET"}},
|
|
{name: "unknown field", reference: npcRegistryReference([]byte(`{"npcs":[],"UNKNOWN_FIELD_SECRET":true}`), "file:///private.json"), wantError: "invalid approved NPC JSON", forbidden: []string{"UNKNOWN_FIELD_SECRET"}},
|
|
{name: "invalid ID", reference: npcRegistryReference(marshalNPCRegistry(t, invalidID), "file:///private.json"), wantError: "decode NPC registry"},
|
|
{name: "alias collision", reference: npcRegistryReference(encodeNPCRegistry(t, valueWithAliasCollision), "file:///private.json"), wantError: string(identity.IssueAliasOwnershipCollision)},
|
|
{name: "byte limit", reference: npcRegistryReference(bytes.Repeat([]byte("x"), NPCRegistryMaxBytes+1), "file:///private.json"), wantError: "limit"},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
_, err := resolveNPCRegistry(test.reference)
|
|
if err == nil || !strings.Contains(err.Error(), test.wantError) {
|
|
t.Fatalf("resolveNPCRegistry() error = %v, want %q", err, test.wantError)
|
|
}
|
|
for _, forbidden := range append(test.forbidden, "Mira Thorn", "The Greencloak", "private.json") {
|
|
if strings.Contains(err.Error(), forbidden) {
|
|
t.Fatalf("error leaked registry content or provenance %q: %v", forbidden, err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResolveNPCRegistryBoundsIdentityDiagnosticsWithoutContent(t *testing.T) {
|
|
const recordCount = 30
|
|
value := dnd.NPCList{NPCs: make([]dnd.NPC, recordCount)}
|
|
for index := range value.NPCs {
|
|
value.NPCs[index] = dnd.NPC{
|
|
ID: "npc:sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
|
Name: fmt.Sprintf("PRIVATE NPC %d", index),
|
|
Aliases: []string{"PRIVATE SHARED ALIAS"},
|
|
Description: "PRIVATE DESCRIPTION",
|
|
Relationships: []dnd.NPCRelationship{},
|
|
SourceRefs: []source.SourceRef{{SourceID: "private-source", StartUnitID: 1, EndUnitID: 1}},
|
|
}
|
|
}
|
|
issues := identity.ValidateList(value)
|
|
if len(issues) <= diagnostics.MaxIssues {
|
|
t.Fatalf("identity issues = %d, want more than display limit", len(issues))
|
|
}
|
|
|
|
_, err := resolveNPCRegistry(npcRegistryReference(marshalNPCRegistry(t, value), "file:///private-registry.json"))
|
|
if err == nil {
|
|
t.Fatal("resolveNPCRegistry() error = nil, want bounded identity rejection")
|
|
}
|
|
message := err.Error()
|
|
if !utf8.ValidString(message) || len([]byte(message)) > diagnostics.MaxMessageBytes {
|
|
t.Fatalf("identity error has invalid encoding or size: bytes=%d message=%q", len([]byte(message)), message)
|
|
}
|
|
wantOmitted := fmt.Sprintf("%d additional issue(s) omitted", len(issues)-diagnostics.MaxIssues)
|
|
if !strings.Contains(message, wantOmitted) {
|
|
t.Fatalf("identity error = %q, want %q", message, wantOmitted)
|
|
}
|
|
for _, forbidden := range []string{"PRIVATE NPC", "PRIVATE SHARED ALIAS", "PRIVATE DESCRIPTION", "private-source", "private-registry.json"} {
|
|
if strings.Contains(message, forbidden) {
|
|
t.Fatalf("identity error leaked %q: %s", forbidden, message)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNPCRegistryFingerprintIsSemanticAndDefensive(t *testing.T) {
|
|
value := validNPCRegistryList()
|
|
canonical := encodeNPCRegistry(t, value)
|
|
pretty, err := json.MarshalIndent(value, "", " ")
|
|
if err != nil {
|
|
t.Fatalf("MarshalIndent() error = %v", err)
|
|
}
|
|
first := newExtractor(t, &fakeSpellsLLMClient{}, npcRegistryReference(canonical, "file:///one.json"))
|
|
second := newExtractor(t, &fakeSpellsLLMClient{}, npcRegistryReference(pretty, "file:///two.json"))
|
|
firstFingerprints := checkpointFingerprintMap(first.CheckpointFingerprints())
|
|
secondFingerprints := checkpointFingerprintMap(second.CheckpointFingerprints())
|
|
if firstFingerprints["npc_registry"] == "" || firstFingerprints["npc_registry"] != secondFingerprints["npc_registry"] {
|
|
t.Fatalf("semantic NPC fingerprints = %#v and %#v, want same npc_registry value", firstFingerprints, secondFingerprints)
|
|
}
|
|
returned := first.CheckpointFingerprints()
|
|
returned[0].Name = "caller-mutated"
|
|
if first.CheckpointFingerprints()[0].Name == "caller-mutated" {
|
|
t.Fatal("CheckpointFingerprints() returned caller-mutable slice state")
|
|
}
|
|
|
|
changed := validNPCRegistryList()
|
|
changed.NPCs[0].Description = "A different description."
|
|
changedFingerprint := checkpointFingerprintMap(newExtractor(t, &fakeSpellsLLMClient{}, npcRegistryReference(encodeNPCRegistry(t, changed), "file:///three.json")).CheckpointFingerprints())
|
|
if changedFingerprint["npc_registry"] == firstFingerprints["npc_registry"] {
|
|
t.Fatalf("semantic NPC fingerprint did not change: %#v", changedFingerprint)
|
|
}
|
|
|
|
metadata := first.ManifestMetadata()
|
|
encoded, err := json.Marshal(map[string]any{"metadata": metadata, "fingerprints": firstFingerprints})
|
|
if err != nil {
|
|
t.Fatalf("marshal metadata: %v", err)
|
|
}
|
|
for _, forbidden := range []string{"Mira Thorn", "The Greencloak", "another-session", "one.json"} {
|
|
if strings.Contains(string(encoded), forbidden) {
|
|
t.Fatalf("metadata or fingerprints leaked %q: %s", forbidden, encoded)
|
|
}
|
|
}
|
|
if metadata["npc_registry_digest"] != firstFingerprints["npc_registry"] || metadata["npc_count"] != 1 {
|
|
t.Fatalf("NPC registry metadata = %#v, want digest and count only", metadata)
|
|
}
|
|
|
|
}
|
|
|
|
func TestExtractPassesCanonicalNPCRegistryToLLMWithoutProvenance(t *testing.T) {
|
|
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
|
|
canonical := encodeNPCRegistry(t, validNPCRegistryList())
|
|
extractor := newExtractor(t, client, npcRegistryReference(append([]byte("\n"), canonical...), "file:///npc-session.json"))
|
|
if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil {
|
|
t.Fatalf("Extract() error = %v, want nil", err)
|
|
}
|
|
input := client.requests[0].Inputs[NPCRegistryReferenceSlot]
|
|
if input.Name != NPCRegistryReferenceSlot || input.MediaType != "application/json" || input.Digest != semanticNPCRegistryDigest(canonical) || input.OriginURI != "" {
|
|
t.Fatalf("NPC prompt input metadata = %#v, want semantic metadata without provenance", input)
|
|
}
|
|
if !bytes.Equal(input.Content, canonical) {
|
|
t.Fatalf("NPC prompt input = %s, want canonical JSON %s", input.Content, canonical)
|
|
}
|
|
}
|
|
|
|
func validNPCRegistryList() dnd.NPCList {
|
|
return dnd.NPCList{NPCs: []dnd.NPC{{
|
|
ID: identity.DeriveID("Mira Thorn"),
|
|
Name: "Mira Thorn",
|
|
Aliases: []string{"The Greencloak"},
|
|
Description: "A guarded ranger who watches the northern road.",
|
|
Relationships: []dnd.NPCRelationship{{
|
|
Target: "Captain Vale", Relationship: "reports to",
|
|
}},
|
|
SourceRefs: []source.SourceRef{{SourceID: "npc-session", StartUnitID: 41, EndUnitID: 43}},
|
|
}}}
|
|
}
|
|
|
|
func encodeNPCRegistry(t *testing.T, value dnd.NPCList) []byte {
|
|
t.Helper()
|
|
content, err := npccodec.New().Encode(value)
|
|
if err != nil {
|
|
t.Fatalf("encode NPC registry: %v", err)
|
|
}
|
|
return content
|
|
}
|
|
|
|
func marshalNPCRegistry(t *testing.T, value dnd.NPCList) []byte {
|
|
t.Helper()
|
|
content, err := json.Marshal(value)
|
|
if err != nil {
|
|
t.Fatalf("marshal NPC registry: %v", err)
|
|
}
|
|
return content
|
|
}
|
|
|
|
func npcRegistryReference(content []byte, origin string) contracts.ReferenceSet {
|
|
references := npcRegistryReferenceWithMedia(content, "application/json; charset=utf-8")
|
|
item := references.Slots[NPCRegistryReferenceSlot].Items[0]
|
|
item.Origin.URI = origin
|
|
slot := references.Slots[NPCRegistryReferenceSlot]
|
|
slot.Items[0] = item
|
|
references.Slots[NPCRegistryReferenceSlot] = slot
|
|
return references
|
|
}
|
|
|
|
func npcRegistryReferenceWithMedia(content []byte, mediaType string) contracts.ReferenceSet {
|
|
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
|
NPCRegistryReferenceSlot: {
|
|
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot, AcceptedMediaTypes: []string{"application/json"}, MaxBytes: NPCRegistryMaxBytes},
|
|
Items: []contracts.ReferenceItem{{
|
|
SlotName: NPCRegistryReferenceSlot,
|
|
MediaType: mediaType,
|
|
Content: append([]byte(nil), content...),
|
|
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///npc-registry.json"},
|
|
}},
|
|
},
|
|
}}
|
|
}
|
|
|
|
func checkpointFingerprintMap(values []pipeline.CheckpointFingerprint) map[string]string {
|
|
result := make(map[string]string, len(values))
|
|
for _, value := range values {
|
|
result[value.Name] = value.Value
|
|
}
|
|
return result
|
|
}
|