Verify spell catalog provenance and checkpoint identity
This commit is contained in:
@@ -83,6 +83,13 @@ func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
||||
if lane.ID != "spells" || lane.Extractor != "dnd/spells" || lane.Merger != "appendorder" || lane.Normalizer != "noop" {
|
||||
t.Fatalf("manifest lane = %#v, want production spells composition", lane)
|
||||
}
|
||||
if len(manifest.References) != 0 {
|
||||
t.Fatalf("base-only manifest references = %#v, want no overlay provenance", manifest.References)
|
||||
}
|
||||
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
||||
if !ok || len(stringValues(extractorMetadata["catalog_overlay_ids"])) != 0 {
|
||||
t.Fatalf("base-only extractor metadata = %#v, want no overlay IDs", lane.Metadata)
|
||||
}
|
||||
|
||||
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, entry.File))
|
||||
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Cure Wounds" || artifact.SpellCasts[0].SourceRefs[0].SourceID != "session-alpha" {
|
||||
|
||||
@@ -455,8 +455,9 @@ func readProductionJSON[T any](t *testing.T, path string) T {
|
||||
}
|
||||
|
||||
type productionFakeLLMClient struct {
|
||||
mu sync.Mutex
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
mu sync.Mutex
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
spellResponse string
|
||||
}
|
||||
|
||||
func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
@@ -468,7 +469,11 @@ func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, r
|
||||
case scenes.PromptID:
|
||||
content = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Opening scene","primary_mode":"Narrative","main_participants":["Aria"],"summary":"The session opens.","boundary_note":"The opening covers the available transcript.","boundary_confidence":"High"}],"boundary_caveats":["The opening boundary is inferred from the short transcript."]}`)
|
||||
case spells.PromptID:
|
||||
content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Heals an injured ally.","narrative_description":"Aria restores the fighter after the fight.","source_refs":[{"source_id":"session-alpha","start_unit_id":1,"end_unit_id":1}]}]}`)
|
||||
if client.spellResponse != "" {
|
||||
content = []byte(client.spellResponse)
|
||||
} else {
|
||||
content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Heals an injured ally.","narrative_description":"Aria restores the fighter after the fight.","source_refs":[{"source_id":"session-alpha","start_unit_id":1,"end_unit_id":1}]}]}`)
|
||||
}
|
||||
default:
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
|
||||
}
|
||||
|
||||
255
internal/cli/spell_catalog_identity_contract_test.go
Normal file
255
internal/cli/spell_catalog_identity_contract_test.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||
)
|
||||
|
||||
func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
|
||||
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve production configuration: %v", err)
|
||||
}
|
||||
overlayPath := filepath.Join(t.TempDir(), "catalog.json")
|
||||
resolved := effective.ResolvedPipeline
|
||||
bindings := resolved.ArtifactLanes[0].ExtractReferences.Bindings
|
||||
catalogBindingIndex := -1
|
||||
for index, binding := range bindings {
|
||||
if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
||||
catalogBindingIndex = index
|
||||
break
|
||||
}
|
||||
}
|
||||
if catalogBindingIndex < 0 {
|
||||
t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings)
|
||||
}
|
||||
resolved.ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath
|
||||
|
||||
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize first catalog: %v", err)
|
||||
}
|
||||
identityA := catalogCheckpointIdentity(t, materializedA)
|
||||
metadataA := catalogExtractorMetadata(t, materializedA)
|
||||
referenceA := catalogReference(t, materializedA)
|
||||
|
||||
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize reordered catalog: %v", err)
|
||||
}
|
||||
identityB := catalogCheckpointIdentity(t, materializedB)
|
||||
metadataB := catalogExtractorMetadata(t, materializedB)
|
||||
referenceB := catalogReference(t, materializedB)
|
||||
|
||||
if identityA.Digest == identityB.Digest {
|
||||
t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest)
|
||||
}
|
||||
if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI {
|
||||
t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB)
|
||||
}
|
||||
digestA, ok := metadataA["catalog_digest"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA)
|
||||
}
|
||||
digestB, ok := metadataB["catalog_digest"].(string)
|
||||
if !ok || digestA != digestB {
|
||||
t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB)
|
||||
}
|
||||
if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) {
|
||||
t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) {
|
||||
base := string(readRepositoryFile(t, "examples", "dnd-spells-production.config.yml"))
|
||||
changed := strings.Replace(base, "./dnd-spells-catalog.json", "./alternate-spell-catalog.json", 1)
|
||||
if changed == base {
|
||||
t.Fatal("production configuration did not contain the maintained catalog binding")
|
||||
}
|
||||
root := t.TempDir()
|
||||
firstPath := filepath.Join(root, "first.yml")
|
||||
secondPath := filepath.Join(root, "second.yml")
|
||||
if err := os.WriteFile(firstPath, []byte(base), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(secondPath, []byte(changed), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
components := productionTestComponents(t)
|
||||
first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve first configuration: %v", err)
|
||||
}
|
||||
second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve changed configuration: %v", err)
|
||||
}
|
||||
if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest {
|
||||
t.Fatalf("resolved pipeline digest = %q for different catalog bindings, want change", first.ResolvedPipeline.Digest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}
|
||||
options := productionRunOptions(t, fake)
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session",
|
||||
"--config", repositoryPath("examples", "dnd-spells-production.config.yml"),
|
||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
||||
}, &stdout, &stderr, options)
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
runRoot := filepath.Join(outputRoot, productionRunID)
|
||||
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
|
||||
if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 {
|
||||
t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest)
|
||||
}
|
||||
lane := manifest.ArtifactLanes[0]
|
||||
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata)
|
||||
}
|
||||
if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") {
|
||||
t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata)
|
||||
}
|
||||
if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) {
|
||||
t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got)
|
||||
}
|
||||
|
||||
var catalogProvenance *artifacts.ReferenceProvenance
|
||||
for index := range manifest.References {
|
||||
reference := &manifest.References[index]
|
||||
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
||||
catalogProvenance = reference
|
||||
break
|
||||
}
|
||||
}
|
||||
if catalogProvenance == nil {
|
||||
t.Fatalf("manifest references = %#v, want spell catalog provenance", manifest.References)
|
||||
}
|
||||
overlayBytes := readRepositoryFile(t, "examples", "dnd-spells-catalog.json")
|
||||
if catalogProvenance.Stage != "extract" || catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spells-catalog.json") {
|
||||
t.Fatalf("catalog provenance = %#v, want extract origin, media, size, and raw digest", catalogProvenance)
|
||||
}
|
||||
manifestBytes, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} {
|
||||
if strings.Contains(string(manifestBytes), leaked) {
|
||||
t.Fatalf("manifest leaked overlay content %q", leaked)
|
||||
}
|
||||
}
|
||||
|
||||
requests := fake.requestsFor(spells.PromptID)
|
||||
if len(requests) != 1 {
|
||||
t.Fatalf("spell requests = %d, want one", len(requests))
|
||||
}
|
||||
catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
||||
if !ok || !strings.Contains(string(catalogInput.Content), "Aegis of Emberfall") || strings.Contains(string(catalogInput.Content), "Emberfall Aegis") {
|
||||
t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name without alias", catalogInput)
|
||||
}
|
||||
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json"))
|
||||
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
|
||||
t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact)
|
||||
}
|
||||
rejected := readProductionJSON[struct {
|
||||
Rejected []json.RawMessage `json:"rejected"`
|
||||
}](t, filepath.Join(runRoot, "rejected.json"))
|
||||
if len(rejected.Rejected) != 0 {
|
||||
t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity {
|
||||
t.Helper()
|
||||
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
||||
Pipeline: resolved,
|
||||
InputKey: resolved.Input.Module,
|
||||
RawInputDigest: "sha256:catalog-test-input",
|
||||
References: pipeline.ReferenceProvenance(resolved),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create checkpoint identity: %v", err)
|
||||
}
|
||||
return identity
|
||||
}
|
||||
|
||||
func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
|
||||
t.Helper()
|
||||
lane := resolved.ArtifactLanes[0]
|
||||
extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet)
|
||||
if err != nil {
|
||||
t.Fatalf("construct extractor: %v", err)
|
||||
}
|
||||
return extractor.ManifestMetadata()
|
||||
}
|
||||
|
||||
func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance {
|
||||
t.Helper()
|
||||
for _, reference := range pipeline.ReferenceProvenance(resolved) {
|
||||
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" {
|
||||
return reference
|
||||
}
|
||||
}
|
||||
t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved))
|
||||
return artifacts.ReferenceProvenance{}
|
||||
}
|
||||
|
||||
func stringValue(value any) string {
|
||||
result, _ := value.(string)
|
||||
return result
|
||||
}
|
||||
|
||||
func stringValues(value any) []string {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var values []string
|
||||
if err := json.Unmarshal(raw, &values); err != nil {
|
||||
return nil
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func digestBytes(value []byte) string {
|
||||
sum := sha256.Sum256(value)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
const reorderedOverlayA = `{
|
||||
"schema_version": "notarius.dnd.spell-catalog-overlay.v1",
|
||||
"catalogs": [
|
||||
{"id":"campaign.a","ruleset":"dnd-5e-2014","source":{"title":"Campaign A"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]},
|
||||
{"id":"campaign.b","ruleset":"dnd-5e-2014","source":{"title":"Campaign B"},"spells":[{"name":"Cinder Veil","aliases":["Veil of Cinder","Cinder Shroud"]}]}
|
||||
]
|
||||
}`
|
||||
|
||||
const reorderedOverlayB = `{"catalogs":[{"spells":[{"aliases":["Cinder Shroud","Veil of Cinder"],"name":"Cinder Veil"}],"source":{"title":"Campaign B"},"ruleset":"dnd-5e-2014","id":"campaign.b"},{"spells":[{"aliases":["Emberfall Aegis"],"name":"Aegis of Emberfall"}],"source":{"title":"Campaign A"},"ruleset":"dnd-5e-2014","id":"campaign.a"}],"schema_version":"notarius.dnd.spell-catalog-overlay.v1"}`
|
||||
Reference in New Issue
Block a user