491 lines
23 KiB
Go
491 lines
23 KiB
Go
package cli
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
|
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
|
)
|
|
|
|
func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
|
|
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
|
if err != nil {
|
|
t.Fatalf("resolve production configuration: %v", err)
|
|
}
|
|
overlayPath := filepath.Join(t.TempDir(), "catalog.json")
|
|
resolved := effective.ResolvedPipeline
|
|
bindings := resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings
|
|
catalogBindingIndex := -1
|
|
for index, binding := range bindings {
|
|
if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
|
catalogBindingIndex = index
|
|
break
|
|
}
|
|
}
|
|
if catalogBindingIndex < 0 {
|
|
t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings)
|
|
}
|
|
resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath
|
|
normalizeBindings := resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings
|
|
normalizeCatalogBindingIndex := -1
|
|
for index, binding := range normalizeBindings {
|
|
if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
|
normalizeCatalogBindingIndex = index
|
|
break
|
|
}
|
|
}
|
|
if normalizeCatalogBindingIndex < 0 {
|
|
t.Fatalf("normalize spell catalog bindings = %#v, want catalog binding", normalizeBindings)
|
|
}
|
|
resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings[normalizeCatalogBindingIndex].Source = overlayPath
|
|
|
|
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
|
if err != nil {
|
|
t.Fatalf("materialize first catalog: %v", err)
|
|
}
|
|
identityA := catalogCheckpointIdentity(t, materializedA)
|
|
metadataA := catalogExtractorMetadata(t, materializedA)
|
|
normalizerMetadataA := catalogNormalizerMetadata(t, materializedA)
|
|
referenceA := catalogReference(t, materializedA)
|
|
|
|
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
|
if err != nil {
|
|
t.Fatalf("materialize reordered catalog: %v", err)
|
|
}
|
|
identityB := catalogCheckpointIdentity(t, materializedB)
|
|
metadataB := catalogExtractorMetadata(t, materializedB)
|
|
normalizerMetadataB := catalogNormalizerMetadata(t, materializedB)
|
|
referenceB := catalogReference(t, materializedB)
|
|
|
|
if identityA.Digest == identityB.Digest {
|
|
t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest)
|
|
}
|
|
if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI {
|
|
t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB)
|
|
}
|
|
digestA, ok := metadataA["catalog_digest"].(string)
|
|
if !ok {
|
|
t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA)
|
|
}
|
|
digestB, ok := metadataB["catalog_digest"].(string)
|
|
if !ok || digestA != digestB {
|
|
t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB)
|
|
}
|
|
if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) {
|
|
t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want)
|
|
}
|
|
normalizerDigestA, ok := normalizerMetadataA["catalog_digest"].(string)
|
|
normalizerDigestB, okB := normalizerMetadataB["catalog_digest"].(string)
|
|
if !ok || !okB || normalizerDigestA != digestA || normalizerDigestB != digestB {
|
|
t.Fatalf("normalizer catalog digests = %#v and %#v, want extractor semantic digests %q and %q", normalizerMetadataA["catalog_digest"], normalizerMetadataB["catalog_digest"], digestA, digestB)
|
|
}
|
|
if got, want := normalizerMetadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(normalizerMetadataB["catalog_overlay_ids"], want) {
|
|
t.Fatalf("normalizer overlay IDs = %#v and %#v, want %#v", got, normalizerMetadataB["catalog_overlay_ids"], want)
|
|
}
|
|
}
|
|
|
|
func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) {
|
|
base := string(readRepositoryFile(t, "examples", "dnd-spells-production.config.yml"))
|
|
changed := strings.Replace(base, "./dnd-spells-catalog.json", "./alternate-spell-catalog.json", 1)
|
|
if changed == base {
|
|
t.Fatal("production configuration did not contain the maintained catalog binding")
|
|
}
|
|
root := t.TempDir()
|
|
firstPath := filepath.Join(root, "first.yml")
|
|
secondPath := filepath.Join(root, "second.yml")
|
|
if err := os.WriteFile(firstPath, []byte(base), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(secondPath, []byte(changed), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
components := productionTestComponents(t)
|
|
first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
|
if err != nil {
|
|
t.Fatalf("resolve first configuration: %v", err)
|
|
}
|
|
second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
|
if err != nil {
|
|
t.Fatalf("resolve changed configuration: %v", err)
|
|
}
|
|
if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest {
|
|
t.Fatalf("resolved pipeline digest = %q for different catalog bindings, want change", first.ResolvedPipeline.Digest)
|
|
}
|
|
}
|
|
|
|
func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
|
|
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fingerprints := prepared.CheckpointFingerprints()
|
|
wantNames := map[string]struct{}{
|
|
"extract:spells:" + spells.Key + ":effective_catalog": {},
|
|
"extract:spells:" + spells.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {},
|
|
"normalize:spells:" + spellnormalize.Key + ":effective_catalog": {},
|
|
"normalize:spells:" + spellnormalize.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {},
|
|
}
|
|
seen := make(map[string]string, len(fingerprints))
|
|
for _, fingerprint := range fingerprints {
|
|
if _, ok := wantNames[fingerprint.Name]; ok {
|
|
seen[fingerprint.Name] = fingerprint.Value
|
|
}
|
|
}
|
|
if len(seen) != len(wantNames) {
|
|
t.Fatalf("prepared fingerprints = %#v, want scoped extractor and normalize catalog identities", fingerprints)
|
|
}
|
|
var catalogDigest string
|
|
for name, value := range seen {
|
|
if catalogDigest == "" {
|
|
catalogDigest = value
|
|
} else if value != catalogDigest {
|
|
t.Fatalf("prepared fingerprint %q = %q, want shared semantic catalog digest %q", name, value, catalogDigest)
|
|
}
|
|
}
|
|
|
|
identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity {
|
|
identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
|
Pipeline: materialized,
|
|
InputKey: materialized.Input.Module,
|
|
RawInputDigest: "sha256:unchanged-input",
|
|
References: pipeline.ReferenceProvenance(materialized),
|
|
ProvenanceFingerprints: checkpointIdentityFingerprints(values),
|
|
})
|
|
if identityErr != nil {
|
|
t.Fatal(identityErr)
|
|
}
|
|
return identity
|
|
}
|
|
first := identityFor(fingerprints)
|
|
changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
|
|
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
|
|
second := identityFor(changed)
|
|
if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) {
|
|
t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second)
|
|
}
|
|
}
|
|
|
|
func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
|
|
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fingerprints := prepared.CheckpointFingerprints()
|
|
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
|
|
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"}
|
|
doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
|
doc.Digest, err = source.DigestDocument(&doc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
normalizeDependencies := []pipeline.CheckpointFingerprint{{Name: "artifact[0]", Value: "sha256:merged-artifact"}}
|
|
normalizeSchema := contracts.ArtifactSchema{ID: "notarius.dnd.spells", Name: "notarius_dnd_spells", Version: "v1"}
|
|
normalizeArtifact := pipeline.CheckpointArtifact{
|
|
LaneID: "spells", ModuleKey: spellnormalize.Key, SourceID: doc.ID,
|
|
SchemaDigest: contracts.DigestArtifactSchema(normalizeSchema),
|
|
Artifact: contracts.SerializedArtifact{
|
|
Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
|
|
},
|
|
}
|
|
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused {
|
|
t.Fatalf("same fingerprint decision = %#v, want reuse", decision)
|
|
}
|
|
if restored, decision := sameLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); !decision.Reused || string(restored.Output.Artifact.Content) != `{"spell_casts":[]}` {
|
|
t.Fatalf("same normalize checkpoint = %#v, decision=%#v, want reuse", restored, decision)
|
|
}
|
|
changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
|
|
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
|
|
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused {
|
|
t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision)
|
|
}
|
|
if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
|
t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision)
|
|
}
|
|
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.v2")
|
|
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName())
|
|
_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, []byte("same input"), nil, nil, "", "", true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, decision := mappingLoader.Source(materialized.Input.Module); decision.Reused {
|
|
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
|
|
}
|
|
}
|
|
|
|
func normalizeSpellCatalogFingerprintName() string {
|
|
return "normalize:spells:" + spellnormalize.Key + ":effective_catalog"
|
|
}
|
|
|
|
func extractSpellMappingFingerprintName() string {
|
|
return "extract:spells:" + spells.Key + ":mapping_policy"
|
|
}
|
|
|
|
func replaceCheckpointFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, value string) []pipeline.CheckpointFingerprint {
|
|
t.Helper()
|
|
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
|
|
matches := 0
|
|
for index := range changed {
|
|
if changed[index].Name == name {
|
|
changed[index].Value = value
|
|
matches++
|
|
}
|
|
}
|
|
if matches != 1 {
|
|
t.Fatalf("checkpoint fingerprints = %#v, want exactly one fingerprint named %q", fingerprints, name)
|
|
}
|
|
return changed
|
|
}
|
|
|
|
func assertOnlyCheckpointFingerprintChanged(t *testing.T, before, after []pipeline.CheckpointFingerprint, changedName string) {
|
|
t.Helper()
|
|
if len(before) != len(after) {
|
|
t.Fatalf("fingerprint lengths = %d and %d, want equal", len(before), len(after))
|
|
}
|
|
changes := 0
|
|
for index := range before {
|
|
if before[index].Name != after[index].Name {
|
|
t.Fatalf("fingerprint[%d] name changed from %q to %q", index, before[index].Name, after[index].Name)
|
|
}
|
|
if before[index].Value == after[index].Value {
|
|
continue
|
|
}
|
|
changes++
|
|
if before[index].Name != changedName {
|
|
t.Fatalf("fingerprint %q changed unexpectedly", before[index].Name)
|
|
}
|
|
}
|
|
if changes != 1 {
|
|
t.Fatalf("fingerprints changed %d values, want exactly %q", changes, changedName)
|
|
}
|
|
}
|
|
|
|
func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
|
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
|
fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}
|
|
options := productionRunOptions(t, fake)
|
|
var stdout, stderr strings.Builder
|
|
code := RunWithOptions([]string{
|
|
"run", "dnd-session",
|
|
"--config", repositoryPath("examples", "dnd-spells-production.config.yml"),
|
|
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
|
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
|
}, &stdout, &stderr, options)
|
|
if code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
runRoot := filepath.Join(outputRoot, productionRunID)
|
|
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
|
|
if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 {
|
|
t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest)
|
|
}
|
|
lane := manifest.ArtifactLanes[0]
|
|
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata)
|
|
}
|
|
if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") {
|
|
t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata)
|
|
}
|
|
if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) {
|
|
t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got)
|
|
}
|
|
normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("lane metadata = %#v, want normalizer metadata", lane.Metadata)
|
|
}
|
|
if normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), []string{"notarius.example-campaign"}) {
|
|
t.Fatalf("normalizer catalog metadata = %#v, want base ID, semantic digest, and overlay IDs", normalizerMetadata)
|
|
}
|
|
if normalizerMetadata["catalog_digest"] != extractorMetadata["catalog_digest"] || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), stringValues(extractorMetadata["catalog_overlay_ids"])) {
|
|
t.Fatalf("extractor metadata = %#v, normalizer metadata = %#v, want shared catalog identity", extractorMetadata, normalizerMetadata)
|
|
}
|
|
|
|
var catalogProvenances []artifacts.ReferenceProvenance
|
|
for index := range manifest.References {
|
|
reference := &manifest.References[index]
|
|
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
|
catalogProvenances = append(catalogProvenances, *reference)
|
|
}
|
|
}
|
|
if len(catalogProvenances) != 2 {
|
|
t.Fatalf("manifest references = %#v, want independently materialized extract and normalize catalog provenance", manifest.References)
|
|
}
|
|
overlayBytes := readRepositoryFile(t, "examples", "dnd-spells-catalog.json")
|
|
for _, catalogProvenance := range catalogProvenances {
|
|
if catalogProvenance.Stage != "extract" && catalogProvenance.Stage != "normalize" {
|
|
t.Fatalf("catalog provenance = %#v, want extract or normalize scope", catalogProvenance)
|
|
}
|
|
if catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spells-catalog.json") {
|
|
t.Fatalf("catalog provenance = %#v, want raw overlay provenance in both scopes", catalogProvenance)
|
|
}
|
|
}
|
|
manifestBytes, err := json.Marshal(manifest)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} {
|
|
if strings.Contains(string(manifestBytes), leaked) {
|
|
t.Fatalf("manifest leaked overlay content %q", leaked)
|
|
}
|
|
}
|
|
|
|
requests := fake.requestsFor(spells.PromptID)
|
|
if len(requests) != 1 {
|
|
t.Fatalf("spell requests = %d, want one", len(requests))
|
|
}
|
|
catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
|
if !ok || !strings.Contains(string(catalogInput.Content), "Aegis of Emberfall") || strings.Contains(string(catalogInput.Content), "Emberfall Aegis") {
|
|
t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name without alias", catalogInput)
|
|
}
|
|
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json"))
|
|
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
|
|
t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact)
|
|
}
|
|
rejected := readProductionJSON[struct {
|
|
Rejected []json.RawMessage `json:"rejected"`
|
|
}](t, filepath.Join(runRoot, "rejected.json"))
|
|
if len(rejected.Rejected) != 0 {
|
|
t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected)
|
|
}
|
|
}
|
|
|
|
func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity {
|
|
t.Helper()
|
|
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
|
Pipeline: resolved,
|
|
InputKey: resolved.Input.Module,
|
|
RawInputDigest: "sha256:catalog-test-input",
|
|
References: pipeline.ReferenceProvenance(resolved),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create checkpoint identity: %v", err)
|
|
}
|
|
return identity
|
|
}
|
|
|
|
func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
|
|
t.Helper()
|
|
lane := resolved.Steps[0].ArtifactLanes[0]
|
|
extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet)
|
|
if err != nil {
|
|
t.Fatalf("construct extractor: %v", err)
|
|
}
|
|
return extractor.ManifestMetadata()
|
|
}
|
|
|
|
func catalogNormalizerMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
|
|
t.Helper()
|
|
lane := resolved.Steps[0].ArtifactLanes[0]
|
|
normalizer, err := spellnormalize.New(spellnormalize.Options{}, lane.NormalizeReferences.ReferenceSet)
|
|
if err != nil {
|
|
t.Fatalf("construct normalizer: %v", err)
|
|
}
|
|
return normalizer.ManifestMetadata()
|
|
}
|
|
|
|
func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance {
|
|
t.Helper()
|
|
for _, reference := range pipeline.ReferenceProvenance(resolved) {
|
|
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" {
|
|
return reference
|
|
}
|
|
}
|
|
t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved))
|
|
return artifacts.ReferenceProvenance{}
|
|
}
|
|
|
|
func stringValue(value any) string {
|
|
result, _ := value.(string)
|
|
return result
|
|
}
|
|
|
|
func stringValues(value any) []string {
|
|
raw, err := json.Marshal(value)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var values []string
|
|
if err := json.Unmarshal(raw, &values); err != nil {
|
|
return nil
|
|
}
|
|
return values
|
|
}
|
|
|
|
func digestBytes(value []byte) string {
|
|
sum := sha256.Sum256(value)
|
|
return "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
const reorderedOverlayA = `{
|
|
"schema_version": "notarius.dnd.spell-catalog-overlay.v1",
|
|
"catalogs": [
|
|
{"id":"campaign.a","ruleset":"dnd-5e-2014","source":{"title":"Campaign A"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]},
|
|
{"id":"campaign.b","ruleset":"dnd-5e-2014","source":{"title":"Campaign B"},"spells":[{"name":"Cinder Veil","aliases":["Veil of Cinder","Cinder Shroud"]}]}
|
|
]
|
|
}`
|
|
|
|
const reorderedOverlayB = `{"catalogs":[{"spells":[{"aliases":["Cinder Shroud","Veil of Cinder"],"name":"Cinder Veil"}],"source":{"title":"Campaign B"},"ruleset":"dnd-5e-2014","id":"campaign.b"},{"spells":[{"aliases":["Emberfall Aegis"],"name":"Aegis of Emberfall"}],"source":{"title":"Campaign A"},"ruleset":"dnd-5e-2014","id":"campaign.a"}],"schema_version":"notarius.dnd.spell-catalog-overlay.v1"}`
|