349 lines
16 KiB
Go
349 lines
16 KiB
Go
package cli
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
|
)
|
|
|
|
func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
|
|
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
|
if err != nil {
|
|
t.Fatalf("resolve production configuration: %v", err)
|
|
}
|
|
overlayPath := filepath.Join(t.TempDir(), "catalog.json")
|
|
resolved := effective.ResolvedPipeline
|
|
bindings := resolved.ArtifactLanes[0].ExtractReferences.Bindings
|
|
catalogBindingIndex := -1
|
|
for index, binding := range bindings {
|
|
if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
|
catalogBindingIndex = index
|
|
break
|
|
}
|
|
}
|
|
if catalogBindingIndex < 0 {
|
|
t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings)
|
|
}
|
|
resolved.ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath
|
|
|
|
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
|
if err != nil {
|
|
t.Fatalf("materialize first catalog: %v", err)
|
|
}
|
|
identityA := catalogCheckpointIdentity(t, materializedA)
|
|
metadataA := catalogExtractorMetadata(t, materializedA)
|
|
referenceA := catalogReference(t, materializedA)
|
|
|
|
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
|
if err != nil {
|
|
t.Fatalf("materialize reordered catalog: %v", err)
|
|
}
|
|
identityB := catalogCheckpointIdentity(t, materializedB)
|
|
metadataB := catalogExtractorMetadata(t, materializedB)
|
|
referenceB := catalogReference(t, materializedB)
|
|
|
|
if identityA.Digest == identityB.Digest {
|
|
t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest)
|
|
}
|
|
if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI {
|
|
t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB)
|
|
}
|
|
digestA, ok := metadataA["catalog_digest"].(string)
|
|
if !ok {
|
|
t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA)
|
|
}
|
|
digestB, ok := metadataB["catalog_digest"].(string)
|
|
if !ok || digestA != digestB {
|
|
t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB)
|
|
}
|
|
if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) {
|
|
t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want)
|
|
}
|
|
}
|
|
|
|
func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) {
|
|
base := string(readRepositoryFile(t, "examples", "dnd-spells-production.config.yml"))
|
|
changed := strings.Replace(base, "./dnd-spells-catalog.json", "./alternate-spell-catalog.json", 1)
|
|
if changed == base {
|
|
t.Fatal("production configuration did not contain the maintained catalog binding")
|
|
}
|
|
root := t.TempDir()
|
|
firstPath := filepath.Join(root, "first.yml")
|
|
secondPath := filepath.Join(root, "second.yml")
|
|
if err := os.WriteFile(firstPath, []byte(base), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(secondPath, []byte(changed), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
components := productionTestComponents(t)
|
|
first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
|
if err != nil {
|
|
t.Fatalf("resolve first configuration: %v", err)
|
|
}
|
|
second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
|
if err != nil {
|
|
t.Fatalf("resolve changed configuration: %v", err)
|
|
}
|
|
if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest {
|
|
t.Fatalf("resolved pipeline digest = %q for different catalog bindings, want change", first.ResolvedPipeline.Digest)
|
|
}
|
|
}
|
|
|
|
func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
|
|
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fingerprints := prepared.CheckpointFingerprints()
|
|
if len(fingerprints) != 2 || fingerprints[0].Value != fingerprints[1].Value {
|
|
t.Fatalf("prepared fingerprints = %#v, want matching extractor and validator catalog identities", fingerprints)
|
|
}
|
|
|
|
identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity {
|
|
identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
|
Pipeline: materialized,
|
|
InputKey: materialized.Input.Module,
|
|
RawInputDigest: "sha256:unchanged-input",
|
|
References: pipeline.ReferenceProvenance(materialized),
|
|
ProvenanceFingerprints: checkpointIdentityFingerprints(values),
|
|
})
|
|
if identityErr != nil {
|
|
t.Fatal(identityErr)
|
|
}
|
|
return identity
|
|
}
|
|
first := identityFor(fingerprints)
|
|
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
|
|
changed[0].Value = "sha256:changed-effective-catalog"
|
|
second := identityFor(changed)
|
|
if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) {
|
|
t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second)
|
|
}
|
|
}
|
|
|
|
func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
|
|
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fingerprints := prepared.CheckpointFingerprints()
|
|
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
|
|
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"}
|
|
doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
|
doc.Digest, err = source.DigestDocument(&doc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, []byte("same input"), nil, nil, "", "", true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused {
|
|
t.Fatalf("same fingerprint decision = %#v, want reuse", decision)
|
|
}
|
|
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
|
|
changed[0].Value = "sha256:changed-effective-catalog"
|
|
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, []byte("same input"), nil, nil, "", "", true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused {
|
|
t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision)
|
|
}
|
|
}
|
|
|
|
func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
|
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
|
fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}
|
|
options := productionRunOptions(t, fake)
|
|
var stdout, stderr strings.Builder
|
|
code := RunWithOptions([]string{
|
|
"run", "dnd-session",
|
|
"--config", repositoryPath("examples", "dnd-spells-production.config.yml"),
|
|
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
|
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
|
}, &stdout, &stderr, options)
|
|
if code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
runRoot := filepath.Join(outputRoot, productionRunID)
|
|
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
|
|
if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 {
|
|
t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest)
|
|
}
|
|
lane := manifest.ArtifactLanes[0]
|
|
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata)
|
|
}
|
|
if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") {
|
|
t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata)
|
|
}
|
|
if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) {
|
|
t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got)
|
|
}
|
|
|
|
var catalogProvenance *artifacts.ReferenceProvenance
|
|
for index := range manifest.References {
|
|
reference := &manifest.References[index]
|
|
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
|
catalogProvenance = reference
|
|
break
|
|
}
|
|
}
|
|
if catalogProvenance == nil {
|
|
t.Fatalf("manifest references = %#v, want spell catalog provenance", manifest.References)
|
|
}
|
|
overlayBytes := readRepositoryFile(t, "examples", "dnd-spells-catalog.json")
|
|
if catalogProvenance.Stage != "extract" || catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spells-catalog.json") {
|
|
t.Fatalf("catalog provenance = %#v, want extract origin, media, size, and raw digest", catalogProvenance)
|
|
}
|
|
manifestBytes, err := json.Marshal(manifest)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} {
|
|
if strings.Contains(string(manifestBytes), leaked) {
|
|
t.Fatalf("manifest leaked overlay content %q", leaked)
|
|
}
|
|
}
|
|
|
|
requests := fake.requestsFor(spells.PromptID)
|
|
if len(requests) != 1 {
|
|
t.Fatalf("spell requests = %d, want one", len(requests))
|
|
}
|
|
catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
|
if !ok || !strings.Contains(string(catalogInput.Content), "Aegis of Emberfall") || strings.Contains(string(catalogInput.Content), "Emberfall Aegis") {
|
|
t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name without alias", catalogInput)
|
|
}
|
|
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json"))
|
|
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
|
|
t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact)
|
|
}
|
|
rejected := readProductionJSON[struct {
|
|
Rejected []json.RawMessage `json:"rejected"`
|
|
}](t, filepath.Join(runRoot, "rejected.json"))
|
|
if len(rejected.Rejected) != 0 {
|
|
t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected)
|
|
}
|
|
}
|
|
|
|
func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity {
|
|
t.Helper()
|
|
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
|
Pipeline: resolved,
|
|
InputKey: resolved.Input.Module,
|
|
RawInputDigest: "sha256:catalog-test-input",
|
|
References: pipeline.ReferenceProvenance(resolved),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create checkpoint identity: %v", err)
|
|
}
|
|
return identity
|
|
}
|
|
|
|
func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
|
|
t.Helper()
|
|
lane := resolved.ArtifactLanes[0]
|
|
extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet)
|
|
if err != nil {
|
|
t.Fatalf("construct extractor: %v", err)
|
|
}
|
|
return extractor.ManifestMetadata()
|
|
}
|
|
|
|
func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance {
|
|
t.Helper()
|
|
for _, reference := range pipeline.ReferenceProvenance(resolved) {
|
|
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" {
|
|
return reference
|
|
}
|
|
}
|
|
t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved))
|
|
return artifacts.ReferenceProvenance{}
|
|
}
|
|
|
|
func stringValue(value any) string {
|
|
result, _ := value.(string)
|
|
return result
|
|
}
|
|
|
|
func stringValues(value any) []string {
|
|
raw, err := json.Marshal(value)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var values []string
|
|
if err := json.Unmarshal(raw, &values); err != nil {
|
|
return nil
|
|
}
|
|
return values
|
|
}
|
|
|
|
func digestBytes(value []byte) string {
|
|
sum := sha256.Sum256(value)
|
|
return "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
const reorderedOverlayA = `{
|
|
"schema_version": "notarius.dnd.spell-catalog-overlay.v1",
|
|
"catalogs": [
|
|
{"id":"campaign.a","ruleset":"dnd-5e-2014","source":{"title":"Campaign A"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]},
|
|
{"id":"campaign.b","ruleset":"dnd-5e-2014","source":{"title":"Campaign B"},"spells":[{"name":"Cinder Veil","aliases":["Veil of Cinder","Cinder Shroud"]}]}
|
|
]
|
|
}`
|
|
|
|
const reorderedOverlayB = `{"catalogs":[{"spells":[{"aliases":["Cinder Shroud","Veil of Cinder"],"name":"Cinder Veil"}],"source":{"title":"Campaign B"},"ruleset":"dnd-5e-2014","id":"campaign.b"},{"spells":[{"aliases":["Emberfall Aegis"],"name":"Aegis of Emberfall"}],"source":{"title":"Campaign A"},"ruleset":"dnd-5e-2014","id":"campaign.a"}],"schema_version":"notarius.dnd.spell-catalog-overlay.v1"}`
|