Integrate extraction artifacts into analysis catalog
This commit is contained in:
222
internal/artifacts/extraction_catalog_test.go
Normal file
222
internal/artifacts/extraction_catalog_test.go
Normal file
@@ -0,0 +1,222 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
)
|
||||
|
||||
func TestArtifactCatalogRegistersExtractionArtifactsDeterministically(t *testing.T) {
|
||||
catalog := NewArtifactCatalog()
|
||||
if err := catalog.RegisterConfiguredArtifacts(map[string]ConfiguredArtifactDefinition{
|
||||
"summary": {Enabled: true},
|
||||
}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := catalog.RegisterExtractionArtifacts(map[string]ExtractionArtifactDefinition{
|
||||
"zeta": {LaneID: "zeta"},
|
||||
"summary": {LaneID: "summary"},
|
||||
"alpha": {LaneID: "alpha"},
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterExtractionArtifacts() error = %v", err)
|
||||
}
|
||||
|
||||
entries := catalog.ListExtraction()
|
||||
if len(entries) != 3 || entries[0].ExtractionKey != "alpha" || entries[1].ExtractionKey != "summary" || entries[2].ExtractionKey != "zeta" {
|
||||
t.Fatalf("ListExtraction() = %#v, want alpha, summary, zeta", entries)
|
||||
}
|
||||
extractionID, ok := catalog.SourceIDForExtractionKey("summary")
|
||||
if !ok || extractionID != "narratio.extraction.summary" {
|
||||
t.Fatalf("SourceIDForExtractionKey(summary) = %q, %v", extractionID, ok)
|
||||
}
|
||||
configuredID, _ := catalog.SourceIDForConfiguredKey("summary")
|
||||
if configuredID == extractionID {
|
||||
t.Fatalf("configured and extraction source families collided at %q", extractionID)
|
||||
}
|
||||
entry, ok := catalog.Lookup(extractionID)
|
||||
if !ok || entry.ProducerStage != "extract" || entry.OutputKind != "notarius_lane" || !entry.Planned {
|
||||
t.Fatalf("Lookup(%q) = %#v, %v", extractionID, entry, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactCatalogRejectsDuplicateExtractionRegistration(t *testing.T) {
|
||||
catalog := NewArtifactCatalog()
|
||||
definitions := map[string]ExtractionArtifactDefinition{"summary": {LaneID: "summary"}}
|
||||
if err := catalog.RegisterExtractionArtifacts(definitions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := catalog.RegisterExtractionArtifacts(definitions); err == nil {
|
||||
t.Fatal("second RegisterExtractionArtifacts() error = nil, want collision error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHydrateExtractionArtifactsAcceptsOnlyCompleteCurrentBundle(t *testing.T) {
|
||||
paths, currentManifest, definitions := validExtractionCatalogFixture(t)
|
||||
catalog := registeredExtractionCatalog(t, definitions)
|
||||
catalog.HydrateExtractionArtifacts(paths, currentManifest, definitions)
|
||||
|
||||
entry, ok := catalog.Lookup(ExtractionArtifactSourceID("encounters"))
|
||||
if !ok || !entry.Available {
|
||||
t.Fatalf("hydrated entry = %#v, %v; want available", entry, ok)
|
||||
}
|
||||
if entry.Provenance != ArtifactProvenanceCurrentExtractManifest || entry.ProducerRunID != "extract-run-1" {
|
||||
t.Fatalf("hydrated provenance = %#v", entry)
|
||||
}
|
||||
resolved, err := ResolveSessionArtifactWithCatalog(paths, currentManifest, entry.SourceID, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveSessionArtifactWithCatalog() error = %v", err)
|
||||
}
|
||||
if resolved.Path != entry.Path || resolved.ProducerRunID != "extract-run-1" {
|
||||
t.Fatalf("resolved = %#v", resolved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHydrateExtractionArtifactsRejectsUntrustedManifestState(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(t *testing.T, paths SessionPaths, m *manifest.Manifest)
|
||||
}{
|
||||
{name: "missing stage", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) { delete(m.Stages, "extract") }},
|
||||
{name: "skipped", mutate: setExtractionStatus(manifest.StatusSkipped)},
|
||||
{name: "failed", mutate: setExtractionStatus(manifest.StatusFailed)},
|
||||
{name: "stale", mutate: setExtractionStatus(manifest.StatusStale)},
|
||||
{name: "interrupted", mutate: setExtractionStatus(manifest.StatusInterrupted)},
|
||||
{name: "missing source", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs = m.Stages["extract"].Outputs[1:]
|
||||
}},
|
||||
{name: "inconsistent producer identity", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].ProducerRunID = "another-run"
|
||||
}},
|
||||
{name: "incompatible contract", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].Contract.SchemaVersion = "99"
|
||||
}},
|
||||
{name: "incompatible provenance", mutate: func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
m.Stages["extract"].Outputs[0].ExternalProvenance.RunID = "another-notarius-run"
|
||||
}},
|
||||
{name: "missing file", mutate: func(t *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "tampered checksum", mutate: func(t *testing.T, _ SessionPaths, m *manifest.Manifest) {
|
||||
if err := os.WriteFile(m.Stages["extract"].Outputs[0].LocalPath, []byte(`{"tampered":true}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{name: "unsafe path", mutate: func(t *testing.T, paths SessionPaths, m *manifest.Manifest) {
|
||||
outside := filepath.Join(paths.Root, "incidental.json")
|
||||
writeExtractionFixtureFile(t, outside, `{"incidental":true}`)
|
||||
m.Stages["extract"].Outputs[0].LocalPath = outside
|
||||
m.Stages["extract"].Outputs[0].Checksum = extractionFixtureChecksum(t, outside)
|
||||
}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
paths, currentManifest, definitions := validExtractionCatalogFixture(t)
|
||||
test.mutate(t, paths, currentManifest)
|
||||
catalog := registeredExtractionCatalog(t, definitions)
|
||||
catalog.HydrateExtractionArtifacts(paths, currentManifest, definitions)
|
||||
entry, _ := catalog.Lookup(ExtractionArtifactSourceID("encounters"))
|
||||
if entry.Available {
|
||||
t.Fatalf("entry became available from %s manifest", test.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveExtractionArtifactNeverDiscoversIncidentalBundleFile(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
paths := SessionPaths{Root: root, ArtifactsDir: filepath.Join(root, "artifacts")}
|
||||
incidental := filepath.Join(paths.ArtifactsDir, "notarius", "incidental", "lanes", "encounters.json")
|
||||
writeExtractionFixtureFile(t, incidental, `{"encounters":[]}`)
|
||||
definitions := extractionFixtureDefinitions()
|
||||
catalog := registeredExtractionCatalog(t, definitions)
|
||||
|
||||
_, err := ResolveSessionArtifactWithCatalog(paths, manifest.New("session", fixtureTime), ExtractionArtifactSourceID("encounters"), catalog)
|
||||
if err == nil || !errors.Is(err, ErrSessionArtifactNotFound) {
|
||||
t.Fatalf("ResolveSessionArtifactWithCatalog() error = %v, want not found", err)
|
||||
}
|
||||
}
|
||||
|
||||
var fixtureTime = mustFixtureTime()
|
||||
|
||||
func mustFixtureTime() (value time.Time) {
|
||||
return time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
func setExtractionStatus(status manifest.StageStatus) func(*testing.T, SessionPaths, *manifest.Manifest) {
|
||||
return func(_ *testing.T, _ SessionPaths, m *manifest.Manifest) { m.Stages["extract"].Status = status }
|
||||
}
|
||||
|
||||
func extractionFixtureDefinitions() map[string]ExtractionArtifactDefinition {
|
||||
return map[string]ExtractionArtifactDefinition{
|
||||
"encounters": {
|
||||
LaneID: "encounters", PipelineID: "campaign.extract", MediaType: "application/json",
|
||||
SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func registeredExtractionCatalog(t *testing.T, definitions map[string]ExtractionArtifactDefinition) *ArtifactCatalog {
|
||||
t.Helper()
|
||||
catalog := NewArtifactCatalog()
|
||||
if err := catalog.RegisterExtractionArtifacts(definitions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return catalog
|
||||
}
|
||||
|
||||
func validExtractionCatalogFixture(t *testing.T) (SessionPaths, *manifest.Manifest, map[string]ExtractionArtifactDefinition) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
paths := SessionPaths{Root: root, ArtifactsDir: filepath.Join(root, "artifacts")}
|
||||
bundleRoot := filepath.Join(paths.ArtifactsDir, "notarius", "extract-run-1")
|
||||
lanePath := filepath.Join(bundleRoot, "lanes", "encounters.json")
|
||||
indexPath := filepath.Join(bundleRoot, "index.json")
|
||||
writeExtractionFixtureFile(t, lanePath, `{"encounters":[]}`)
|
||||
writeExtractionFixtureFile(t, indexPath, `{"lanes":[]}`)
|
||||
definitions := extractionFixtureDefinitions()
|
||||
m := manifest.New("session", fixtureTime)
|
||||
m.Stages["extract"] = &manifest.StageRecord{
|
||||
Name: "extract", Status: manifest.StatusSucceeded,
|
||||
Metadata: map[string]any{
|
||||
"narratio_run_id": "extract-run-1", "bundle_root": bundleRoot,
|
||||
"receipt": map[string]any{"run_id": "notarius-run-1", "pipeline_id": "campaign.extract"},
|
||||
},
|
||||
Outputs: []manifest.ArtifactRecord{
|
||||
{
|
||||
Kind: "notarius_lane", SourceID: ExtractionArtifactSourceID("encounters"), LocalPath: lanePath,
|
||||
ProducerRunID: "extract-run-1", Checksum: extractionFixtureChecksum(t, lanePath),
|
||||
Contract: &artifactmodel.ContractMetadata{MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters"},
|
||||
ExternalProvenance: &artifactmodel.ExternalProvenance{System: "notarius", RunID: "notarius-run-1", PipelineID: "campaign.extract", ArtifactID: "encounters"},
|
||||
},
|
||||
{Kind: "notarius_index", LocalPath: indexPath, ProducerRunID: "extract-run-1", Checksum: extractionFixtureChecksum(t, indexPath)},
|
||||
},
|
||||
}
|
||||
return paths, m, definitions
|
||||
}
|
||||
|
||||
func writeExtractionFixtureFile(t *testing.T, path, body string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionFixtureChecksum(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
checksum, err := SHA256File(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return checksum
|
||||
}
|
||||
Reference in New Issue
Block a user