Centralize extraction bundle evidence

internal/artifacts/extraction_catalog.go

@@ -1,13 +1,11 @@
 package artifacts
 
 import (
-	"encoding/json"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
-	"gitea.maximumdirect.net/eric/narratio/internal/fileops"
 	"gitea.maximumdirect.net/eric/narratio/internal/manifest"
 )
 
@@ -22,11 +20,6 @@ const (
 	extractionMetadataRoot = "bundle_root"
 )
 
-type hydratedExtraction struct {
-	sourceID string
-	path     string
-}
-
 // HydrateExtractionArtifacts marks extraction sources available only when the current
 // manifest contains one complete, internally consistent, succeeded extraction bundle.
 // Invalid, stale, incomplete, or unsafe records leave every extraction source unavailable.
@@ -38,71 +31,12 @@ func (c *ArtifactCatalog) HydrateExtractionArtifacts(
 	if c == nil || m == nil || len(configured) == 0 {
 		return
 	}
-	record := m.Stages[extractStageName]
-	if record == nil || record.Name != extractStageName || record.Status != manifest.StatusSucceeded {
+	proof := InspectExtractionEvidence(paths, m, configured)
+	if proof.State != ExtractionEvidenceValid {
 		return
 	}
-	producerRunID := extractionMetadataString(record.Metadata, extractionMetadataRun)
-	if !safeExtractionPathSegment(producerRunID) {
-		return
-	}
-	bundleRoot := filepath.Clean(filepath.Join(paths.ArtifactsDir, "notarius", producerRunID))
-	if !filepath.IsAbs(bundleRoot) || extractionMetadataString(record.Metadata, extractionMetadataRoot) != bundleRoot {
-		return
-	}
-	if !safeExistingExtractionDirectory(paths.Root, bundleRoot) {
-		return
-	}
-	receiptRunID, receiptPipelineID := extractionReceiptIdentity(record.Metadata)
-	if receiptRunID == "" || receiptPipelineID == "" {
-		return
-	}
-
-	expected := make(map[string]ExtractionArtifactDefinition, len(configured))
-	for key, definition := range configured {
-		sourceID, ok := c.SourceIDForExtractionKey(key)
-		if !ok {
-			return
-		}
-		expected[sourceID] = definition
-	}
-
-	seen := make(map[string]struct{}, len(expected))
-	hydrated := make([]hydratedExtraction, 0, len(expected))
-	indexSeen := false
-	for _, output := range record.Outputs {
-		if strings.TrimSpace(output.ProducerRunID) != producerRunID {
-			return
-		}
-		if output.SourceID == "" {
-			if indexSeen || output.Kind != extractionIndexKind || filepath.Clean(output.LocalPath) != filepath.Join(bundleRoot, "index.json") ||
-				!validExtractionPayload(bundleRoot, output.LocalPath, output.Checksum) {
-				return
-			}
-			indexSeen = true
-			continue
-		}
-
-		definition, ok := expected[output.SourceID]
-		if !ok || output.Kind != extractionLaneKind {
-			return
-		}
-		if _, duplicate := seen[output.SourceID]; duplicate {
-			return
-		}
-		if !compatibleCatalogExtractionContract(output.Contract, definition) ||
-			!compatibleCatalogExtractionProvenance(output.ExternalProvenance, receiptRunID, receiptPipelineID, definition) ||
-			!validExtractionPayload(bundleRoot, output.LocalPath, output.Checksum) {
-			return
-		}
-		seen[output.SourceID] = struct{}{}
-		hydrated = append(hydrated, hydratedExtraction{sourceID: output.SourceID, path: output.LocalPath})
-	}
-	if !indexSeen || len(seen) != len(expected) || len(record.Outputs) != len(expected)+1 {
-		return
-	}
-	for _, item := range hydrated {
-		_ = c.markAvailableFromExtractManifest(item.sourceID, item.path, producerRunID)
+	for sourceID, path := range proof.Outputs {
+		_ = c.markAvailableFromExtractManifest(sourceID, path, proof.ProducerRunID)
 	}
 }
 
@@ -120,25 +54,6 @@ func compatibleCatalogExtractionProvenance(
 		pipelineID == strings.TrimSpace(want.PipelineID) && got.ArtifactID == want.LaneID
 }
 
-func validExtractionPayload(bundleRoot, path, checksum string) bool {
-	if !filepath.IsAbs(path) || !pathWithinExtractionRoot(bundleRoot, path) || strings.TrimSpace(checksum) == "" {
-		return false
-	}
-	info, err := os.Lstat(path)
-	if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
-		return false
-	}
-	if !safeExtractionComponents(bundleRoot, path) {
-		return false
-	}
-	actual, err := SHA256File(path)
-	if err != nil || actual != checksum {
-		return false
-	}
-	body, err := fileops.ReadRegularFile(path, MaxExtractionPayloadBytes)
-	return err == nil && json.Valid(body)
-}
-
 func safeExistingExtractionDirectory(sessionRoot, bundleRoot string) bool {
 	if !pathWithinExtractionRoot(sessionRoot, bundleRoot) || !safeExtractionComponents(sessionRoot, bundleRoot) {
 		return false

internal/artifacts/extraction_catalog_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 	"time"
 
@@ -76,6 +77,60 @@ func TestHydrateExtractionArtifactsAcceptsOnlyCompleteCurrentBundle(t *testing.T
 	}
 }
 
+func TestInspectExtractionEvidenceClassifiesBundleStates(t *testing.T) {
+	tests := []struct {
+		name   string
+		state  ExtractionEvidenceState
+		mutate func(*testing.T, *SessionPaths, *manifest.Manifest)
+	}{
+		{name: "valid", state: ExtractionEvidenceValid, mutate: func(_ *testing.T, _ *SessionPaths, _ *manifest.Manifest) {}},
+		{name: "absent", state: ExtractionEvidenceAbsent, mutate: func(_ *testing.T, _ *SessionPaths, m *manifest.Manifest) {
+			m.Stages["extract"].Status = manifest.StatusFailed
+		}},
+		{name: "obsolete version", state: ExtractionEvidenceObsolete, mutate: func(_ *testing.T, _ *SessionPaths, m *manifest.Manifest) {
+			m.Stages["extract"].Outputs[0].Contract.SchemaVersion = "99"
+		}},
+		{name: "incomplete", state: ExtractionEvidenceObsolete, mutate: func(_ *testing.T, _ *SessionPaths, m *manifest.Manifest) {
+			m.Stages["extract"].Outputs = m.Stages["extract"].Outputs[1:]
+		}},
+		{name: "unsafe root", state: ExtractionEvidenceUnsafe, mutate: func(t *testing.T, paths *SessionPaths, _ *manifest.Manifest) {
+			paths.Root = t.TempDir()
+		}},
+		{name: "unsafe symlink", state: ExtractionEvidenceUnsafe, mutate: func(t *testing.T, _ *SessionPaths, m *manifest.Manifest) {
+			if runtime.GOOS == "windows" {
+				t.Skip("symlink creation requires privileges on Windows")
+			}
+			path := m.Stages["extract"].Outputs[0].LocalPath
+			outside := filepath.Join(t.TempDir(), "outside.json")
+			writeExtractionFixtureFile(t, outside, `{"outside":true}`)
+			if err := os.Remove(path); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.Symlink(outside, path); err != nil {
+				t.Fatal(err)
+			}
+		}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			paths, currentManifest, definitions := validExtractionCatalogFixture(t)
+			test.mutate(t, &paths, currentManifest)
+			proof := InspectExtractionEvidence(paths, currentManifest, definitions)
+			if proof.State != test.state {
+				t.Fatalf("proof = %#v, want %q", proof, test.state)
+			}
+
+			catalog := registeredExtractionCatalog(t, definitions)
+			catalog.HydrateExtractionArtifacts(paths, currentManifest, definitions)
+			entry, _ := catalog.Lookup(ExtractionArtifactSourceID("encounters"))
+			if entry.Available != (test.state == ExtractionEvidenceValid) {
+				t.Fatalf("catalog availability for %s = %v", test.state, entry.Available)
+			}
+		})
+	}
+}
+
 func TestHydrateExtractionArtifactsRejectsUntrustedManifestState(t *testing.T) {
 	tests := []struct {
 		name   string

internal/artifacts/extraction_evidence.go

@@ -0,0 +1,131 @@
+package artifacts
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/narratio/internal/fileops"
+	"gitea.maximumdirect.net/eric/narratio/internal/manifest"
+)
+
+type ExtractionEvidenceState string
+
+const (
+	ExtractionEvidenceValid    ExtractionEvidenceState = "valid"
+	ExtractionEvidenceAbsent   ExtractionEvidenceState = "absent"
+	ExtractionEvidenceObsolete ExtractionEvidenceState = "obsolete"
+	ExtractionEvidenceUnsafe   ExtractionEvidenceState = "unsafe"
+)
+
+// ExtractionEvidence is a policy-neutral proof of configured extraction output.
+type ExtractionEvidence struct {
+	State                 ExtractionEvidenceState
+	Reason, ProducerRunID string
+	Outputs               map[string]string
+}
+
+// InspectExtractionEvidence verifies structure, confinement, identity, contracts, and payload bytes.
+func InspectExtractionEvidence(paths SessionPaths, m *manifest.Manifest, configured map[string]ExtractionArtifactDefinition) ExtractionEvidence {
+	if m == nil || len(configured) == 0 {
+		return ExtractionEvidence{State: ExtractionEvidenceAbsent, Reason: "extraction evidence is absent"}
+	}
+	r := m.Stages[extractStageName]
+	if r == nil || r.Name != extractStageName || r.Status != manifest.StatusSucceeded {
+		return ExtractionEvidence{State: ExtractionEvidenceAbsent, Reason: "extract stage has no succeeded result"}
+	}
+	runID := extractionMetadataString(r.Metadata, extractionMetadataRun)
+	if !safeExtractionPathSegment(runID) {
+		return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result has an invalid producing run ID"}
+	}
+	root := filepath.Clean(filepath.Join(paths.ArtifactsDir, "notarius", runID))
+	if !filepath.IsAbs(root) || extractionMetadataString(r.Metadata, extractionMetadataRoot) != root {
+		return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result does not identify its canonical immutable bundle"}
+	}
+	info, err := os.Lstat(root)
+	if os.IsNotExist(err) {
+		return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "immutable Notarius bundle is missing"}
+	}
+	if err != nil || info.Mode()&os.ModeSymlink != 0 {
+		return ExtractionEvidence{State: ExtractionEvidenceUnsafe, Reason: "immutable Notarius bundle is unsafe"}
+	}
+	if !info.IsDir() {
+		return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "immutable Notarius bundle is not a directory"}
+	}
+	if !safeExistingExtractionDirectory(paths.Root, root) {
+		return ExtractionEvidence{State: ExtractionEvidenceUnsafe, Reason: "immutable Notarius bundle is unsafe"}
+	}
+	receiptRunID, receiptPipelineID := extractionReceiptIdentity(r.Metadata)
+	if receiptRunID == "" || receiptPipelineID == "" {
+		return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result has incompatible Notarius receipt identity"}
+	}
+	expected := make(map[string]ExtractionArtifactDefinition, len(configured))
+	for key, d := range configured {
+		expected[ExtractionArtifactSourceID(key)] = d
+	}
+	seen, outputs := map[string]struct{}{}, map[string]string{}
+	indexSeen := false
+	for _, out := range r.Outputs {
+		if strings.TrimSpace(out.ProducerRunID) != runID {
+			return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract output producer identity is inconsistent"}
+		}
+		if out.SourceID == "" {
+			if indexSeen || out.Kind != extractionIndexKind || filepath.Clean(out.LocalPath) != filepath.Join(root, "index.json") {
+				return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract index path is not canonical"}
+			}
+			if state, reason := inspectExtractionPayload(root, out.LocalPath, out.Checksum); state != ExtractionEvidenceValid {
+				return ExtractionEvidence{State: state, Reason: reason}
+			}
+			indexSeen = true
+			continue
+		}
+		d, ok := expected[out.SourceID]
+		if !ok || out.Kind != extractionLaneKind {
+			return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result source set differs from current configuration"}
+		}
+		if _, duplicate := seen[out.SourceID]; duplicate {
+			return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result contains a duplicate configured source"}
+		}
+		if !compatibleCatalogExtractionContract(out.Contract, d) || !compatibleCatalogExtractionProvenance(out.ExternalProvenance, receiptRunID, receiptPipelineID, d) {
+			return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract output contract or provenance is incompatible"}
+		}
+		if state, reason := inspectExtractionPayload(root, out.LocalPath, out.Checksum); state != ExtractionEvidenceValid {
+			return ExtractionEvidence{State: state, Reason: reason}
+		}
+		seen[out.SourceID] = struct{}{}
+		outputs[out.SourceID] = out.LocalPath
+	}
+	if !indexSeen || len(seen) != len(expected) || len(r.Outputs) != len(expected)+1 {
+		return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result is incomplete"}
+	}
+	return ExtractionEvidence{State: ExtractionEvidenceValid, ProducerRunID: runID, Outputs: outputs}
+}
+
+func inspectExtractionPayload(root, path, checksum string) (ExtractionEvidenceState, string) {
+	if !filepath.IsAbs(path) || !pathWithinExtractionRoot(root, path) || strings.TrimSpace(checksum) == "" {
+		return ExtractionEvidenceUnsafe, "extract output path or checksum is unsafe"
+	}
+	info, err := os.Lstat(path)
+	if os.IsNotExist(err) {
+		return ExtractionEvidenceObsolete, "extract output is missing"
+	}
+	if !safeExtractionComponents(root, path) {
+		return ExtractionEvidenceUnsafe, "extract output path contains unsafe components"
+	}
+	if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
+		return ExtractionEvidenceUnsafe, "extract output is not a regular file"
+	}
+	actual, err := SHA256File(path)
+	if err != nil {
+		return ExtractionEvidenceUnsafe, "extract output checksum cannot be read"
+	}
+	if actual != checksum {
+		return ExtractionEvidenceObsolete, "extract output checksum does not match durable bytes"
+	}
+	body, err := fileops.ReadRegularFile(path, MaxExtractionPayloadBytes)
+	if err != nil || !json.Valid(body) {
+		return ExtractionEvidenceObsolete, "extract output is not valid JSON"
+	}
+	return ExtractionEvidenceValid, ""
+}