Integrate extraction artifacts into analysis catalog

internal/artifacts/extraction_catalog.go

@@ -0,0 +1,180 @@
+package artifacts
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
+	"gitea.maximumdirect.net/eric/narratio/internal/manifest"
+)
+
+const (
+	extractStageName       = "extract"
+	extractionLaneKind     = "notarius_lane"
+	extractionIndexKind    = "notarius_index"
+	extractionMetadataRun  = "narratio_run_id"
+	extractionMetadataRoot = "bundle_root"
+)
+
+type hydratedExtraction struct {
+	sourceID string
+	path     string
+}
+
+// HydrateExtractionArtifacts marks extraction sources available only when the current
+// manifest contains one complete, internally consistent, succeeded extraction bundle.
+// Invalid, stale, incomplete, or unsafe records leave every extraction source unavailable.
+func (c *ArtifactCatalog) HydrateExtractionArtifacts(
+	paths SessionPaths,
+	m *manifest.Manifest,
+	configured map[string]ExtractionArtifactDefinition,
+) {
+	if c == nil || m == nil || len(configured) == 0 {
+		return
+	}
+	record := m.Stages[extractStageName]
+	if record == nil || record.Name != extractStageName || record.Status != manifest.StatusSucceeded {
+		return
+	}
+	producerRunID := extractionMetadataString(record.Metadata, extractionMetadataRun)
+	if !safeExtractionPathSegment(producerRunID) {
+		return
+	}
+	bundleRoot := filepath.Clean(filepath.Join(paths.ArtifactsDir, "notarius", producerRunID))
+	if !filepath.IsAbs(bundleRoot) || extractionMetadataString(record.Metadata, extractionMetadataRoot) != bundleRoot {
+		return
+	}
+	if !safeExistingExtractionDirectory(paths.Root, bundleRoot) {
+		return
+	}
+	receiptRunID, receiptPipelineID := extractionReceiptIdentity(record.Metadata)
+	if receiptRunID == "" || receiptPipelineID == "" {
+		return
+	}
+
+	expected := make(map[string]ExtractionArtifactDefinition, len(configured))
+	for key, definition := range configured {
+		sourceID, ok := c.SourceIDForExtractionKey(key)
+		if !ok {
+			return
+		}
+		expected[sourceID] = definition
+	}
+
+	seen := make(map[string]struct{}, len(expected))
+	hydrated := make([]hydratedExtraction, 0, len(expected))
+	indexSeen := false
+	for _, output := range record.Outputs {
+		if strings.TrimSpace(output.ProducerRunID) != producerRunID {
+			return
+		}
+		if output.SourceID == "" {
+			if indexSeen || output.Kind != extractionIndexKind || filepath.Clean(output.LocalPath) != filepath.Join(bundleRoot, "index.json") ||
+				!validExtractionPayload(bundleRoot, output.LocalPath, output.Checksum) {
+				return
+			}
+			indexSeen = true
+			continue
+		}
+
+		definition, ok := expected[output.SourceID]
+		if !ok || output.Kind != extractionLaneKind {
+			return
+		}
+		if _, duplicate := seen[output.SourceID]; duplicate {
+			return
+		}
+		if !compatibleCatalogExtractionContract(output.Contract, definition) ||
+			!compatibleCatalogExtractionProvenance(output.ExternalProvenance, receiptRunID, receiptPipelineID, definition) ||
+			!validExtractionPayload(bundleRoot, output.LocalPath, output.Checksum) {
+			return
+		}
+		seen[output.SourceID] = struct{}{}
+		hydrated = append(hydrated, hydratedExtraction{sourceID: output.SourceID, path: output.LocalPath})
+	}
+	if !indexSeen || len(seen) != len(expected) || len(record.Outputs) != len(expected)+1 {
+		return
+	}
+	for _, item := range hydrated {
+		_ = c.markAvailableFromExtractManifest(item.sourceID, item.path, producerRunID)
+	}
+}
+
+func compatibleCatalogExtractionContract(got *artifactmodel.ContractMetadata, want ExtractionArtifactDefinition) bool {
+	return got != nil && got.MediaType == want.MediaType && got.SchemaID == want.SchemaID &&
+		got.SchemaVersion == want.SchemaVersion && (want.ModuleKey == "" || got.ModuleKey == want.ModuleKey)
+}
+
+func compatibleCatalogExtractionProvenance(
+	got *artifactmodel.ExternalProvenance,
+	runID, pipelineID string,
+	want ExtractionArtifactDefinition,
+) bool {
+	return got != nil && got.System == "notarius" && got.RunID == runID && got.PipelineID == pipelineID &&
+		pipelineID == strings.TrimSpace(want.PipelineID) && got.ArtifactID == want.LaneID
+}
+
+func validExtractionPayload(bundleRoot, path, checksum string) bool {
+	if !filepath.IsAbs(path) || !pathWithinExtractionRoot(bundleRoot, path) || strings.TrimSpace(checksum) == "" {
+		return false
+	}
+	info, err := os.Lstat(path)
+	if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
+		return false
+	}
+	if !safeExtractionComponents(bundleRoot, path) {
+		return false
+	}
+	actual, err := SHA256File(path)
+	if err != nil || actual != checksum {
+		return false
+	}
+	body, err := os.ReadFile(path)
+	return err == nil && json.Valid(body)
+}
+
+func safeExistingExtractionDirectory(sessionRoot, bundleRoot string) bool {
+	if !pathWithinExtractionRoot(sessionRoot, bundleRoot) || !safeExtractionComponents(sessionRoot, bundleRoot) {
+		return false
+	}
+	info, err := os.Lstat(bundleRoot)
+	return err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0
+}
+
+func safeExtractionComponents(root, target string) bool {
+	relative, err := filepath.Rel(filepath.Clean(root), filepath.Clean(target))
+	if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
+		return false
+	}
+	current := filepath.Clean(root)
+	for _, part := range strings.Split(relative, string(filepath.Separator)) {
+		current = filepath.Join(current, part)
+		info, err := os.Lstat(current)
+		if err != nil || info.Mode()&os.ModeSymlink != 0 {
+			return false
+		}
+	}
+	return true
+}
+
+func pathWithinExtractionRoot(root, target string) bool {
+	relative, err := filepath.Rel(filepath.Clean(root), filepath.Clean(target))
+	return err == nil && relative != "." && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
+}
+
+func safeExtractionPathSegment(value string) bool {
+	return value != "" && value != "." && value != ".." && filepath.Base(value) == value &&
+		!strings.ContainsAny(value, `/\\`)
+}
+
+func extractionMetadataString(metadata map[string]any, key string) string {
+	value, _ := metadata[key].(string)
+	return strings.TrimSpace(value)
+}
+
+func extractionReceiptIdentity(metadata map[string]any) (string, string) {
+	receipt, _ := metadata["receipt"].(map[string]any)
+	return extractionMetadataString(receipt, "run_id"), extractionMetadataString(receipt, "pipeline_id")
+}