Centralize extraction bundle evidence
This commit is contained in:
@@ -65,11 +65,13 @@ Configured sources (`narratio.artifact.*`):
|
|||||||
|
|
||||||
Extraction sources (`narratio.extraction.*`):
|
Extraction sources (`narratio.extraction.*`):
|
||||||
|
|
||||||
- use the shared registration and manifest hydration path in
|
- use the shared typed bundle evidence inspection in `extraction_evidence.go`;
|
||||||
`extraction_catalog.go`;
|
|
||||||
- require a current successful extract record with the exact configured source,
|
- require a current successful extract record with the exact configured source,
|
||||||
compatible contract and Notarius provenance, a confined regular durable
|
compatible contract and Notarius provenance, a confined regular durable
|
||||||
payload, and matching checksum; and
|
payload, and matching checksum;
|
||||||
|
- remain unavailable unless catalog hydration receives valid evidence. Resume
|
||||||
|
treats absent or obsolete evidence as a rerun decision and unsafe evidence as
|
||||||
|
an error; and
|
||||||
- are never inferred by scanning the Notarius bundle directory.
|
- are never inferred by scanning the Notarius bundle directory.
|
||||||
|
|
||||||
Previous-session sources (`narratio.previous_session.artifact.*`):
|
Previous-session sources (`narratio.previous_session.artifact.*`):
|
||||||
@@ -166,7 +168,8 @@ physical layout.
|
|||||||
|
|
||||||
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
- Registry and resolution: `internal/artifacts/artifact_resolver.go`,
|
||||||
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`,
|
`internal/artifacts/catalog.go`, `internal/artifacts/transcripts.go`,
|
||||||
`internal/artifacts/extraction_catalog.go`
|
`internal/artifacts/extraction_catalog.go`,
|
||||||
|
`internal/artifacts/extraction_evidence.go`
|
||||||
- Current state: `internal/artifacts/current_state.go`
|
- Current state: `internal/artifacts/current_state.go`
|
||||||
- Paths and keys: `internal/artifacts/paths.go`,
|
- Paths and keys: `internal/artifacts/paths.go`,
|
||||||
`internal/artifacts/s3_keys.go`
|
`internal/artifacts/s3_keys.go`
|
||||||
|
|||||||
@@ -896,6 +896,8 @@ policy decisions made by resume and catalog consumers.
|
|||||||
consumers and assert their intentionally different outcomes for every typed state.
|
consumers and assert their intentionally different outcomes for every typed state.
|
||||||
Include unsafe root, symlink, incomplete, obsolete-version, and valid cases.
|
Include unsafe root, symlink, incomplete, obsolete-version, and valid cases.
|
||||||
|
|
||||||
|
**Status:** Completed.
|
||||||
|
|
||||||
## Stage 27 — Bind extraction reuse to direct transcript identity
|
## Stage 27 — Bind extraction reuse to direct transcript identity
|
||||||
|
|
||||||
**Read first:** `audit-findings.md` lines 2380–2423 (COR-020) and 3744–3774
|
**Read first:** `audit-findings.md` lines 2380–2423 (COR-020) and 3744–3774
|
||||||
|
|||||||
@@ -1,13 +1,11 @@
|
|||||||
package artifacts
|
package artifacts
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -22,11 +20,6 @@ const (
|
|||||||
extractionMetadataRoot = "bundle_root"
|
extractionMetadataRoot = "bundle_root"
|
||||||
)
|
)
|
||||||
|
|
||||||
type hydratedExtraction struct {
|
|
||||||
sourceID string
|
|
||||||
path string
|
|
||||||
}
|
|
||||||
|
|
||||||
// HydrateExtractionArtifacts marks extraction sources available only when the current
|
// HydrateExtractionArtifacts marks extraction sources available only when the current
|
||||||
// manifest contains one complete, internally consistent, succeeded extraction bundle.
|
// manifest contains one complete, internally consistent, succeeded extraction bundle.
|
||||||
// Invalid, stale, incomplete, or unsafe records leave every extraction source unavailable.
|
// Invalid, stale, incomplete, or unsafe records leave every extraction source unavailable.
|
||||||
@@ -38,71 +31,12 @@ func (c *ArtifactCatalog) HydrateExtractionArtifacts(
|
|||||||
if c == nil || m == nil || len(configured) == 0 {
|
if c == nil || m == nil || len(configured) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
record := m.Stages[extractStageName]
|
proof := InspectExtractionEvidence(paths, m, configured)
|
||||||
if record == nil || record.Name != extractStageName || record.Status != manifest.StatusSucceeded {
|
if proof.State != ExtractionEvidenceValid {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
producerRunID := extractionMetadataString(record.Metadata, extractionMetadataRun)
|
for sourceID, path := range proof.Outputs {
|
||||||
if !safeExtractionPathSegment(producerRunID) {
|
_ = c.markAvailableFromExtractManifest(sourceID, path, proof.ProducerRunID)
|
||||||
return
|
|
||||||
}
|
|
||||||
bundleRoot := filepath.Clean(filepath.Join(paths.ArtifactsDir, "notarius", producerRunID))
|
|
||||||
if !filepath.IsAbs(bundleRoot) || extractionMetadataString(record.Metadata, extractionMetadataRoot) != bundleRoot {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !safeExistingExtractionDirectory(paths.Root, bundleRoot) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
receiptRunID, receiptPipelineID := extractionReceiptIdentity(record.Metadata)
|
|
||||||
if receiptRunID == "" || receiptPipelineID == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
expected := make(map[string]ExtractionArtifactDefinition, len(configured))
|
|
||||||
for key, definition := range configured {
|
|
||||||
sourceID, ok := c.SourceIDForExtractionKey(key)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
expected[sourceID] = definition
|
|
||||||
}
|
|
||||||
|
|
||||||
seen := make(map[string]struct{}, len(expected))
|
|
||||||
hydrated := make([]hydratedExtraction, 0, len(expected))
|
|
||||||
indexSeen := false
|
|
||||||
for _, output := range record.Outputs {
|
|
||||||
if strings.TrimSpace(output.ProducerRunID) != producerRunID {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if output.SourceID == "" {
|
|
||||||
if indexSeen || output.Kind != extractionIndexKind || filepath.Clean(output.LocalPath) != filepath.Join(bundleRoot, "index.json") ||
|
|
||||||
!validExtractionPayload(bundleRoot, output.LocalPath, output.Checksum) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
indexSeen = true
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
definition, ok := expected[output.SourceID]
|
|
||||||
if !ok || output.Kind != extractionLaneKind {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if _, duplicate := seen[output.SourceID]; duplicate {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !compatibleCatalogExtractionContract(output.Contract, definition) ||
|
|
||||||
!compatibleCatalogExtractionProvenance(output.ExternalProvenance, receiptRunID, receiptPipelineID, definition) ||
|
|
||||||
!validExtractionPayload(bundleRoot, output.LocalPath, output.Checksum) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
seen[output.SourceID] = struct{}{}
|
|
||||||
hydrated = append(hydrated, hydratedExtraction{sourceID: output.SourceID, path: output.LocalPath})
|
|
||||||
}
|
|
||||||
if !indexSeen || len(seen) != len(expected) || len(record.Outputs) != len(expected)+1 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for _, item := range hydrated {
|
|
||||||
_ = c.markAvailableFromExtractManifest(item.sourceID, item.path, producerRunID)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -120,25 +54,6 @@ func compatibleCatalogExtractionProvenance(
|
|||||||
pipelineID == strings.TrimSpace(want.PipelineID) && got.ArtifactID == want.LaneID
|
pipelineID == strings.TrimSpace(want.PipelineID) && got.ArtifactID == want.LaneID
|
||||||
}
|
}
|
||||||
|
|
||||||
func validExtractionPayload(bundleRoot, path, checksum string) bool {
|
|
||||||
if !filepath.IsAbs(path) || !pathWithinExtractionRoot(bundleRoot, path) || strings.TrimSpace(checksum) == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
info, err := os.Lstat(path)
|
|
||||||
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if !safeExtractionComponents(bundleRoot, path) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
actual, err := SHA256File(path)
|
|
||||||
if err != nil || actual != checksum {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
body, err := fileops.ReadRegularFile(path, MaxExtractionPayloadBytes)
|
|
||||||
return err == nil && json.Valid(body)
|
|
||||||
}
|
|
||||||
|
|
||||||
func safeExistingExtractionDirectory(sessionRoot, bundleRoot string) bool {
|
func safeExistingExtractionDirectory(sessionRoot, bundleRoot string) bool {
|
||||||
if !pathWithinExtractionRoot(sessionRoot, bundleRoot) || !safeExtractionComponents(sessionRoot, bundleRoot) {
|
if !pathWithinExtractionRoot(sessionRoot, bundleRoot) || !safeExtractionComponents(sessionRoot, bundleRoot) {
|
||||||
return false
|
return false
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -76,6 +77,60 @@ func TestHydrateExtractionArtifactsAcceptsOnlyCompleteCurrentBundle(t *testing.T
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInspectExtractionEvidenceClassifiesBundleStates(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
state ExtractionEvidenceState
|
||||||
|
mutate func(*testing.T, *SessionPaths, *manifest.Manifest)
|
||||||
|
}{
|
||||||
|
{name: "valid", state: ExtractionEvidenceValid, mutate: func(_ *testing.T, _ *SessionPaths, _ *manifest.Manifest) {}},
|
||||||
|
{name: "absent", state: ExtractionEvidenceAbsent, mutate: func(_ *testing.T, _ *SessionPaths, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Status = manifest.StatusFailed
|
||||||
|
}},
|
||||||
|
{name: "obsolete version", state: ExtractionEvidenceObsolete, mutate: func(_ *testing.T, _ *SessionPaths, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs[0].Contract.SchemaVersion = "99"
|
||||||
|
}},
|
||||||
|
{name: "incomplete", state: ExtractionEvidenceObsolete, mutate: func(_ *testing.T, _ *SessionPaths, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Outputs = m.Stages["extract"].Outputs[1:]
|
||||||
|
}},
|
||||||
|
{name: "unsafe root", state: ExtractionEvidenceUnsafe, mutate: func(t *testing.T, paths *SessionPaths, _ *manifest.Manifest) {
|
||||||
|
paths.Root = t.TempDir()
|
||||||
|
}},
|
||||||
|
{name: "unsafe symlink", state: ExtractionEvidenceUnsafe, mutate: func(t *testing.T, _ *SessionPaths, m *manifest.Manifest) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("symlink creation requires privileges on Windows")
|
||||||
|
}
|
||||||
|
path := m.Stages["extract"].Outputs[0].LocalPath
|
||||||
|
outside := filepath.Join(t.TempDir(), "outside.json")
|
||||||
|
writeExtractionFixtureFile(t, outside, `{"outside":true}`)
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(outside, path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
paths, currentManifest, definitions := validExtractionCatalogFixture(t)
|
||||||
|
test.mutate(t, &paths, currentManifest)
|
||||||
|
proof := InspectExtractionEvidence(paths, currentManifest, definitions)
|
||||||
|
if proof.State != test.state {
|
||||||
|
t.Fatalf("proof = %#v, want %q", proof, test.state)
|
||||||
|
}
|
||||||
|
|
||||||
|
catalog := registeredExtractionCatalog(t, definitions)
|
||||||
|
catalog.HydrateExtractionArtifacts(paths, currentManifest, definitions)
|
||||||
|
entry, _ := catalog.Lookup(ExtractionArtifactSourceID("encounters"))
|
||||||
|
if entry.Available != (test.state == ExtractionEvidenceValid) {
|
||||||
|
t.Fatalf("catalog availability for %s = %v", test.state, entry.Available)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHydrateExtractionArtifactsRejectsUntrustedManifestState(t *testing.T) {
|
func TestHydrateExtractionArtifactsRejectsUntrustedManifestState(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
131
internal/artifacts/extraction_evidence.go
Normal file
131
internal/artifacts/extraction_evidence.go
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
package artifacts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
|
||||||
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ExtractionEvidenceState string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ExtractionEvidenceValid ExtractionEvidenceState = "valid"
|
||||||
|
ExtractionEvidenceAbsent ExtractionEvidenceState = "absent"
|
||||||
|
ExtractionEvidenceObsolete ExtractionEvidenceState = "obsolete"
|
||||||
|
ExtractionEvidenceUnsafe ExtractionEvidenceState = "unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ExtractionEvidence is a policy-neutral proof of configured extraction output.
|
||||||
|
type ExtractionEvidence struct {
|
||||||
|
State ExtractionEvidenceState
|
||||||
|
Reason, ProducerRunID string
|
||||||
|
Outputs map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectExtractionEvidence verifies structure, confinement, identity, contracts, and payload bytes.
|
||||||
|
func InspectExtractionEvidence(paths SessionPaths, m *manifest.Manifest, configured map[string]ExtractionArtifactDefinition) ExtractionEvidence {
|
||||||
|
if m == nil || len(configured) == 0 {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceAbsent, Reason: "extraction evidence is absent"}
|
||||||
|
}
|
||||||
|
r := m.Stages[extractStageName]
|
||||||
|
if r == nil || r.Name != extractStageName || r.Status != manifest.StatusSucceeded {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceAbsent, Reason: "extract stage has no succeeded result"}
|
||||||
|
}
|
||||||
|
runID := extractionMetadataString(r.Metadata, extractionMetadataRun)
|
||||||
|
if !safeExtractionPathSegment(runID) {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result has an invalid producing run ID"}
|
||||||
|
}
|
||||||
|
root := filepath.Clean(filepath.Join(paths.ArtifactsDir, "notarius", runID))
|
||||||
|
if !filepath.IsAbs(root) || extractionMetadataString(r.Metadata, extractionMetadataRoot) != root {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result does not identify its canonical immutable bundle"}
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(root)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "immutable Notarius bundle is missing"}
|
||||||
|
}
|
||||||
|
if err != nil || info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceUnsafe, Reason: "immutable Notarius bundle is unsafe"}
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "immutable Notarius bundle is not a directory"}
|
||||||
|
}
|
||||||
|
if !safeExistingExtractionDirectory(paths.Root, root) {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceUnsafe, Reason: "immutable Notarius bundle is unsafe"}
|
||||||
|
}
|
||||||
|
receiptRunID, receiptPipelineID := extractionReceiptIdentity(r.Metadata)
|
||||||
|
if receiptRunID == "" || receiptPipelineID == "" {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result has incompatible Notarius receipt identity"}
|
||||||
|
}
|
||||||
|
expected := make(map[string]ExtractionArtifactDefinition, len(configured))
|
||||||
|
for key, d := range configured {
|
||||||
|
expected[ExtractionArtifactSourceID(key)] = d
|
||||||
|
}
|
||||||
|
seen, outputs := map[string]struct{}{}, map[string]string{}
|
||||||
|
indexSeen := false
|
||||||
|
for _, out := range r.Outputs {
|
||||||
|
if strings.TrimSpace(out.ProducerRunID) != runID {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract output producer identity is inconsistent"}
|
||||||
|
}
|
||||||
|
if out.SourceID == "" {
|
||||||
|
if indexSeen || out.Kind != extractionIndexKind || filepath.Clean(out.LocalPath) != filepath.Join(root, "index.json") {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract index path is not canonical"}
|
||||||
|
}
|
||||||
|
if state, reason := inspectExtractionPayload(root, out.LocalPath, out.Checksum); state != ExtractionEvidenceValid {
|
||||||
|
return ExtractionEvidence{State: state, Reason: reason}
|
||||||
|
}
|
||||||
|
indexSeen = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
d, ok := expected[out.SourceID]
|
||||||
|
if !ok || out.Kind != extractionLaneKind {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result source set differs from current configuration"}
|
||||||
|
}
|
||||||
|
if _, duplicate := seen[out.SourceID]; duplicate {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result contains a duplicate configured source"}
|
||||||
|
}
|
||||||
|
if !compatibleCatalogExtractionContract(out.Contract, d) || !compatibleCatalogExtractionProvenance(out.ExternalProvenance, receiptRunID, receiptPipelineID, d) {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract output contract or provenance is incompatible"}
|
||||||
|
}
|
||||||
|
if state, reason := inspectExtractionPayload(root, out.LocalPath, out.Checksum); state != ExtractionEvidenceValid {
|
||||||
|
return ExtractionEvidence{State: state, Reason: reason}
|
||||||
|
}
|
||||||
|
seen[out.SourceID] = struct{}{}
|
||||||
|
outputs[out.SourceID] = out.LocalPath
|
||||||
|
}
|
||||||
|
if !indexSeen || len(seen) != len(expected) || len(r.Outputs) != len(expected)+1 {
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result is incomplete"}
|
||||||
|
}
|
||||||
|
return ExtractionEvidence{State: ExtractionEvidenceValid, ProducerRunID: runID, Outputs: outputs}
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectExtractionPayload(root, path, checksum string) (ExtractionEvidenceState, string) {
|
||||||
|
if !filepath.IsAbs(path) || !pathWithinExtractionRoot(root, path) || strings.TrimSpace(checksum) == "" {
|
||||||
|
return ExtractionEvidenceUnsafe, "extract output path or checksum is unsafe"
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return ExtractionEvidenceObsolete, "extract output is missing"
|
||||||
|
}
|
||||||
|
if !safeExtractionComponents(root, path) {
|
||||||
|
return ExtractionEvidenceUnsafe, "extract output path contains unsafe components"
|
||||||
|
}
|
||||||
|
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return ExtractionEvidenceUnsafe, "extract output is not a regular file"
|
||||||
|
}
|
||||||
|
actual, err := SHA256File(path)
|
||||||
|
if err != nil {
|
||||||
|
return ExtractionEvidenceUnsafe, "extract output checksum cannot be read"
|
||||||
|
}
|
||||||
|
if actual != checksum {
|
||||||
|
return ExtractionEvidenceObsolete, "extract output checksum does not match durable bytes"
|
||||||
|
}
|
||||||
|
body, err := fileops.ReadRegularFile(path, MaxExtractionPayloadBytes)
|
||||||
|
if err != nil || !json.Valid(body) {
|
||||||
|
return ExtractionEvidenceObsolete, "extract output is not valid JSON"
|
||||||
|
}
|
||||||
|
return ExtractionEvidenceValid, ""
|
||||||
|
}
|
||||||
@@ -3,16 +3,11 @@ package stage
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func (extractStage) ValidateResume(_ context.Context, env *Env, m *manifest.Manifest) (ResumeValidation, error) {
|
func (extractStage) ValidateResume(_ context.Context, env *Env, m *manifest.Manifest) (ResumeValidation, error) {
|
||||||
@@ -26,22 +21,6 @@ func (extractStage) ValidateResume(_ context.Context, env *Env, m *manifest.Mani
|
|||||||
if m == nil {
|
if m == nil {
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: session manifest is required")
|
return ResumeValidation{}, fmt.Errorf("extract resume: session manifest is required")
|
||||||
}
|
}
|
||||||
record := m.Stages[(extractStage{}).Name()]
|
|
||||||
if record == nil || record.Status != manifest.StatusSucceeded {
|
|
||||||
return NonResumable("extract stage has no succeeded result"), nil
|
|
||||||
}
|
|
||||||
if record.Name != (extractStage{}).Name() {
|
|
||||||
return NonResumable("extract stage record identity is inconsistent"), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
producerRunID := metadataString(record.Metadata, "narratio_run_id")
|
|
||||||
if producerRunID == "" {
|
|
||||||
return NonResumable("extract result is missing its producing run ID"), nil
|
|
||||||
}
|
|
||||||
if !safePathSegment(producerRunID) {
|
|
||||||
return NonResumable("extract result has an invalid producing run ID"), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
timeout, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout))
|
timeout, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout))
|
||||||
if err != nil || timeout <= 0 {
|
if err != nil || timeout <= 0 {
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: invalid Notarius timeout %q", cfg.Timeout)
|
return ResumeValidation{}, fmt.Errorf("extract resume: invalid Notarius timeout %q", cfg.Timeout)
|
||||||
@@ -62,7 +41,8 @@ func (extractStage) ValidateResume(_ context.Context, env *Env, m *manifest.Mani
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: build configuration fingerprint: %w", err)
|
return ResumeValidation{}, fmt.Errorf("extract resume: build configuration fingerprint: %w", err)
|
||||||
}
|
}
|
||||||
if metadataString(record.Metadata, "configuration_fingerprint") != fingerprint {
|
record := m.Stages[(extractStage{}).Name()]
|
||||||
|
if record == nil || metadataString(record.Metadata, "configuration_fingerprint") != fingerprint {
|
||||||
return NonResumable("Notarius invocation contract changed"), nil
|
return NonResumable("Notarius invocation contract changed"), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -70,169 +50,27 @@ func (extractStage) ValidateResume(_ context.Context, env *Env, m *manifest.Mani
|
|||||||
if sessionID == "" {
|
if sessionID == "" {
|
||||||
sessionID = strings.TrimSpace(env.Config.Session.SessionID)
|
sessionID = strings.TrimSpace(env.Config.Session.SessionID)
|
||||||
}
|
}
|
||||||
campaign := strings.TrimSpace(m.Campaign)
|
if sessionID == "" {
|
||||||
if campaign == "" {
|
return ResumeValidation{}, fmt.Errorf("extract resume: session ID is required")
|
||||||
campaign = strings.TrimSpace(env.Config.Session.Campaign)
|
|
||||||
}
|
|
||||||
if sessionID == "" || campaign == "" {
|
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: session ID and campaign are required")
|
|
||||||
}
|
}
|
||||||
paths := sessionPathsForEnv(env, sessionID)
|
paths := sessionPathsForEnv(env, sessionID)
|
||||||
workspaceRoot := strings.TrimSpace(paths.WorkspaceRoot)
|
definitions := make(map[string]artifacts.ExtractionArtifactDefinition, len(cfg.Outputs))
|
||||||
if workspaceRoot == "" {
|
|
||||||
workspaceRoot = env.Config.Pipeline.Workspace.Root
|
|
||||||
}
|
|
||||||
bundleRoot, err := absolutePath(artifacts.SessionNotariusBundleDirForCampaign(workspaceRoot, campaign, sessionID, producerRunID))
|
|
||||||
if err != nil {
|
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: resolve durable bundle path: %w", err)
|
|
||||||
}
|
|
||||||
storedBundleRoot := metadataString(record.Metadata, "bundle_root")
|
|
||||||
if storedBundleRoot == "" || !filepath.IsAbs(storedBundleRoot) || filepath.Clean(storedBundleRoot) != bundleRoot {
|
|
||||||
return NonResumable("extract result does not identify its canonical immutable bundle"), nil
|
|
||||||
}
|
|
||||||
bundleRelative, err := pathsafe.SlashRelativeFromRoot(paths.Root, bundleRoot)
|
|
||||||
if err != nil {
|
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: immutable bundle path is unsafe: %w", err)
|
|
||||||
}
|
|
||||||
if err := rejectSymlinkComponents(paths.Root, bundleRelative); err != nil {
|
|
||||||
return ResumeValidation{}, err
|
|
||||||
}
|
|
||||||
bundleInfo, err := os.Lstat(bundleRoot)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return NonResumable("immutable Notarius bundle is missing"), nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: inspect immutable bundle: %w", err)
|
|
||||||
}
|
|
||||||
if bundleInfo.Mode()&os.ModeSymlink != 0 {
|
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: immutable bundle must not be a symlink")
|
|
||||||
}
|
|
||||||
if !bundleInfo.IsDir() {
|
|
||||||
return NonResumable("immutable Notarius bundle is not a directory"), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
receiptRunID, receiptPipelineID := receiptIdentity(record.Metadata)
|
|
||||||
if receiptRunID == "" || receiptPipelineID != cfg.PipelineID {
|
|
||||||
return NonResumable("extract result has incompatible Notarius receipt identity"), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
expectedSources := make(map[string]config.NotariusOutputConfig, len(cfg.Outputs))
|
|
||||||
for key, output := range cfg.Outputs {
|
for key, output := range cfg.Outputs {
|
||||||
expectedSources[artifacts.ExtractionArtifactSourceID(key)] = output
|
definitions[key] = artifacts.ExtractionArtifactDefinition{
|
||||||
}
|
LaneID: output.LaneID, PipelineID: cfg.PipelineID, MediaType: output.MediaType,
|
||||||
seenSources := make(map[string]struct{}, len(expectedSources))
|
SchemaID: output.SchemaID, SchemaVersion: output.SchemaVersion, ModuleKey: output.ModuleKey,
|
||||||
indexSeen := false
|
|
||||||
for _, output := range record.Outputs {
|
|
||||||
if output.ProducerRunID != producerRunID {
|
|
||||||
return NonResumable("extract output producer identity is inconsistent"), nil
|
|
||||||
}
|
|
||||||
if output.SourceID == "" {
|
|
||||||
if indexSeen || output.Kind != extractIndexOutputKind {
|
|
||||||
return NonResumable("extract result has an unexpected non-selectable output"), nil
|
|
||||||
}
|
|
||||||
indexSeen = true
|
|
||||||
expectedIndex := filepath.Join(bundleRoot, "index.json")
|
|
||||||
if filepath.Clean(output.LocalPath) != expectedIndex {
|
|
||||||
return NonResumable("extract index path is not canonical"), nil
|
|
||||||
}
|
|
||||||
validation, err := validateResumePayload(bundleRoot, output.LocalPath, output.Checksum)
|
|
||||||
if err != nil || !validation.Resumable {
|
|
||||||
return validation, err
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
expected, ok := expectedSources[output.SourceID]
|
|
||||||
if !ok || output.Kind != extractLaneOutputKind {
|
|
||||||
return NonResumable("extract result source set differs from current configuration"), nil
|
|
||||||
}
|
|
||||||
if _, duplicate := seenSources[output.SourceID]; duplicate {
|
|
||||||
return NonResumable("extract result contains a duplicate configured source"), nil
|
|
||||||
}
|
|
||||||
seenSources[output.SourceID] = struct{}{}
|
|
||||||
if !compatibleExtractionContract(output.Contract, expected) {
|
|
||||||
return NonResumable("extract output contract is incompatible with current configuration"), nil
|
|
||||||
}
|
|
||||||
if !compatibleExtractionProvenance(output.ExternalProvenance, receiptRunID, receiptPipelineID, expected.LaneID) {
|
|
||||||
return NonResumable("extract output has incompatible Notarius provenance"), nil
|
|
||||||
}
|
|
||||||
validation, err := validateResumePayload(bundleRoot, output.LocalPath, output.Checksum)
|
|
||||||
if err != nil || !validation.Resumable {
|
|
||||||
return validation, err
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !indexSeen {
|
proof := artifacts.InspectExtractionEvidence(paths, m, definitions)
|
||||||
return NonResumable("extract result is missing its canonical index"), nil
|
if proof.State == artifacts.ExtractionEvidenceUnsafe {
|
||||||
|
return ResumeValidation{}, fmt.Errorf("extract resume: %s", proof.Reason)
|
||||||
}
|
}
|
||||||
if len(seenSources) != len(expectedSources) || len(record.Outputs) != len(expectedSources)+1 {
|
if proof.State != artifacts.ExtractionEvidenceValid {
|
||||||
return NonResumable("extract result is missing configured sources"), nil
|
return NonResumable(proof.Reason), nil
|
||||||
}
|
}
|
||||||
return Resumable(), nil
|
return Resumable(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateResumePayload(bundleRoot, path, checksum string) (ResumeValidation, error) {
|
|
||||||
if !filepath.IsAbs(path) {
|
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: persisted output path must be absolute")
|
|
||||||
}
|
|
||||||
relative, err := pathsafe.SlashRelativeFromRoot(bundleRoot, path)
|
|
||||||
if err != nil {
|
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: persisted output path is unsafe: %w", err)
|
|
||||||
}
|
|
||||||
if err := rejectSymlinkComponents(bundleRoot, relative); err != nil {
|
|
||||||
return ResumeValidation{}, err
|
|
||||||
}
|
|
||||||
info, err := os.Lstat(path)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return NonResumable("extract output is missing"), nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: inspect output %q: %w", path, err)
|
|
||||||
}
|
|
||||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: output %q must be a regular file without symlinks", path)
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(checksum) == "" {
|
|
||||||
return NonResumable("extract output is missing its checksum"), nil
|
|
||||||
}
|
|
||||||
actual, err := artifacts.SHA256File(path)
|
|
||||||
if err != nil {
|
|
||||||
return ResumeValidation{}, fmt.Errorf("extract resume: checksum output %q: %w", path, err)
|
|
||||||
}
|
|
||||||
if actual != checksum {
|
|
||||||
return NonResumable("extract output checksum does not match durable bytes"), nil
|
|
||||||
}
|
|
||||||
return Resumable(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func rejectSymlinkComponents(root, slashRelative string) error {
|
|
||||||
current := filepath.Clean(root)
|
|
||||||
parts := strings.Split(filepath.FromSlash(slashRelative), string(filepath.Separator))
|
|
||||||
for _, part := range parts[:len(parts)-1] {
|
|
||||||
current = filepath.Join(current, part)
|
|
||||||
info, err := os.Lstat(current)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("extract resume: inspect output directory %q: %w", current, err)
|
|
||||||
}
|
|
||||||
if info.Mode()&os.ModeSymlink != 0 {
|
|
||||||
return fmt.Errorf("extract resume: output directory %q must not be a symlink", current)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func compatibleExtractionContract(got *artifactmodel.ContractMetadata, want config.NotariusOutputConfig) bool {
|
|
||||||
return got != nil && got.MediaType == want.MediaType && got.SchemaID == want.SchemaID &&
|
|
||||||
got.SchemaVersion == want.SchemaVersion && (want.ModuleKey == "" || got.ModuleKey == want.ModuleKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
func compatibleExtractionProvenance(got *artifactmodel.ExternalProvenance, runID, pipelineID, laneID string) bool {
|
|
||||||
return got != nil && got.System == "notarius" && got.RunID == runID &&
|
|
||||||
got.PipelineID == pipelineID && got.ArtifactID == laneID
|
|
||||||
}
|
|
||||||
|
|
||||||
func metadataString(metadata map[string]any, key string) string {
|
func metadataString(metadata map[string]any, key string) string {
|
||||||
if metadata == nil {
|
if metadata == nil {
|
||||||
return ""
|
return ""
|
||||||
@@ -240,11 +78,3 @@ func metadataString(metadata map[string]any, key string) string {
|
|||||||
value, _ := metadata[key].(string)
|
value, _ := metadata[key].(string)
|
||||||
return strings.TrimSpace(value)
|
return strings.TrimSpace(value)
|
||||||
}
|
}
|
||||||
|
|
||||||
func receiptIdentity(metadata map[string]any) (string, string) {
|
|
||||||
if metadata == nil {
|
|
||||||
return "", ""
|
|
||||||
}
|
|
||||||
receipt, _ := metadata["receipt"].(map[string]any)
|
|
||||||
return metadataString(receipt, "run_id"), metadataString(receipt, "pipeline_id")
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -361,7 +362,7 @@ func TestExtractStageAdapterResultIsImmediatelyReusableAndCatalogVisible(t *test
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExtractStageResumeValidationRejectsObsoleteResults(t *testing.T) {
|
func TestExtractStageResumeValidationRequiresFreshEvidence(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
mutate func(*testing.T, *Env, *manifest.Manifest)
|
mutate func(*testing.T, *Env, *manifest.Manifest)
|
||||||
@@ -372,6 +373,9 @@ func TestExtractStageResumeValidationRejectsObsoleteResults(t *testing.T) {
|
|||||||
{name: "config changed", mutate: func(_ *testing.T, env *Env, _ *manifest.Manifest) {
|
{name: "config changed", mutate: func(_ *testing.T, env *Env, _ *manifest.Manifest) {
|
||||||
env.Config.Pipeline.Notarius.PipelineID = "changed"
|
env.Config.Pipeline.Notarius.PipelineID = "changed"
|
||||||
}},
|
}},
|
||||||
|
{name: "no succeeded extract record", mutate: func(_ *testing.T, _ *Env, m *manifest.Manifest) {
|
||||||
|
m.Stages["extract"].Status = manifest.StatusFailed
|
||||||
|
}},
|
||||||
{name: "missing lane", mutate: func(t *testing.T, _ *Env, m *manifest.Manifest) {
|
{name: "missing lane", mutate: func(t *testing.T, _ *Env, m *manifest.Manifest) {
|
||||||
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
if err := os.Remove(m.Stages["extract"].Outputs[0].LocalPath); err != nil {
|
||||||
t.Fatalf("Remove(lane) error = %v", err)
|
t.Fatalf("Remove(lane) error = %v", err)
|
||||||
@@ -419,16 +423,45 @@ func TestExtractStageResumeValidationRejectsObsoleteResults(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestExtractStageResumeValidationRejectsUnsafePathWithError(t *testing.T) {
|
func TestExtractStageResumeValidationRejectsUnsafePathWithError(t *testing.T) {
|
||||||
env, m, _ := setupExtractEnv(t)
|
tests := []struct {
|
||||||
seedSucceededExtractResult(t, env, m)
|
name string
|
||||||
prior := *m.Stages["extract"]
|
mutate func(*testing.T, *Env, *manifest.Manifest)
|
||||||
m.Stages["extract"].Outputs[0].LocalPath = filepath.Join(env.Config.Pipeline.Workspace.Root, "outside.json")
|
}{
|
||||||
|
{name: "path outside bundle", mutate: func(_ *testing.T, env *Env, m *manifest.Manifest) {
|
||||||
if _, err := (extractStage{}).ValidateResume(context.Background(), env, m); err == nil || !strings.Contains(err.Error(), "unsafe") {
|
m.Stages["extract"].Outputs[0].LocalPath = filepath.Join(env.Config.Pipeline.Workspace.Root, "outside.json")
|
||||||
t.Fatalf("ValidateResume() error = %v, want unsafe path error", err)
|
}},
|
||||||
|
{name: "symlink", mutate: func(t *testing.T, _ *Env, m *manifest.Manifest) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("symlink creation requires privileges on Windows")
|
||||||
|
}
|
||||||
|
path := m.Stages["extract"].Outputs[0].LocalPath
|
||||||
|
outside := filepath.Join(t.TempDir(), "outside.json")
|
||||||
|
if err := os.WriteFile(outside, []byte(`{"outside":true}`), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(outside, path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}},
|
||||||
}
|
}
|
||||||
if m.Stages["extract"].Status != prior.Status || m.Stages["extract"].Error != prior.Error {
|
|
||||||
t.Fatalf("validation mutated stage record: %#v", m.Stages["extract"])
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
env, m, _ := setupExtractEnv(t)
|
||||||
|
seedSucceededExtractResult(t, env, m)
|
||||||
|
prior := *m.Stages["extract"]
|
||||||
|
test.mutate(t, env, m)
|
||||||
|
|
||||||
|
if _, err := (extractStage{}).ValidateResume(context.Background(), env, m); err == nil || !strings.Contains(err.Error(), "unsafe") {
|
||||||
|
t.Fatalf("ValidateResume() error = %v, want unsafe evidence error", err)
|
||||||
|
}
|
||||||
|
if m.Stages["extract"].Status != prior.Status || m.Stages["extract"].Error != prior.Error {
|
||||||
|
t.Fatalf("validation mutated stage record: %#v", m.Stages["extract"])
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user