Add deterministic analysis input identities

This commit is contained in:
2026-08-29 19:02:16 +00:00
parent 62de6abdbf
commit ab5751459a
12 changed files with 815 additions and 150 deletions

View File

@@ -8,6 +8,7 @@ import (
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
@@ -30,6 +31,9 @@ type AnalyzeEvidence struct {
SourceID string
Path string
ProducerRunID string
Contract *artifactmodel.ContractMetadata
Checksum string
Size int64
}
// InspectAnalyzeEvidence verifies that one configured artifact has supported,
@@ -107,6 +111,9 @@ func InspectAnalyzeEvidence(
SourceID: sourceID,
Path: filepath.Join(paths.Root, filepath.FromSlash(configuredPath)),
ProducerRunID: record.ProducerRunID,
Contract: cloneArtifactContract(record.Output.Contract),
Checksum: record.Output.Checksum,
Size: record.OutputSize,
}
}
@@ -129,6 +136,9 @@ func (c *ArtifactCatalog) HydrateAnalyzeArtifacts(
if evidence.State != AnalyzeEvidenceCurrent {
continue
}
_ = c.markAvailableFromAnalyzeManifest(evidence.SourceID, evidence.Path, evidence.ProducerRunID)
_ = c.markAvailableFromAnalyzeManifest(
evidence.SourceID, evidence.Path, evidence.ProducerRunID,
evidence.Checksum, evidence.Size, evidence.Contract,
)
}
}

View File

@@ -157,6 +157,18 @@ func TestHydrateAnalyzeArtifactsUsesOnlyCurrentConfiguredKeys(t *testing.T) {
if !entry.Available || entry.Provenance != ArtifactProvenanceCurrentAnalyzeManifest || entry.ProducerRunID != "run-1" {
t.Fatalf("entry = %#v", entry)
}
record := m.Stages["analyze"].AnalyzeArtifacts["session_recap"]
if entry.Checksum != record.Output.Checksum || entry.Size != record.OutputSize {
t.Fatalf("entry content identity = (%q, %d), want (%q, %d)", entry.Checksum, entry.Size, record.Output.Checksum, record.OutputSize)
}
if entry.Contract == nil || *entry.Contract != *record.Output.Contract {
t.Fatalf("entry contract = %#v, want %#v", entry.Contract, record.Output.Contract)
}
entry.Contract.SchemaVersion = "mutated"
again, _ := catalog.Lookup(ConfiguredArtifactSourceID("session_recap"))
if again.Contract == nil || again.Contract.SchemaVersion != "1" {
t.Fatalf("catalog contract was mutated through lookup: %#v", again.Contract)
}
if _, ok := catalog.Lookup(ConfiguredArtifactSourceID("removed")); ok {
t.Fatal("removed manifest record was advertised in current catalog")
}

View File

@@ -107,6 +107,9 @@ type ResolvedSessionArtifact struct {
OutputKind string
ProducerRunID string
Provenance string
Contract *artifactmodel.ContractMetadata
Checksum string
Size int64
}
// SessionArtifactNotFoundError includes context when a known artifact cannot be read.
@@ -259,6 +262,9 @@ func ResolveSessionArtifactWithCatalog(paths SessionPaths, m *manifest.Manifest,
OutputKind: entry.OutputKind,
ProducerRunID: entry.ProducerRunID,
Provenance: entry.Provenance,
Contract: cloneArtifactContract(entry.Contract),
Checksum: entry.Checksum,
Size: entry.Size,
}, nil
}

View File

@@ -5,6 +5,7 @@ import (
"sort"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
"gitea.maximumdirect.net/eric/narratio/internal/config"
)
@@ -64,6 +65,9 @@ type CatalogEntry struct {
Path string
Provenance string
ProducerRunID string
Contract *artifactmodel.ContractMetadata
Checksum string
Size int64
}
// ArtifactCatalog tracks built-in, configured, and extraction artifact definitions and runtime state.
@@ -98,6 +102,7 @@ func (c *ArtifactCatalog) RegisterExtractionArtifacts(configured map[string]Extr
if _, exists := c.extractionIndex[trimmed]; exists {
return fmt.Errorf("duplicate extraction artifact key %q", trimmed)
}
def := configured[key]
sourceID := ExtractionArtifactSourceID(trimmed)
if err := c.addEntry(CatalogEntry{
SourceID: sourceID,
@@ -105,6 +110,10 @@ func (c *ArtifactCatalog) RegisterExtractionArtifacts(configured map[string]Extr
ProducerStage: "extract",
OutputKind: "notarius_lane",
Planned: true,
Contract: &artifactmodel.ContractMetadata{
MediaType: def.MediaType, SchemaID: def.SchemaID,
SchemaVersion: def.SchemaVersion, ModuleKey: def.ModuleKey,
},
}); err != nil {
return fmt.Errorf("register extraction artifact %q: %w", trimmed, err)
}
@@ -221,7 +230,7 @@ func (c *ArtifactCatalog) Lookup(sourceID string) (CatalogEntry, bool) {
return CatalogEntry{}, false
}
entry, ok := c.entries[strings.TrimSpace(sourceID)]
return entry, ok
return cloneCatalogEntry(entry), ok
}
// SourceIDForConfiguredKey returns canonical source ID for one configured key.
@@ -255,7 +264,7 @@ func (c *ArtifactCatalog) ListConfigured() []CatalogEntry {
out := make([]CatalogEntry, 0, len(keys))
for _, key := range keys {
sourceID := c.configuredIndex[key]
out = append(out, c.entries[sourceID])
out = append(out, cloneCatalogEntry(c.entries[sourceID]))
}
return out
}
@@ -272,7 +281,7 @@ func (c *ArtifactCatalog) ListExtraction() []CatalogEntry {
sort.Strings(keys)
out := make([]CatalogEntry, 0, len(keys))
for _, key := range keys {
out = append(out, c.entries[c.extractionIndex[key]])
out = append(out, cloneCatalogEntry(c.entries[c.extractionIndex[key]]))
}
return out
}
@@ -282,22 +291,36 @@ func (c *ArtifactCatalog) MarkAvailableGenerated(sourceID, path string) error {
return c.markAvailable(sourceID, path, ArtifactProvenanceGeneratedCurrentAnalyzeRun)
}
func (c *ArtifactCatalog) markAvailableFromExtractManifest(sourceID, path, producerRunID string) error {
func (c *ArtifactCatalog) markAvailableFromExtractManifest(
sourceID, path, producerRunID, checksum string,
size int64,
contract *artifactmodel.ContractMetadata,
) error {
if err := c.markAvailable(sourceID, path, ArtifactProvenanceCurrentExtractManifest); err != nil {
return err
}
entry := c.entries[strings.TrimSpace(sourceID)]
entry.ProducerRunID = strings.TrimSpace(producerRunID)
entry.Checksum = strings.TrimSpace(checksum)
entry.Size = size
entry.Contract = cloneArtifactContract(contract)
c.entries[entry.SourceID] = entry
return nil
}
func (c *ArtifactCatalog) markAvailableFromAnalyzeManifest(sourceID, path, producerRunID string) error {
func (c *ArtifactCatalog) markAvailableFromAnalyzeManifest(
sourceID, path, producerRunID, checksum string,
size int64,
contract *artifactmodel.ContractMetadata,
) error {
if err := c.markAvailable(sourceID, path, ArtifactProvenanceCurrentAnalyzeManifest); err != nil {
return err
}
entry := c.entries[strings.TrimSpace(sourceID)]
entry.ProducerRunID = strings.TrimSpace(producerRunID)
entry.Checksum = strings.TrimSpace(checksum)
entry.Size = size
entry.Contract = cloneArtifactContract(contract)
c.entries[entry.SourceID] = entry
return nil
}
@@ -318,10 +341,29 @@ func (c *ArtifactCatalog) markAvailable(sourceID, path, provenance string) error
entry.Available = true
entry.Path = trimmedPath
entry.Provenance = provenance
entry.ProducerRunID = ""
entry.Checksum = ""
entry.Size = 0
if provenance == ArtifactProvenanceGeneratedCurrentAnalyzeRun {
entry.Contract = nil
}
c.entries[normalizedID] = entry
return nil
}
func cloneCatalogEntry(entry CatalogEntry) CatalogEntry {
entry.Contract = cloneArtifactContract(entry.Contract)
return entry
}
func cloneArtifactContract(contract *artifactmodel.ContractMetadata) *artifactmodel.ContractMetadata {
if contract == nil {
return nil
}
clone := *contract
return &clone
}
func (c *ArtifactCatalog) addEntry(entry CatalogEntry) error {
if c == nil {
return fmt.Errorf("artifact catalog is nil")

View File

@@ -39,8 +39,11 @@ func (c *ArtifactCatalog) HydrateExtractionArtifacts(
if proof.State != ExtractionEvidenceValid {
return
}
for sourceID, path := range proof.Outputs {
_ = c.markAvailableFromExtractManifest(sourceID, path, proof.ProducerRunID)
for sourceID, output := range proof.Outputs {
_ = c.markAvailableFromExtractManifest(
sourceID, output.Path, proof.ProducerRunID,
output.Checksum, output.Size, output.Contract,
)
}
}

View File

@@ -68,6 +68,17 @@ func TestHydrateExtractionArtifactsAcceptsOnlyCompleteCurrentBundle(t *testing.T
if entry.Provenance != ArtifactProvenanceCurrentExtractManifest || entry.ProducerRunID != "extract-run-1" {
t.Fatalf("hydrated provenance = %#v", entry)
}
wantOutput := currentManifest.Stages["extract"].Outputs[0]
info, err := os.Stat(wantOutput.LocalPath)
if err != nil {
t.Fatal(err)
}
if entry.Checksum != wantOutput.Checksum || entry.Size != info.Size() {
t.Fatalf("hydrated content identity = (%q, %d), want (%q, %d)", entry.Checksum, entry.Size, wantOutput.Checksum, info.Size())
}
if entry.Contract == nil || *entry.Contract != *wantOutput.Contract {
t.Fatalf("hydrated contract = %#v, want %#v", entry.Contract, wantOutput.Contract)
}
resolved, err := ResolveSessionArtifactWithCatalog(paths, currentManifest, entry.SourceID, catalog)
if err != nil {
t.Fatalf("ResolveSessionArtifactWithCatalog() error = %v", err)

View File

@@ -6,6 +6,7 @@ import (
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
@@ -23,7 +24,15 @@ const (
type ExtractionEvidence struct {
State ExtractionEvidenceState
Reason, ProducerRunID string
Outputs map[string]string
Outputs map[string]ExtractionEvidenceOutput
}
// ExtractionEvidenceOutput is the verified semantic identity of one lane.
type ExtractionEvidenceOutput struct {
Path string
Checksum string
Size int64
Contract *artifactmodel.ContractMetadata
}
// InspectExtractionEvidence verifies structure, confinement, identities, contracts, and payload bytes.
@@ -72,7 +81,7 @@ func InspectExtractionEvidence(
for key, d := range configured {
expected[ExtractionArtifactSourceID(key)] = d
}
seen, outputs := map[string]struct{}{}, map[string]string{}
seen, outputs := map[string]struct{}{}, map[string]ExtractionEvidenceOutput{}
indexSeen := false
for _, out := range r.Outputs {
if strings.TrimSpace(out.ProducerRunID) != runID {
@@ -82,7 +91,7 @@ func InspectExtractionEvidence(
if indexSeen || out.Kind != extractionIndexKind || filepath.Clean(out.LocalPath) != filepath.Join(root, "index.json") {
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract index path is not canonical"}
}
if state, reason := inspectExtractionPayload(root, out.LocalPath, out.Checksum); state != ExtractionEvidenceValid {
if state, reason, _ := inspectExtractionPayload(root, out.LocalPath, out.Checksum); state != ExtractionEvidenceValid {
return ExtractionEvidence{State: state, Reason: reason}
}
indexSeen = true
@@ -98,11 +107,15 @@ func InspectExtractionEvidence(
if !compatibleCatalogExtractionContract(out.Contract, d) || !compatibleCatalogExtractionProvenance(out.ExternalProvenance, receiptRunID, receiptPipelineID, d) {
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract output contract or provenance is incompatible"}
}
if state, reason := inspectExtractionPayload(root, out.LocalPath, out.Checksum); state != ExtractionEvidenceValid {
state, reason, size := inspectExtractionPayload(root, out.LocalPath, out.Checksum)
if state != ExtractionEvidenceValid {
return ExtractionEvidence{State: state, Reason: reason}
}
seen[out.SourceID] = struct{}{}
outputs[out.SourceID] = out.LocalPath
outputs[out.SourceID] = ExtractionEvidenceOutput{
Path: out.LocalPath, Checksum: out.Checksum, Size: size,
Contract: cloneArtifactContract(out.Contract),
}
}
if !indexSeen || len(seen) != len(expected) || len(r.Outputs) != len(expected)+1 {
return ExtractionEvidence{State: ExtractionEvidenceObsolete, Reason: "extract result is incomplete"}
@@ -110,30 +123,30 @@ func InspectExtractionEvidence(
return ExtractionEvidence{State: ExtractionEvidenceValid, ProducerRunID: runID, Outputs: outputs}
}
func inspectExtractionPayload(root, path, checksum string) (ExtractionEvidenceState, string) {
func inspectExtractionPayload(root, path, checksum string) (ExtractionEvidenceState, string, int64) {
if !filepath.IsAbs(path) || !pathWithinExtractionRoot(root, path) || strings.TrimSpace(checksum) == "" {
return ExtractionEvidenceUnsafe, "extract output path or checksum is unsafe"
return ExtractionEvidenceUnsafe, "extract output path or checksum is unsafe", 0
}
info, err := os.Lstat(path)
if os.IsNotExist(err) {
return ExtractionEvidenceObsolete, "extract output is missing"
return ExtractionEvidenceObsolete, "extract output is missing", 0
}
if !safeExtractionComponents(root, path) {
return ExtractionEvidenceUnsafe, "extract output path contains unsafe components"
return ExtractionEvidenceUnsafe, "extract output path contains unsafe components", 0
}
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
return ExtractionEvidenceUnsafe, "extract output is not a regular file"
return ExtractionEvidenceUnsafe, "extract output is not a regular file", 0
}
actual, err := SHA256File(path)
if err != nil {
return ExtractionEvidenceUnsafe, "extract output checksum cannot be read"
return ExtractionEvidenceUnsafe, "extract output checksum cannot be read", 0
}
if actual != checksum {
return ExtractionEvidenceObsolete, "extract output checksum does not match durable bytes"
return ExtractionEvidenceObsolete, "extract output checksum does not match durable bytes", 0
}
body, err := fileops.ReadRegularFile(path, MaxExtractionPayloadBytes)
if err != nil || !json.Valid(body) {
return ExtractionEvidenceObsolete, "extract output is not valid JSON"
return ExtractionEvidenceObsolete, "extract output is not valid JSON", 0
}
return ExtractionEvidenceValid, ""
return ExtractionEvidenceValid, "", info.Size()
}

View File

@@ -44,21 +44,6 @@ type analyzeExecutionContext struct {
Catalog *artifacts.ArtifactCatalog
}
type analyzeInputResolutionState uint8
const (
analyzeInputPresent analyzeInputResolutionState = iota
analyzeInputAbsent
analyzeInputError
)
type analyzeInputResolution struct {
State analyzeInputResolutionState
Path string
Artifact *artifacts.ResolvedSessionArtifact
Err error
}
func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
if env == nil || env.Config == nil {
return nil, fmt.Errorf("analyze: stage environment config is required")
@@ -335,34 +320,25 @@ func executeAnalyzeArtifact(
artifactName := plan.Name
artifactCfg := plan.Cfg
inputPaths := map[string]string{}
resolvedInputs, err := resolveAnalyzeInputIdentities(artifactCfg.Inputs, execution)
if err != nil {
return nil, fmt.Errorf("analyze: resolve inputs for artifact %q: %w", artifactName, err)
}
inputPaths := resolvedInputs.Paths()
omittedOptionalInputs := []string{}
reusedArtifacts := []map[string]any{}
inputNames := sortedScriptoriumInputNames(artifactCfg.Inputs)
for _, inputName := range inputNames {
inputCfg := artifactCfg.Inputs[inputName]
resolution := resolveScriptoriumInput(inputCfg, execution)
switch resolution.State {
case analyzeInputError:
return nil, fmt.Errorf("analyze: resolve input %q for artifact %q: %w", inputName, artifactName, resolution.Err)
case analyzeInputAbsent:
if inputCfg.Required {
return nil, fmt.Errorf("analyze: required input %q for artifact %q could not be resolved", inputName, artifactName)
}
omittedOptionalInputs = append(omittedOptionalInputs, inputName)
for _, identity := range resolvedInputs.Ordered {
if !identity.Present {
omittedOptionalInputs = append(omittedOptionalInputs, identity.Name)
continue
case analyzeInputPresent:
inputPaths[inputName] = resolution.Path
default:
return nil, fmt.Errorf("analyze: resolve input %q for artifact %q: invalid resolution state", inputName, artifactName)
}
if resolution.Artifact != nil && resolution.Artifact.Provenance == artifacts.ArtifactProvenanceCurrentAnalyzeManifest {
resolvedArtifact := resolvedInputs.Artifact(identity.Name)
if resolvedArtifact != nil && resolvedArtifact.Provenance == artifacts.ArtifactProvenanceCurrentAnalyzeManifest {
reusedArtifacts = append(reusedArtifacts, map[string]any{
"name": configuredArtifactNameFromSourceID(resolution.Artifact.ID),
"source_id": resolution.Artifact.ID,
"path": resolution.Artifact.Path,
"provenance": resolution.Artifact.Provenance,
"name": configuredArtifactNameFromSourceID(resolvedArtifact.ID),
"source_id": resolvedArtifact.ID,
"path": resolvedArtifact.Path,
"provenance": resolvedArtifact.Provenance,
})
}
}
@@ -620,92 +596,6 @@ func discoverAnalyzeArtifactRef(m *manifest.Manifest, paths artifacts.SessionPat
return resolved.Path, resolved.Provenance
}
func resolveScriptoriumInput(inputCfg config.ScriptoriumInputConfig, execution analyzeExecutionContext) analyzeInputResolution {
source := strings.TrimSpace(inputCfg.Source)
descriptor, describeErr := artifactpolicy.DescribeScriptoriumInputSource(source)
if describeErr != nil {
return analyzeInputFailure(describeErr)
}
if descriptor.Source.Kind == artifactpolicy.SourceKindStableInput {
identity, err := artifacts.ResolvePreparedInput(execution.Paths, execution.Manifest, descriptor.Source.ID)
if err == nil {
return analyzeInputFound(identity.Path, nil)
}
if errors.Is(err, artifacts.ErrPreparedInputAbsent) {
if inputCfg.Required {
return analyzeInputFailure(fmt.Errorf(
"required prepared input source %q is unavailable; run narratio run-stage prepare %s --force",
descriptor.Source.ID,
execution.SessionID,
))
}
return analyzeInputMissing()
}
return analyzeInputFailure(fmt.Errorf(
"prepared input source %q is invalid; run narratio run-stage prepare %s --force: %w",
descriptor.Source.ID,
execution.SessionID,
err,
))
}
if descriptor.Source.Kind == artifactpolicy.SourceKindPreviousArtifact {
resolved, err := artifacts.ResolvePreviousSessionArtifactWithCatalog(execution.Paths, execution.Manifest, source, execution.Catalog)
if err == nil {
copy := resolved
return analyzeInputFound(resolved.Path, &copy)
}
if errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
if inputCfg.Required {
return analyzeInputFailure(fmt.Errorf(
"required previous-session input source %q is unavailable; run narratio run-stage prepare %s --force",
source,
execution.SessionID,
))
}
return analyzeInputMissing()
}
return analyzeInputFailure(err)
}
resolved, err := artifacts.ResolveSessionArtifactWithCatalog(execution.Paths, execution.Manifest, source, execution.Catalog)
if err == nil {
copy := resolved
return analyzeInputFound(resolved.Path, &copy)
}
if !errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
return analyzeInputFailure(err)
}
if !inputCfg.Required {
return analyzeInputMissing()
}
switch descriptor.Source.Kind {
case artifactpolicy.SourceKindExtraction:
return analyzeInputFailure(fmt.Errorf(
"required extraction source %q is unavailable; enable and configure pipeline.notarius output %q, then run narratio run-stage extract %s --force",
source,
descriptor.Source.ConfiguredKey,
execution.SessionID,
))
case artifactpolicy.SourceKindConfiguredArtifact:
return analyzeInputFailure(fmt.Errorf("configured artifact source %q is unavailable", source))
default:
return analyzeInputFailure(requiredBuiltInInputError(descriptor.Source.ID, execution))
}
}
func analyzeInputFound(path string, artifact *artifacts.ResolvedSessionArtifact) analyzeInputResolution {
return analyzeInputResolution{State: analyzeInputPresent, Path: path, Artifact: artifact}
}
func analyzeInputMissing() analyzeInputResolution {
return analyzeInputResolution{State: analyzeInputAbsent}
}
func analyzeInputFailure(err error) analyzeInputResolution {
return analyzeInputResolution{State: analyzeInputError, Err: err}
}
func requiredBuiltInInputError(source string, execution analyzeExecutionContext) error {
entry, ok := execution.Catalog.Lookup(source)
if !ok || strings.TrimSpace(entry.ProducerStage) == "" {

View File

@@ -0,0 +1,294 @@
package stage
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/fileops"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
)
// analyzeInputContract captures the Narratio-visible content contract without
// producer identity or local placement.
type analyzeInputContract struct {
OutputKind string
ManifestKind string
MediaType string
SchemaID string
SchemaVersion string
ModuleKey string
}
// analyzeInputIdentity is the stable semantic identity of one configured
// Scriptorium input. It deliberately contains no filesystem path or run ID.
type analyzeInputIdentity struct {
Name string
SourceID string
Required bool
Present bool
LogicalID string
Contract analyzeInputContract
Checksum string
Size int64
}
type resolvedAnalyzeInputs struct {
Ordered []analyzeInputIdentity
paths map[string]string
artifacts map[string]*artifacts.ResolvedSessionArtifact
}
func (r resolvedAnalyzeInputs) Paths() map[string]string {
if len(r.paths) == 0 {
return map[string]string{}
}
out := make(map[string]string, len(r.paths))
for name, path := range r.paths {
out[name] = path
}
return out
}
func (r resolvedAnalyzeInputs) Artifact(name string) *artifacts.ResolvedSessionArtifact {
artifact := r.artifacts[name]
if artifact == nil {
return nil
}
copy := *artifact
if artifact.Contract != nil {
contract := *artifact.Contract
copy.Contract = &contract
}
return &copy
}
func resolveAnalyzeInputIdentities(
inputs map[string]config.ScriptoriumInputConfig,
execution analyzeExecutionContext,
) (resolvedAnalyzeInputs, error) {
result := resolvedAnalyzeInputs{
Ordered: make([]analyzeInputIdentity, 0, len(inputs)),
paths: make(map[string]string, len(inputs)),
artifacts: make(map[string]*artifacts.ResolvedSessionArtifact, len(inputs)),
}
for _, name := range sortedScriptoriumInputNames(inputs) {
identity, path, artifact, err := resolveAnalyzeInputIdentity(name, inputs[name], execution)
if err != nil {
return resolvedAnalyzeInputs{}, fmt.Errorf("resolve input %q: %w", name, err)
}
result.Ordered = append(result.Ordered, identity)
if !identity.Present {
continue
}
result.paths[name] = path
if artifact != nil {
result.artifacts[name] = artifact
}
}
return result, nil
}
func resolveAnalyzeInputIdentity(
name string,
inputCfg config.ScriptoriumInputConfig,
execution analyzeExecutionContext,
) (analyzeInputIdentity, string, *artifacts.ResolvedSessionArtifact, error) {
descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(inputCfg.Source)
if err != nil {
return analyzeInputIdentity{}, "", nil, err
}
identity := analyzeInputIdentity{
Name: name,
SourceID: descriptor.Source.ID,
Required: inputCfg.Required,
LogicalID: descriptor.Source.ID,
Contract: declaredAnalyzeInputContract(descriptor, execution.Catalog),
}
if descriptor.Source.Kind == artifactpolicy.SourceKindStableInput {
prepared, preparedErr := artifacts.ResolvePreparedInput(execution.Paths, execution.Manifest, descriptor.Source.ID)
if preparedErr == nil {
identity.Present = true
identity.Checksum = prepared.Checksum
identity.Size = prepared.Size
identity.Contract.ManifestKind = prepared.ManifestKind
return identity, prepared.Path, nil, nil
}
if errors.Is(preparedErr, artifacts.ErrPreparedInputAbsent) {
if !inputCfg.Required {
return identity, "", nil, nil
}
return analyzeInputIdentity{}, "", nil, fmt.Errorf(
"required prepared input source %q is unavailable; run narratio run-stage prepare %s --force",
descriptor.Source.ID,
execution.SessionID,
)
}
return analyzeInputIdentity{}, "", nil, fmt.Errorf(
"prepared input source %q is invalid; run narratio run-stage prepare %s --force: %w",
descriptor.Source.ID,
execution.SessionID,
preparedErr,
)
}
var resolved artifacts.ResolvedSessionArtifact
if descriptor.Source.Kind == artifactpolicy.SourceKindPreviousArtifact {
resolved, err = artifacts.ResolvePreviousSessionArtifactWithCatalog(
execution.Paths, execution.Manifest, descriptor.Source.ID, execution.Catalog,
)
} else {
resolved, err = artifacts.ResolveSessionArtifactWithCatalog(
execution.Paths, execution.Manifest, descriptor.Source.ID, execution.Catalog,
)
}
if err != nil {
if !errors.Is(err, artifacts.ErrSessionArtifactNotFound) {
return analyzeInputIdentity{}, "", nil, err
}
if !inputCfg.Required {
return identity, "", nil, nil
}
return analyzeInputIdentity{}, "", nil, missingAnalyzeInputError(descriptor, execution)
}
identity.Present = true
identity.Contract = contractForResolvedAnalyzeInput(resolved)
if strings.TrimSpace(resolved.Checksum) != "" && resolved.Size > 0 {
identity.Checksum = resolved.Checksum
identity.Size = resolved.Size
} else {
identity.Checksum, identity.Size, err = hashAnalyzeInputFile(execution.Paths, resolved.Path)
if err != nil {
return analyzeInputIdentity{}, "", nil, fmt.Errorf("verify source %q: %w", descriptor.Source.ID, err)
}
}
copy := resolved
return identity, resolved.Path, &copy, nil
}
func missingAnalyzeInputError(
descriptor artifactpolicy.ScriptoriumInputSourceDescriptor,
execution analyzeExecutionContext,
) error {
source := descriptor.Source.ID
switch descriptor.Source.Kind {
case artifactpolicy.SourceKindPreviousArtifact:
return fmt.Errorf(
"required previous-session input source %q is unavailable; run narratio run-stage prepare %s --force",
source,
execution.SessionID,
)
case artifactpolicy.SourceKindExtraction:
return fmt.Errorf(
"required extraction source %q is unavailable; enable and configure pipeline.notarius output %q, then run narratio run-stage extract %s --force",
source,
descriptor.Source.ConfiguredKey,
execution.SessionID,
)
case artifactpolicy.SourceKindConfiguredArtifact:
return fmt.Errorf("configured artifact source %q is unavailable", source)
default:
return requiredBuiltInInputError(source, execution)
}
}
func declaredAnalyzeInputContract(
descriptor artifactpolicy.ScriptoriumInputSourceDescriptor,
catalog *artifacts.ArtifactCatalog,
) analyzeInputContract {
if descriptor.Source.Kind == artifactpolicy.SourceKindStableInput {
if prepared, ok := artifactpolicy.DescribePreparedInputSource(descriptor.Source.ID); ok {
return analyzeInputContract{OutputKind: "prepared_input", ManifestKind: prepared.ManifestKind}
}
}
if descriptor.Source.Kind == artifactpolicy.SourceKindPreviousArtifact {
return analyzeInputContract{OutputKind: "previous_session_cache"}
}
if catalog != nil {
if entry, ok := catalog.Lookup(descriptor.Source.ID); ok {
return analyzeInputContractFromMetadata(entry.OutputKind, "", entry.Contract)
}
}
return analyzeInputContract{}
}
func contractForResolvedAnalyzeInput(resolved artifacts.ResolvedSessionArtifact) analyzeInputContract {
return analyzeInputContractFromMetadata(resolved.OutputKind, "", resolved.Contract)
}
func analyzeInputContractFromMetadata(
outputKind, manifestKind string,
contract *artifactmodel.ContractMetadata,
) analyzeInputContract {
result := analyzeInputContract{OutputKind: outputKind, ManifestKind: manifestKind}
if contract != nil {
result.MediaType = contract.MediaType
result.SchemaID = contract.SchemaID
result.SchemaVersion = contract.SchemaVersion
result.ModuleKey = contract.ModuleKey
}
return result
}
func hashAnalyzeInputFile(paths artifacts.SessionPaths, path string) (string, int64, error) {
root, err := filepath.Abs(strings.TrimSpace(paths.Root))
if err != nil || strings.TrimSpace(paths.Root) == "" {
if err == nil {
err = fmt.Errorf("session root is required")
}
return "", 0, err
}
target, err := filepath.Abs(strings.TrimSpace(path))
if err != nil || strings.TrimSpace(path) == "" {
if err == nil {
err = fmt.Errorf("input path is required")
}
return "", 0, err
}
relative, err := filepath.Rel(root, target)
if err != nil {
return "", 0, fmt.Errorf("resolve input below session root: %w", err)
}
relative, err = pathsafe.NormalizeRelativeDestination(filepath.ToSlash(relative))
if err != nil {
return "", 0, fmt.Errorf("input path is outside session root: %w", err)
}
file, err := fileops.OpenConfinedRegularFile(root, relative)
if err != nil {
return "", 0, err
}
info, err := file.Stat()
if err != nil {
_ = file.Close()
return "", 0, fmt.Errorf("inspect input: %w", err)
}
digest := sha256.New()
size, readErr := io.Copy(digest, io.LimitReader(file, artifacts.MaxResolvedArtifactBytes+1))
closeErr := file.Close()
if readErr != nil {
return "", 0, fmt.Errorf("checksum input: %w", readErr)
}
if closeErr != nil {
return "", 0, fmt.Errorf("close input: %w", closeErr)
}
if size > artifacts.MaxResolvedArtifactBytes {
return "", 0, fmt.Errorf("input exceeds %d-byte limit", artifacts.MaxResolvedArtifactBytes)
}
if size == 0 {
return "", 0, fmt.Errorf("input is empty")
}
if size != info.Size() {
return "", 0, fmt.Errorf("input size changed while hashing")
}
return hex.EncodeToString(digest.Sum(nil)), size, nil
}

View File

@@ -0,0 +1,365 @@
package stage
import (
"math/rand"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestResolveAnalyzeInputIdentitiesSupportsEverySourceKind(t *testing.T) {
tests := []struct {
name string
sourceID string
setup func(*testing.T, *Env, *manifest.Manifest) string
wantOutputKind string
wantContract *artifactmodel.ContractMetadata
}{
{
name: "transcript", sourceID: "narratio.transcript.final_trimmed",
setup: func(t *testing.T, env *Env, m *manifest.Manifest) string {
path := filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsDir, "final.trimmed.json")
writeAnalyzeFile(t, path, `{"segments":[]}`)
return path
},
wantOutputKind: "transcript_final_trimmed",
},
{
name: "prepared", sourceID: artifactpolicy.SourceInputPlayers,
setup: func(t *testing.T, env *Env, m *manifest.Manifest) string {
path := filepath.Join(sessionPathsForEnv(env, m.SessionID).InputsDir, "players.yml")
recordPreparedAnalyzeInput(t, m, artifactpolicy.SourceInputPlayers, path, "players:\n - Hrank\n")
return path
},
wantOutputKind: "prepared_input",
},
{
name: "extraction", sourceID: artifacts.ExtractionArtifactSourceID("encounters"),
setup: func(t *testing.T, env *Env, m *manifest.Manifest) string {
paths := sessionPathsForEnv(env, m.SessionID)
writeAnalyzeFile(t, filepath.Join(paths.TranscriptsDir, "final.trimmed.json"), `{"segments":[]}`)
return configureAnalyzeExtractionFixture(t, env, m)["encounters"]
},
wantOutputKind: "notarius_lane",
wantContract: &artifactmodel.ContractMetadata{
MediaType: "application/json", SchemaID: "encounters", SchemaVersion: "1", ModuleKey: "encounters",
},
},
{
name: "previous", sourceID: "narratio.previous_session.artifact.session_recap",
setup: func(t *testing.T, env *Env, m *manifest.Manifest) string {
path := mustPreviousArtifactPath(t, sessionPathsForEnv(env, m.SessionID), "artifacts/session_recap.md")
writeAnalyzeFile(t, path, "previous recap\n")
m.Inputs = append(m.Inputs, manifest.InputRecord{Kind: "previous_artifact", Path: path})
return path
},
wantOutputKind: "previous_session_cache",
},
{
name: "configured", sourceID: artifacts.ConfiguredArtifactSourceID("player_handout"),
setup: func(t *testing.T, env *Env, m *manifest.Manifest) string {
env.Config.Pipeline.Scriptorium.Artifacts["player_handout"] = config.ScriptoriumArtifactConfig{
Enabled: false, OutputPath: "artifacts/player_handout.md",
}
path := filepath.Join(sessionPathsForEnv(env, m.SessionID).ArtifactsDir, "player_handout.md")
writeAnalyzeFile(t, path, "player handout\n")
setCurrentAnalyzeEvidence(t, m, "player_handout", "artifacts/player_handout.md", path)
return path
},
wantOutputKind: "scriptorium_artifact",
wantContract: &artifactmodel.ContractMetadata{
MediaType: "text/markdown", SchemaID: "narratio.player_handout", SchemaVersion: "1",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
wantPath := tt.setup(t, env, m)
execution := newAnalyzeIdentityExecution(t, env, m)
resolved, err := resolveAnalyzeInputIdentities(map[string]config.ScriptoriumInputConfig{
"subject": {Source: tt.sourceID, Required: true},
}, execution)
if err != nil {
t.Fatalf("resolveAnalyzeInputIdentities() error = %v", err)
}
if len(resolved.Ordered) != 1 {
t.Fatalf("identities = %#v, want one", resolved.Ordered)
}
identity := resolved.Ordered[0]
if identity.Name != "subject" || identity.SourceID != tt.sourceID || identity.LogicalID != tt.sourceID {
t.Fatalf("identity names = %#v", identity)
}
if !identity.Required || !identity.Present {
t.Fatalf("identity availability = %#v", identity)
}
if identity.Contract.OutputKind != tt.wantOutputKind {
t.Fatalf("output kind = %q, want %q", identity.Contract.OutputKind, tt.wantOutputKind)
}
if tt.name == "prepared" && identity.Contract.ManifestKind != "players" {
t.Fatalf("manifest kind = %q, want players", identity.Contract.ManifestKind)
}
if tt.wantContract != nil {
got := artifactmodel.ContractMetadata{
MediaType: identity.Contract.MediaType, SchemaID: identity.Contract.SchemaID,
SchemaVersion: identity.Contract.SchemaVersion, ModuleKey: identity.Contract.ModuleKey,
}
if !reflect.DeepEqual(got, *tt.wantContract) {
t.Fatalf("contract = %#v, want %#v", got, *tt.wantContract)
}
}
checksum, err := artifacts.SHA256File(wantPath)
if err != nil {
t.Fatal(err)
}
info, err := os.Stat(wantPath)
if err != nil {
t.Fatal(err)
}
if identity.Checksum != checksum || identity.Size != info.Size() {
t.Fatalf("content identity = (%q, %d), want (%q, %d)", identity.Checksum, identity.Size, checksum, info.Size())
}
if got := resolved.Paths()["subject"]; got != wantPath {
t.Fatalf("runtime path = %q, want %q", got, wantPath)
}
})
}
}
func TestResolveAnalyzeInputIdentitiesRecordsOptionalAbsenceAndRejectsRequiredAbsence(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
execution := newAnalyzeIdentityExecution(t, env, m)
sourceID := artifactpolicy.SourceInputPlayers
resolved, err := resolveAnalyzeInputIdentities(map[string]config.ScriptoriumInputConfig{
"players": {Source: sourceID, Required: false},
}, execution)
if err != nil {
t.Fatalf("optional input error = %v", err)
}
if len(resolved.Ordered) != 1 {
t.Fatalf("identities = %#v, want one", resolved.Ordered)
}
identity := resolved.Ordered[0]
if identity.Present || identity.Required || identity.SourceID != sourceID || identity.Checksum != "" || identity.Size != 0 {
t.Fatalf("optional absent identity = %#v", identity)
}
if len(resolved.Paths()) != 0 {
t.Fatalf("optional absent paths = %#v, want none", resolved.Paths())
}
_, err = resolveAnalyzeInputIdentities(map[string]config.ScriptoriumInputConfig{
"players": {Source: sourceID, Required: true},
}, execution)
if err == nil || !strings.Contains(err.Error(), "run narratio run-stage prepare") {
t.Fatalf("required input error = %v, want prepare guidance", err)
}
}
func TestResolveAnalyzeInputIdentitiesRejectsUnsafePathsAndFileTypes(t *testing.T) {
tests := []struct {
name string
sourceID string
setup func(*testing.T, *Env, *manifest.Manifest)
}{
{
name: "directory", sourceID: "narratio.transcript.final_trimmed",
setup: func(t *testing.T, env *Env, m *manifest.Manifest) {
t.Helper()
path := filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsDir, "final.trimmed.json")
if err := os.MkdirAll(path, 0o755); err != nil {
t.Fatal(err)
}
},
},
{
name: "symlink", sourceID: "narratio.transcript.final_trimmed",
setup: func(t *testing.T, env *Env, m *manifest.Manifest) {
t.Helper()
path := filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsDir, "final.trimmed.json")
outside := filepath.Join(t.TempDir(), "outside.json")
writeAnalyzeFile(t, outside, `{"segments":[]}`)
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, path); err != nil {
t.Fatal(err)
}
},
},
{
name: "manifest path outside session", sourceID: artifactpolicy.SourceInputPlayers,
setup: func(t *testing.T, _ *Env, m *manifest.Manifest) {
t.Helper()
outside := filepath.Join(t.TempDir(), "players.yml")
recordPreparedAnalyzeInput(t, m, artifactpolicy.SourceInputPlayers, outside, "players:\n - Hrank\n")
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
tt.setup(t, env, m)
execution := newAnalyzeIdentityExecution(t, env, m)
_, err := resolveAnalyzeInputIdentities(map[string]config.ScriptoriumInputConfig{
"subject": {Source: tt.sourceID, Required: true},
}, execution)
if err == nil {
t.Fatal("resolveAnalyzeInputIdentities() error = nil, want unsafe-file rejection")
}
})
}
}
func TestAnalyzeInputIdentityIsStableAcrossWorkspaceRelocationAndChangesWithBytes(t *testing.T) {
resolve := func(t *testing.T, body string) analyzeInputIdentity {
t.Helper()
env, m, _ := setupAnalyzeEnv(t)
path := filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsDir, "final.trimmed.json")
writeAnalyzeFile(t, path, body)
resolved, err := resolveAnalyzeInputIdentities(map[string]config.ScriptoriumInputConfig{
"transcript": {Source: "narratio.transcript.final_trimmed", Required: true},
}, newAnalyzeIdentityExecution(t, env, m))
if err != nil {
t.Fatal(err)
}
return resolved.Ordered[0]
}
first := resolve(t, `{"segments":[]}`)
relocated := resolve(t, `{"segments":[]}`)
if !reflect.DeepEqual(first, relocated) {
t.Fatalf("relocated identity changed:\nfirst: %#v\nsecond: %#v", first, relocated)
}
changed := resolve(t, `{"segments":[1]}`)
if first.Checksum == changed.Checksum {
t.Fatalf("checksum did not change with bytes: %#v", changed)
}
}
func TestAnalyzeInputIdentityChangesWithContract(t *testing.T) {
resolve := func(t *testing.T, schemaVersion string) analyzeInputIdentity {
t.Helper()
env, m, _ := setupAnalyzeEnv(t)
env.Config.Pipeline.Scriptorium.Artifacts["player_handout"] = config.ScriptoriumArtifactConfig{
Enabled: false, OutputPath: "artifacts/player_handout.md",
}
path := filepath.Join(sessionPathsForEnv(env, m.SessionID).ArtifactsDir, "player_handout.md")
writeAnalyzeFile(t, path, "same bytes\n")
setCurrentAnalyzeEvidence(t, m, "player_handout", "artifacts/player_handout.md", path)
record := m.Stages["analyze"].AnalyzeArtifacts["player_handout"]
record.Output.Contract.SchemaVersion = schemaVersion
m.Stages["analyze"].AnalyzeArtifacts["player_handout"] = record
resolved, err := resolveAnalyzeInputIdentities(map[string]config.ScriptoriumInputConfig{
"handout": {Source: artifacts.ConfiguredArtifactSourceID("player_handout"), Required: true},
}, newAnalyzeIdentityExecution(t, env, m))
if err != nil {
t.Fatal(err)
}
return resolved.Ordered[0]
}
first := resolve(t, "1")
changed := resolve(t, "2")
if first.Checksum != changed.Checksum || first.Size != changed.Size {
t.Fatalf("test fixture content changed: %#v vs %#v", first, changed)
}
if reflect.DeepEqual(first, changed) {
t.Fatalf("identity did not change with contract: %#v", changed)
}
}
func TestConfiguredAnalyzeInputRequiresCurrentManifestEvidence(t *testing.T) {
tests := []struct {
name string
stale bool
}{
{name: "incidental file"},
{name: "stale evidence", stale: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
env.Config.Pipeline.Scriptorium.Artifacts["player_handout"] = config.ScriptoriumArtifactConfig{
Enabled: false, OutputPath: "artifacts/player_handout.md",
}
path := filepath.Join(sessionPathsForEnv(env, m.SessionID).ArtifactsDir, "player_handout.md")
writeAnalyzeFile(t, path, "untrusted handout\n")
if tt.stale {
setCurrentAnalyzeEvidence(t, m, "player_handout", "artifacts/player_handout.md", path)
record := m.Stages["analyze"].AnalyzeArtifacts["player_handout"]
record.Status = manifest.AnalyzeArtifactStale
record.Output = nil
record.OutputSize = 0
m.Stages["analyze"].AnalyzeArtifacts["player_handout"] = record
}
_, err := resolveAnalyzeInputIdentities(map[string]config.ScriptoriumInputConfig{
"handout": {Source: artifacts.ConfiguredArtifactSourceID("player_handout"), Required: true},
}, newAnalyzeIdentityExecution(t, env, m))
if err == nil || !strings.Contains(err.Error(), "configured artifact source") {
t.Fatalf("error = %v, want configured source unavailable", err)
}
})
}
}
func TestResolveAnalyzeInputIdentitiesOrdersConfiguredNamesDeterministically(t *testing.T) {
env, m, _ := setupAnalyzeEnv(t)
path := filepath.Join(sessionPathsForEnv(env, m.SessionID).TranscriptsDir, "final.trimmed.json")
writeAnalyzeFile(t, path, `{"segments":[]}`)
execution := newAnalyzeIdentityExecution(t, env, m)
names := []string{"zeta", "alpha", "middle", "beta"}
want := append([]string(nil), names...)
slices.Sort(want)
random := rand.New(rand.NewSource(42))
for attempt := 0; attempt < 20; attempt++ {
inputs := make(map[string]config.ScriptoriumInputConfig, len(names))
for _, index := range random.Perm(len(names)) {
inputs[names[index]] = config.ScriptoriumInputConfig{
Source: "narratio.transcript.final_trimmed", Required: true,
}
}
resolved, err := resolveAnalyzeInputIdentities(inputs, execution)
if err != nil {
t.Fatal(err)
}
got := make([]string, 0, len(resolved.Ordered))
for _, identity := range resolved.Ordered {
got = append(got, identity.Name)
}
if !slices.Equal(got, want) {
t.Fatalf("ordered names = %#v, want %#v", got, want)
}
}
}
func newAnalyzeIdentityExecution(t *testing.T, env *Env, m *manifest.Manifest) analyzeExecutionContext {
t.Helper()
configured := artifacts.ConfiguredArtifactDefinitions(env.Config.Pipeline.Scriptorium.Artifacts)
effective, err := artifacts.ResolveEffectiveArtifactSet(configured, nil)
if err != nil {
t.Fatal(err)
}
paths := sessionPathsForEnv(env, m.SessionID)
catalog, err := buildAnalyzeRuntimeArtifactCatalog(
paths, m, env.Config.Pipeline.Scriptorium, env.Config.Pipeline.Notarius, effective,
)
if err != nil {
t.Fatal(err)
}
return analyzeExecutionContext{
Env: env, Manifest: m, Paths: paths, SessionID: m.SessionID, Catalog: catalog,
}
}