465 lines
18 KiB
Go
465 lines
18 KiB
Go
package checkpoint
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"path"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
)
|
|
|
|
type WorkspaceRecorder struct {
|
|
root string
|
|
identityDigest string
|
|
now func() time.Time
|
|
}
|
|
|
|
func NewWorkspaceRecorder(settings coreworkspace.Settings, identity coreworkspace.CheckpointIdentity) (pipeline.CheckpointRecorder, error) {
|
|
root, err := settings.CheckpointDirectory(identity)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if strings.TrimSpace(root) == "" {
|
|
return pipeline.NoopCheckpointRecorder(), nil
|
|
}
|
|
return &WorkspaceRecorder{root: root, identityDigest: identity.Digest, now: time.Now}, nil
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) SourceRunning(moduleKey string) error {
|
|
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusRunning)
|
|
manifest.ModuleKey = moduleKey
|
|
manifest.StartedAt = timePtr(r.timestamp())
|
|
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) SourceSucceeded(moduleKey string, doc *source.SourceDocument) error {
|
|
if doc == nil {
|
|
return fmt.Errorf("checkpoint source document must not be nil")
|
|
}
|
|
if err := r.writePayload("source/source-document.json", sourceDocumentEnvelope{Document: cloneSourceDocument(*doc)}); err != nil {
|
|
return err
|
|
}
|
|
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusSucceeded)
|
|
manifest.ModuleKey = moduleKey
|
|
manifest.OutputDigests = workspaceFingerprints(digestFingerprints("source_document", doc.Digest))
|
|
manifest.CompletedAt = timePtr(r.timestamp())
|
|
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{
|
|
StageManifest: manifest,
|
|
SourceID: doc.ID,
|
|
})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) SourceFailed(moduleKey string, err error) error {
|
|
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusFailed)
|
|
manifest.ModuleKey = moduleKey
|
|
manifest.CompletedAt = timePtr(r.timestamp())
|
|
manifest.Metadata = errorMetadata(err)
|
|
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) ExtractRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
|
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
|
manifest.StartedAt = timePtr(r.timestamp())
|
|
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) ExtractSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []pipeline.CheckpointArtifact, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
|
|
payload := artifactExtractEnvelope{Outputs: artifactCheckpointEnvelopes(outputs), Rejected: cloneRejectedOutputs(rejected), Warnings: cloneWarnings(warnings)}
|
|
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
|
|
return err
|
|
}
|
|
manifest := r.laneManifest(coreworkspace.StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
|
|
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests(outputs))
|
|
manifest.ValidationStatus = validationStatusString(warnings, rejected)
|
|
manifest.Rejections = rejectionSummaries(rejected)
|
|
manifest.CompletedAt = timePtr(r.timestamp())
|
|
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest, ChunkCount: len(outputs) + len(rejected), OutputCount: len(outputs)})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) ExtractFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
|
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
|
manifest.CompletedAt = timePtr(r.timestamp())
|
|
manifest.Metadata = errorMetadata(err)
|
|
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) MergeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
|
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
|
manifest.StartedAt = timePtr(r.timestamp())
|
|
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) MergeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
|
|
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
|
|
return err
|
|
}
|
|
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
|
|
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
|
|
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
|
manifest.CompletedAt = timePtr(r.timestamp())
|
|
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) MergeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
|
|
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
|
|
manifest.ValidationStatus = "rejected"
|
|
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
|
manifest.CompletedAt = timePtr(r.timestamp())
|
|
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) MergeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
|
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
|
manifest.CompletedAt = timePtr(r.timestamp())
|
|
manifest.Metadata = errorMetadata(err)
|
|
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) NormalizeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
|
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
|
manifest.StartedAt = timePtr(r.timestamp())
|
|
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) NormalizeSucceeded(laneID, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output pipeline.CheckpointArtifact, warnings []contracts.Warning) error {
|
|
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), artifactSingleEnvelope{Output: artifactCheckpointEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}); err != nil {
|
|
return err
|
|
}
|
|
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
|
|
manifest.OutputDigests = workspaceFingerprints(artifactOutputDigests([]pipeline.CheckpointArtifact{output}))
|
|
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
|
manifest.CompletedAt = timePtr(r.timestamp())
|
|
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) NormalizeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
|
|
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
|
|
manifest.ValidationStatus = "rejected"
|
|
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
|
manifest.CompletedAt = timePtr(r.timestamp())
|
|
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) NormalizeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
|
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
|
manifest.CompletedAt = timePtr(r.timestamp())
|
|
manifest.Metadata = errorMetadata(err)
|
|
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) writeManifest(name string, payload any) error {
|
|
return r.writeJSON(name, payload)
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) writePayload(name string, payload any) error {
|
|
return r.writeJSON(name, payload)
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) writeJSON(name string, payload any) error {
|
|
if r == nil || strings.TrimSpace(r.root) == "" {
|
|
return nil
|
|
}
|
|
return coreworkspace.WriteJSON(r.root, name, payload)
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) timestamp() time.Time {
|
|
if r == nil || r.now == nil {
|
|
return time.Now().UTC()
|
|
}
|
|
return r.now().UTC()
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) newStageManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus) coreworkspace.StageManifest {
|
|
manifest := coreworkspace.NewStageManifest(stage, status)
|
|
if strings.TrimSpace(r.identityDigest) != "" {
|
|
manifest.Metadata = map[string]string{"checkpoint_identity_digest": r.identityDigest}
|
|
}
|
|
return manifest
|
|
}
|
|
|
|
func (r *WorkspaceRecorder) laneManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) coreworkspace.StageManifest {
|
|
manifest := r.newStageManifest(stage, status)
|
|
manifest.LaneID = laneID
|
|
manifest.ModuleKey = moduleKey
|
|
manifest.DependencyFingerprints = workspaceFingerprints(dependencies)
|
|
return manifest
|
|
}
|
|
|
|
type sourceDocumentEnvelope struct {
|
|
Document source.SourceDocument `json:"document"`
|
|
}
|
|
|
|
type binaryEnvelope struct {
|
|
ContentBase64 string `json:"content_base64,omitempty"`
|
|
ContentDigest string `json:"content_digest,omitempty"`
|
|
MediaType string `json:"media_type,omitempty"`
|
|
Metadata map[string]any `json:"metadata,omitempty"`
|
|
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
|
}
|
|
|
|
type artifactCheckpointEnvelope struct {
|
|
LaneID string `json:"lane_id"`
|
|
ModuleKey string `json:"module_key"`
|
|
SourceID string `json:"source_id,omitempty"`
|
|
ChunkID string `json:"chunk_id,omitempty"`
|
|
ChunkIndex int `json:"chunk_index,omitempty"`
|
|
ChunkRef source.SourceRef `json:"chunk_ref,omitempty"`
|
|
Kind contracts.ArtifactKind `json:"artifact_kind"`
|
|
Schema contracts.ArtifactSchema `json:"schema"`
|
|
SchemaDigest string `json:"schema_digest"`
|
|
Content binaryEnvelope `json:"content"`
|
|
}
|
|
type artifactExtractEnvelope struct {
|
|
Outputs []artifactCheckpointEnvelope `json:"outputs"`
|
|
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
|
|
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
|
}
|
|
type artifactSingleEnvelope struct {
|
|
Output artifactCheckpointEnvelope `json:"output"`
|
|
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
|
}
|
|
|
|
func artifactCheckpointEnvelopeFromOutput(output pipeline.CheckpointArtifact) artifactCheckpointEnvelope {
|
|
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
|
|
schema.JSONSchema = nil
|
|
return artifactCheckpointEnvelope{LaneID: output.LaneID, ModuleKey: output.ModuleKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Kind: output.Artifact.Kind, Schema: schema, SchemaDigest: output.SchemaDigest, Content: binaryEnvelopeFromContent(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)}
|
|
}
|
|
func artifactCheckpointEnvelopes(outputs []pipeline.CheckpointArtifact) []artifactCheckpointEnvelope {
|
|
if len(outputs) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]artifactCheckpointEnvelope, 0, len(outputs))
|
|
for _, v := range outputs {
|
|
out = append(out, artifactCheckpointEnvelopeFromOutput(v))
|
|
}
|
|
return out
|
|
}
|
|
func artifactOutputDigests(outputs []pipeline.CheckpointArtifact) []pipeline.CheckpointFingerprint {
|
|
values := make([]pipeline.CheckpointFingerprint, 0, len(outputs))
|
|
for i, v := range outputs {
|
|
values = append(values, pipeline.CheckpointFingerprint{Name: fmt.Sprintf("artifact[%d]", i), Value: contentDigest(v.Artifact.Content)})
|
|
}
|
|
return normalizeFingerprints(values)
|
|
}
|
|
|
|
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
|
|
return binaryEnvelope{
|
|
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
|
ContentDigest: contentDigest(content),
|
|
MediaType: mediaType,
|
|
Metadata: cloneMetadata(metadata),
|
|
Warnings: cloneWarnings(warnings),
|
|
}
|
|
}
|
|
|
|
func cloneSourceDocument(doc source.SourceDocument) source.SourceDocument {
|
|
doc.Units = cloneSourceUnits(doc.Units)
|
|
doc.Metadata = cloneMetadata(doc.Metadata)
|
|
return doc
|
|
}
|
|
|
|
func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
|
|
if len(units) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]source.SourceUnit, 0, len(units))
|
|
for _, unit := range units {
|
|
out = append(out, source.SourceUnit{
|
|
ID: unit.ID,
|
|
Kind: unit.Kind,
|
|
Text: unit.Text,
|
|
Ref: unit.Ref,
|
|
Metadata: cloneMetadata(unit.Metadata),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
|
|
if len(warnings) == 0 {
|
|
return nil
|
|
}
|
|
return append([]contracts.Warning(nil), warnings...)
|
|
}
|
|
|
|
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
|
|
if len(rejected) == 0 {
|
|
return nil
|
|
}
|
|
return append([]contracts.RejectedOutput(nil), rejected...)
|
|
}
|
|
|
|
func cloneMetadata(metadata map[string]any) map[string]any {
|
|
if len(metadata) == 0 {
|
|
return nil
|
|
}
|
|
out := make(map[string]any, len(metadata))
|
|
for key, value := range metadata {
|
|
out[key] = value
|
|
}
|
|
return out
|
|
}
|
|
|
|
func digestFingerprints(name string, digest string) []pipeline.CheckpointFingerprint {
|
|
digest = strings.TrimSpace(digest)
|
|
if digest == "" {
|
|
return nil
|
|
}
|
|
return []pipeline.CheckpointFingerprint{{Name: name, Value: digest}}
|
|
}
|
|
|
|
func workspaceFingerprints(values []pipeline.CheckpointFingerprint) []coreworkspace.Fingerprint {
|
|
normalized := normalizeFingerprints(values)
|
|
if len(normalized) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]coreworkspace.Fingerprint, 0, len(normalized))
|
|
for _, value := range normalized {
|
|
out = append(out, coreworkspace.Fingerprint{Name: value.Name, Value: value.Value})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func normalizeFingerprints(values []pipeline.CheckpointFingerprint) []pipeline.CheckpointFingerprint {
|
|
if len(values) == 0 {
|
|
return nil
|
|
}
|
|
byName := make(map[string]string, len(values))
|
|
for _, value := range values {
|
|
name := strings.TrimSpace(value.Name)
|
|
fingerprint := strings.TrimSpace(value.Value)
|
|
if name == "" || fingerprint == "" {
|
|
continue
|
|
}
|
|
byName[name] = fingerprint
|
|
}
|
|
if len(byName) == 0 {
|
|
return nil
|
|
}
|
|
names := make([]string, 0, len(byName))
|
|
for name := range byName {
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
out := make([]pipeline.CheckpointFingerprint, 0, len(names))
|
|
for _, name := range names {
|
|
out = append(out, pipeline.CheckpointFingerprint{Name: name, Value: byName[name]})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func rejectionSummaries(rejected []contracts.RejectedOutput) []coreworkspace.RejectionSummary {
|
|
if len(rejected) == 0 {
|
|
return nil
|
|
}
|
|
type key struct {
|
|
validatorName string
|
|
reasonCode string
|
|
message string
|
|
}
|
|
counts := make(map[key]int, len(rejected))
|
|
for _, item := range rejected {
|
|
k := key{validatorName: item.ValidatorName, reasonCode: item.ReasonCode, message: item.Message}
|
|
counts[k]++
|
|
}
|
|
keys := make([]key, 0, len(counts))
|
|
for k := range counts {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Slice(keys, func(i, j int) bool {
|
|
if keys[i].validatorName != keys[j].validatorName {
|
|
return keys[i].validatorName < keys[j].validatorName
|
|
}
|
|
if keys[i].reasonCode != keys[j].reasonCode {
|
|
return keys[i].reasonCode < keys[j].reasonCode
|
|
}
|
|
return keys[i].message < keys[j].message
|
|
})
|
|
out := make([]coreworkspace.RejectionSummary, 0, len(keys))
|
|
for _, k := range keys {
|
|
out = append(out, coreworkspace.RejectionSummary{
|
|
ValidatorName: k.validatorName,
|
|
ReasonCode: k.reasonCode,
|
|
Message: k.message,
|
|
Count: counts[k],
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func statusForRejected(rejected []contracts.RejectedOutput) coreworkspace.StageStatus {
|
|
if len(rejected) > 0 {
|
|
return coreworkspace.StatusSucceededWithRejections
|
|
}
|
|
return coreworkspace.StatusSucceeded
|
|
}
|
|
|
|
func validationStatusString(warnings []contracts.Warning, rejected []contracts.RejectedOutput) string {
|
|
if len(rejected) > 0 {
|
|
return "rejected"
|
|
}
|
|
if len(warnings) > 0 {
|
|
return "approved_with_warnings"
|
|
}
|
|
return "approved"
|
|
}
|
|
|
|
func errorMetadata(err error) map[string]string {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
return map[string]string{"error": err.Error()}
|
|
}
|
|
|
|
func laneManifestPath(stage string, laneID string) string {
|
|
return lanePayloadPath(stage, laneID, "manifest.json")
|
|
}
|
|
|
|
func lanePayloadPath(stage string, laneID string, file string) string {
|
|
return path.Join(stage, checkpointPathComponent(laneID), file)
|
|
}
|
|
|
|
func checkpointPathComponent(value string) string {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return "_"
|
|
}
|
|
var b strings.Builder
|
|
for _, r := range value {
|
|
switch {
|
|
case r >= 'a' && r <= 'z':
|
|
b.WriteRune(r)
|
|
case r >= 'A' && r <= 'Z':
|
|
b.WriteRune(r)
|
|
case r >= '0' && r <= '9':
|
|
b.WriteRune(r)
|
|
case r == '-' || r == '_' || r == '.':
|
|
b.WriteRune(r)
|
|
default:
|
|
b.WriteString(fmt.Sprintf("~%x", r))
|
|
}
|
|
}
|
|
out := b.String()
|
|
if out == "." || out == ".." || strings.Contains(out, "..") {
|
|
return "_"
|
|
}
|
|
return out
|
|
}
|
|
|
|
func contentDigest(content []byte) string {
|
|
sum := sha256.Sum256(content)
|
|
return "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func timePtr(t time.Time) *time.Time {
|
|
return &t
|
|
}
|