package pipeline import ( "crypto/sha256" "encoding/hex" "fmt" "sort" "strings" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) type CheckpointFingerprint struct { Name string `json:"name"` Value string `json:"value"` } type CheckpointRecorder interface { SourceRunning(moduleKey string) error SourceSucceeded(moduleKey string, doc *source.SourceDocument) error SourceFailed(moduleKey string, err error) error ChunkRunning(moduleKey string, sourceDigest string) error ChunkSucceeded(moduleKey string, sourceDigest string, chunks []contracts.SourceChunk, warnings []contracts.Warning) error ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error ChunkFailed(moduleKey string, sourceDigest string, err error) error ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error } type CheckpointDecision struct { Reused bool `json:"reused"` Reason string `json:"reason,omitempty"` } type CheckpointEvent struct { Stage string `json:"stage"` LaneID string `json:"lane_id,omitempty"` ModuleKey string `json:"module_key,omitempty"` Action string `json:"action"` Reason string `json:"reason,omitempty"` } type SourceCheckpoint struct { Document *source.SourceDocument } type ChunkCheckpoint struct { Chunks []contracts.SourceChunk Warnings []contracts.Warning } type ExtractCheckpoint struct { Outputs []contracts.ExtractOutput Rejected []contracts.RejectedOutput Warnings []contracts.Warning } type MergeCheckpoint struct { Output contracts.MergeOutput Warnings []contracts.Warning } type NormalizeCheckpoint struct { Output contracts.NormalizeOutput Warnings []contracts.Warning } type CheckpointLoader interface { Enabled() bool Source(moduleKey string) (SourceCheckpoint, CheckpointDecision) Chunk(moduleKey string, sourceDigest string) (ChunkCheckpoint, CheckpointDecision) Extract(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) Merge(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) Normalize(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) } type noopCheckpointRecorder struct{} type noopCheckpointLoader struct{} func NoopCheckpointRecorder() CheckpointRecorder { return noopCheckpointRecorder{} } func NoopCheckpointLoader() CheckpointLoader { return noopCheckpointLoader{} } func (noopCheckpointRecorder) SourceRunning(string) error { return nil } func (noopCheckpointRecorder) SourceSucceeded(string, *source.SourceDocument) error { return nil } func (noopCheckpointRecorder) SourceFailed(string, error) error { return nil } func (noopCheckpointRecorder) ChunkRunning(string, string) error { return nil } func (noopCheckpointRecorder) ChunkSucceeded(string, string, []contracts.SourceChunk, []contracts.Warning) error { return nil } func (noopCheckpointRecorder) ChunkRejected(string, string, contracts.RejectedOutput) error { return nil } func (noopCheckpointRecorder) ChunkFailed(string, string, error) error { return nil } func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error { return nil } func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []contracts.ExtractOutput, []contracts.RejectedOutput, []contracts.Warning) error { return nil } func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error { return nil } func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil } func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, contracts.MergeOutput, []contracts.Warning) error { return nil } func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error { return nil } func (noopCheckpointRecorder) MergeFailed(string, string, []CheckpointFingerprint, error) error { return nil } func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error { return nil } func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, contracts.NormalizeOutput, []contracts.Warning) error { return nil } func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error { return nil } func (noopCheckpointRecorder) NormalizeFailed(string, string, []CheckpointFingerprint, error) error { return nil } func (noopCheckpointLoader) Enabled() bool { return false } func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) { return SourceCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"} } func (noopCheckpointLoader) Chunk(string, string) (ChunkCheckpoint, CheckpointDecision) { return ChunkCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"} } func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) { return ExtractCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"} } func (noopCheckpointLoader) Merge(string, string, []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) { return MergeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"} } func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) { return NormalizeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"} } func rawOutputDigests(payloads []contracts.RawPayload) []CheckpointFingerprint { values := make([]CheckpointFingerprint, 0, len(payloads)) for i, payload := range payloads { values = append(values, CheckpointFingerprint{ Name: fmt.Sprintf("payload[%d]", i), Value: checkpointContentDigest(payload.Content), }) } return normalizeCheckpointFingerprints(values) } func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload { if len(outputs) == 0 { return nil } payloads := make([]contracts.RawPayload, 0, len(outputs)) for _, output := range outputs { payloads = append(payloads, output.Payload) } return payloads } func digestFingerprints(name string, digest string) []CheckpointFingerprint { digest = strings.TrimSpace(digest) if digest == "" { return nil } return []CheckpointFingerprint{{Name: name, Value: digest}} } func joinedChunkDigest(chunks []contracts.SourceChunk) string { if len(chunks) == 0 { return "" } values := make([]string, 0, len(chunks)) for _, chunk := range chunks { values = append(values, chunk.ID+"="+checkpointContentDigest(chunk.Content)) } sort.Strings(values) sum := sha256.Sum256([]byte(strings.Join(values, "\n"))) return "sha256:" + hex.EncodeToString(sum[:]) } func normalizeCheckpointFingerprints(values []CheckpointFingerprint) []CheckpointFingerprint { if len(values) == 0 { return nil } byName := make(map[string]string, len(values)) for _, value := range values { name := strings.TrimSpace(value.Name) fingerprint := strings.TrimSpace(value.Value) if name == "" || fingerprint == "" { continue } byName[name] = fingerprint } if len(byName) == 0 { return nil } names := make([]string, 0, len(byName)) for name := range byName { names = append(names, name) } sort.Strings(names) out := make([]CheckpointFingerprint, 0, len(names)) for _, name := range names { out = append(out, CheckpointFingerprint{Name: name, Value: byName[name]}) } return out } func checkpointContentDigest(content []byte) string { sum := sha256.Sum256(content) return "sha256:" + hex.EncodeToString(sum[:]) }