Write workspace checkpoints during runs
This commit is contained in:
625
internal/framework/checkpoint/recorder.go
Normal file
625
internal/framework/checkpoint/recorder.go
Normal file
@@ -0,0 +1,625 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
type WorkspaceRecorder struct {
|
||||
root string
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewWorkspaceRecorder(settings coreworkspace.Settings, identity coreworkspace.CheckpointIdentity) (pipeline.CheckpointRecorder, error) {
|
||||
root, err := settings.CheckpointDirectory(identity)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return pipeline.NoopCheckpointRecorder(), nil
|
||||
}
|
||||
return &WorkspaceRecorder{root: root, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) SourceRunning(moduleKey string) error {
|
||||
manifest := coreworkspace.NewStageManifest(coreworkspace.StageSource, coreworkspace.StatusRunning)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) SourceSucceeded(moduleKey string, doc *source.SourceDocument) error {
|
||||
if doc == nil {
|
||||
return fmt.Errorf("checkpoint source document must not be nil")
|
||||
}
|
||||
if err := r.writePayload("source/source-document.json", sourceDocumentEnvelope{Document: cloneSourceDocument(*doc)}); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := coreworkspace.NewStageManifest(coreworkspace.StageSource, coreworkspace.StatusSucceeded)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.OutputDigests = workspaceFingerprints(digestFingerprints("source_document", doc.Digest))
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{
|
||||
StageManifest: manifest,
|
||||
SourceID: doc.ID,
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) SourceFailed(moduleKey string, err error) error {
|
||||
manifest := coreworkspace.NewStageManifest(coreworkspace.StageSource, coreworkspace.StatusFailed)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkRunning(moduleKey string, sourceDigest string) error {
|
||||
manifest := coreworkspace.NewStageManifest(coreworkspace.StageChunk, coreworkspace.StatusRunning)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkSucceeded(moduleKey string, sourceDigest string, chunks []contracts.SourceChunk, warnings []contracts.Warning) error {
|
||||
payload := chunksEnvelope{Chunks: chunkEnvelopes(chunks), Warnings: cloneWarnings(warnings)}
|
||||
if err := r.writePayload("chunk/chunks.json", payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := coreworkspace.NewStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceeded)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.OutputDigests = workspaceFingerprints(chunkOutputDigests(chunks))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{
|
||||
StageManifest: manifest,
|
||||
ChunkCount: len(chunks),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error {
|
||||
manifest := coreworkspace.NewStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceededWithRejections)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.ValidationStatus = "rejected"
|
||||
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkFailed(moduleKey string, sourceDigest string, err error) error {
|
||||
manifest := coreworkspace.NewStageManifest(coreworkspace.StageChunk, coreworkspace.StatusFailed)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ExtractRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
||||
manifest := laneManifest(coreworkspace.StageExtract, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ExtractSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
|
||||
payload := extractOutputsEnvelope{
|
||||
Outputs: extractOutputEnvelopes(outputs),
|
||||
Rejected: cloneRejectedOutputs(rejected),
|
||||
Warnings: cloneWarnings(warnings),
|
||||
}
|
||||
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := laneManifest(coreworkspace.StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests(extractPayloads(outputs)))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, rejected)
|
||||
manifest.Rejections = rejectionSummaries(rejected)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{
|
||||
StageManifest: manifest,
|
||||
ChunkCount: len(outputs) + len(rejected),
|
||||
OutputCount: len(outputs),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ExtractFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
||||
manifest := laneManifest(coreworkspace.StageExtract, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
||||
manifest := laneManifest(coreworkspace.StageMerge, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error {
|
||||
payload := mergeOutputEnvelope{Output: mergeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
|
||||
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{
|
||||
StageManifest: manifest,
|
||||
InputCount: len(dependencies),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
|
||||
manifest := laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
|
||||
manifest.ValidationStatus = "rejected"
|
||||
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
||||
manifest := laneManifest(coreworkspace.StageMerge, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
||||
manifest := laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error {
|
||||
payload := normalizeOutputEnvelope{Output: normalizeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
|
||||
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
|
||||
manifest := laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
|
||||
manifest.ValidationStatus = "rejected"
|
||||
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
||||
manifest := laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) writeManifest(name string, payload any) error {
|
||||
return r.writeJSON(name, payload)
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) writePayload(name string, payload any) error {
|
||||
return r.writeJSON(name, payload)
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) writeJSON(name string, payload any) error {
|
||||
if r == nil || strings.TrimSpace(r.root) == "" {
|
||||
return nil
|
||||
}
|
||||
return coreworkspace.WriteJSON(r.root, name, payload)
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) timestamp() time.Time {
|
||||
if r == nil || r.now == nil {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
return r.now().UTC()
|
||||
}
|
||||
|
||||
func laneManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) coreworkspace.StageManifest {
|
||||
manifest := coreworkspace.NewStageManifest(stage, status)
|
||||
manifest.LaneID = laneID
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(dependencies)
|
||||
return manifest
|
||||
}
|
||||
|
||||
type sourceDocumentEnvelope struct {
|
||||
Document source.SourceDocument `json:"document"`
|
||||
}
|
||||
|
||||
type chunksEnvelope struct {
|
||||
Chunks []chunkEnvelope `json:"chunks"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type chunkEnvelope struct {
|
||||
ID string `json:"id"`
|
||||
SourceID string `json:"source_id"`
|
||||
Index int `json:"index"`
|
||||
StartUnitID int `json:"start_unit_id"`
|
||||
EndUnitID int `json:"end_unit_id"`
|
||||
Content binaryEnvelope `json:"content"`
|
||||
Units []source.SourceUnit `json:"units,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type extractOutputsEnvelope struct {
|
||||
Outputs []extractOutputEnvelope `json:"outputs"`
|
||||
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type extractOutputEnvelope struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
SourceID string `json:"source_id"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ChunkIndex int `json:"chunk_index"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload binaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type mergeOutputEnvelope struct {
|
||||
Output mergeOutputPayload `json:"output"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type mergeOutputPayload struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
MergerKey string `json:"merger_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload binaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type normalizeOutputEnvelope struct {
|
||||
Output normalizeOutputPayload `json:"output"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type normalizeOutputPayload struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
NormalizerKey string `json:"normalizer_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload binaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type binaryEnvelope struct {
|
||||
ContentBase64 string `json:"content_base64,omitempty"`
|
||||
ContentDigest string `json:"content_digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
func chunkEnvelopes(chunks []contracts.SourceChunk) []chunkEnvelope {
|
||||
if len(chunks) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]chunkEnvelope, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
out = append(out, chunkEnvelope{
|
||||
ID: chunk.ID,
|
||||
SourceID: chunk.SourceID,
|
||||
Index: chunk.Index,
|
||||
StartUnitID: chunk.StartUnitID,
|
||||
EndUnitID: chunk.EndUnitID,
|
||||
Content: binaryEnvelopeFromContent(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
|
||||
Units: cloneSourceUnits(chunk.Units),
|
||||
Metadata: cloneMetadata(chunk.Metadata),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func extractOutputEnvelopes(outputs []contracts.ExtractOutput) []extractOutputEnvelope {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]extractOutputEnvelope, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
out = append(out, extractOutputEnvelope{
|
||||
LaneID: output.LaneID,
|
||||
ExtractorKey: output.ExtractorKey,
|
||||
SourceID: output.SourceID,
|
||||
ChunkID: output.ChunkID,
|
||||
ChunkIndex: output.ChunkIndex,
|
||||
Schema: schemaEnvelope(output.Schema),
|
||||
Payload: binaryEnvelopeFromPayload(output.Payload),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeOutputEnvelopeFromOutput(output contracts.MergeOutput) mergeOutputPayload {
|
||||
return mergeOutputPayload{
|
||||
LaneID: output.LaneID,
|
||||
MergerKey: output.MergerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: schemaEnvelope(output.Schema),
|
||||
Payload: binaryEnvelopeFromPayload(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeOutputEnvelopeFromOutput(output contracts.NormalizeOutput) normalizeOutputPayload {
|
||||
return normalizeOutputPayload{
|
||||
LaneID: output.LaneID,
|
||||
NormalizerKey: output.NormalizerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: schemaEnvelope(output.Schema),
|
||||
Payload: binaryEnvelopeFromPayload(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func schemaEnvelope(schema contracts.ResponseSchema) contracts.ResponseSchema {
|
||||
schema.JSONSchema = nil
|
||||
return schema
|
||||
}
|
||||
|
||||
func binaryEnvelopeFromPayload(payload contracts.RawPayload) binaryEnvelope {
|
||||
return binaryEnvelopeFromContent(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
|
||||
}
|
||||
|
||||
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
|
||||
return binaryEnvelope{
|
||||
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
||||
ContentDigest: contentDigest(content),
|
||||
MediaType: mediaType,
|
||||
Metadata: cloneMetadata(metadata),
|
||||
Warnings: cloneWarnings(warnings),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneSourceDocument(doc source.SourceDocument) source.SourceDocument {
|
||||
doc.Units = cloneSourceUnits(doc.Units)
|
||||
doc.Metadata = cloneMetadata(doc.Metadata)
|
||||
return doc
|
||||
}
|
||||
|
||||
func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
|
||||
if len(units) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]source.SourceUnit, 0, len(units))
|
||||
for _, unit := range units {
|
||||
out = append(out, source.SourceUnit{
|
||||
ID: unit.ID,
|
||||
Kind: unit.Kind,
|
||||
Text: unit.Text,
|
||||
Metadata: cloneMetadata(unit.Metadata),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
|
||||
if len(warnings) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]contracts.Warning(nil), warnings...)
|
||||
}
|
||||
|
||||
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
|
||||
if len(rejected) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]contracts.RejectedOutput(nil), rejected...)
|
||||
}
|
||||
|
||||
func cloneMetadata(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(metadata))
|
||||
for key, value := range metadata {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rawOutputDigests(payloads []contracts.RawPayload) []pipeline.CheckpointFingerprint {
|
||||
values := make([]pipeline.CheckpointFingerprint, 0, len(payloads))
|
||||
for i, payload := range payloads {
|
||||
values = append(values, pipeline.CheckpointFingerprint{
|
||||
Name: fmt.Sprintf("payload[%d]", i),
|
||||
Value: contentDigest(payload.Content),
|
||||
})
|
||||
}
|
||||
return normalizeFingerprints(values)
|
||||
}
|
||||
|
||||
func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
payloads := make([]contracts.RawPayload, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
payloads = append(payloads, output.Payload)
|
||||
}
|
||||
return payloads
|
||||
}
|
||||
|
||||
func chunkOutputDigests(chunks []contracts.SourceChunk) []pipeline.CheckpointFingerprint {
|
||||
values := make([]pipeline.CheckpointFingerprint, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
values = append(values, pipeline.CheckpointFingerprint{
|
||||
Name: chunk.ID,
|
||||
Value: contentDigest(chunk.Content),
|
||||
})
|
||||
}
|
||||
return normalizeFingerprints(values)
|
||||
}
|
||||
|
||||
func digestFingerprints(name string, digest string) []pipeline.CheckpointFingerprint {
|
||||
digest = strings.TrimSpace(digest)
|
||||
if digest == "" {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{{Name: name, Value: digest}}
|
||||
}
|
||||
|
||||
func workspaceFingerprints(values []pipeline.CheckpointFingerprint) []coreworkspace.Fingerprint {
|
||||
normalized := normalizeFingerprints(values)
|
||||
if len(normalized) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]coreworkspace.Fingerprint, 0, len(normalized))
|
||||
for _, value := range normalized {
|
||||
out = append(out, coreworkspace.Fingerprint{Name: value.Name, Value: value.Value})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeFingerprints(values []pipeline.CheckpointFingerprint) []pipeline.CheckpointFingerprint {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
byName := make(map[string]string, len(values))
|
||||
for _, value := range values {
|
||||
name := strings.TrimSpace(value.Name)
|
||||
fingerprint := strings.TrimSpace(value.Value)
|
||||
if name == "" || fingerprint == "" {
|
||||
continue
|
||||
}
|
||||
byName[name] = fingerprint
|
||||
}
|
||||
if len(byName) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := make([]string, 0, len(byName))
|
||||
for name := range byName {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
out := make([]pipeline.CheckpointFingerprint, 0, len(names))
|
||||
for _, name := range names {
|
||||
out = append(out, pipeline.CheckpointFingerprint{Name: name, Value: byName[name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rejectionSummaries(rejected []contracts.RejectedOutput) []coreworkspace.RejectionSummary {
|
||||
if len(rejected) == 0 {
|
||||
return nil
|
||||
}
|
||||
type key struct {
|
||||
validatorName string
|
||||
reasonCode string
|
||||
message string
|
||||
}
|
||||
counts := make(map[key]int, len(rejected))
|
||||
for _, item := range rejected {
|
||||
k := key{validatorName: item.ValidatorName, reasonCode: item.ReasonCode, message: item.Message}
|
||||
counts[k]++
|
||||
}
|
||||
keys := make([]key, 0, len(counts))
|
||||
for k := range counts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].validatorName != keys[j].validatorName {
|
||||
return keys[i].validatorName < keys[j].validatorName
|
||||
}
|
||||
if keys[i].reasonCode != keys[j].reasonCode {
|
||||
return keys[i].reasonCode < keys[j].reasonCode
|
||||
}
|
||||
return keys[i].message < keys[j].message
|
||||
})
|
||||
out := make([]coreworkspace.RejectionSummary, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
out = append(out, coreworkspace.RejectionSummary{
|
||||
ValidatorName: k.validatorName,
|
||||
ReasonCode: k.reasonCode,
|
||||
Message: k.message,
|
||||
Count: counts[k],
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func statusForRejected(rejected []contracts.RejectedOutput) coreworkspace.StageStatus {
|
||||
if len(rejected) > 0 {
|
||||
return coreworkspace.StatusSucceededWithRejections
|
||||
}
|
||||
return coreworkspace.StatusSucceeded
|
||||
}
|
||||
|
||||
func validationStatusString(warnings []contracts.Warning, rejected []contracts.RejectedOutput) string {
|
||||
if len(rejected) > 0 {
|
||||
return "rejected"
|
||||
}
|
||||
if len(warnings) > 0 {
|
||||
return "approved_with_warnings"
|
||||
}
|
||||
return "approved"
|
||||
}
|
||||
|
||||
func errorMetadata(err error) map[string]string {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{"error": err.Error()}
|
||||
}
|
||||
|
||||
func laneManifestPath(stage string, laneID string) string {
|
||||
return lanePayloadPath(stage, laneID, "manifest.json")
|
||||
}
|
||||
|
||||
func lanePayloadPath(stage string, laneID string, file string) string {
|
||||
return path.Join(stage, checkpointPathComponent(laneID), file)
|
||||
}
|
||||
|
||||
func checkpointPathComponent(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "_"
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
b.WriteRune(r)
|
||||
case r >= 'A' && r <= 'Z':
|
||||
b.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '-' || r == '_' || r == '.':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteString(fmt.Sprintf("~%x", r))
|
||||
}
|
||||
}
|
||||
out := b.String()
|
||||
if out == "." || out == ".." || strings.Contains(out, "..") {
|
||||
return "_"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func contentDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func timePtr(t time.Time) *time.Time {
|
||||
return &t
|
||||
}
|
||||
195
internal/framework/checkpoint/recorder_test.go
Normal file
195
internal/framework/checkpoint/recorder_test.go
Normal file
@@ -0,0 +1,195 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
doc := &source.SourceDocument{
|
||||
ID: "source-1",
|
||||
Kind: "document",
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:source",
|
||||
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello"}},
|
||||
}
|
||||
chunks := []contracts.SourceChunk{
|
||||
{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Index: 0,
|
||||
StartUnitID: 1,
|
||||
EndUnitID: 1,
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
Units: doc.Units,
|
||||
},
|
||||
}
|
||||
|
||||
if err := recorder.SourceRunning("seriatim"); err != nil {
|
||||
t.Fatalf("SourceRunning: %v", err)
|
||||
}
|
||||
assertManifestStatus(t, filepath.Join(root, "source", "manifest.json"), coreworkspace.StatusRunning)
|
||||
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
|
||||
t.Fatalf("SourceSucceeded: %v", err)
|
||||
}
|
||||
assertManifestStatus(t, filepath.Join(root, "source", "manifest.json"), coreworkspace.StatusSucceeded)
|
||||
if _, err := os.Stat(filepath.Join(root, "source", "source-document.json")); err != nil {
|
||||
t.Fatalf("expected source checkpoint payload: %v", err)
|
||||
}
|
||||
|
||||
if err := recorder.ChunkRunning("generic", doc.Digest); err != nil {
|
||||
t.Fatalf("ChunkRunning: %v", err)
|
||||
}
|
||||
if err := recorder.ChunkSucceeded("generic", doc.Digest, chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
assertManifestStatus(t, filepath.Join(root, "chunk", "manifest.json"), coreworkspace.StatusSucceeded)
|
||||
var chunkPayload struct {
|
||||
Chunks []struct {
|
||||
Content struct {
|
||||
ContentBase64 string `json:"content_base64"`
|
||||
ContentDigest string `json:"content_digest"`
|
||||
} `json:"content"`
|
||||
} `json:"chunks"`
|
||||
}
|
||||
readJSON(t, filepath.Join(root, "chunk", "chunks.json"), &chunkPayload)
|
||||
if len(chunkPayload.Chunks) != 1 {
|
||||
t.Fatalf("checkpoint chunks = %#v, want one", chunkPayload.Chunks)
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(chunkPayload.Chunks[0].Content.ContentBase64)
|
||||
if err != nil {
|
||||
t.Fatalf("decode chunk content: %v", err)
|
||||
}
|
||||
if string(decoded) != "chunk content" {
|
||||
t.Fatalf("chunk content = %q, want original content", decoded)
|
||||
}
|
||||
if got, want := chunkPayload.Chunks[0].Content.ContentDigest, contentDigest([]byte("chunk content")); got != want {
|
||||
t.Fatalf("content digest = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceRecorderRecordsRejectedExtractOutputs(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
rejected := []contracts.RejectedOutput{
|
||||
{
|
||||
Stage: string(pipeline.StageExtract),
|
||||
LaneID: "spells",
|
||||
ModuleKey: "dnd/spells",
|
||||
ChunkID: "chunk-1",
|
||||
ValidatorName: "shape",
|
||||
ReasonCode: "invalid_shape",
|
||||
Message: "bad shape",
|
||||
},
|
||||
}
|
||||
|
||||
if err := recorder.ExtractRunning("spells", "dnd/spells", []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}); err != nil {
|
||||
t.Fatalf("ExtractRunning: %v", err)
|
||||
}
|
||||
if err := recorder.ExtractSucceeded("spells", "dnd/spells", nil, nil, rejected, nil); err != nil {
|
||||
t.Fatalf("ExtractSucceeded: %v", err)
|
||||
}
|
||||
|
||||
var manifest coreworkspace.ExtractLaneManifest
|
||||
readJSON(t, filepath.Join(root, "extract", "spells", "manifest.json"), &manifest)
|
||||
if manifest.Status != coreworkspace.StatusSucceededWithRejections || manifest.ValidationStatus != "rejected" {
|
||||
t.Fatalf("extract manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus)
|
||||
}
|
||||
if len(manifest.Rejections) != 1 || manifest.Rejections[0].Count != 1 || manifest.Rejections[0].ReasonCode != "invalid_shape" {
|
||||
t.Fatalf("rejections = %#v", manifest.Rejections)
|
||||
}
|
||||
var payload struct {
|
||||
Rejected []contracts.RejectedOutput `json:"rejected"`
|
||||
}
|
||||
readJSON(t, filepath.Join(root, "extract", "spells", "outputs.json"), &payload)
|
||||
if len(payload.Rejected) != 1 || payload.Rejected[0].ChunkID != "chunk-1" {
|
||||
t.Fatalf("checkpoint rejected payload = %#v", payload.Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceRecorderRecordsFailedStages(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
|
||||
if err := recorder.MergeRunning("spells", "appendorder", nil); err != nil {
|
||||
t.Fatalf("MergeRunning: %v", err)
|
||||
}
|
||||
if err := recorder.MergeFailed("spells", "appendorder", nil, assertErr("merge failed")); err != nil {
|
||||
t.Fatalf("MergeFailed: %v", err)
|
||||
}
|
||||
|
||||
var manifest coreworkspace.MergeLaneManifest
|
||||
readJSON(t, filepath.Join(root, "merge", "spells", "manifest.json"), &manifest)
|
||||
if manifest.Status != coreworkspace.StatusFailed {
|
||||
t.Fatalf("status = %q, want failed", manifest.Status)
|
||||
}
|
||||
if !strings.Contains(manifest.Metadata["error"], "merge failed") {
|
||||
t.Fatalf("metadata = %#v, want error", manifest.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceRecorderRecordsWarningOnlyValidation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
output := contracts.NormalizeOutput{
|
||||
LaneID: "spells",
|
||||
NormalizerKey: "noop",
|
||||
SourceID: "source-1",
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"ok":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
warnings := []contracts.Warning{{ReasonCode: "note", Message: "warning"}}
|
||||
|
||||
if err := recorder.NormalizeSucceeded("spells", "noop", nil, output, warnings); err != nil {
|
||||
t.Fatalf("NormalizeSucceeded: %v", err)
|
||||
}
|
||||
|
||||
var manifest coreworkspace.NormalizeLaneManifest
|
||||
readJSON(t, filepath.Join(root, "normalize", "spells", "manifest.json"), &manifest)
|
||||
if manifest.Status != coreworkspace.StatusSucceeded || manifest.ValidationStatus != "approved_with_warnings" {
|
||||
t.Fatalf("normalize manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestRecorder(t *testing.T, root string) *WorkspaceRecorder {
|
||||
t.Helper()
|
||||
return &WorkspaceRecorder{root: root}
|
||||
}
|
||||
|
||||
func assertManifestStatus(t *testing.T, path string, want coreworkspace.StageStatus) {
|
||||
t.Helper()
|
||||
var manifest coreworkspace.StageManifest
|
||||
readJSON(t, path, &manifest)
|
||||
if manifest.Status != want {
|
||||
t.Fatalf("%s status = %q, want %q", path, manifest.Status, want)
|
||||
}
|
||||
}
|
||||
|
||||
func readJSON(t *testing.T, path string, out any) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %q: %v", path, err)
|
||||
}
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
t.Fatalf("decode %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
type assertErr string
|
||||
|
||||
func (e assertErr) Error() string { return string(e) }
|
||||
Reference in New Issue
Block a user