Add filesystem checkpoint compatibility tests
This commit is contained in:
415
internal/framework/checkpoint/filesystem_test.go
Normal file
415
internal/framework/checkpoint/filesystem_test.go
Normal file
@@ -0,0 +1,415 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
type filesystemCheckpointFixture struct {
|
||||
root string
|
||||
identity Identity
|
||||
loader pipeline.CheckpointLoader
|
||||
doc source.SourceDocument
|
||||
extract pipeline.CheckpointArtifact
|
||||
merge pipeline.CheckpointArtifact
|
||||
normalize pipeline.CheckpointArtifact
|
||||
dependencies []pipeline.CheckpointFingerprint
|
||||
warnings []contracts.Warning
|
||||
rejected []contracts.RejectedOutput
|
||||
}
|
||||
|
||||
func TestFilesystemCheckpointRoundTripsAllStages(t *testing.T) {
|
||||
fixture := seedFilesystemCheckpoints(t)
|
||||
|
||||
// Recording owns its inputs. These mutations must not change the durable values.
|
||||
fixture.doc.Units[0].Text = "caller mutation"
|
||||
fixture.doc.Metadata["owner"] = "caller mutation"
|
||||
fixture.extract.Artifact.Content[0] = 'x'
|
||||
fixture.extract.Artifact.Metadata["content"] = "caller mutation"
|
||||
fixture.merge.Artifact.Content[0] = 'x'
|
||||
fixture.normalize.Artifact.Content[0] = 'x'
|
||||
fixture.warnings[0].Message = "caller mutation"
|
||||
fixture.rejected[0].Message = "caller mutation"
|
||||
|
||||
t.Run("source", func(t *testing.T) {
|
||||
got, decision := fixture.loader.Source("source-module")
|
||||
if !decision.Reused {
|
||||
t.Fatalf("source decision = %#v", decision)
|
||||
}
|
||||
if got.Document == nil || got.Document.ID != "document-1" || got.Document.Units[0].Text != "original source" || got.Document.Metadata["owner"] != "fixture" {
|
||||
t.Fatalf("source was not restored: %#v", got)
|
||||
}
|
||||
|
||||
got.Document.Units[0].Text = "loaded mutation"
|
||||
got.Document.Metadata["owner"] = "loaded mutation"
|
||||
reloaded, decision := fixture.loader.Source("source-module")
|
||||
if !decision.Reused || reloaded.Document.Units[0].Text != "original source" || reloaded.Document.Metadata["owner"] != "fixture" {
|
||||
t.Fatalf("source reload changed after loaded mutation: %#v decision=%#v", reloaded, decision)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("extract", func(t *testing.T) {
|
||||
got, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies)
|
||||
if !decision.Reused || len(got.Outputs) != 1 || len(got.Rejected) != 1 || len(got.Warnings) != 1 {
|
||||
t.Fatalf("extract result=%#v decision=%#v", got, decision)
|
||||
}
|
||||
output := got.Outputs[0]
|
||||
if !bytes.Equal(output.Artifact.Content, []byte(`{"spell":"fire"}`)) || output.Artifact.Kind != "spell" || output.Artifact.Schema.ID != "spell-schema" || output.Artifact.Schema.Version != "1" || output.Artifact.MediaType != "application/json" || output.Artifact.Metadata["chunk"] != "chunk-a" || output.ChunkRef.StartUnitID != 1 || got.Warnings[0].ReasonCode != "partial" || got.Rejected[0].ReasonCode != "invalid_source" {
|
||||
t.Fatalf("extract values were not restored: %#v", got)
|
||||
}
|
||||
manifest := readManifest[ExtractLaneManifest](t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "extract", "lane-a", "manifest.json"))
|
||||
if manifest.Status != StatusSucceededWithRejections {
|
||||
t.Fatalf("extract status = %q, want succeeded with rejections", manifest.Status)
|
||||
}
|
||||
|
||||
got.Outputs[0].Artifact.Content[0] = 'y'
|
||||
got.Warnings[0].Message = "loaded mutation"
|
||||
reloaded, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies)
|
||||
if !decision.Reused || !bytes.Equal(reloaded.Outputs[0].Artifact.Content, []byte(`{"spell":"fire"}`)) || reloaded.Warnings[0].Message != "partial output" {
|
||||
t.Fatalf("extract reload changed after loaded mutation: %#v decision=%#v", reloaded, decision)
|
||||
}
|
||||
})
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
load func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision)
|
||||
want []byte
|
||||
}{
|
||||
{name: "merge", load: func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision) {
|
||||
got, decision := fixture.loader.Merge("lane-a", "merge-module", fixture.dependencies)
|
||||
return got.Output, got.Warnings, decision
|
||||
}, want: []byte(`{"spells":["fire"]}`)},
|
||||
{name: "normalize", load: func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision) {
|
||||
got, decision := fixture.loader.Normalize("lane-a", "normalize-module", fixture.dependencies)
|
||||
return got.Output, got.Warnings, decision
|
||||
}, want: []byte(`{"spells":["fire"],"normalized":true}`)},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, warnings, decision := tt.load()
|
||||
if !decision.Reused || !bytes.Equal(got.Artifact.Content, tt.want) || got.Artifact.Kind != "spell" || got.Artifact.Schema.ID != "spell-schema" || got.Artifact.Schema.Version != "1" || got.Artifact.Metadata["lane"] != "lane-a" || len(warnings) != 1 || warnings[0].ReasonCode != "review" {
|
||||
t.Fatalf("%s result=%#v warnings=%#v decision=%#v", tt.name, got, warnings, decision)
|
||||
}
|
||||
|
||||
got.Artifact.Content[0] = 'z'
|
||||
reloaded, warnings, decision := tt.load()
|
||||
if !decision.Reused || !bytes.Equal(reloaded.Artifact.Content, tt.want) || warnings[0].Message != "review manually" {
|
||||
t.Fatalf("%s reload changed after loaded mutation: %#v warnings=%#v decision=%#v", tt.name, reloaded, warnings, decision)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if runtime.GOOS != "windows" {
|
||||
t.Run("restrictive permissions", func(t *testing.T) {
|
||||
root := filepath.Join(fixture.root, mustRelativePath(t, fixture.identity))
|
||||
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
want := os.FileMode(0o600)
|
||||
if info.IsDir() {
|
||||
want = 0o700
|
||||
}
|
||||
if got := info.Mode().Perm(); got != want {
|
||||
t.Errorf("%s permissions = %o, want %o", path, got, want)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemCheckpointRejectsMissingAndCorruptState(t *testing.T) {
|
||||
for _, stage := range checkpointStages() {
|
||||
t.Run(stage.name+" missing manifest", func(t *testing.T) {
|
||||
fixture := seedFilesystemCheckpoints(t)
|
||||
if err := os.Remove(stage.manifest(fixture)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertStageNotReused(t, stage, fixture, "missing")
|
||||
})
|
||||
|
||||
t.Run(stage.name+" malformed manifest", func(t *testing.T) {
|
||||
fixture := seedFilesystemCheckpoints(t)
|
||||
if err := os.WriteFile(stage.manifest(fixture), []byte("{"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertStageNotReused(t, stage, fixture, "decode")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemCheckpointRejectsIncompatibleManifests(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
edit func(map[string]any)
|
||||
want string
|
||||
}{
|
||||
{"v1 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV1 }, "workspace schema"},
|
||||
{"unknown schema", func(m map[string]any) { m["workspace_schema_version"] = "notarius.workspace.future" }, "workspace schema"},
|
||||
{"identity", func(m map[string]any) { m["metadata"].(map[string]any)["checkpoint_identity_digest"] = "sha256:other" }, "identity"},
|
||||
{"stage", func(m map[string]any) { m["stage"] = string(StageMerge) }, "stage"},
|
||||
{"lane", func(m map[string]any) { m["lane_id"] = "lane-other" }, "lane"},
|
||||
{"module", func(m map[string]any) { m["module_key"] = "module-other" }, "module"},
|
||||
{"dependency", func(m map[string]any) {
|
||||
m["dependency_fingerprints"] = []map[string]string{{"name": "input", "value": "other"}}
|
||||
}, "dependency"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fixture := seedFilesystemCheckpoints(t)
|
||||
editManifest(t, checkpointStages()[1].manifest(fixture), tt.edit)
|
||||
assertStageNotReused(t, checkpointStages()[1], fixture, tt.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemCheckpointRejectsNonTerminalStatuses(t *testing.T) {
|
||||
for _, status := range []StageStatus{StatusRunning, StatusFailed, StatusPending, StatusInvalidated} {
|
||||
t.Run(string(status), func(t *testing.T) {
|
||||
fixture := seedFilesystemCheckpoints(t)
|
||||
editManifest(t, checkpointStages()[1].manifest(fixture), func(m map[string]any) { m["status"] = string(status) })
|
||||
assertStageNotReused(t, checkpointStages()[1], fixture, "status")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemCheckpointRejectsIncompleteArtifactsAndContent(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
edit func(map[string]any)
|
||||
want string
|
||||
}{
|
||||
{"artifact kind", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["artifact_kind"] = "" }, "artifact codec identity"},
|
||||
{"schema id", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["schema"].(map[string]any)["id"] = "" }, "artifact codec identity"},
|
||||
{"schema version", func(m map[string]any) {
|
||||
m["outputs"].([]any)[0].(map[string]any)["schema"].(map[string]any)["version"] = ""
|
||||
}, "artifact codec identity"},
|
||||
{"schema digest", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["schema_digest"] = "" }, "artifact codec identity"},
|
||||
{"base64", func(m map[string]any) {
|
||||
m["outputs"].([]any)[0].(map[string]any)["content"].(map[string]any)["content_base64"] = "%"
|
||||
}, "base64"},
|
||||
{"content digest", func(m map[string]any) {
|
||||
m["outputs"].([]any)[0].(map[string]any)["content"].(map[string]any)["content_digest"] = "sha256:other"
|
||||
}, "content digest"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fixture := seedFilesystemCheckpoints(t)
|
||||
editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "extract", "lane-a", "outputs.json"), tt.edit)
|
||||
assertStageNotReused(t, checkpointStages()[1], fixture, tt.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemCheckpointRejectsSourceAndOutputDigestMismatches(t *testing.T) {
|
||||
t.Run("invalid source document", func(t *testing.T) {
|
||||
fixture := seedFilesystemCheckpoints(t)
|
||||
editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "source", "source-document.json"), func(m map[string]any) {
|
||||
m["document"].(map[string]any)["units"].([]any)[0].(map[string]any)["text"] = ""
|
||||
})
|
||||
assertStageNotReused(t, checkpointStages()[0], fixture, "source checkpoint document")
|
||||
})
|
||||
|
||||
t.Run("source output digest", func(t *testing.T) {
|
||||
fixture := seedFilesystemCheckpoints(t)
|
||||
editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "source", "source-document.json"), func(m map[string]any) {
|
||||
m["document"].(map[string]any)["digest"] = "sha256:other"
|
||||
})
|
||||
assertStageNotReused(t, checkpointStages()[0], fixture, "output digest")
|
||||
})
|
||||
|
||||
for _, stage := range checkpointStages()[1:] {
|
||||
t.Run(stage.name+" output digest", func(t *testing.T) {
|
||||
fixture := seedFilesystemCheckpoints(t)
|
||||
editManifest(t, stage.manifest(fixture), func(m map[string]any) {
|
||||
m["output_digests"].([]any)[0].(map[string]any)["value"] = "sha256:other"
|
||||
})
|
||||
assertStageNotReused(t, stage, fixture, "output digest")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesystemCheckpointReusesExtractWithRejections(t *testing.T) {
|
||||
fixture := seedFilesystemCheckpoints(t)
|
||||
result, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies)
|
||||
if !decision.Reused || len(result.Rejected) != 1 || result.Rejected[0].Message != "source reference is invalid" {
|
||||
t.Fatalf("result=%#v decision=%#v", result, decision)
|
||||
}
|
||||
}
|
||||
|
||||
type checkpointStage struct {
|
||||
name string
|
||||
manifest func(filesystemCheckpointFixture) string
|
||||
load func(filesystemCheckpointFixture) pipeline.CheckpointDecision
|
||||
}
|
||||
|
||||
func checkpointStages() []checkpointStage {
|
||||
return []checkpointStage{
|
||||
{
|
||||
name: "source",
|
||||
manifest: func(f filesystemCheckpointFixture) string {
|
||||
return filepath.Join(f.root, mustRelativePathForTest(f.identity), "source", "manifest.json")
|
||||
},
|
||||
load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision {
|
||||
_, d := f.loader.Source("source-module")
|
||||
return d
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "extract",
|
||||
manifest: func(f filesystemCheckpointFixture) string {
|
||||
return filepath.Join(f.root, mustRelativePathForTest(f.identity), "extract", "lane-a", "manifest.json")
|
||||
},
|
||||
load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision {
|
||||
_, d := f.loader.Extract("lane-a", "extract-module", f.dependencies)
|
||||
return d
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "merge",
|
||||
manifest: func(f filesystemCheckpointFixture) string {
|
||||
return filepath.Join(f.root, mustRelativePathForTest(f.identity), "merge", "lane-a", "manifest.json")
|
||||
},
|
||||
load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision {
|
||||
_, d := f.loader.Merge("lane-a", "merge-module", f.dependencies)
|
||||
return d
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize",
|
||||
manifest: func(f filesystemCheckpointFixture) string {
|
||||
return filepath.Join(f.root, mustRelativePathForTest(f.identity), "normalize", "lane-a", "manifest.json")
|
||||
},
|
||||
load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision {
|
||||
_, d := f.loader.Normalize("lane-a", "normalize-module", f.dependencies)
|
||||
return d
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func seedFilesystemCheckpoints(t *testing.T) filesystemCheckpointFixture {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
identity, err := NewIdentity(representativeIdentityInput())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorder, err := NewFilesystemRecorder(root, identity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture := filesystemCheckpointFixture{
|
||||
root: root,
|
||||
identity: identity,
|
||||
doc: checkpointDocument(),
|
||||
extract: checkpointArtifact("extract", `{"spell":"fire"}`),
|
||||
merge: checkpointArtifact("merge", `{"spells":["fire"]}`),
|
||||
normalize: checkpointArtifact("normalize", `{"spells":["fire"],"normalized":true}`),
|
||||
dependencies: []pipeline.CheckpointFingerprint{{Name: "source", Value: "sha256:source"}, {Name: "chunk-plan", Value: "sha256:plan"}},
|
||||
warnings: []contracts.Warning{{Scope: "extract", ReasonCode: "partial", Message: "partial output"}},
|
||||
rejected: []contracts.RejectedOutput{{Stage: "extract", LaneID: "lane-a", ModuleKey: "extract-module", ChunkID: "chunk-a", ValidatorName: "source_refs", ReasonCode: "invalid_source", Message: "source reference is invalid", AttemptCount: 1}},
|
||||
}
|
||||
if err := recorder.SourceSucceeded("source-module", &fixture.doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.ExtractSucceeded("lane-a", "extract-module", fixture.dependencies, []pipeline.CheckpointArtifact{fixture.extract}, fixture.rejected, fixture.warnings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mergeWarnings := []contracts.Warning{{Scope: "merge", ReasonCode: "review", Message: "review manually"}}
|
||||
if err := recorder.MergeSucceeded("lane-a", "merge-module", fixture.dependencies, fixture.merge, mergeWarnings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.NormalizeSucceeded("lane-a", "normalize-module", fixture.dependencies, fixture.normalize, mergeWarnings); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixture.loader, err = NewFilesystemLoader(root, identity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return fixture
|
||||
}
|
||||
|
||||
func checkpointDocument() source.SourceDocument {
|
||||
return source.SourceDocument{
|
||||
ID: "document-1", Kind: "transcript", Format: "text", Digest: "sha256:document", Metadata: map[string]any{"owner": "fixture", "number": float64(1)},
|
||||
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "original source", Ref: source.SourceRef{SourceID: "document-1", StartUnitID: 1, EndUnitID: 1}, Metadata: map[string]any{"speaker": "narrator"}}},
|
||||
}
|
||||
}
|
||||
|
||||
func checkpointArtifact(module, content string) pipeline.CheckpointArtifact {
|
||||
return pipeline.CheckpointArtifact{
|
||||
LaneID: "lane-a", ModuleKey: module, SourceID: "document-1", ChunkID: "chunk-a", ChunkIndex: 0,
|
||||
ChunkRef: source.SourceRef{SourceID: "document-1", StartUnitID: 1, EndUnitID: 1}, SchemaDigest: "sha256:schema",
|
||||
Artifact: contracts.SerializedArtifact{Kind: "spell", Schema: contracts.ArtifactSchema{ID: "spell-schema", Name: "Spell", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}, MediaType: "application/json", Content: []byte(content), Metadata: map[string]any{"chunk": "chunk-a", "lane": "lane-a"}},
|
||||
}
|
||||
}
|
||||
|
||||
func assertStageNotReused(t *testing.T, stage checkpointStage, fixture filesystemCheckpointFixture, want string) {
|
||||
t.Helper()
|
||||
decision := stage.load(fixture)
|
||||
if decision.Reused || !strings.Contains(strings.ToLower(decision.Reason), strings.ToLower(want)) {
|
||||
t.Fatalf("%s decision=%#v, want non-reused reason containing %q", stage.name, decision, want)
|
||||
}
|
||||
}
|
||||
|
||||
func editManifest(t *testing.T, path string, edit func(map[string]any)) {
|
||||
t.Helper()
|
||||
editJSON(t, path, edit)
|
||||
}
|
||||
|
||||
func editJSON(t *testing.T, path string, edit func(map[string]any)) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value map[string]any
|
||||
if err := json.Unmarshal(data, &value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
edit(value)
|
||||
data, err = json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func readManifest[T any](t *testing.T, path string) T {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value T
|
||||
if err := json.Unmarshal(data, &value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func mustRelativePath(t *testing.T, identity Identity) string {
|
||||
t.Helper()
|
||||
return mustRelativePathForTest(identity)
|
||||
}
|
||||
|
||||
func mustRelativePathForTest(identity Identity) string {
|
||||
path, err := identity.RelativePath()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
Reference in New Issue
Block a user