package checkpoint import ( "bytes" "encoding/json" "os" "path/filepath" "runtime" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) type filesystemCheckpointFixture struct { root string identity Identity loader pipeline.CheckpointLoader doc source.SourceDocument extract pipeline.CheckpointArtifact merge pipeline.CheckpointArtifact normalize pipeline.CheckpointArtifact dependencies []pipeline.CheckpointFingerprint warnings []contracts.Warning rejected []contracts.RejectedOutput } func TestFilesystemCheckpointRoundTripsAllStages(t *testing.T) { fixture := seedFilesystemCheckpoints(t) // Recording owns its inputs. These mutations must not change the durable values. fixture.doc.Units[0].Text = "caller mutation" fixture.doc.Metadata["owner"] = "caller mutation" fixture.extract.Artifact.Content[0] = 'x' fixture.extract.Artifact.Metadata["content"] = "caller mutation" fixture.merge.Artifact.Content[0] = 'x' fixture.normalize.Artifact.Content[0] = 'x' fixture.warnings[0].Message = "caller mutation" fixture.rejected[0].Message = "caller mutation" t.Run("source", func(t *testing.T) { got, decision := fixture.loader.Source("source-module") if !decision.Reused { t.Fatalf("source decision = %#v", decision) } if got.Document == nil || got.Document.ID != "document-1" || got.Document.Units[0].Text != "original source" || got.Document.Metadata["owner"] != "fixture" { t.Fatalf("source was not restored: %#v", got) } got.Document.Units[0].Text = "loaded mutation" got.Document.Metadata["owner"] = "loaded mutation" reloaded, decision := fixture.loader.Source("source-module") if !decision.Reused || reloaded.Document.Units[0].Text != "original source" || reloaded.Document.Metadata["owner"] != "fixture" { t.Fatalf("source reload changed after loaded mutation: %#v decision=%#v", reloaded, decision) } }) t.Run("extract", func(t *testing.T) { got, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies) if !decision.Reused || len(got.Outputs) != 1 || len(got.Rejected) != 1 || len(got.Warnings) != 1 { t.Fatalf("extract result=%#v decision=%#v", got, decision) } output := got.Outputs[0] if !bytes.Equal(output.Artifact.Content, []byte(`{"spell":"fire"}`)) || output.Artifact.Kind != "spell" || output.Artifact.Schema.ID != "spell-schema" || output.Artifact.Schema.Version != "1" || output.Artifact.MediaType != "application/json" || output.Artifact.Metadata["chunk"] != "chunk-a" || output.ChunkRef.StartUnitID != 1 || got.Warnings[0].ReasonCode != "partial" || got.Rejected[0].ReasonCode != "invalid_source" { t.Fatalf("extract values were not restored: %#v", got) } manifest := readManifest[ExtractLaneManifest](t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "extract", "lane-a", "manifest.json")) if manifest.Status != StatusSucceededWithRejections { t.Fatalf("extract status = %q, want succeeded with rejections", manifest.Status) } got.Outputs[0].Artifact.Content[0] = 'y' got.Warnings[0].Message = "loaded mutation" reloaded, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies) if !decision.Reused || !bytes.Equal(reloaded.Outputs[0].Artifact.Content, []byte(`{"spell":"fire"}`)) || reloaded.Warnings[0].Message != "partial output" { t.Fatalf("extract reload changed after loaded mutation: %#v decision=%#v", reloaded, decision) } }) for _, tt := range []struct { name string load func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision) want []byte }{ {name: "merge", load: func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision) { got, decision := fixture.loader.Merge("lane-a", "merge-module", fixture.dependencies) return got.Output, got.Warnings, decision }, want: []byte(`{"spells":["fire"]}`)}, {name: "normalize", load: func() (pipeline.CheckpointArtifact, []contracts.Warning, pipeline.CheckpointDecision) { got, decision := fixture.loader.Normalize("lane-a", "normalize-module", fixture.dependencies) return got.Output, got.Warnings, decision }, want: []byte(`{"spells":["fire"],"normalized":true}`)}, } { t.Run(tt.name, func(t *testing.T) { got, warnings, decision := tt.load() if !decision.Reused || !bytes.Equal(got.Artifact.Content, tt.want) || got.Artifact.Kind != "spell" || got.Artifact.Schema.ID != "spell-schema" || got.Artifact.Schema.Version != "1" || got.Artifact.Metadata["lane"] != "lane-a" || len(warnings) != 1 || warnings[0].ReasonCode != "review" { t.Fatalf("%s result=%#v warnings=%#v decision=%#v", tt.name, got, warnings, decision) } got.Artifact.Content[0] = 'z' reloaded, warnings, decision := tt.load() if !decision.Reused || !bytes.Equal(reloaded.Artifact.Content, tt.want) || warnings[0].Message != "review manually" { t.Fatalf("%s reload changed after loaded mutation: %#v warnings=%#v decision=%#v", tt.name, reloaded, warnings, decision) } }) } if runtime.GOOS != "windows" { t.Run("restrictive permissions", func(t *testing.T) { root := filepath.Join(fixture.root, mustRelativePath(t, fixture.identity)) err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { return err } want := os.FileMode(0o600) if info.IsDir() { want = 0o700 } if got := info.Mode().Perm(); got != want { t.Errorf("%s permissions = %o, want %o", path, got, want) } return nil }) if err != nil { t.Fatal(err) } }) } } func TestFilesystemCheckpointRejectsMissingAndCorruptState(t *testing.T) { for _, stage := range checkpointStages() { t.Run(stage.name+" missing manifest", func(t *testing.T) { fixture := seedFilesystemCheckpoints(t) if err := os.Remove(stage.manifest(fixture)); err != nil { t.Fatal(err) } assertStageNotReused(t, stage, fixture, "missing") }) t.Run(stage.name+" malformed manifest", func(t *testing.T) { fixture := seedFilesystemCheckpoints(t) if err := os.WriteFile(stage.manifest(fixture), []byte("{"), 0o600); err != nil { t.Fatal(err) } assertStageNotReused(t, stage, fixture, "decode") }) } } func TestFilesystemCheckpointRejectsIncompatibleManifests(t *testing.T) { for _, tt := range []struct { name string edit func(map[string]any) want string }{ {"v1 schema", func(m map[string]any) { m["workspace_schema_version"] = WorkspaceSchemaVersionV1 }, "workspace schema"}, {"unknown schema", func(m map[string]any) { m["workspace_schema_version"] = "notarius.workspace.future" }, "workspace schema"}, {"identity", func(m map[string]any) { m["metadata"].(map[string]any)["checkpoint_identity_digest"] = "sha256:other" }, "identity"}, {"stage", func(m map[string]any) { m["stage"] = string(StageMerge) }, "stage"}, {"lane", func(m map[string]any) { m["lane_id"] = "lane-other" }, "lane"}, {"module", func(m map[string]any) { m["module_key"] = "module-other" }, "module"}, {"dependency", func(m map[string]any) { m["dependency_fingerprints"] = []map[string]string{{"name": "input", "value": "other"}} }, "dependency"}, } { t.Run(tt.name, func(t *testing.T) { fixture := seedFilesystemCheckpoints(t) editManifest(t, checkpointStages()[1].manifest(fixture), tt.edit) assertStageNotReused(t, checkpointStages()[1], fixture, tt.want) }) } } func TestFilesystemCheckpointRejectsNonTerminalStatuses(t *testing.T) { for _, status := range []StageStatus{StatusRunning, StatusFailed, StatusPending, StatusInvalidated} { t.Run(string(status), func(t *testing.T) { fixture := seedFilesystemCheckpoints(t) editManifest(t, checkpointStages()[1].manifest(fixture), func(m map[string]any) { m["status"] = string(status) }) assertStageNotReused(t, checkpointStages()[1], fixture, "status") }) } } func TestFilesystemCheckpointRejectsIncompleteArtifactsAndContent(t *testing.T) { for _, tt := range []struct { name string edit func(map[string]any) want string }{ {"artifact kind", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["artifact_kind"] = "" }, "artifact codec identity"}, {"schema id", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["schema"].(map[string]any)["id"] = "" }, "artifact codec identity"}, {"schema version", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["schema"].(map[string]any)["version"] = "" }, "artifact codec identity"}, {"schema digest", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["schema_digest"] = "" }, "artifact codec identity"}, {"base64", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["content"].(map[string]any)["content_base64"] = "%" }, "base64"}, {"content digest", func(m map[string]any) { m["outputs"].([]any)[0].(map[string]any)["content"].(map[string]any)["content_digest"] = "sha256:other" }, "content digest"}, } { t.Run(tt.name, func(t *testing.T) { fixture := seedFilesystemCheckpoints(t) editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "extract", "lane-a", "outputs.json"), tt.edit) assertStageNotReused(t, checkpointStages()[1], fixture, tt.want) }) } } func TestFilesystemCheckpointRejectsSourceAndOutputDigestMismatches(t *testing.T) { t.Run("invalid source document", func(t *testing.T) { fixture := seedFilesystemCheckpoints(t) editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "source", "source-document.json"), func(m map[string]any) { m["document"].(map[string]any)["units"].([]any)[0].(map[string]any)["text"] = "" }) assertStageNotReused(t, checkpointStages()[0], fixture, "source checkpoint document") }) t.Run("source output digest", func(t *testing.T) { fixture := seedFilesystemCheckpoints(t) editJSON(t, filepath.Join(fixture.root, mustRelativePath(t, fixture.identity), "source", "source-document.json"), func(m map[string]any) { m["document"].(map[string]any)["digest"] = "sha256:other" }) assertStageNotReused(t, checkpointStages()[0], fixture, "output digest") }) for _, stage := range checkpointStages()[1:] { t.Run(stage.name+" output digest", func(t *testing.T) { fixture := seedFilesystemCheckpoints(t) editManifest(t, stage.manifest(fixture), func(m map[string]any) { m["output_digests"].([]any)[0].(map[string]any)["value"] = "sha256:other" }) assertStageNotReused(t, stage, fixture, "output digest") }) } } func TestFilesystemCheckpointReusesExtractWithRejections(t *testing.T) { fixture := seedFilesystemCheckpoints(t) result, decision := fixture.loader.Extract("lane-a", "extract-module", fixture.dependencies) if !decision.Reused || len(result.Rejected) != 1 || result.Rejected[0].Message != "source reference is invalid" { t.Fatalf("result=%#v decision=%#v", result, decision) } } type checkpointStage struct { name string manifest func(filesystemCheckpointFixture) string load func(filesystemCheckpointFixture) pipeline.CheckpointDecision } func checkpointStages() []checkpointStage { return []checkpointStage{ { name: "source", manifest: func(f filesystemCheckpointFixture) string { return filepath.Join(f.root, mustRelativePathForTest(f.identity), "source", "manifest.json") }, load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision { _, d := f.loader.Source("source-module") return d }, }, { name: "extract", manifest: func(f filesystemCheckpointFixture) string { return filepath.Join(f.root, mustRelativePathForTest(f.identity), "extract", "lane-a", "manifest.json") }, load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision { _, d := f.loader.Extract("lane-a", "extract-module", f.dependencies) return d }, }, { name: "merge", manifest: func(f filesystemCheckpointFixture) string { return filepath.Join(f.root, mustRelativePathForTest(f.identity), "merge", "lane-a", "manifest.json") }, load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision { _, d := f.loader.Merge("lane-a", "merge-module", f.dependencies) return d }, }, { name: "normalize", manifest: func(f filesystemCheckpointFixture) string { return filepath.Join(f.root, mustRelativePathForTest(f.identity), "normalize", "lane-a", "manifest.json") }, load: func(f filesystemCheckpointFixture) pipeline.CheckpointDecision { _, d := f.loader.Normalize("lane-a", "normalize-module", f.dependencies) return d }, }, } } func seedFilesystemCheckpoints(t *testing.T) filesystemCheckpointFixture { t.Helper() root := t.TempDir() identity, err := NewIdentity(representativeIdentityInput()) if err != nil { t.Fatal(err) } recorder, err := NewFilesystemRecorder(root, identity) if err != nil { t.Fatal(err) } fixture := filesystemCheckpointFixture{ root: root, identity: identity, doc: checkpointDocument(), extract: checkpointArtifact("extract", `{"spell":"fire"}`), merge: checkpointArtifact("merge", `{"spells":["fire"]}`), normalize: checkpointArtifact("normalize", `{"spells":["fire"],"normalized":true}`), dependencies: []pipeline.CheckpointFingerprint{{Name: "source", Value: "sha256:source"}, {Name: "chunk-plan", Value: "sha256:plan"}}, warnings: []contracts.Warning{{Scope: "extract", ReasonCode: "partial", Message: "partial output"}}, rejected: []contracts.RejectedOutput{{Stage: "extract", LaneID: "lane-a", ModuleKey: "extract-module", ChunkID: "chunk-a", ValidatorName: "source_refs", ReasonCode: "invalid_source", Message: "source reference is invalid", AttemptCount: 1}}, } if err := recorder.SourceSucceeded("source-module", &fixture.doc); err != nil { t.Fatal(err) } if err := recorder.ExtractSucceeded("lane-a", "extract-module", fixture.dependencies, []pipeline.CheckpointArtifact{fixture.extract}, fixture.rejected, fixture.warnings); err != nil { t.Fatal(err) } mergeWarnings := []contracts.Warning{{Scope: "merge", ReasonCode: "review", Message: "review manually"}} if err := recorder.MergeSucceeded("lane-a", "merge-module", fixture.dependencies, fixture.merge, mergeWarnings); err != nil { t.Fatal(err) } if err := recorder.NormalizeSucceeded("lane-a", "normalize-module", fixture.dependencies, fixture.normalize, mergeWarnings); err != nil { t.Fatal(err) } fixture.loader, err = NewFilesystemLoader(root, identity) if err != nil { t.Fatal(err) } return fixture } func checkpointDocument() source.SourceDocument { return source.SourceDocument{ ID: "document-1", Kind: "transcript", Format: "text", Digest: "sha256:document", Metadata: map[string]any{"owner": "fixture", "number": float64(1)}, Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "original source", Ref: source.SourceRef{SourceID: "document-1", StartUnitID: 1, EndUnitID: 1}, Metadata: map[string]any{"speaker": "narrator"}}}, } } func checkpointArtifact(module, content string) pipeline.CheckpointArtifact { return pipeline.CheckpointArtifact{ LaneID: "lane-a", ModuleKey: module, SourceID: "document-1", ChunkID: "chunk-a", ChunkIndex: 0, ChunkRef: source.SourceRef{SourceID: "document-1", StartUnitID: 1, EndUnitID: 1}, SchemaDigest: "sha256:schema", Artifact: contracts.SerializedArtifact{Kind: "spell", Schema: contracts.ArtifactSchema{ID: "spell-schema", Name: "Spell", Version: "1", JSONSchema: []byte(`{"type":"object"}`)}, MediaType: "application/json", Content: []byte(content), Metadata: map[string]any{"chunk": "chunk-a", "lane": "lane-a"}}, } } func assertStageNotReused(t *testing.T, stage checkpointStage, fixture filesystemCheckpointFixture, want string) { t.Helper() decision := stage.load(fixture) if decision.Reused || !strings.Contains(strings.ToLower(decision.Reason), strings.ToLower(want)) { t.Fatalf("%s decision=%#v, want non-reused reason containing %q", stage.name, decision, want) } } func editManifest(t *testing.T, path string, edit func(map[string]any)) { t.Helper() editJSON(t, path, edit) } func editJSON(t *testing.T, path string, edit func(map[string]any)) { t.Helper() data, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var value map[string]any if err := json.Unmarshal(data, &value); err != nil { t.Fatal(err) } edit(value) data, err = json.Marshal(value) if err != nil { t.Fatal(err) } if err := os.WriteFile(path, data, 0o600); err != nil { t.Fatal(err) } } func readManifest[T any](t *testing.T, path string) T { t.Helper() data, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var value T if err := json.Unmarshal(data, &value); err != nil { t.Fatal(err) } return value } func mustRelativePath(t *testing.T, identity Identity) string { t.Helper() return mustRelativePathForTest(identity) } func mustRelativePathForTest(identity Identity) string { path, err := identity.RelativePath() if err != nil { panic(err) } return path }