Integrate chunk plan caching into the runner
This commit is contained in:
@@ -1,11 +1,9 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -25,24 +23,6 @@ func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
|
||||
Digest: "sha256:source",
|
||||
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}},
|
||||
}
|
||||
chunks := []source.Chunk{
|
||||
{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Index: 0,
|
||||
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1},
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
Units: doc.Units,
|
||||
Annotations: source.ChunkAnnotations{
|
||||
"same": json.RawMessage(`{"range":1}`),
|
||||
},
|
||||
PlanAnnotations: source.ChunkAnnotations{
|
||||
"same": json.RawMessage(`{"plan":2}`),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := recorder.SourceRunning("seriatim"); err != nil {
|
||||
t.Fatalf("SourceRunning: %v", err)
|
||||
}
|
||||
@@ -54,60 +34,6 @@ func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
|
||||
if _, err := os.Stat(filepath.Join(root, "source", "source-document.json")); err != nil {
|
||||
t.Fatalf("expected source checkpoint payload: %v", err)
|
||||
}
|
||||
|
||||
if err := recorder.ChunkRunning("generic", doc.Digest); err != nil {
|
||||
t.Fatalf("ChunkRunning: %v", err)
|
||||
}
|
||||
if err := recorder.ChunkSucceeded("generic", doc.Digest, chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
assertManifestStatus(t, filepath.Join(root, "chunk", "manifest.json"), coreworkspace.StatusSucceeded)
|
||||
var chunkPayload struct {
|
||||
Chunks []struct {
|
||||
Annotations source.ChunkAnnotations `json:"annotations"`
|
||||
PlanAnnotations source.ChunkAnnotations `json:"plan_annotations"`
|
||||
Content struct {
|
||||
ContentBase64 string `json:"content_base64"`
|
||||
ContentDigest string `json:"content_digest"`
|
||||
} `json:"content"`
|
||||
} `json:"chunks"`
|
||||
}
|
||||
readJSON(t, filepath.Join(root, "chunk", "chunks.json"), &chunkPayload)
|
||||
if len(chunkPayload.Chunks) != 1 {
|
||||
t.Fatalf("checkpoint chunks = %#v, want one", chunkPayload.Chunks)
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(chunkPayload.Chunks[0].Content.ContentBase64)
|
||||
if err != nil {
|
||||
t.Fatalf("decode chunk content: %v", err)
|
||||
}
|
||||
if string(decoded) != "chunk content" {
|
||||
t.Fatalf("chunk content = %q, want original content", decoded)
|
||||
}
|
||||
if got, want := chunkPayload.Chunks[0].Content.ContentDigest, contentDigest([]byte("chunk content")); got != want {
|
||||
t.Fatalf("content digest = %q, want %q", got, want)
|
||||
}
|
||||
if !jsonEqual(t, chunkPayload.Chunks[0].Annotations["same"], json.RawMessage(`{"range":1}`)) || !jsonEqual(t, chunkPayload.Chunks[0].PlanAnnotations["same"], json.RawMessage(`{"plan":2}`)) {
|
||||
t.Fatalf("checkpoint annotations = %s / %s", chunkPayload.Chunks[0].Annotations["same"], chunkPayload.Chunks[0].PlanAnnotations["same"])
|
||||
}
|
||||
loaded, decision := (&WorkspaceLoader{root: root}).Chunk("generic", doc.Digest)
|
||||
if !decision.Reused || len(loaded.Chunks) != 1 {
|
||||
t.Fatalf("loaded chunk checkpoint = %#v, decision = %#v", loaded, decision)
|
||||
}
|
||||
if string(loaded.Chunks[0].Annotations["same"]) != `{"range":1}` || string(loaded.Chunks[0].PlanAnnotations["same"]) != `{"plan":2}` {
|
||||
t.Fatalf("loaded canonical annotations = %s / %s", loaded.Chunks[0].Annotations["same"], loaded.Chunks[0].PlanAnnotations["same"])
|
||||
}
|
||||
}
|
||||
|
||||
func jsonEqual(t *testing.T, left, right []byte) bool {
|
||||
t.Helper()
|
||||
var leftValue, rightValue any
|
||||
if err := json.Unmarshal(left, &leftValue); err != nil {
|
||||
t.Fatalf("decode left JSON: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(right, &rightValue); err != nil {
|
||||
t.Fatalf("decode right JSON: %v", err)
|
||||
}
|
||||
return reflect.DeepEqual(leftValue, rightValue)
|
||||
}
|
||||
|
||||
func TestWorkspaceArtifactCheckpointsRoundTripCodecIdentityAndBytes(t *testing.T) {
|
||||
@@ -158,24 +84,6 @@ func TestWorkspaceLoaderInvalidatesMissingCorruptAndMismatchedCheckpoints(t *tes
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dependency mismatch", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
chunks := []source.Chunk{{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
}}
|
||||
if err := recorder.ChunkSucceeded("generic", "sha256:source-a", chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
loader := &WorkspaceLoader{root: root}
|
||||
if _, decision := loader.Chunk("generic", "sha256:source-b"); decision.Reused || !strings.Contains(decision.Reason, "dependency") {
|
||||
t.Fatalf("decision = %#v, want dependency invalidation", decision)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("incompatible workspace schema remains untouched", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
@@ -207,28 +115,41 @@ func TestWorkspaceLoaderInvalidatesMissingCorruptAndMismatchedCheckpoints(t *tes
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("corrupt payload", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
chunks := []source.Chunk{{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
}}
|
||||
if err := recorder.ChunkSucceeded("generic", "sha256:source", chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
payloadPath := filepath.Join(root, "chunk", "chunks.json")
|
||||
data := strings.ReplaceAll(string(readFile(t, payloadPath)), contentDigest([]byte("chunk content")), "sha256:bad")
|
||||
if err := os.WriteFile(payloadPath, []byte(data), 0o644); err != nil {
|
||||
t.Fatalf("corrupt chunk payload: %v", err)
|
||||
}
|
||||
loader := &WorkspaceLoader{root: root}
|
||||
if _, decision := loader.Chunk("generic", "sha256:source"); decision.Reused || !strings.Contains(decision.Reason, "invalid") {
|
||||
t.Fatalf("decision = %#v, want corrupt payload invalidation", decision)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestWorkspaceCheckpointsIgnoreLegacyChunkFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
legacyManifest := []byte(`{"legacy":"manifest"}`)
|
||||
legacyPayload := []byte(`{"legacy":"chunks"}`)
|
||||
legacyDir := filepath.Join(root, "chunk")
|
||||
if err := os.MkdirAll(legacyDir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifestPath := filepath.Join(legacyDir, "manifest.json")
|
||||
payloadPath := filepath.Join(legacyDir, "chunks.json")
|
||||
if err := os.WriteFile(manifestPath, legacyManifest, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(payloadPath, legacyPayload, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
doc := &source.SourceDocument{ID: "source-1", Kind: "document", Format: "text/plain", Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello", Ref: source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 1}}}}
|
||||
doc.Digest, _ = source.DigestDocument(doc)
|
||||
recorder := newTestRecorder(t, root)
|
||||
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, decision := (&WorkspaceLoader{root: root}).Source("seriatim")
|
||||
if !decision.Reused || loaded.Document == nil || loaded.Document.Digest != doc.Digest {
|
||||
t.Fatalf("source checkpoint = %#v decision = %#v", loaded, decision)
|
||||
}
|
||||
if got := readFile(t, manifestPath); string(got) != string(legacyManifest) {
|
||||
t.Fatalf("legacy manifest changed: %s", got)
|
||||
}
|
||||
if got := readFile(t, payloadPath); string(got) != string(legacyPayload) {
|
||||
t.Fatalf("legacy payload changed: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceRecorderRecordsRejectedExtractOutputs(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user