Add chunk plan cache configuration and storage

2026-07-18 00:07:53 +00:00
parent 7844c0a93f
commit ebd449d847
13 changed files with 843 additions and 1 deletions
--- a/internal/framework/chunkplan/store.go
+++ b/internal/framework/chunkplan/store.go
@@ -0,0 +1,207 @@
+package chunkplan
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+const SchemaVersion = "notarius.chunk-plan.v1"
+
+type filesystemStore struct {
+	root string
+}
+
+func NewFilesystemStore(root string) (pipeline.ChunkPlanStore, error) {
+	root = strings.TrimSpace(root)
+	if root == "" {
+		return nil, fmt.Errorf("chunk plan root must not be empty")
+	}
+	if strings.ContainsRune(root, '\x00') {
+		return nil, fmt.Errorf("chunk plan root must not contain NUL")
+	}
+	return &filesystemStore{root: filepath.Clean(root)}, nil
+}
+
+func (s *filesystemStore) Load(sourceDigest string) (pipeline.ChunkPlanRecord, pipeline.ChunkPlanDecision, error) {
+	target, err := s.planPath(sourceDigest)
+	if err != nil {
+		return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{}, err
+	}
+	data, err := os.ReadFile(target)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanMissing, Reason: "chunk plan not found"}, nil
+		}
+		return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{}, fmt.Errorf("read chunk plan: %w", err)
+	}
+
+	var record pipeline.ChunkPlanRecord
+	decoder := json.NewDecoder(bytes.NewReader(data))
+	decoder.DisallowUnknownFields()
+	decoder.UseNumber()
+	if err := decoder.Decode(&record); err != nil {
+		return invalidDecision(fmt.Sprintf("decode stored chunk plan: %v", err))
+	}
+	var trailing any
+	if err := decoder.Decode(&trailing); err != io.EOF {
+		if err == nil {
+			return invalidDecision("stored chunk plan contains trailing JSON")
+		}
+		return invalidDecision(fmt.Sprintf("decode stored chunk plan trailer: %v", err))
+	}
+	if err := validateRecord(record, sourceDigest); err != nil {
+		return invalidDecision(err.Error())
+	}
+	return record, pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanHit, Reason: "stored chunk plan is valid"}, nil
+}
+
+func (s *filesystemStore) Save(record pipeline.ChunkPlanRecord) error {
+	if err := validateRecord(record, record.SourceDigest); err != nil {
+		return fmt.Errorf("validate chunk plan record: %w", err)
+	}
+	target, err := s.planPath(record.SourceDigest)
+	if err != nil {
+		return err
+	}
+	data, err := json.Marshal(record)
+	if err != nil {
+		return fmt.Errorf("encode chunk plan record: %w", err)
+	}
+	data = append(data, '\n')
+	if err := writeAtomic(target, data); err != nil {
+		return fmt.Errorf("write chunk plan: %w", err)
+	}
+	return nil
+}
+
+func (s *filesystemStore) planPath(sourceDigest string) (string, error) {
+	if s == nil || strings.TrimSpace(s.root) == "" {
+		return "", fmt.Errorf("chunk plan store must not be nil")
+	}
+	hexDigest, err := digestPathSegment(sourceDigest)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(s.root, hexDigest, "plan.json"), nil
+}
+
+func digestPathSegment(digest string) (string, error) {
+	if !strings.HasPrefix(digest, "sha256:") {
+		return "", fmt.Errorf("source digest must use sha256:<64 lowercase hex> format")
+	}
+	hexDigest := strings.TrimPrefix(digest, "sha256:")
+	if len(hexDigest) != 64 || strings.ToLower(hexDigest) != hexDigest {
+		return "", fmt.Errorf("source digest must use sha256:<64 lowercase hex> format")
+	}
+	decoded, err := hex.DecodeString(hexDigest)
+	if err != nil || len(decoded) != 32 {
+		return "", fmt.Errorf("source digest must use sha256:<64 lowercase hex> format")
+	}
+	return hexDigest, nil
+}
+
+func validateRecord(record pipeline.ChunkPlanRecord, requestedDigest string) error {
+	if record.SchemaVersion != SchemaVersion {
+		return fmt.Errorf("schema_version %q is not supported", record.SchemaVersion)
+	}
+	if _, err := digestPathSegment(requestedDigest); err != nil {
+		return err
+	}
+	if record.SourceDigest != requestedDigest {
+		return fmt.Errorf("source_digest does not match requested source")
+	}
+	if record.Plan.SourceDigest != record.SourceDigest {
+		return fmt.Errorf("plan source_digest does not match record source_digest")
+	}
+	if len(record.Plan.Ranges) == 0 {
+		return fmt.Errorf("plan ranges must not be empty")
+	}
+	if err := source.ValidateChunkAnnotations(record.Plan.Annotations); err != nil {
+		return fmt.Errorf("plan annotations: %w", err)
+	}
+	seenStarts := make(map[int]struct{}, len(record.Plan.Ranges))
+	for i, chunkRange := range record.Plan.Ranges {
+		if chunkRange.StartUnitID <= 0 || chunkRange.EndUnitID <= 0 {
+			return fmt.Errorf("plan range[%d] boundaries must be positive", i)
+		}
+		if _, exists := seenStarts[chunkRange.StartUnitID]; exists {
+			return fmt.Errorf("plan range[%d] duplicates start_unit_id %d", i, chunkRange.StartUnitID)
+		}
+		seenStarts[chunkRange.StartUnitID] = struct{}{}
+		if err := source.ValidateChunkAnnotations(chunkRange.Annotations); err != nil {
+			return fmt.Errorf("plan range[%d] annotations: %w", i, err)
+		}
+	}
+	wantDigest, err := source.DigestChunkPlan(record.Plan)
+	if err != nil {
+		return fmt.Errorf("digest plan: %w", err)
+	}
+	if record.PlanDigest != wantDigest {
+		return fmt.Errorf("plan_digest does not match plan")
+	}
+	if strings.TrimSpace(record.Producer.InputModule) == "" {
+		return fmt.Errorf("producer input_module must not be empty")
+	}
+	if strings.TrimSpace(record.Producer.ChunkModule) == "" {
+		return fmt.Errorf("producer chunk_module must not be empty")
+	}
+	if record.CreatedAt.IsZero() {
+		return fmt.Errorf("created_at must not be zero")
+	}
+	return nil
+}
+
+func invalidDecision(reason string) (pipeline.ChunkPlanRecord, pipeline.ChunkPlanDecision, error) {
+	return pipeline.ChunkPlanRecord{}, pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanInvalid, Reason: reason}, nil
+}
+
+func writeAtomic(target string, data []byte) error {
+	dir := filepath.Dir(target)
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return err
+	}
+	if err := os.Chmod(dir, 0o700); err != nil {
+		return err
+	}
+
+	temp, err := os.CreateTemp(dir, ".plan.json.tmp-*")
+	if err != nil {
+		return err
+	}
+	tempPath := temp.Name()
+	removeTemp := true
+	defer func() {
+		if removeTemp {
+			_ = os.Remove(tempPath)
+		}
+	}()
+	if err := temp.Chmod(0o600); err != nil {
+		_ = temp.Close()
+		return err
+	}
+	if _, err := temp.Write(data); err != nil {
+		_ = temp.Close()
+		return err
+	}
+	if err := temp.Sync(); err != nil {
+		_ = temp.Close()
+		return err
+	}
+	if err := temp.Close(); err != nil {
+		return err
+	}
+	if err := os.Rename(tempPath, target); err != nil {
+		return err
+	}
+	removeTemp = false
+	return nil
+}
--- a/internal/framework/chunkplan/store_test.go
+++ b/internal/framework/chunkplan/store_test.go
@@ -0,0 +1,283 @@
+package chunkplan
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
+	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+const testSourceDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+
+func TestFilesystemStoreRoundTripAndExactPath(t *testing.T) {
+	root := filepath.Join(t.TempDir(), "plans")
+	store := newStore(t, root)
+	record := testRecord(t, 1)
+	if err := store.Save(record); err != nil {
+		t.Fatalf("Save() error = %v", err)
+	}
+
+	target := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"), "plan.json")
+	data, err := os.ReadFile(target)
+	if err != nil {
+		t.Fatalf("read exact plan path: %v", err)
+	}
+	if bytes.Contains(data, []byte("RAW REFERENCE CONTENT")) {
+		t.Fatal("stored record contains raw reference content")
+	}
+	var envelope struct {
+		Producer struct {
+			References []map[string]any `json:"references"`
+		} `json:"producer"`
+	}
+	if err := json.Unmarshal(data, &envelope); err != nil {
+		t.Fatal(err)
+	}
+	if len(envelope.Producer.References) != 1 {
+		t.Fatalf("stored references = %#v", envelope.Producer.References)
+	}
+	if _, exists := envelope.Producer.References[0]["content"]; exists {
+		t.Fatalf("stored reference contains content field: %#v", envelope.Producer.References[0])
+	}
+
+	got, decision, err := store.Load(testSourceDigest)
+	if err != nil || decision.Status != pipeline.ChunkPlanHit {
+		t.Fatalf("Load() decision=%#v error=%v", decision, err)
+	}
+	if !reflect.DeepEqual(got, record) {
+		t.Fatalf("round trip record = %#v, want %#v", got, record)
+	}
+}
+
+func TestFilesystemStorePermissions(t *testing.T) {
+	root := filepath.Join(t.TempDir(), "plans")
+	store := newStore(t, root)
+	if err := store.Save(testRecord(t, 1)); err != nil {
+		t.Fatal(err)
+	}
+	digestDir := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"))
+	for path, want := range map[string]os.FileMode{
+		root:                                  0o700,
+		digestDir:                             0o700,
+		filepath.Join(digestDir, "plan.json"): 0o600,
+	} {
+		info, err := os.Stat(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got := info.Mode().Perm(); got != want {
+			t.Fatalf("%s mode = %04o, want %04o", path, got, want)
+		}
+	}
+}
+
+func TestFilesystemStoreMissingAndOperationalErrors(t *testing.T) {
+	root := t.TempDir()
+	store := newStore(t, root)
+	_, decision, err := store.Load(testSourceDigest)
+	if err != nil || decision.Status != pipeline.ChunkPlanMissing {
+		t.Fatalf("missing decision=%#v error=%v", decision, err)
+	}
+
+	target := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"), "plan.json")
+	if err := os.MkdirAll(target, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := store.Load(testSourceDigest); err == nil {
+		t.Fatal("Load(plan.json directory) error = nil")
+	}
+}
+
+func TestFilesystemStoreRejectsMalformedSourceDigests(t *testing.T) {
+	store := newStore(t, t.TempDir())
+	for _, digest := range []string{"", "sha1:" + strings.Repeat("a", 64), "sha256:../escape", "sha256:" + strings.Repeat("A", 64), "sha256:" + strings.Repeat("a", 63)} {
+		t.Run(digest, func(t *testing.T) {
+			if _, _, err := store.Load(digest); err == nil {
+				t.Fatal("Load() error = nil")
+			}
+			record := testRecord(t, 1)
+			record.SourceDigest = digest
+			record.Plan.SourceDigest = digest
+			record.PlanDigest, _ = source.DigestChunkPlan(record.Plan)
+			if err := store.Save(record); err == nil {
+				t.Fatal("Save() error = nil")
+			}
+		})
+	}
+}
+
+func TestFilesystemStoreReportsInvalidRecordsAsRecoverable(t *testing.T) {
+	tests := []struct {
+		name   string
+		mutate func([]byte) []byte
+		want   string
+	}{
+		{name: "unknown field", mutate: func(data []byte) []byte {
+			return bytes.Replace(data, []byte(`{"schema_version"`), []byte(`{"unknown":true,"schema_version"`), 1)
+		}, want: "unknown"},
+		{name: "schema mismatch", mutate: replaceJSON(`notarius.chunk-plan.v1`, `notarius.chunk-plan.v2`), want: "schema_version"},
+		{name: "source mismatch", mutate: replaceJSON(testSourceDigest, "sha256:"+strings.Repeat("b", 64)), want: "source_digest"},
+		{name: "plan digest mismatch", mutate: func(data []byte) []byte {
+			prefix := []byte(`"plan_digest":"sha256:`)
+			index := bytes.Index(data, prefix)
+			if index >= 0 {
+				data[index+len(prefix)] = '0'
+			}
+			return data
+		}, want: "plan_digest"},
+		{name: "noncanonical annotation", mutate: func(data []byte) []byte {
+			return bytes.Replace(data, []byte(`{"value":1}`), []byte(`{ "value": 1 }`), 1)
+		}, want: "canonical JSON"},
+		{name: "trailing JSON", mutate: func(data []byte) []byte { return append(data, []byte(` {}`)...) }, want: "trailing"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			root := t.TempDir()
+			store := newStore(t, root)
+			record := testRecord(t, 1)
+			if err := store.Save(record); err != nil {
+				t.Fatal(err)
+			}
+			path := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"), "plan.json")
+			data, err := os.ReadFile(path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := os.WriteFile(path, tc.mutate(data), 0o600); err != nil {
+				t.Fatal(err)
+			}
+			got, decision, err := store.Load(testSourceDigest)
+			if err != nil || decision.Status != pipeline.ChunkPlanInvalid || !reflect.DeepEqual(got, pipeline.ChunkPlanRecord{}) || !strings.Contains(decision.Reason, tc.want) {
+				t.Fatalf("record=%#v decision=%#v error=%v", got, decision, err)
+			}
+		})
+	}
+}
+
+func TestFilesystemStoreAtomicallyReplacesAndPreservesValidRecordOnFailure(t *testing.T) {
+	root := t.TempDir()
+	store := newStore(t, root)
+	first := testRecord(t, 1)
+	second := testRecord(t, 2)
+	if err := store.Save(first); err != nil {
+		t.Fatal(err)
+	}
+	invalid := second
+	invalid.PlanDigest = "sha256:" + strings.Repeat("0", 64)
+	if err := store.Save(invalid); err == nil {
+		t.Fatal("Save(invalid) error = nil")
+	}
+	got, _, _ := store.Load(testSourceDigest)
+	if !reflect.DeepEqual(got, first) {
+		t.Fatalf("record after failed replacement = %#v", got)
+	}
+	if err := store.Save(second); err != nil {
+		t.Fatal(err)
+	}
+	got, _, _ = store.Load(testSourceDigest)
+	if !reflect.DeepEqual(got, second) {
+		t.Fatalf("record after replacement = %#v", got)
+	}
+	assertNoTemps(t, filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:")))
+}
+
+func TestFilesystemStoreConcurrentWritersExposeCompleteRecord(t *testing.T) {
+	store := newStore(t, t.TempDir())
+	const writers = 24
+	records := make([]pipeline.ChunkPlanRecord, writers)
+	for i := range records {
+		records[i] = testRecord(t, i+1)
+	}
+	var wg sync.WaitGroup
+	errs := make(chan error, writers)
+	for i := range records {
+		wg.Add(1)
+		go func(record pipeline.ChunkPlanRecord) {
+			defer wg.Done()
+			errs <- store.Save(record)
+		}(records[i])
+	}
+	wg.Wait()
+	close(errs)
+	for err := range errs {
+		if err != nil {
+			t.Fatalf("concurrent Save() error = %v", err)
+		}
+	}
+	got, decision, err := store.Load(testSourceDigest)
+	if err != nil || decision.Status != pipeline.ChunkPlanHit {
+		t.Fatalf("Load() decision=%#v error=%v", decision, err)
+	}
+	var annotation struct {
+		Value int `json:"value"`
+	}
+	if err := json.Unmarshal(got.Plan.Annotations["test/value"], &annotation); err != nil || annotation.Value < 1 || annotation.Value > writers {
+		t.Fatalf("final annotation=%#v error=%v", annotation, err)
+	}
+}
+
+func newStore(t *testing.T, root string) pipeline.ChunkPlanStore {
+	t.Helper()
+	store, err := NewFilesystemStore(root)
+	if err != nil {
+		t.Fatalf("NewFilesystemStore() error = %v", err)
+	}
+	return store
+}
+
+func testRecord(t *testing.T, value int) pipeline.ChunkPlanRecord {
+	t.Helper()
+	annotation, err := json.Marshal(map[string]int{"value": value})
+	if err != nil {
+		t.Fatal(err)
+	}
+	plan := source.ChunkPlan{
+		SourceDigest: testSourceDigest,
+		Ranges:       []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2, Annotations: source.ChunkAnnotations{"test/range": json.RawMessage(`{"range":true}`)}}},
+		Annotations:  source.ChunkAnnotations{"test/value": annotation},
+	}
+	planDigest, err := source.DigestChunkPlan(plan)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return pipeline.ChunkPlanRecord{
+		SchemaVersion: SchemaVersion,
+		SourceDigest:  testSourceDigest,
+		PlanDigest:    planDigest,
+		Plan:          plan,
+		Producer: pipeline.ChunkPlanProducer{
+			InputModule: "input/test", ChunkModule: "chunk/test", LLMProfile: "profile/test",
+			References: []artifacts.ReferenceProvenance{{Stage: "chunk", SlotName: "guide", OriginType: "file", OriginURI: "file:///guide.txt", Digest: "sha256:reference"}},
+			Metadata:   map[string]any{"prompt_id": "test/prompt", "enabled": true},
+		},
+		Warnings:  []contracts.Warning{{Scope: "chunk/test", ReasonCode: "observed", Message: "warning"}},
+		CreatedAt: time.Date(2026, 7, 18, 12, 0, 0, 0, time.UTC),
+	}
+}
+
+func replaceJSON(old, replacement string) func([]byte) []byte {
+	return func(data []byte) []byte { return bytes.Replace(data, []byte(old), []byte(replacement), 1) }
+}
+
+func assertNoTemps(t *testing.T, dir string) {
+	t.Helper()
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, entry := range entries {
+		if strings.Contains(entry.Name(), ".tmp-") {
+			t.Fatalf("temporary file remains: %s", entry.Name())
+		}
+	}
+}
--- a/internal/framework/pipeline/chunk_cache.go
+++ b/internal/framework/pipeline/chunk_cache.go
@@ -0,0 +1,29 @@
+package pipeline
+
+import (
+	"fmt"
+	"strings"
+)
+
+type ChunkCacheMode string
+
+const (
+	ChunkCacheAuto    ChunkCacheMode = "auto"
+	ChunkCacheBypass  ChunkCacheMode = "bypass"
+	ChunkCacheRefresh ChunkCacheMode = "refresh"
+)
+
+func ParseChunkCacheMode(raw string) (ChunkCacheMode, error) {
+	mode := ChunkCacheMode(strings.TrimSpace(raw))
+	switch mode {
+	case ChunkCacheAuto, ChunkCacheBypass, ChunkCacheRefresh:
+		return mode, nil
+	default:
+		return "", fmt.Errorf("chunk cache mode %q is not supported", strings.TrimSpace(raw))
+	}
+}
+
+func (m ChunkCacheMode) Validate() error {
+	_, err := ParseChunkCacheMode(string(m))
+	return err
+}
--- a/internal/framework/pipeline/chunk_cache_test.go
+++ b/internal/framework/pipeline/chunk_cache_test.go
@@ -0,0 +1,20 @@
+package pipeline
+
+import "testing"
+
+func TestParseChunkCacheMode(t *testing.T) {
+	for _, value := range []ChunkCacheMode{ChunkCacheAuto, ChunkCacheBypass, ChunkCacheRefresh} {
+		got, err := ParseChunkCacheMode(string(value))
+		if err != nil || got != value {
+			t.Fatalf("ParseChunkCacheMode(%q) = %q, %v", value, got, err)
+		}
+		if err := value.Validate(); err != nil {
+			t.Fatalf("%q.Validate() error = %v", value, err)
+		}
+	}
+	for _, value := range []string{"", "enabled", "AUTO", "auto,refresh"} {
+		if _, err := ParseChunkCacheMode(value); err == nil {
+			t.Fatalf("ParseChunkCacheMode(%q) error = nil, want error", value)
+		}
+	}
+}
--- a/internal/framework/pipeline/chunk_plan_store.go
+++ b/internal/framework/pipeline/chunk_plan_store.go
@@ -0,0 +1,47 @@
+package pipeline
+
+import (
+	"time"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
+	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+)
+
+type ChunkPlanProducer struct {
+	InputModule string                          `json:"input_module"`
+	ChunkModule string                          `json:"chunk_module"`
+	LLMProfile  string                          `json:"llm_profile,omitempty"`
+	References  []artifacts.ReferenceProvenance `json:"references,omitempty"`
+	Metadata    map[string]any                  `json:"metadata,omitempty"`
+}
+
+type ChunkPlanRecord struct {
+	SchemaVersion string              `json:"schema_version"`
+	SourceDigest  string              `json:"source_digest"`
+	PlanDigest    string              `json:"plan_digest"`
+	Plan          source.ChunkPlan    `json:"plan"`
+	Producer      ChunkPlanProducer   `json:"producer"`
+	Warnings      []contracts.Warning `json:"warnings,omitempty"`
+	CreatedAt     time.Time           `json:"created_at"`
+}
+
+type ChunkPlanStore interface {
+	Load(sourceDigest string) (ChunkPlanRecord, ChunkPlanDecision, error)
+	Save(ChunkPlanRecord) error
+}
+
+type ChunkPlanStoreFactory func(root string) (ChunkPlanStore, error)
+
+type ChunkPlanDecision struct {
+	Status ChunkPlanStatus `json:"status"`
+	Reason string          `json:"reason,omitempty"`
+}
+
+type ChunkPlanStatus string
+
+const (
+	ChunkPlanHit     ChunkPlanStatus = "hit"
+	ChunkPlanMissing ChunkPlanStatus = "missing"
+	ChunkPlanInvalid ChunkPlanStatus = "invalid"
+)