eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add chunk plan cache configuration and storage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-18 00:07:53 +00:00
parent 7844c0a93f
commit ebd449d847
13 changed files with 843 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
internal/core/config/chunk_cache_test.go Normal file
View File

@@ -0,0 +1,126 @@
package config
import (
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestChunkCacheDefaults(t *testing.T) {
cfg := Default()
if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheAuto || cfg.Workspace.ChunkCache.Directory != "" {
t.Fatalf("chunk cache defaults = %#v", cfg.Workspace.ChunkCache)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestChunkCacheFileConfiguration(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
workspace:
chunk_cache:
mode: refresh
directory: " ./state/../plans "
`)
if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheRefresh {
t.Fatalf("mode = %q", cfg.Workspace.ChunkCache.Mode)
}
if got, want := cfg.Workspace.ChunkCache.Directory, filepath.Clean("./state/../plans"); got != want {
t.Fatalf("directory = %q, want %q", got, want)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestChunkCacheEnvironmentOverridesFile(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 2
workspace:
chunk_cache:
mode: bypass
directory: /file/plans
`))
if err != nil {
t.Fatal(err)
}
cfg := Default()
if err := cfg.ApplyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
t.Fatal(err)
}
if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{
"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "refresh",
"NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR": " /environment/../cache/plans ",
})); err != nil {
t.Fatal(err)
}
if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheRefresh || cfg.Workspace.ChunkCache.Directory != filepath.Clean("/environment/../cache/plans") {
t.Fatalf("effective chunk cache = %#v", cfg.Workspace.ChunkCache)
}
}
func TestChunkCacheEmptyDirectoryEnvironmentSelectsDefault(t *testing.T) {
cfg := Default()
cfg.Workspace.ChunkCache.Directory = "/file/plans"
if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR": " \t "})); err != nil {
t.Fatal(err)
}
if cfg.Workspace.ChunkCache.Directory != "" {
t.Fatalf("directory = %q, want unset", cfg.Workspace.ChunkCache.Directory)
}
}
func TestChunkCacheRejectsInvalidSuppliedModes(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte("version: 2\nworkspace:\n chunk_cache:\n mode: sometimes\n"))
if err != nil {
t.Fatal(err)
}
cfg := Default()
if err := cfg.ApplyFileConfigWithLookup(fileCfg, emptyLookup); err == nil || !strings.Contains(err.Error(), "workspace.chunk_cache.mode") {
t.Fatalf("file mode error = %v", err)
}
cfg = Default()
if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "sometimes"})); err == nil || !strings.Contains(err.Error(), "NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE") {
t.Fatalf("environment mode error = %v", err)
}
cfg = Default()
cfg.Workspace.ChunkCache.Mode = "sometimes"
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "chunk cache") {
t.Fatalf("Validate() error = %v", err)
}
}
func TestChunkCacheConfigurationClonesAndRedacts(t *testing.T) {
cfg := Default()
cfg.Workspace.ChunkCache = WorkspaceChunkCacheConfig{Mode: pipeline.ChunkCacheRefresh, Directory: "/var/cache/notarius/chunk-plans"}
cloned := cloneConfig(cfg)
redacted := cfg.Redacted()
if cloned.Workspace.ChunkCache != cfg.Workspace.ChunkCache || redacted.Workspace.ChunkCache != cfg.Workspace.ChunkCache {
t.Fatalf("cloned=%#v redacted=%#v", cloned.Workspace.ChunkCache, redacted.Workspace.ChunkCache)
}
redacted.Workspace.ChunkCache.Directory = "/changed"
if cfg.Workspace.ChunkCache.Directory != "/var/cache/notarius/chunk-plans" {
t.Fatal("redacted mutation changed original")
}
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestChunkCacheDirectoryValidation(t *testing.T) {
cfg := Default()
cfg.Workspace.ChunkCache.Directory = "/var/cache/notarius/chunk-plans"
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate(system root) error = %v", err)
}
cfg.Workspace.ChunkCache.Directory = "bad\x00path"
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "NUL") {
t.Fatalf("Validate(NUL directory) error = %v", err)
}
}

7
internal/core/config/config.go
View File

@@ -38,11 +38,17 @@ type DiagnosticsConfig struct {
type WorkspaceConfig struct {
Directory string `json:"directory,omitempty"`
ChunkCache WorkspaceChunkCacheConfig `json:"chunk_cache"`
Diagnostics WorkspaceDiagnosticsConfig `json:"diagnostics"`
Resume WorkspaceResumeConfig `json:"resume"`
Debug WorkspaceDebugConfig `json:"debug"`
}
type WorkspaceChunkCacheConfig struct {
Mode pipeline.ChunkCacheMode `json:"mode"`
Directory string `json:"directory,omitempty"`
}
type WorkspaceDiagnosticsConfig struct {
Enabled bool `json:"enabled"`
Retention diagnostics.RetentionMode `json:"retention,omitempty"`
@@ -71,6 +77,7 @@ func Default() Config {
Retention: diagnostics.RetentionAuto,
},
Workspace: WorkspaceConfig{
ChunkCache: WorkspaceChunkCacheConfig{Mode: pipeline.ChunkCacheAuto},
Diagnostics: WorkspaceDiagnosticsConfig{
Enabled: true,
},

11
internal/core/config/env.go
View File

@@ -7,6 +7,7 @@ import (
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func LoadFromEnv() (Config, error) {
@@ -57,6 +58,16 @@ func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool))
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIR"); ok {
c.Workspace.Directory = strings.TrimSpace(raw)
}
if raw, ok := lookup("NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE"); ok {
mode, err := pipeline.ParseChunkCacheMode(raw)
if err != nil {
return fmt.Errorf("NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE: %w", err)
}
c.Workspace.ChunkCache.Mode = mode
}
if raw, ok := lookup("NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR"); ok {
c.Workspace.ChunkCache.Directory = cleanOptionalPath(raw)
}
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED"); ok {
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED", raw)
if err != nil {

27
internal/core/config/file_config.go
View File

@@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
@@ -54,11 +55,17 @@ type FileDiagnosticsConfig struct {
type FileWorkspaceConfig struct {
Directory *string `yaml:"directory,omitempty"`
ChunkCache *FileWorkspaceChunkCacheConfig `yaml:"chunk_cache,omitempty"`
Diagnostics *FileWorkspaceDiagnosticsConfig `yaml:"diagnostics,omitempty"`
Resume *FileWorkspaceEnabledConfig `yaml:"resume,omitempty"`
Debug *FileWorkspaceEnabledConfig `yaml:"debug,omitempty"`
}
type FileWorkspaceChunkCacheConfig struct {
Mode *string `yaml:"mode,omitempty"`
Directory *string `yaml:"directory,omitempty"`
}
type FileWorkspaceDiagnosticsConfig struct {
Enabled *bool `yaml:"enabled,omitempty"`
Retention *string `yaml:"retention,omitempty"`
@@ -338,6 +345,18 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
if fileCfg.Workspace.Directory != nil {
c.Workspace.Directory = strings.TrimSpace(*fileCfg.Workspace.Directory)
}
if fileCfg.Workspace.ChunkCache != nil {
if fileCfg.Workspace.ChunkCache.Mode != nil {
mode, err := pipeline.ParseChunkCacheMode(*fileCfg.Workspace.ChunkCache.Mode)
if err != nil {
return fmt.Errorf("workspace.chunk_cache.mode: %w", err)
}
c.Workspace.ChunkCache.Mode = mode
}
if fileCfg.Workspace.ChunkCache.Directory != nil {
c.Workspace.ChunkCache.Directory = cleanOptionalPath(*fileCfg.Workspace.ChunkCache.Directory)
}
}
if fileCfg.Workspace.Diagnostics != nil {
if fileCfg.Workspace.Diagnostics.Enabled != nil {
c.Workspace.Diagnostics.Enabled = *fileCfg.Workspace.Diagnostics.Enabled
@@ -360,6 +379,14 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
return nil
}
func cleanOptionalPath(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
return filepath.Clean(value)
}
func normalizeStageWorkers(values map[string]int) (map[string]int, bool, error) {
workers := make(map[string]int, len(values))
configured := false

6
internal/core/config/validation.go
View File

@@ -61,6 +61,12 @@ func validateScriptorium(cfg ScriptoriumConfig) error {
}
func validateWorkspace(cfg WorkspaceConfig) error {
if err := cfg.ChunkCache.Mode.Validate(); err != nil {
return fmt.Errorf("workspace chunk cache: %w", err)
}
if strings.ContainsRune(cfg.ChunkCache.Directory, '\x00') {
return fmt.Errorf("workspace chunk cache directory must not contain NUL")
}
if cfg.Diagnostics.retentionSet {
switch cfg.Diagnostics.Retention {
case "", diagnostics.RetentionAuto, diagnostics.RetentionAlways, diagnostics.RetentionNever:

22
internal/core/workspace/chunk_plan.go Normal file
View File

@@ -0,0 +1,22 @@
package workspace
import (
"fmt"
"path/filepath"
"strings"
)
func DefaultChunkPlanRoot(userCacheDir func() (string, error)) (string, error) {
if userCacheDir == nil {
return "", fmt.Errorf("user cache directory resolver must not be nil")
}
root, err := userCacheDir()
if err != nil {
return "", fmt.Errorf("resolve user cache directory: %w", err)
}
root = strings.TrimSpace(root)
if root == "" {
return "", fmt.Errorf("user cache directory must not be empty")
}
return filepath.Join(filepath.Clean(root), "notarius", "chunk-plans"), nil
}

57
internal/core/workspace/chunk_plan_test.go Normal file
View File

@@ -0,0 +1,57 @@
package workspace
import (
"errors"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
)
func TestDefaultChunkPlanRoot(t *testing.T) {
got, err := DefaultChunkPlanRoot(func() (string, error) { return "/cache/user", nil })
if err != nil {
t.Fatalf("DefaultChunkPlanRoot() error = %v", err)
}
if want := filepath.Join("/cache/user", "notarius", "chunk-plans"); got != want {
t.Fatalf("root = %q, want %q", got, want)
}
}
func TestDefaultChunkPlanRootRejectsResolverFailures(t *testing.T) {
boom := errors.New("resolver failed")
if _, err := DefaultChunkPlanRoot(func() (string, error) { return "", boom }); !errors.Is(err, boom) {
t.Fatalf("resolver error = %v", err)
}
if _, err := DefaultChunkPlanRoot(func() (string, error) { return " \t ", nil }); err == nil || !strings.Contains(err.Error(), "empty") {
t.Fatalf("empty result error = %v", err)
}
if _, err := DefaultChunkPlanRoot(nil); err == nil {
t.Fatal("nil resolver error = nil")
}
}
func TestChunkPlanDirectoryIsIndependentFromWorkspaceSettings(t *testing.T) {
base := config.Default()
base.Workspace.Directory = "/workspace/one"
base.Workspace.ChunkCache.Directory = "/cache/plans"
first := FromConfig(base)
changedWorkspace := base
changedWorkspace.Workspace.Directory = "/workspace/two"
second := FromConfig(changedWorkspace)
if base.Workspace.ChunkCache.Directory != changedWorkspace.Workspace.ChunkCache.Directory {
t.Fatal("workspace directory changed chunk plan directory")
}
if first.RootDir == second.RootDir || first.CheckpointsRoot == second.CheckpointsRoot || first.DebugRoot == second.DebugRoot {
t.Fatalf("workspace settings did not follow workspace directory: %#v %#v", first, second)
}
changedCache := base
changedCache.Workspace.ChunkCache.Directory = "/cache/other"
third := FromConfig(changedCache)
if first != third {
t.Fatalf("chunk plan directory changed workspace settings: %#v %#v", first, third)
}
}