From 5086596f945a09b683cc29e4e340f1d35a432323 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 16 Sep 2026 17:42:16 +0000 Subject: [PATCH] Add typed Notarius cache controls --- docs/roadmap/implementation.md | 7 +- internal/config/config.go | 2 + internal/config/credential_projection.go | 3 + internal/config/notarius_test.go | 89 ++++++++++++++++++++++ internal/config/pipeline_configkit_test.go | 51 +++++++++++++ internal/config/pipeline_normalize.go | 1 + internal/config/validate_notarius.go | 14 +++- 7 files changed, 165 insertions(+), 2 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index e0dd77c..56a3947 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -59,7 +59,12 @@ Do not create a general cache framework or change unrelated safety policies. ## Stage 1 — Add Typed Cache Controls -Status: Not started +Status: Complete (2026-09-16) + +Completion: Added typed, zero-default `resume` and `chunk_cache` controls with +enabled-section normalization and validation, safe YAML/JSON projection, and +ConfigKit profile/provenance coverage. Verified with +`GOWORK=off go test ./internal/config` and `GOWORK=off go test ./...`. Read: `internal/config/config.go`, `credential_projection.go`, `pipeline_normalize.go`, `validate_notarius.go`, `notarius_test.go`, diff --git a/internal/config/config.go b/internal/config/config.go index 6335c74..cb0f8e1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -310,6 +310,8 @@ type NotariusConfig struct { Enabled bool `yaml:"enabled" json:"enabled"` Binary string `yaml:"binary" json:"binary"` LLMAPIKey *configkit.Secret `yaml:"llm_api_key,omitempty" json:"llm_api_key,omitempty"` + Resume bool `yaml:"resume,omitempty" json:"resume,omitempty"` + ChunkCache string `yaml:"chunk_cache,omitempty" json:"chunk_cache,omitempty"` ConfigPath string `yaml:"config_path" json:"config_path"` PipelineID string `yaml:"pipeline_id" json:"pipeline_id"` Timeout string `yaml:"timeout" json:"timeout"` diff --git a/internal/config/credential_projection.go b/internal/config/credential_projection.go index 6e93277..3c27b04 100644 --- a/internal/config/credential_projection.go +++ b/internal/config/credential_projection.go @@ -56,6 +56,8 @@ func (cfg NotariusConfig) MarshalYAML() (any, error) { Enabled bool `yaml:"enabled"` Binary string `yaml:"binary"` LLMAPIKey *string `yaml:"llm_api_key,omitempty"` + Resume bool `yaml:"resume,omitempty"` + ChunkCache string `yaml:"chunk_cache,omitempty"` ConfigPath string `yaml:"config_path"` PipelineID string `yaml:"pipeline_id"` Timeout string `yaml:"timeout"` @@ -65,6 +67,7 @@ func (cfg NotariusConfig) MarshalYAML() (any, error) { } return safeNotarius{ Enabled: cfg.Enabled, Binary: cfg.Binary, LLMAPIKey: secretReferenceName(cfg.LLMAPIKey), + Resume: cfg.Resume, ChunkCache: cfg.ChunkCache, ConfigPath: cfg.ConfigPath, PipelineID: cfg.PipelineID, Timeout: cfg.Timeout, WorkingDirectory: cfg.WorkingDirectory, References: cfg.References, Outputs: cfg.Outputs, }, nil diff --git a/internal/config/notarius_test.go b/internal/config/notarius_test.go index 42a5a58..6ec8f26 100644 --- a/internal/config/notarius_test.go +++ b/internal/config/notarius_test.go @@ -1,12 +1,15 @@ package config import ( + "encoding/json" "fmt" "os" "path/filepath" "reflect" "strings" "testing" + + "gopkg.in/yaml.v3" ) func TestNotariusOmittedAndDisabledBehavior(t *testing.T) { @@ -45,6 +48,9 @@ notarius: if disabled.Notarius.Binary != DefaultNotariusBinary || disabled.Notarius.Timeout != DefaultNotariusTimeout { t.Fatalf("disabled defaults = %#v", disabled.Notarius) } + if disabled.Notarius.Resume || disabled.Notarius.ChunkCache != "" { + t.Fatalf("disabled cache-control defaults = %#v", disabled.Notarius) + } if disabled.Notarius.ConfigPath != "relative/notarius.yml" || disabled.Notarius.WorkingDirectory != "" { t.Fatalf("disabled paths were resolved unexpectedly: %#v", disabled.Notarius) } @@ -187,6 +193,89 @@ notarius: } } +func TestNotariusCacheControlNormalizationAndValidation(t *testing.T) { + for _, mode := range []string{"", "auto", "bypass", "refresh"} { + t.Run("mode "+mode, func(t *testing.T) { + cfg := validNotariusConfig() + cfg.Resume = true + if mode != "" { + cfg.ChunkCache = " " + mode + " " + } + if err := NormalizePipelineConfig(&PipelineConfig{Notarius: cfg}); err != nil { + t.Fatalf("NormalizePipelineConfig() error = %v", err) + } + if cfg.ChunkCache != mode { + t.Fatalf("ChunkCache = %q, want %q", cfg.ChunkCache, mode) + } + if !cfg.Resume { + t.Fatal("Resume = false, want true") + } + if err := validateNotarius(cfg, nil); err != nil { + t.Fatalf("validateNotarius() error = %v", err) + } + }) + } + + for _, mode := range []string{"AUTO", "invalid"} { + t.Run("invalid "+mode, func(t *testing.T) { + cfg := validNotariusConfig() + cfg.ChunkCache = mode + err := validateNotarius(cfg, nil) + if err == nil || !strings.Contains(err.Error(), "pipeline.notarius.chunk_cache") { + t.Fatalf("validateNotarius() error = %v, want chunk_cache error", err) + } + }) + } + + disabled := &NotariusConfig{Enabled: false, Resume: true, ChunkCache: " INVALID "} + if err := NormalizePipelineConfig(&PipelineConfig{Notarius: disabled}); err != nil { + t.Fatalf("NormalizePipelineConfig(disabled) error = %v", err) + } + if disabled.ChunkCache != " INVALID " { + t.Fatalf("disabled ChunkCache = %q, want unchanged", disabled.ChunkCache) + } + if err := validateNotarius(disabled, nil); err != nil { + t.Fatalf("validateNotarius(disabled) error = %v", err) + } +} + +func TestNotariusCacheControlsSafeSerialization(t *testing.T) { + cfg := validNotariusConfig() + cfg.Resume = true + cfg.ChunkCache = "refresh" + + yamlData, err := yaml.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + jsonData, err := json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(yamlData), "resume: true") || !strings.Contains(string(yamlData), "chunk_cache: refresh") { + t.Fatalf("YAML omitted cache controls: %s", yamlData) + } + if !strings.Contains(string(jsonData), `"resume":true`) || !strings.Contains(string(jsonData), `"chunk_cache":"refresh"`) { + t.Fatalf("JSON omitted cache controls: %s", jsonData) + } + + cfg.Resume = false + cfg.ChunkCache = "" + yamlData, err = yaml.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + jsonData, err = json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + for _, serialized := range []string{string(yamlData), string(jsonData)} { + if strings.Contains(serialized, "resume") || strings.Contains(serialized, "chunk_cache") { + t.Fatalf("zero-value cache controls were serialized: %s", serialized) + } + } +} + func TestNotariusRelativePathsDoNotMakeEffectiveDigestLocationDependent(t *testing.T) { pipelineYAML := testPipelineBaseYAML + ` notarius: diff --git a/internal/config/pipeline_configkit_test.go b/internal/config/pipeline_configkit_test.go index ef97a24..cd492d1 100644 --- a/internal/config/pipeline_configkit_test.go +++ b/internal/config/pipeline_configkit_test.go @@ -182,6 +182,57 @@ func TestNotariusCredentialRootProvenance(t *testing.T) { t.Fatal("Notarius credential provenance did not identify the root pipeline") } +func TestNotariusCacheControlProfileReplacementAndNullReset(t *testing.T) { + tests := []struct { + name string + overlay string + wantResume bool + wantMode string + wantSource bool + wantNil bool + }{ + {name: "replacement", overlay: "notarius:\n resume: true\n chunk_cache: refresh\n", wantResume: true, wantMode: "refresh", wantSource: true}, + {name: "null reset", overlay: "notarius: null\n", wantNil: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + root := writePipelineFixture(t, dir, "pipeline.yml", "whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n resume: false\n chunk_cache: bypass\n") + overlay := writePipelineFixture(t, dir, "profiles/testing.yml", test.overlay) + profile := "testing" + cfg, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Profile: &profile}) + if err != nil { + t.Fatal(err) + } + if test.wantNil { + if cfg.Notarius != nil { + t.Fatalf("Notarius = %#v, want null reset", cfg.Notarius) + } + return + } + if cfg.Notarius == nil || cfg.Notarius.Resume != test.wantResume || cfg.Notarius.ChunkCache != test.wantMode { + t.Fatalf("cache controls = resume %v, mode %q; want resume %v, mode %q", cfg.Notarius.Resume, cfg.Notarius.ChunkCache, test.wantResume, test.wantMode) + } + if !test.wantSource { + return + } + sources, err := EffectivePipelineSources(cfg, ResolvedParty{}) + if err != nil { + t.Fatal(err) + } + found := map[string]bool{} + for _, source := range sources { + if source.Path == "notarius.resume" || source.Path == "notarius.chunk_cache" { + found[source.Path] = source.Source == overlay + } + } + if !found["notarius.resume"] || !found["notarius.chunk_cache"] { + t.Fatalf("cache-control provenance did not identify profile: %#v", found) + } + }) + } +} + func TestPipelineConfigKitAllowsOneBaseDeclarationWithProfileOverride(t *testing.T) { dir := t.TempDir() writePipelineFixture(t, dir, "pipeline.yml", `configkit: diff --git a/internal/config/pipeline_normalize.go b/internal/config/pipeline_normalize.go index cbcd5d1..bd4c1d4 100644 --- a/internal/config/pipeline_normalize.go +++ b/internal/config/pipeline_normalize.go @@ -14,6 +14,7 @@ func NormalizePipelineConfig(cfg *PipelineConfig) error { } notarius := *cfg.Notarius notarius.Binary = strings.TrimSpace(notarius.Binary) + notarius.ChunkCache = strings.TrimSpace(notarius.ChunkCache) notarius.ConfigPath = cleanNotariusPath(notarius.ConfigPath) notarius.PipelineID = strings.TrimSpace(notarius.PipelineID) notarius.Timeout = strings.TrimSpace(notarius.Timeout) diff --git a/internal/config/validate_notarius.go b/internal/config/validate_notarius.go index d967279..2af357a 100644 --- a/internal/config/validate_notarius.go +++ b/internal/config/validate_notarius.go @@ -39,10 +39,12 @@ func validateNotarius(cfg *NotariusConfig, scriptorium *ScriptoriumConfig) error if len(cfg.References) > MaxNotariusReferenceBindings { return fmt.Errorf("pipeline.notarius.references must contain at most %d bindings", MaxNotariusReferenceBindings) } - if err := validateNotariusCanonicalScalars(cfg); err != nil { return err } + if !validNotariusChunkCacheMode(cfg.ChunkCache) { + return fmt.Errorf("pipeline.notarius.chunk_cache must be empty, auto, bypass, or refresh") + } if err := validateNotariusReferences(cfg.References); err != nil { return err } @@ -56,6 +58,7 @@ func validateNotariusCanonicalScalars(cfg *NotariusConfig) error { want string }{ {name: "binary", got: cfg.Binary, want: strings.TrimSpace(cfg.Binary)}, + {name: "chunk_cache", got: cfg.ChunkCache, want: strings.TrimSpace(cfg.ChunkCache)}, {name: "config_path", got: cfg.ConfigPath, want: cleanNotariusPath(cfg.ConfigPath)}, {name: "pipeline_id", got: cfg.PipelineID, want: strings.TrimSpace(cfg.PipelineID)}, {name: "timeout", got: cfg.Timeout, want: strings.TrimSpace(cfg.Timeout)}, @@ -69,6 +72,15 @@ func validateNotariusCanonicalScalars(cfg *NotariusConfig) error { return nil } +func validNotariusChunkCacheMode(mode string) bool { + switch mode { + case "", "auto", "bypass", "refresh": + return true + default: + return false + } +} + func normalizeNotariusReferences(references map[string]string) (map[string]string, error) { selectors := make([]string, 0, len(references)) for selector := range references {