eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make PromptKit profile handling safer and more consistent

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-03 18:35:40 +00:00
parent 12ac25bd63
commit 39388e96d4
19 changed files with 220 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
internal/framework/llm/promptkit_client_test.go
View File

@@ -429,6 +429,33 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
if strings.TrimSpace(fresh[0].Value) == "" {
t.Fatal("built-in profile fingerprint is empty")
}
t.Run("directory layout", func(t *testing.T) {
profileDir := t.TempDir()
firstPath := filepath.Join(profileDir, "first-profile.yaml")
secondPath := filepath.Join(profileDir, "second-profile.yaml")
content := []byte("id: directory-profile\nendpoint: http://promptkit.test/v1\nmodel: directory-model\n")
if err := os.WriteFile(firstPath, content, 0o600); err != nil {
t.Fatal(err)
}
first, err := promptKitProfileFingerprint(profileDir, "", "")
if err != nil {
t.Fatal(err)
}
if err := os.Rename(firstPath, secondPath); err != nil {
t.Fatal(err)
}
second, err := promptKitProfileFingerprint(profileDir, "", "")
if err != nil {
t.Fatal(err)
}
if first == second {
t.Fatalf("profile-source fingerprint = %#v after source filename changed", first)
}
if strings.Contains(first.Value, firstPath) || strings.Contains(second.Value, secondPath) {
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
}
})
}
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {

35
internal/framework/llm/promptkit_profile_fingerprint.go
View File

@@ -27,18 +27,21 @@ func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest
switch {
case strings.TrimSpace(profileFile) != "":
data, err := os.ReadFile(strings.TrimSpace(profileFile))
cleanProfileFile := strings.TrimSpace(profileFile)
data, err := os.ReadFile(cleanProfileFile)
if err != nil {
return CheckpointFingerprint{}, fmt.Errorf("read PromptKit profile file for checkpoint identity")
}
writeFingerprintPart(hasher, []byte(filepath.ToSlash(filepath.Base(cleanProfileFile))))
writeFingerprintPart(hasher, data)
case strings.TrimSpace(profileDir) != "":
digests, err := promptKitProfileFileDigests(strings.TrimSpace(profileDir))
files, err := promptKitProfileFiles(strings.TrimSpace(profileDir))
if err != nil {
return CheckpointFingerprint{}, err
}
for _, digest := range digests {
writeFingerprintPart(hasher, digest)
for _, file := range files {
writeFingerprintPart(hasher, []byte(file.path))
writeFingerprintPart(hasher, file.digest)
}
}
@@ -58,8 +61,13 @@ func promptKitLocalBackendFingerprint(endpoint string) CheckpointFingerprint {
}
}
func promptKitProfileFileDigests(root string) ([][]byte, error) {
var digests [][]byte
type promptKitProfileFile struct {
path string
digest []byte
}
func promptKitProfileFiles(root string) ([]promptKitProfileFile, error) {
var files []promptKitProfileFile
err := filepath.WalkDir(root, func(name string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
@@ -75,17 +83,24 @@ func promptKitProfileFileDigests(root string) ([][]byte, error) {
if err != nil {
return err
}
relativePath, err := filepath.Rel(root, name)
if err != nil {
return err
}
sum := sha256.Sum256(data)
digests = append(digests, append([]byte(nil), sum[:]...))
files = append(files, promptKitProfileFile{
path: filepath.ToSlash(relativePath),
digest: append([]byte(nil), sum[:]...),
})
return nil
})
if err != nil {
return nil, fmt.Errorf("read PromptKit profile directory for checkpoint identity")
}
sort.Slice(digests, func(i, j int) bool {
return string(digests[i]) < string(digests[j])
sort.Slice(files, func(i, j int) bool {
return files[i].path < files[j].path
})
return digests, nil
return files, nil
}
func writeFingerprintPart(hasher interface{ Write([]byte) (int, error) }, value []byte) {

24
internal/framework/llm/promptkit_profiles.go
View File

@@ -45,16 +45,34 @@ type PromptKitProfileInspectionError struct {
}
func (e *PromptKitProfileInspectionError) Error() string {
if errors.Is(e.err, promptkit.ErrProfileNotFound) {
switch {
case errors.Is(e.err, promptkit.ErrProfileNotFound):
return fmt.Sprintf("PromptKit profile %q is not configured", e.ProfileID)
case errors.Is(e.err, promptkit.ErrInvalidRequest):
return fmt.Sprintf("PromptKit profile ID %q is invalid", e.ProfileID)
case errors.Is(e.err, promptkit.ErrProfileLoad):
return fmt.Sprintf("PromptKit profile %q is invalid or unreadable", e.ProfileID)
default:
return fmt.Sprintf("PromptKit profile %q could not be inspected", e.ProfileID)
}
return fmt.Sprintf("inspect PromptKit profile %q: %v", e.ProfileID, e.err)
}
func (e *PromptKitProfileInspectionError) Unwrap() error {
return e.err
}
type promptKitProfileConfigurationError struct {
err error
}
func (e *promptKitProfileConfigurationError) Error() string {
return "PromptKit profile configuration is invalid or unreadable"
}
func (e *promptKitProfileConfigurationError) Unwrap() error {
return e.err
}
func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) {
source, options, err := promptKitProfileSourceEngineOptions(cfg.Source)
if err != nil {
@@ -74,7 +92,7 @@ func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptK
ProfileDir: source.ProfileDir,
}, options...)
if err != nil {
return nil, fmt.Errorf("create PromptKit profile inspector: %w", err)
return nil, &promptKitProfileConfigurationError{err: err}
}
return &PromptKitProfileInspector{engine: engine}, nil
}

4
internal/framework/pipeline/chunker_registry.go
View File

@@ -24,10 +24,6 @@ func NewChunkerRegistry() *ChunkerRegistry {
}
}
func (r *ChunkerRegistry) Register(key string, constructor ChunkerConstructor) error {
return r.RegisterWithSpec(defaultModuleSpec(key, StageChunk), constructor)
}
func (r *ChunkerRegistry) RegisterWithSpec(spec ModuleSpec, constructor ChunkerConstructor) error {
if constructor == nil {
return fmt.Errorf("chunker constructor for %q must not be nil", strings.TrimSpace(spec.Key))

14
internal/framework/pipeline/chunker_registry_test.go
View File

@@ -29,6 +29,10 @@ type registryBehaviorCase[M any] struct {
moduleKey func(M) string
}
func testModuleSpec(key string, stage ModuleStage) ModuleSpec {
return ModuleSpec{Key: key, Stage: stage, ExecutionClass: contracts.ExecutionClassDeterministic}
}
func TestChunkerRegistryBehavior(t *testing.T) {
runRegistryBehaviorTests(t, registryBehaviorCase[contracts.Chunker]{
name: "ChunkerRegistry",
@@ -39,7 +43,9 @@ func TestChunkerRegistryBehavior(t *testing.T) {
return NewChunkerRegistry()
},
register: func(registry any, key string, constructor func() (contracts.Chunker, error)) error {
return registry.(*ChunkerRegistry).Register(key, constructor)
return registry.(*ChunkerRegistry).RegisterWithSpec(ModuleSpec{
Key: key, Stage: StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic,
}, constructor)
},
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.Chunker, error)) error {
return registry.(*ChunkerRegistry).RegisterWithSpec(spec, constructor)
@@ -55,7 +61,9 @@ func TestChunkerRegistryBehavior(t *testing.T) {
},
nilRegister: func(key string, constructor func() (contracts.Chunker, error)) error {
var registry *ChunkerRegistry
return registry.Register(key, constructor)
return registry.RegisterWithSpec(ModuleSpec{
Key: key, Stage: StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic,
}, constructor)
},
nilBuild: func(key string) (contracts.Chunker, error) {
var registry *ChunkerRegistry
@@ -136,7 +144,7 @@ func runRegistryBehaviorTests[M any](t *testing.T, testCase registryBehaviorCase
}
})
t.Run(testCase.name+"/default spec from register", func(t *testing.T) {
t.Run(testCase.name+"/minimal explicit spec registration", func(t *testing.T) {
registry := testCase.newRegistry()
if err := testCase.register(registry, " "+testCase.key+" ", testCase.constructor(testCase.key)); err != nil {
t.Fatalf("Register() error = %v, want nil", err)

2
internal/framework/pipeline/evidence_preparation_test.go
View File

@@ -108,7 +108,7 @@ func registerTestEvidenceOutput(t *testing.T, registries *Registries, policy Evi
func registerTestEvidenceOutputWithProfileValidation(t *testing.T, registries *Registries, policy EvidenceContextPolicy, validateProfile OutputProfileOptionValidator) {
t.Helper()
registry := NewOutputEncoderRegistry()
if err := registry.RegisterBuilderWithProfileValidation(defaultModuleSpec("output", StageOutput), func(options map[string]any) error {
if err := registry.RegisterBuilderWithProfileValidation(testModuleSpec("output", StageOutput), func(options map[string]any) error {
return RejectUnknownOptions(options, "known")
}, validateProfile, func(BuildRequest) (contracts.OutputEncoder, error) {
return testEvidenceOutput{policy: cloneEvidenceContextPolicy(policy)}, nil

4
internal/framework/pipeline/input_registry.go
View File

@@ -24,10 +24,6 @@ func NewInputAdapterRegistry() *InputAdapterRegistry {
}
}
func (r *InputAdapterRegistry) Register(key string, constructor InputAdapterConstructor) error {
return r.RegisterWithSpec(defaultModuleSpec(key, StageInput), constructor)
}
func (r *InputAdapterRegistry) RegisterWithSpec(spec ModuleSpec, constructor InputAdapterConstructor) error {
if constructor == nil {
return fmt.Errorf("input adapter constructor for %q must not be nil", strings.TrimSpace(spec.Key))

30
internal/framework/pipeline/input_registry_test.go
View File

@@ -11,10 +11,14 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func registerTestInput(registry *InputAdapterRegistry, key string, constructor InputAdapterConstructor) error {
return registry.RegisterWithSpec(testModuleSpec(key, StageInput), constructor)
}
func TestInputAdapterRegistryRegisterAndBuild(t *testing.T) {
registry := NewInputAdapterRegistry()
if err := registry.Register("generic-input", fakeInputAdapterConstructor("generic-input")); err != nil {
if err := registerTestInput(registry, "generic-input", fakeInputAdapterConstructor("generic-input")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
@@ -30,7 +34,7 @@ func TestInputAdapterRegistryRegisterAndBuild(t *testing.T) {
func TestInputAdapterRegistryRegisterAndBuildTrimKeys(t *testing.T) {
registry := NewInputAdapterRegistry()
if err := registry.Register(" generic-input ", fakeInputAdapterConstructor("generic-input")); err != nil {
if err := registerTestInput(registry, " generic-input ", fakeInputAdapterConstructor("generic-input")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
@@ -82,10 +86,10 @@ func TestInputAdapterRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
}
}
func TestInputAdapterRegistryRegisterStoresDefaultSpec(t *testing.T) {
func TestInputAdapterRegistryRegisterWithSpecStoresMinimalMetadata(t *testing.T) {
registry := NewInputAdapterRegistry()
if err := registry.Register(" generic-input ", fakeInputAdapterConstructor("generic-input")); err != nil {
if err := registerTestInput(registry, " generic-input ", fakeInputAdapterConstructor("generic-input")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
@@ -123,7 +127,7 @@ func TestInputAdapterRegistrySpecRejectsUnknownKey(t *testing.T) {
func TestInputAdapterRegistryRegisterRejectsEmptyKey(t *testing.T) {
registry := NewInputAdapterRegistry()
err := registry.Register(" \t", fakeInputAdapterConstructor("generic-input"))
err := registerTestInput(registry, " \t", fakeInputAdapterConstructor("generic-input"))
if err == nil {
t.Fatal("Register() error = nil, want error")
@@ -135,11 +139,11 @@ func TestInputAdapterRegistryRegisterRejectsEmptyKey(t *testing.T) {
func TestInputAdapterRegistryRegisterRejectsDuplicateKey(t *testing.T) {
registry := NewInputAdapterRegistry()
if err := registry.Register("generic-input", fakeInputAdapterConstructor("generic-input")); err != nil {
if err := registerTestInput(registry, "generic-input", fakeInputAdapterConstructor("generic-input")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
err := registry.Register(" generic-input ", fakeInputAdapterConstructor("generic-input"))
err := registerTestInput(registry, " generic-input ", fakeInputAdapterConstructor("generic-input"))
if err == nil {
t.Fatal("Register() error = nil, want error")
@@ -152,7 +156,7 @@ func TestInputAdapterRegistryRegisterRejectsDuplicateKey(t *testing.T) {
func TestInputAdapterRegistryRegisterRejectsNilConstructor(t *testing.T) {
registry := NewInputAdapterRegistry()
err := registry.Register("generic-input", nil)
err := registerTestInput(registry, "generic-input", nil)
if err == nil {
t.Fatal("Register() error = nil, want error")
@@ -178,7 +182,7 @@ func TestInputAdapterRegistryBuildRejectsUnknownKey(t *testing.T) {
func TestInputAdapterRegistryBuildWrapsConstructorError(t *testing.T) {
registry := NewInputAdapterRegistry()
constructorErr := errors.New("constructor failed")
if err := registry.Register("generic-input", func() (contracts.InputAdapter, error) {
if err := registerTestInput(registry, "generic-input", func() (contracts.InputAdapter, error) {
return nil, constructorErr
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
@@ -199,7 +203,7 @@ func TestInputAdapterRegistryBuildWrapsConstructorError(t *testing.T) {
func TestInputAdapterRegistryBuildRejectsNilAdapter(t *testing.T) {
registry := NewInputAdapterRegistry()
if err := registry.Register("generic-input", func() (contracts.InputAdapter, error) {
if err := registerTestInput(registry, "generic-input", func() (contracts.InputAdapter, error) {
return nil, nil
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
@@ -217,7 +221,7 @@ func TestInputAdapterRegistryBuildRejectsNilAdapter(t *testing.T) {
func TestInputAdapterRegistryBuildRejectsAdapterKeyMismatch(t *testing.T) {
registry := NewInputAdapterRegistry()
if err := registry.Register("generic-input", fakeInputAdapterConstructor("other-input")); err != nil {
if err := registerTestInput(registry, "generic-input", fakeInputAdapterConstructor("other-input")); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
@@ -234,7 +238,7 @@ func TestInputAdapterRegistryBuildRejectsAdapterKeyMismatch(t *testing.T) {
func TestInputAdapterRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
registry := NewInputAdapterRegistry()
for _, key := range []string{"zeta", "alpha", "middle"} {
if err := registry.Register(key, fakeInputAdapterConstructor(key)); err != nil {
if err := registerTestInput(registry, key, fakeInputAdapterConstructor(key)); err != nil {
t.Fatalf("Register(%q) error = %v, want nil", key, err)
}
}
@@ -255,7 +259,7 @@ func TestInputAdapterRegistryRegisteredKeysReturnsSortedCopy(t *testing.T) {
func TestInputAdapterRegistryNilRegistryBehavior(t *testing.T) {
var registry *InputAdapterRegistry
if err := registry.Register("generic-input", fakeInputAdapterConstructor("generic-input")); err == nil {
if err := registerTestInput(registry, "generic-input", fakeInputAdapterConstructor("generic-input")); err == nil {
t.Fatal("Register() error = nil, want error")
}
if _, err := registry.Build("generic-input"); err == nil {

8
internal/framework/pipeline/module.go
View File

@@ -30,14 +30,6 @@ type ModuleSpec struct {
ReferenceSlots []contracts.ReferenceSlot
}
func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec {
return ModuleSpec{
Key: key,
Stage: stage,
ExecutionClass: contracts.ExecutionClassDeterministic,
}
}
func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
executionClass := contracts.ExecutionClass(strings.TrimSpace(string(spec.ExecutionClass)))
return ModuleSpec{

4
internal/framework/pipeline/output_registry.go
View File

@@ -32,10 +32,6 @@ func NewOutputEncoderRegistry() *OutputEncoderRegistry {
}
}
func (r *OutputEncoderRegistry) Register(key string, constructor OutputEncoderConstructor) error {
return r.RegisterWithSpec(defaultModuleSpec(key, StageOutput), constructor)
}
func (r *OutputEncoderRegistry) RegisterWithSpec(spec ModuleSpec, constructor OutputEncoderConstructor) error {
if constructor == nil {
return fmt.Errorf("output encoder constructor for %q must not be nil", strings.TrimSpace(spec.Key))

10
internal/framework/pipeline/output_registry_test.go
View File

@@ -17,7 +17,9 @@ func TestOutputEncoderRegistryBehavior(t *testing.T) {
return NewOutputEncoderRegistry()
},
register: func(registry any, key string, constructor func() (contracts.OutputEncoder, error)) error {
return registry.(*OutputEncoderRegistry).Register(key, constructor)
return registry.(*OutputEncoderRegistry).RegisterWithSpec(ModuleSpec{
Key: key, Stage: StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic,
}, constructor)
},
registerWithSpec: func(registry any, spec ModuleSpec, constructor func() (contracts.OutputEncoder, error)) error {
return registry.(*OutputEncoderRegistry).RegisterWithSpec(spec, constructor)
@@ -33,7 +35,9 @@ func TestOutputEncoderRegistryBehavior(t *testing.T) {
},
nilRegister: func(key string, constructor func() (contracts.OutputEncoder, error)) error {
var registry *OutputEncoderRegistry
return registry.Register(key, constructor)
return registry.RegisterWithSpec(ModuleSpec{
Key: key, Stage: StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic,
}, constructor)
},
nilBuild: func(key string) (contracts.OutputEncoder, error) {
var registry *OutputEncoderRegistry
@@ -60,7 +64,7 @@ func TestOutputEncoderRegistryBehavior(t *testing.T) {
func TestOutputProfileValidationReceivesOwnedOptionsAndLaneIDs(t *testing.T) {
registry := NewOutputEncoderRegistry()
if err := registry.RegisterBuilderWithProfileValidation(defaultModuleSpec("profile-output", StageOutput), func(options map[string]any) error {
if err := registry.RegisterBuilderWithProfileValidation(testModuleSpec("profile-output", StageOutput), func(options map[string]any) error {
options["nested"].(map[string]any)["value"] = "changed"
return nil
}, func(context OutputProfileOptionContext, options map[string]any) error {

12
internal/framework/pipeline/preparation_test.go
View File

@@ -480,19 +480,19 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con
if err := RegisterArtifactCodec(registries.ArtifactCodecs, notesCodec()); err != nil {
t.Fatal(err)
}
if err := registries.Inputs.RegisterBuilderWithSpec(defaultModuleSpec("input", StageInput), strict, func(request BuildRequest) (contracts.InputAdapter, error) {
if err := registries.Inputs.RegisterBuilderWithSpec(testModuleSpec("input", StageInput), strict, func(request BuildRequest) (contracts.InputAdapter, error) {
record("input", &request)
return input, nil
}); err != nil {
t.Fatal(err)
}
if err := registries.Chunkers.RegisterBuilderWithSpec(defaultModuleSpec("chunk", StageChunk), strict, func(request BuildRequest) (contracts.Chunker, error) {
if err := registries.Chunkers.RegisterBuilderWithSpec(testModuleSpec("chunk", StageChunk), strict, func(request BuildRequest) (contracts.Chunker, error) {
record("chunk", &request)
return &typedTestChunker{key: "chunk"}, nil
}); err != nil {
t.Fatal(err)
}
extractSpec := defaultModuleSpec("extract", StageExtract)
extractSpec := testModuleSpec("extract", StageExtract)
extractSpec.ArtifactKind = "test/notes"
if err := RegisterExtractorBuilder(registries.Extractors, extractSpec, strict, func(request BuildRequest) (contracts.Extractor[codecNotes], error) {
record("extract", &request)
@@ -503,7 +503,7 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con
}); err != nil {
t.Fatal(err)
}
mergeSpec := defaultModuleSpec("merge", StageMerge)
mergeSpec := testModuleSpec("merge", StageMerge)
mergeSpec.ArtifactKind = "test/notes"
if err := RegisterMergerBuilder(registries.Mergers, mergeSpec, strict, func(request BuildRequest) (contracts.Merger[codecNotes], error) {
record("merge", &request)
@@ -511,7 +511,7 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con
}); err != nil {
t.Fatal(err)
}
normalizeSpec := defaultModuleSpec("normalize", StageNormalize)
normalizeSpec := testModuleSpec("normalize", StageNormalize)
normalizeSpec.ArtifactKind = "test/notes"
if err := RegisterNormalizerBuilder(registries.Normalizers, normalizeSpec, strict, func(request BuildRequest) (contracts.Normalizer[codecNotes], error) {
record("normalize", &request)
@@ -532,7 +532,7 @@ func constructionRegistriesWithHooks(t *testing.T, built *[]string, failure *con
}); err != nil {
t.Fatal(err)
}
if err := registries.Outputs.RegisterBuilderWithSpec(defaultModuleSpec("output", StageOutput), strict, func(request BuildRequest) (contracts.OutputEncoder, error) {
if err := registries.Outputs.RegisterBuilderWithSpec(testModuleSpec("output", StageOutput), strict, func(request BuildRequest) (contracts.OutputEncoder, error) {
record("output", &request)
if failure.output != nil {
return nil, failure.output