eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add fallback PromptKit profile assets

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-03 16:34:48 +00:00
parent 4829f94157
commit b05634ee86
10 changed files with 349 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/cli/promptkit_profiles.go
View File

@@ -8,11 +8,14 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string, assets *llm.AssetRegistry) error {
if len(profileIDs) == 0 {
return nil
}
inspector, err := llm.NewPromptKitProfileInspector(promptKitProfileSourceConfig(cfg))
inspector, err := llm.NewPromptKitProfileInspector(llm.PromptKitProfileInspectorConfig{
Source: promptKitProfileSourceConfig(cfg),
Assets: assets,
})
if err != nil {
return fmt.Errorf("load PromptKit profiles: %w", err)
}

16
internal/cli/promptkit_profiles_test.go
View File

@@ -10,8 +10,10 @@ import (
"strings"
"sync/atomic"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) {
@@ -112,7 +114,7 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
ctx, cancel = context.WithCancel(ctx)
cancel()
}
err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID})
err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID}, nil)
if len(tt.wantErr) == 0 {
if err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
@@ -136,3 +138,15 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load())
}
}
func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
assets := llm.NewAssetRegistry()
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/fallback.yaml": {Data: []byte("id: fallback-profile\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
}, "profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"fallback-profile"}, assets); err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
}
}

6
internal/cli/run.go
View File

@@ -46,6 +46,7 @@ type Options struct {
ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
promptKitAssets *frameworkllm.AssetRegistry
}
type LLMRuntimeOverrides struct {
@@ -121,6 +122,7 @@ func normalizeOptions(opts Options) (Options, error) {
}
opts.Registries = components.registries
opts.Catalog = catalogFromRegistries(components.registries)
opts.promptKitAssets = components.assets
if opts.LLMClientFactory == nil {
opts.LLMClientFactory = productionLLMClientFactoryWithAssets(components.assets)
}
@@ -340,7 +342,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs); err != nil {
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs, opts.promptKitAssets); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
workingDir, err := os.Getwd()
@@ -1048,7 +1050,7 @@ func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) in
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline)); err != nil {
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline), opts.promptKitAssets); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}

109
internal/framework/llm/asset_registry.go
View File

@@ -21,8 +21,9 @@ type AssetSource struct {
}
type AssetRegistry struct {
prompts []AssetSource
schemas []AssetSource
prompts []AssetSource
schemas []AssetSource
fallbackProfiles []AssetSource
}
type AssetHashPart struct {
@@ -58,6 +59,20 @@ func (r *AssetRegistry) RegisterSchemaFS(fsys fs.FS, root string) error {
return nil
}
// RegisterFallbackProfileFS registers profile assets that PromptKit uses only
// when an operator-configured source does not provide a matching profile.
func (r *AssetRegistry) RegisterFallbackProfileFS(fsys fs.FS, root string) error {
if r == nil {
return fmt.Errorf("asset registry must not be nil")
}
source, err := newAssetSource(fsys, root)
if err != nil {
return fmt.Errorf("register fallback profile assets: %w", err)
}
r.fallbackProfiles = append(r.fallbackProfiles, source)
return nil
}
func (r *AssetRegistry) PromptFS() (fs.FS, error) {
if r == nil {
return nil, fmt.Errorf("asset registry must not be nil")
@@ -72,19 +87,76 @@ func (r *AssetRegistry) SchemaFS() (fs.FS, error) {
return flattenAssetSources(r.schemas)
}
func (r *AssetRegistry) FallbackProfileFS() (fs.FS, error) {
if r == nil {
return nil, fmt.Errorf("asset registry must not be nil")
}
return flattenAssetSources(r.fallbackProfiles)
}
// FallbackProfileDigest returns a deterministic, non-secret identity for the
// flattened fallback profile assets.
func (r *AssetRegistry) FallbackProfileDigest() (string, error) {
_, digest, _, err := r.fallbackProfileAssets()
return digest, err
}
func (r *AssetRegistry) PromptKitOptions() ([]promptkit.Option, error) {
options, _, err := r.promptKitOptions()
return options, err
}
func (r *AssetRegistry) promptKitOptions() ([]promptkit.Option, string, error) {
promptFS, err := r.PromptFS()
if err != nil {
return nil, fmt.Errorf("prepare prompt assets: %w", err)
return nil, "", fmt.Errorf("prepare prompt assets: %w", err)
}
schemaFS, err := r.SchemaFS()
if err != nil {
return nil, fmt.Errorf("prepare schema assets: %w", err)
return nil, "", fmt.Errorf("prepare schema assets: %w", err)
}
return []promptkit.Option{
options := []promptkit.Option{
promptkit.WithPromptFS(promptFS, "."),
promptkit.WithSchemaFS(schemaFS, "."),
}, nil
}
fallbackFS, fallbackDigest, hasFallback, err := r.fallbackProfileAssets()
if err != nil {
return nil, "", err
}
if hasFallback {
options = append(options, promptkit.WithFallbackProfileFS(fallbackFS, "."))
}
return options, fallbackDigest, nil
}
func (r *AssetRegistry) promptKitFallbackProfileOption() (promptkit.Option, bool, error) {
fallbackFS, _, hasFallback, err := r.fallbackProfileAssets()
if err != nil {
return nil, false, err
}
if !hasFallback {
return nil, false, nil
}
return promptkit.WithFallbackProfileFS(fallbackFS, "."), true, nil
}
func (r *AssetRegistry) fallbackProfileAssets() (fs.FS, string, bool, error) {
if r == nil {
return nil, "", false, fmt.Errorf("asset registry must not be nil")
}
if len(r.fallbackProfiles) == 0 {
empty := sha256.Sum256([]byte("notarius:fallback-profile-assets:empty"))
return nil, "sha256:" + hex.EncodeToString(empty[:]), false, nil
}
fallbackFS, err := r.FallbackProfileFS()
if err != nil {
return nil, "", false, fmt.Errorf("prepare fallback profile assets: %w", err)
}
digest, err := hashAssetFilesystem(fallbackFS)
if err != nil {
return nil, "", false, err
}
return fallbackFS, digest, true, nil
}
func HashAssets(parts []AssetHashPart) (string, error) {
@@ -117,6 +189,28 @@ func HashAssets(parts []AssetHashPart) (string, error) {
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil
}
func hashAssetFilesystem(fsys fs.FS) (string, error) {
var parts []AssetHashPart
err := fs.WalkDir(fsys, ".", func(name string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
return nil
}
parts = append(parts, AssetHashPart{FS: fsys, Path: name})
return nil
})
if err != nil {
return "", fmt.Errorf("walk assets for digest: %w", err)
}
if len(parts) > 0 {
return HashAssets(parts)
}
empty := sha256.Sum256([]byte("notarius:fallback-profile-assets:empty"))
return "sha256:" + hex.EncodeToString(empty[:]), nil
}
func newAssetSource(fsys fs.FS, root string) (AssetSource, error) {
if fsys == nil {
return AssetSource{}, fmt.Errorf("filesystem must not be nil")
@@ -248,6 +342,9 @@ func (m assetMapFS) dirEntries(dir string) []fs.DirEntry {
children[childName] = entry
}
if len(children) == 0 {
if dir == "." {
return []fs.DirEntry{}
}
return nil
}
names := make([]string, 0, len(children))

90
internal/framework/llm/asset_registry_test.go
View File

@@ -2,6 +2,7 @@ package llm
import (
"context"
"io/fs"
"strings"
"testing"
"testing/fstest"
@@ -98,6 +99,89 @@ func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
}
}
func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) {
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
"first/profiles/one.yaml": {Data: []byte("id: one\nmodel: first\n")},
}, "first/profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
"second/two.yaml": {Data: []byte("id: two\nmodel: second\n")},
}, "second"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
fallbackFS, err := registry.FallbackProfileFS()
if err != nil {
t.Fatalf("FallbackProfileFS() error = %v, want nil", err)
}
for _, name := range []string{"one.yaml", "two.yaml"} {
if _, err := fs.ReadFile(fallbackFS, name); err != nil {
t.Fatalf("FallbackProfileFS().ReadFile(%q) error = %v, want nil", name, err)
}
}
}
func TestAssetRegistryRejectsInvalidFallbackProfileRoot(t *testing.T) {
registry := NewAssetRegistry()
err := registry.RegisterFallbackProfileFS(fstest.MapFS{}, "../profiles")
if err == nil || !strings.Contains(err.Error(), "invalid path") {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want invalid root error", err)
}
}
func TestAssetRegistryRejectsUnreadableFallbackProfileAssets(t *testing.T) {
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(unreadableAssetFS{}, "."); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
_, err := registry.FallbackProfileFS()
if err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("FallbackProfileFS() error = %v, want unreadable asset error", err)
}
}
func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) {
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("id: first\n")}}, "first"); err != nil {
t.Fatal(err)
}
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("id: second\n")}}, "second"); err != nil {
t.Fatal(err)
}
_, err := registry.FallbackProfileFS()
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
t.Fatalf("FallbackProfileFS() error = %v, want duplicate path error", err)
}
}
func TestAssetRegistryFallbackProfileDigestTracksContentWithoutLeakingIt(t *testing.T) {
digestFor := func(content string) string {
t.Helper()
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/profile.yaml": {Data: []byte(content)},
}, "profiles"); err != nil {
t.Fatal(err)
}
digest, err := registry.FallbackProfileDigest()
if err != nil {
t.Fatal(err)
}
return digest
}
first := digestFor("id: fallback\nmodel: model-one\n")
second := digestFor("id: fallback\nmodel: model-two\n")
if first == second {
t.Fatalf("fallback profile digests = %q and %q, want content change", first, second)
}
if !strings.HasPrefix(first, "sha256:") || strings.Contains(first, "model-one") || strings.Contains(first, "profile.yaml") {
t.Fatalf("fallback profile digest leaked source details: %q", first)
}
}
func TestAssetRegistryCombinesNamespacedPromptSources(t *testing.T) {
registry := NewAssetRegistry()
mustRegisterPromptFS(t, registry, fstest.MapFS{
@@ -196,3 +280,9 @@ output:
repair_attempts: 0
`
}
type unreadableAssetFS struct{}
func (unreadableAssetFS) Open(name string) (fs.File, error) {
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrPermission}
}

30
internal/framework/llm/promptkit_client.go
View File

@@ -37,12 +37,13 @@ type PromptKitClientConfig struct {
}
type PromptKitClient struct {
engine *promptkit.Engine
recorder *LLMProfileRecorder
profileDir string
profileFile string
localEndpoint string
reasoningEffort *string
engine *promptkit.Engine
recorder *LLMProfileRecorder
profileDir string
profileFile string
localEndpoint string
fallbackProfileDigest string
reasoningEffort *string
}
type LLMProfileRecorder struct {
@@ -69,7 +70,7 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
if err != nil {
return nil, err
}
options, err := cfg.Assets.PromptKitOptions()
options, fallbackProfileDigest, err := cfg.Assets.promptKitOptions()
if err != nil {
return nil, err
}
@@ -94,12 +95,13 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
reasoningEffort = &value
}
return &PromptKitClient{
engine: engine,
recorder: recorder,
profileDir: profileSource.ProfileDir,
profileFile: profileSource.ProfileFile,
localEndpoint: profileSource.localEndpoint(),
reasoningEffort: reasoningEffort,
engine: engine,
recorder: recorder,
profileDir: profileSource.ProfileDir,
profileFile: profileSource.ProfileFile,
localEndpoint: profileSource.localEndpoint(),
fallbackProfileDigest: fallbackProfileDigest,
reasoningEffort: reasoningEffort,
}, nil
}
@@ -325,7 +327,7 @@ func (c *PromptKitClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint,
if c == nil {
return nil, nil
}
fingerprint, err := promptKitProfileFingerprint(c.profileDir, c.profileFile)
fingerprint, err := promptKitProfileFingerprint(c.profileDir, c.profileFile, c.fallbackProfileDigest)
if err != nil {
return nil, err
}

78
internal/framework/llm/promptkit_client_test.go
View File

@@ -431,6 +431,84 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
}
}
func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing.T) {
assets := newTestPromptKitAssets(t)
const profileID = "fallback-profile"
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/fallback.yaml": {Data: []byte("id: " + profileID + "\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
}, "profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: assets,
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
ProfileID: profileID,
SessionID: "fallback-profile-test",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", err)
}
if response.ProfileID != profileID || response.Model != "fallback-model" {
t.Fatalf("completion response = %#v, want fallback profile", response)
}
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Assets: assets})
if err != nil {
t.Fatalf("NewPromptKitProfileInspector() error = %v, want nil", err)
}
inspection, err := inspector.InspectProfile(context.Background(), profileID)
if err != nil {
t.Fatalf("InspectProfile() error = %v, want nil", err)
}
if inspection.ProfileID != profileID || inspection.Model != "fallback-model" {
t.Fatalf("profile inspection = %#v, want fallback profile", inspection)
}
}
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
fingerprintFor := func(content string) CheckpointFingerprint {
t.Helper()
assets := newTestPromptKitAssets(t)
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/fallback.yaml": {Data: []byte(content)},
}, "profiles"); err != nil {
t.Fatal(err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: assets})
if err != nil {
t.Fatal(err)
}
fingerprints, err := client.LLMCheckpointFingerprints()
if err != nil {
t.Fatal(err)
}
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitProfileFingerprintName {
t.Fatalf("checkpoint fingerprints = %#v, want profile source identity", fingerprints)
}
return fingerprints[0]
}
first := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-one\n")
second := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-two\n")
if first == second {
t.Fatalf("checkpoint fingerprints = %#v and %#v, want fallback asset change", first, second)
}
if strings.Contains(first.Value, "model-one") || strings.Contains(first.Value, "fallback.yaml") {
t.Fatalf("checkpoint fingerprint leaked fallback source details: %#v", first)
}
}
func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing.T) {
const (
firstEndpoint = "http://localhost:8000/v1"

3
internal/framework/llm/promptkit_profile_fingerprint.go
View File

@@ -20,9 +20,10 @@ const (
promptKitBuiltinProfileCatalogID = "promptkit:v0.5.0:builtin-profiles"
)
func promptKitProfileFingerprint(profileDir, profileFile string) (CheckpointFingerprint, error) {
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
hasher := sha256.New()
writeFingerprintPart(hasher, []byte(promptKitBuiltinProfileCatalogID))
writeFingerprintPart(hasher, []byte(strings.TrimSpace(fallbackProfileDigest)))
switch {
case strings.TrimSpace(profileFile) != "":

18
internal/framework/llm/promptkit_profiles.go
View File

@@ -15,6 +15,11 @@ type PromptKitProfileSourceConfig struct {
LocalBackend *PromptKitLocalBackendConfig
}
type PromptKitProfileInspectorConfig struct {
Source PromptKitProfileSourceConfig
Assets *AssetRegistry
}
func (c PromptKitProfileSourceConfig) localEndpoint() string {
if c.LocalBackend == nil {
return ""
@@ -50,11 +55,20 @@ func (e *PromptKitProfileInspectionError) Unwrap() error {
return e.err
}
func NewPromptKitProfileInspector(cfg PromptKitProfileSourceConfig) (*PromptKitProfileInspector, error) {
source, options, err := promptKitProfileSourceEngineOptions(cfg)
func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) {
source, options, err := promptKitProfileSourceEngineOptions(cfg.Source)
if err != nil {
return nil, err
}
if cfg.Assets != nil {
fallbackOption, hasFallback, err := cfg.Assets.promptKitFallbackProfileOption()
if err != nil {
return nil, err
}
if hasFallback {
options = append(options, fallbackOption)
}
}
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: ".",
ProfileDir: source.ProfileDir,