Add fallback PromptKit profile assets

This commit is contained in:
2026-08-03 16:34:48 +00:00
parent 4829f94157
commit b05634ee86
10 changed files with 349 additions and 42 deletions

View File

@@ -62,21 +62,25 @@ completion responses and recorded profile manifests identify the adapter
provider as `promptkit`.
The CLI's profile-inspection engine and the production adapter use the same
profile-source construction to apply the configured profile directory or file
and register the optional conventional `local` backend. Preflight therefore
resolves the same ordinary profile source and backend membership as runtime
without performing generation. When the registration is absent, a profile
selecting `backend: local` fails inspection instead of falling back to a
built-in or endpoint-only target.
profile-source construction to apply the configured profile directory or file,
the optional registered fallback profile assets, and the optional conventional
`local` backend. Preflight therefore resolves the same profile sources and
backend membership as runtime without performing generation. Fallback assets
are mounted only when at least one source is registered. PromptKit owns source
precedence and profile parsing: an operator-provided matching profile takes
precedence over a fallback profile without Notarius merging either document.
When the registration is absent, a profile selecting `backend: local` fails
inspection instead of falling back to a built-in or endpoint-only target.
Before execution, the adapter also contributes a non-secret checkpoint
fingerprint for the effective PromptKit profile source. It combines the
identity of PromptKit's compiled-in profile catalog with a deterministic digest
of every YAML profile in the configured profile directory, or of the configured
profile file. The fingerprint contains neither profile content nor source
paths. It covers both explicit binding profiles and prompt-selected defaults,
so changing a model or other profile setting cannot reuse checkpoints created
under the prior profile source. This cache identity is independent of durable
profile file, and a deterministic digest of the flattened fallback profile
assets. The fingerprint contains neither profile content nor source paths. It
covers both explicit binding profiles and prompt-selected defaults, so changing
a model or other profile setting cannot reuse checkpoints created under the
prior profile source. This cache identity is independent of durable
profile provenance: run manifests continue to list only profiles actually
observed during LLM calls. When the local backend is registered, a second
fingerprint hashes its trimmed endpoint behind a stable marker. Changing that
@@ -109,10 +113,12 @@ only by the Notarius scheduler.
## Prompt And Schema Assets
An `AssetRegistry` collects prompt and schema filesystems from production module
families. It flattens registered roots into the PromptKit filesystems and
rejects invalid roots, unreadable assets, duplicate paths, and missing prompt
or schema files during preparation. The frameworks `promptfs` helper combines
An `AssetRegistry` collects prompt, schema, and optional fallback-profile
filesystems from production module families. It flattens registered roots into
the corresponding PromptKit filesystems and rejects invalid roots, unreadable
assets, duplicate paths, and missing prompt or schema files during preparation.
Fallback assets receive a safe content digest for checkpoint identity; raw
paths and bytes are never included. The frameworks `promptfs` helper combines
module-owned prompt files with reusable domain fragments without making the
framework depend on D&D content.

View File

@@ -8,11 +8,14 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string, assets *llm.AssetRegistry) error {
if len(profileIDs) == 0 {
return nil
}
inspector, err := llm.NewPromptKitProfileInspector(promptKitProfileSourceConfig(cfg))
inspector, err := llm.NewPromptKitProfileInspector(llm.PromptKitProfileInspectorConfig{
Source: promptKitProfileSourceConfig(cfg),
Assets: assets,
})
if err != nil {
return fmt.Errorf("load PromptKit profiles: %w", err)
}

View File

@@ -10,8 +10,10 @@ import (
"strings"
"sync/atomic"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) {
@@ -112,7 +114,7 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
ctx, cancel = context.WithCancel(ctx)
cancel()
}
err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID})
err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID}, nil)
if len(tt.wantErr) == 0 {
if err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
@@ -136,3 +138,15 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load())
}
}
func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
assets := llm.NewAssetRegistry()
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/fallback.yaml": {Data: []byte("id: fallback-profile\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
}, "profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"fallback-profile"}, assets); err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
}
}

View File

@@ -46,6 +46,7 @@ type Options struct {
ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
promptKitAssets *frameworkllm.AssetRegistry
}
type LLMRuntimeOverrides struct {
@@ -121,6 +122,7 @@ func normalizeOptions(opts Options) (Options, error) {
}
opts.Registries = components.registries
opts.Catalog = catalogFromRegistries(components.registries)
opts.promptKitAssets = components.assets
if opts.LLMClientFactory == nil {
opts.LLMClientFactory = productionLLMClientFactoryWithAssets(components.assets)
}
@@ -340,7 +342,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs); err != nil {
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs, opts.promptKitAssets); err != nil {
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
workingDir, err := os.Getwd()
@@ -1048,7 +1050,7 @@ func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) in
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline)); err != nil {
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline), opts.promptKitAssets); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}

View File

@@ -21,8 +21,9 @@ type AssetSource struct {
}
type AssetRegistry struct {
prompts []AssetSource
schemas []AssetSource
prompts []AssetSource
schemas []AssetSource
fallbackProfiles []AssetSource
}
type AssetHashPart struct {
@@ -58,6 +59,20 @@ func (r *AssetRegistry) RegisterSchemaFS(fsys fs.FS, root string) error {
return nil
}
// RegisterFallbackProfileFS registers profile assets that PromptKit uses only
// when an operator-configured source does not provide a matching profile.
func (r *AssetRegistry) RegisterFallbackProfileFS(fsys fs.FS, root string) error {
if r == nil {
return fmt.Errorf("asset registry must not be nil")
}
source, err := newAssetSource(fsys, root)
if err != nil {
return fmt.Errorf("register fallback profile assets: %w", err)
}
r.fallbackProfiles = append(r.fallbackProfiles, source)
return nil
}
func (r *AssetRegistry) PromptFS() (fs.FS, error) {
if r == nil {
return nil, fmt.Errorf("asset registry must not be nil")
@@ -72,19 +87,76 @@ func (r *AssetRegistry) SchemaFS() (fs.FS, error) {
return flattenAssetSources(r.schemas)
}
func (r *AssetRegistry) FallbackProfileFS() (fs.FS, error) {
if r == nil {
return nil, fmt.Errorf("asset registry must not be nil")
}
return flattenAssetSources(r.fallbackProfiles)
}
// FallbackProfileDigest returns a deterministic, non-secret identity for the
// flattened fallback profile assets.
func (r *AssetRegistry) FallbackProfileDigest() (string, error) {
_, digest, _, err := r.fallbackProfileAssets()
return digest, err
}
func (r *AssetRegistry) PromptKitOptions() ([]promptkit.Option, error) {
options, _, err := r.promptKitOptions()
return options, err
}
func (r *AssetRegistry) promptKitOptions() ([]promptkit.Option, string, error) {
promptFS, err := r.PromptFS()
if err != nil {
return nil, fmt.Errorf("prepare prompt assets: %w", err)
return nil, "", fmt.Errorf("prepare prompt assets: %w", err)
}
schemaFS, err := r.SchemaFS()
if err != nil {
return nil, fmt.Errorf("prepare schema assets: %w", err)
return nil, "", fmt.Errorf("prepare schema assets: %w", err)
}
return []promptkit.Option{
options := []promptkit.Option{
promptkit.WithPromptFS(promptFS, "."),
promptkit.WithSchemaFS(schemaFS, "."),
}, nil
}
fallbackFS, fallbackDigest, hasFallback, err := r.fallbackProfileAssets()
if err != nil {
return nil, "", err
}
if hasFallback {
options = append(options, promptkit.WithFallbackProfileFS(fallbackFS, "."))
}
return options, fallbackDigest, nil
}
func (r *AssetRegistry) promptKitFallbackProfileOption() (promptkit.Option, bool, error) {
fallbackFS, _, hasFallback, err := r.fallbackProfileAssets()
if err != nil {
return nil, false, err
}
if !hasFallback {
return nil, false, nil
}
return promptkit.WithFallbackProfileFS(fallbackFS, "."), true, nil
}
func (r *AssetRegistry) fallbackProfileAssets() (fs.FS, string, bool, error) {
if r == nil {
return nil, "", false, fmt.Errorf("asset registry must not be nil")
}
if len(r.fallbackProfiles) == 0 {
empty := sha256.Sum256([]byte("notarius:fallback-profile-assets:empty"))
return nil, "sha256:" + hex.EncodeToString(empty[:]), false, nil
}
fallbackFS, err := r.FallbackProfileFS()
if err != nil {
return nil, "", false, fmt.Errorf("prepare fallback profile assets: %w", err)
}
digest, err := hashAssetFilesystem(fallbackFS)
if err != nil {
return nil, "", false, err
}
return fallbackFS, digest, true, nil
}
func HashAssets(parts []AssetHashPart) (string, error) {
@@ -117,6 +189,28 @@ func HashAssets(parts []AssetHashPart) (string, error) {
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil
}
func hashAssetFilesystem(fsys fs.FS) (string, error) {
var parts []AssetHashPart
err := fs.WalkDir(fsys, ".", func(name string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
return nil
}
parts = append(parts, AssetHashPart{FS: fsys, Path: name})
return nil
})
if err != nil {
return "", fmt.Errorf("walk assets for digest: %w", err)
}
if len(parts) > 0 {
return HashAssets(parts)
}
empty := sha256.Sum256([]byte("notarius:fallback-profile-assets:empty"))
return "sha256:" + hex.EncodeToString(empty[:]), nil
}
func newAssetSource(fsys fs.FS, root string) (AssetSource, error) {
if fsys == nil {
return AssetSource{}, fmt.Errorf("filesystem must not be nil")
@@ -248,6 +342,9 @@ func (m assetMapFS) dirEntries(dir string) []fs.DirEntry {
children[childName] = entry
}
if len(children) == 0 {
if dir == "." {
return []fs.DirEntry{}
}
return nil
}
names := make([]string, 0, len(children))

View File

@@ -2,6 +2,7 @@ package llm
import (
"context"
"io/fs"
"strings"
"testing"
"testing/fstest"
@@ -98,6 +99,89 @@ func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
}
}
func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) {
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
"first/profiles/one.yaml": {Data: []byte("id: one\nmodel: first\n")},
}, "first/profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
"second/two.yaml": {Data: []byte("id: two\nmodel: second\n")},
}, "second"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
fallbackFS, err := registry.FallbackProfileFS()
if err != nil {
t.Fatalf("FallbackProfileFS() error = %v, want nil", err)
}
for _, name := range []string{"one.yaml", "two.yaml"} {
if _, err := fs.ReadFile(fallbackFS, name); err != nil {
t.Fatalf("FallbackProfileFS().ReadFile(%q) error = %v, want nil", name, err)
}
}
}
func TestAssetRegistryRejectsInvalidFallbackProfileRoot(t *testing.T) {
registry := NewAssetRegistry()
err := registry.RegisterFallbackProfileFS(fstest.MapFS{}, "../profiles")
if err == nil || !strings.Contains(err.Error(), "invalid path") {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want invalid root error", err)
}
}
func TestAssetRegistryRejectsUnreadableFallbackProfileAssets(t *testing.T) {
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(unreadableAssetFS{}, "."); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
_, err := registry.FallbackProfileFS()
if err == nil || !strings.Contains(err.Error(), "permission denied") {
t.Fatalf("FallbackProfileFS() error = %v, want unreadable asset error", err)
}
}
func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) {
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("id: first\n")}}, "first"); err != nil {
t.Fatal(err)
}
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("id: second\n")}}, "second"); err != nil {
t.Fatal(err)
}
_, err := registry.FallbackProfileFS()
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
t.Fatalf("FallbackProfileFS() error = %v, want duplicate path error", err)
}
}
func TestAssetRegistryFallbackProfileDigestTracksContentWithoutLeakingIt(t *testing.T) {
digestFor := func(content string) string {
t.Helper()
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/profile.yaml": {Data: []byte(content)},
}, "profiles"); err != nil {
t.Fatal(err)
}
digest, err := registry.FallbackProfileDigest()
if err != nil {
t.Fatal(err)
}
return digest
}
first := digestFor("id: fallback\nmodel: model-one\n")
second := digestFor("id: fallback\nmodel: model-two\n")
if first == second {
t.Fatalf("fallback profile digests = %q and %q, want content change", first, second)
}
if !strings.HasPrefix(first, "sha256:") || strings.Contains(first, "model-one") || strings.Contains(first, "profile.yaml") {
t.Fatalf("fallback profile digest leaked source details: %q", first)
}
}
func TestAssetRegistryCombinesNamespacedPromptSources(t *testing.T) {
registry := NewAssetRegistry()
mustRegisterPromptFS(t, registry, fstest.MapFS{
@@ -196,3 +280,9 @@ output:
repair_attempts: 0
`
}
type unreadableAssetFS struct{}
func (unreadableAssetFS) Open(name string) (fs.File, error) {
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrPermission}
}

View File

@@ -37,12 +37,13 @@ type PromptKitClientConfig struct {
}
type PromptKitClient struct {
engine *promptkit.Engine
recorder *LLMProfileRecorder
profileDir string
profileFile string
localEndpoint string
reasoningEffort *string
engine *promptkit.Engine
recorder *LLMProfileRecorder
profileDir string
profileFile string
localEndpoint string
fallbackProfileDigest string
reasoningEffort *string
}
type LLMProfileRecorder struct {
@@ -69,7 +70,7 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
if err != nil {
return nil, err
}
options, err := cfg.Assets.PromptKitOptions()
options, fallbackProfileDigest, err := cfg.Assets.promptKitOptions()
if err != nil {
return nil, err
}
@@ -94,12 +95,13 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
reasoningEffort = &value
}
return &PromptKitClient{
engine: engine,
recorder: recorder,
profileDir: profileSource.ProfileDir,
profileFile: profileSource.ProfileFile,
localEndpoint: profileSource.localEndpoint(),
reasoningEffort: reasoningEffort,
engine: engine,
recorder: recorder,
profileDir: profileSource.ProfileDir,
profileFile: profileSource.ProfileFile,
localEndpoint: profileSource.localEndpoint(),
fallbackProfileDigest: fallbackProfileDigest,
reasoningEffort: reasoningEffort,
}, nil
}
@@ -325,7 +327,7 @@ func (c *PromptKitClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint,
if c == nil {
return nil, nil
}
fingerprint, err := promptKitProfileFingerprint(c.profileDir, c.profileFile)
fingerprint, err := promptKitProfileFingerprint(c.profileDir, c.profileFile, c.fallbackProfileDigest)
if err != nil {
return nil, err
}

View File

@@ -431,6 +431,84 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
}
}
func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing.T) {
assets := newTestPromptKitAssets(t)
const profileID = "fallback-profile"
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/fallback.yaml": {Data: []byte("id: " + profileID + "\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
}, "profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
fake := &fakePromptKitLLM{content: `{"ok":true}`}
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: assets,
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
ProfileID: profileID,
SessionID: "fallback-profile-test",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", err)
}
if response.ProfileID != profileID || response.Model != "fallback-model" {
t.Fatalf("completion response = %#v, want fallback profile", response)
}
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Assets: assets})
if err != nil {
t.Fatalf("NewPromptKitProfileInspector() error = %v, want nil", err)
}
inspection, err := inspector.InspectProfile(context.Background(), profileID)
if err != nil {
t.Fatalf("InspectProfile() error = %v, want nil", err)
}
if inspection.ProfileID != profileID || inspection.Model != "fallback-model" {
t.Fatalf("profile inspection = %#v, want fallback profile", inspection)
}
}
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
fingerprintFor := func(content string) CheckpointFingerprint {
t.Helper()
assets := newTestPromptKitAssets(t)
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
"profiles/fallback.yaml": {Data: []byte(content)},
}, "profiles"); err != nil {
t.Fatal(err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: assets})
if err != nil {
t.Fatal(err)
}
fingerprints, err := client.LLMCheckpointFingerprints()
if err != nil {
t.Fatal(err)
}
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitProfileFingerprintName {
t.Fatalf("checkpoint fingerprints = %#v, want profile source identity", fingerprints)
}
return fingerprints[0]
}
first := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-one\n")
second := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-two\n")
if first == second {
t.Fatalf("checkpoint fingerprints = %#v and %#v, want fallback asset change", first, second)
}
if strings.Contains(first.Value, "model-one") || strings.Contains(first.Value, "fallback.yaml") {
t.Fatalf("checkpoint fingerprint leaked fallback source details: %#v", first)
}
}
func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing.T) {
const (
firstEndpoint = "http://localhost:8000/v1"

View File

@@ -20,9 +20,10 @@ const (
promptKitBuiltinProfileCatalogID = "promptkit:v0.5.0:builtin-profiles"
)
func promptKitProfileFingerprint(profileDir, profileFile string) (CheckpointFingerprint, error) {
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
hasher := sha256.New()
writeFingerprintPart(hasher, []byte(promptKitBuiltinProfileCatalogID))
writeFingerprintPart(hasher, []byte(strings.TrimSpace(fallbackProfileDigest)))
switch {
case strings.TrimSpace(profileFile) != "":

View File

@@ -15,6 +15,11 @@ type PromptKitProfileSourceConfig struct {
LocalBackend *PromptKitLocalBackendConfig
}
type PromptKitProfileInspectorConfig struct {
Source PromptKitProfileSourceConfig
Assets *AssetRegistry
}
func (c PromptKitProfileSourceConfig) localEndpoint() string {
if c.LocalBackend == nil {
return ""
@@ -50,11 +55,20 @@ func (e *PromptKitProfileInspectionError) Unwrap() error {
return e.err
}
func NewPromptKitProfileInspector(cfg PromptKitProfileSourceConfig) (*PromptKitProfileInspector, error) {
source, options, err := promptKitProfileSourceEngineOptions(cfg)
func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) {
source, options, err := promptKitProfileSourceEngineOptions(cfg.Source)
if err != nil {
return nil, err
}
if cfg.Assets != nil {
fallbackOption, hasFallback, err := cfg.Assets.promptKitFallbackProfileOption()
if err != nil {
return nil, err
}
if hasFallback {
options = append(options, fallbackOption)
}
}
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: ".",
ProfileDir: source.ProfileDir,