Add fallback PromptKit profile assets
This commit is contained in:
@@ -62,21 +62,25 @@ completion responses and recorded profile manifests identify the adapter
|
||||
provider as `promptkit`.
|
||||
|
||||
The CLI's profile-inspection engine and the production adapter use the same
|
||||
profile-source construction to apply the configured profile directory or file
|
||||
and register the optional conventional `local` backend. Preflight therefore
|
||||
resolves the same ordinary profile source and backend membership as runtime
|
||||
without performing generation. When the registration is absent, a profile
|
||||
selecting `backend: local` fails inspection instead of falling back to a
|
||||
built-in or endpoint-only target.
|
||||
profile-source construction to apply the configured profile directory or file,
|
||||
the optional registered fallback profile assets, and the optional conventional
|
||||
`local` backend. Preflight therefore resolves the same profile sources and
|
||||
backend membership as runtime without performing generation. Fallback assets
|
||||
are mounted only when at least one source is registered. PromptKit owns source
|
||||
precedence and profile parsing: an operator-provided matching profile takes
|
||||
precedence over a fallback profile without Notarius merging either document.
|
||||
When the registration is absent, a profile selecting `backend: local` fails
|
||||
inspection instead of falling back to a built-in or endpoint-only target.
|
||||
|
||||
Before execution, the adapter also contributes a non-secret checkpoint
|
||||
fingerprint for the effective PromptKit profile source. It combines the
|
||||
identity of PromptKit's compiled-in profile catalog with a deterministic digest
|
||||
of every YAML profile in the configured profile directory, or of the configured
|
||||
profile file. The fingerprint contains neither profile content nor source
|
||||
paths. It covers both explicit binding profiles and prompt-selected defaults,
|
||||
so changing a model or other profile setting cannot reuse checkpoints created
|
||||
under the prior profile source. This cache identity is independent of durable
|
||||
profile file, and a deterministic digest of the flattened fallback profile
|
||||
assets. The fingerprint contains neither profile content nor source paths. It
|
||||
covers both explicit binding profiles and prompt-selected defaults, so changing
|
||||
a model or other profile setting cannot reuse checkpoints created under the
|
||||
prior profile source. This cache identity is independent of durable
|
||||
profile provenance: run manifests continue to list only profiles actually
|
||||
observed during LLM calls. When the local backend is registered, a second
|
||||
fingerprint hashes its trimmed endpoint behind a stable marker. Changing that
|
||||
@@ -109,10 +113,12 @@ only by the Notarius scheduler.
|
||||
|
||||
## Prompt And Schema Assets
|
||||
|
||||
An `AssetRegistry` collects prompt and schema filesystems from production module
|
||||
families. It flattens registered roots into the PromptKit filesystems and
|
||||
rejects invalid roots, unreadable assets, duplicate paths, and missing prompt
|
||||
or schema files during preparation. The framework’s `promptfs` helper combines
|
||||
An `AssetRegistry` collects prompt, schema, and optional fallback-profile
|
||||
filesystems from production module families. It flattens registered roots into
|
||||
the corresponding PromptKit filesystems and rejects invalid roots, unreadable
|
||||
assets, duplicate paths, and missing prompt or schema files during preparation.
|
||||
Fallback assets receive a safe content digest for checkpoint identity; raw
|
||||
paths and bytes are never included. The framework’s `promptfs` helper combines
|
||||
module-owned prompt files with reusable domain fragments without making the
|
||||
framework depend on D&D content.
|
||||
|
||||
|
||||
@@ -8,11 +8,14 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
|
||||
func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string, assets *llm.AssetRegistry) error {
|
||||
if len(profileIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
inspector, err := llm.NewPromptKitProfileInspector(promptKitProfileSourceConfig(cfg))
|
||||
inspector, err := llm.NewPromptKitProfileInspector(llm.PromptKitProfileInspectorConfig{
|
||||
Source: promptKitProfileSourceConfig(cfg),
|
||||
Assets: assets,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("load PromptKit profiles: %w", err)
|
||||
}
|
||||
|
||||
@@ -10,8 +10,10 @@ import (
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) {
|
||||
@@ -112,7 +114,7 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
ctx, cancel = context.WithCancel(ctx)
|
||||
cancel()
|
||||
}
|
||||
err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID})
|
||||
err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID}, nil)
|
||||
if len(tt.wantErr) == 0 {
|
||||
if err != nil {
|
||||
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
||||
@@ -136,3 +138,15 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
||||
assets := llm.NewAssetRegistry()
|
||||
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"profiles/fallback.yaml": {Data: []byte("id: fallback-profile\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
|
||||
}, "profiles"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"fallback-profile"}, assets); err != nil {
|
||||
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +46,7 @@ type Options struct {
|
||||
ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory
|
||||
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
|
||||
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
|
||||
promptKitAssets *frameworkllm.AssetRegistry
|
||||
}
|
||||
|
||||
type LLMRuntimeOverrides struct {
|
||||
@@ -121,6 +122,7 @@ func normalizeOptions(opts Options) (Options, error) {
|
||||
}
|
||||
opts.Registries = components.registries
|
||||
opts.Catalog = catalogFromRegistries(components.registries)
|
||||
opts.promptKitAssets = components.assets
|
||||
if opts.LLMClientFactory == nil {
|
||||
opts.LLMClientFactory = productionLLMClientFactoryWithAssets(components.assets)
|
||||
}
|
||||
@@ -340,7 +342,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
|
||||
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs); err != nil {
|
||||
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs, opts.promptKitAssets); err != nil {
|
||||
return failPipelineCommand(stderr, commandState, terminalWriter, err)
|
||||
}
|
||||
workingDir, err := os.Getwd()
|
||||
@@ -1048,7 +1050,7 @@ func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) in
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline)); err != nil {
|
||||
if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline), opts.promptKitAssets); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -21,8 +21,9 @@ type AssetSource struct {
|
||||
}
|
||||
|
||||
type AssetRegistry struct {
|
||||
prompts []AssetSource
|
||||
schemas []AssetSource
|
||||
prompts []AssetSource
|
||||
schemas []AssetSource
|
||||
fallbackProfiles []AssetSource
|
||||
}
|
||||
|
||||
type AssetHashPart struct {
|
||||
@@ -58,6 +59,20 @@ func (r *AssetRegistry) RegisterSchemaFS(fsys fs.FS, root string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterFallbackProfileFS registers profile assets that PromptKit uses only
|
||||
// when an operator-configured source does not provide a matching profile.
|
||||
func (r *AssetRegistry) RegisterFallbackProfileFS(fsys fs.FS, root string) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
source, err := newAssetSource(fsys, root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("register fallback profile assets: %w", err)
|
||||
}
|
||||
r.fallbackProfiles = append(r.fallbackProfiles, source)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) PromptFS() (fs.FS, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("asset registry must not be nil")
|
||||
@@ -72,19 +87,76 @@ func (r *AssetRegistry) SchemaFS() (fs.FS, error) {
|
||||
return flattenAssetSources(r.schemas)
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) FallbackProfileFS() (fs.FS, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
return flattenAssetSources(r.fallbackProfiles)
|
||||
}
|
||||
|
||||
// FallbackProfileDigest returns a deterministic, non-secret identity for the
|
||||
// flattened fallback profile assets.
|
||||
func (r *AssetRegistry) FallbackProfileDigest() (string, error) {
|
||||
_, digest, _, err := r.fallbackProfileAssets()
|
||||
return digest, err
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) PromptKitOptions() ([]promptkit.Option, error) {
|
||||
options, _, err := r.promptKitOptions()
|
||||
return options, err
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) promptKitOptions() ([]promptkit.Option, string, error) {
|
||||
promptFS, err := r.PromptFS()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare prompt assets: %w", err)
|
||||
return nil, "", fmt.Errorf("prepare prompt assets: %w", err)
|
||||
}
|
||||
schemaFS, err := r.SchemaFS()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare schema assets: %w", err)
|
||||
return nil, "", fmt.Errorf("prepare schema assets: %w", err)
|
||||
}
|
||||
return []promptkit.Option{
|
||||
options := []promptkit.Option{
|
||||
promptkit.WithPromptFS(promptFS, "."),
|
||||
promptkit.WithSchemaFS(schemaFS, "."),
|
||||
}, nil
|
||||
}
|
||||
fallbackFS, fallbackDigest, hasFallback, err := r.fallbackProfileAssets()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if hasFallback {
|
||||
options = append(options, promptkit.WithFallbackProfileFS(fallbackFS, "."))
|
||||
}
|
||||
return options, fallbackDigest, nil
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) promptKitFallbackProfileOption() (promptkit.Option, bool, error) {
|
||||
fallbackFS, _, hasFallback, err := r.fallbackProfileAssets()
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !hasFallback {
|
||||
return nil, false, nil
|
||||
}
|
||||
return promptkit.WithFallbackProfileFS(fallbackFS, "."), true, nil
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) fallbackProfileAssets() (fs.FS, string, bool, error) {
|
||||
if r == nil {
|
||||
return nil, "", false, fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
if len(r.fallbackProfiles) == 0 {
|
||||
empty := sha256.Sum256([]byte("notarius:fallback-profile-assets:empty"))
|
||||
return nil, "sha256:" + hex.EncodeToString(empty[:]), false, nil
|
||||
}
|
||||
fallbackFS, err := r.FallbackProfileFS()
|
||||
if err != nil {
|
||||
return nil, "", false, fmt.Errorf("prepare fallback profile assets: %w", err)
|
||||
}
|
||||
digest, err := hashAssetFilesystem(fallbackFS)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
return fallbackFS, digest, true, nil
|
||||
}
|
||||
|
||||
func HashAssets(parts []AssetHashPart) (string, error) {
|
||||
@@ -117,6 +189,28 @@ func HashAssets(parts []AssetHashPart) (string, error) {
|
||||
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func hashAssetFilesystem(fsys fs.FS) (string, error) {
|
||||
var parts []AssetHashPart
|
||||
err := fs.WalkDir(fsys, ".", func(name string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
parts = append(parts, AssetHashPart{FS: fsys, Path: name})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("walk assets for digest: %w", err)
|
||||
}
|
||||
if len(parts) > 0 {
|
||||
return HashAssets(parts)
|
||||
}
|
||||
empty := sha256.Sum256([]byte("notarius:fallback-profile-assets:empty"))
|
||||
return "sha256:" + hex.EncodeToString(empty[:]), nil
|
||||
}
|
||||
|
||||
func newAssetSource(fsys fs.FS, root string) (AssetSource, error) {
|
||||
if fsys == nil {
|
||||
return AssetSource{}, fmt.Errorf("filesystem must not be nil")
|
||||
@@ -248,6 +342,9 @@ func (m assetMapFS) dirEntries(dir string) []fs.DirEntry {
|
||||
children[childName] = entry
|
||||
}
|
||||
if len(children) == 0 {
|
||||
if dir == "." {
|
||||
return []fs.DirEntry{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
names := make([]string, 0, len(children))
|
||||
|
||||
@@ -2,6 +2,7 @@ package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
@@ -98,6 +99,89 @@ func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"first/profiles/one.yaml": {Data: []byte("id: one\nmodel: first\n")},
|
||||
}, "first/profiles"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"second/two.yaml": {Data: []byte("id: two\nmodel: second\n")},
|
||||
}, "second"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
fallbackFS, err := registry.FallbackProfileFS()
|
||||
if err != nil {
|
||||
t.Fatalf("FallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
for _, name := range []string{"one.yaml", "two.yaml"} {
|
||||
if _, err := fs.ReadFile(fallbackFS, name); err != nil {
|
||||
t.Fatalf("FallbackProfileFS().ReadFile(%q) error = %v, want nil", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryRejectsInvalidFallbackProfileRoot(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
err := registry.RegisterFallbackProfileFS(fstest.MapFS{}, "../profiles")
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid path") {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want invalid root error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryRejectsUnreadableFallbackProfileAssets(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
if err := registry.RegisterFallbackProfileFS(unreadableAssetFS{}, "."); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
_, err := registry.FallbackProfileFS()
|
||||
if err == nil || !strings.Contains(err.Error(), "permission denied") {
|
||||
t.Fatalf("FallbackProfileFS() error = %v, want unreadable asset error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("id: first\n")}}, "first"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("id: second\n")}}, "second"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := registry.FallbackProfileFS()
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
|
||||
t.Fatalf("FallbackProfileFS() error = %v, want duplicate path error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryFallbackProfileDigestTracksContentWithoutLeakingIt(t *testing.T) {
|
||||
digestFor := func(content string) string {
|
||||
t.Helper()
|
||||
registry := NewAssetRegistry()
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"profiles/profile.yaml": {Data: []byte(content)},
|
||||
}, "profiles"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest, err := registry.FallbackProfileDigest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
first := digestFor("id: fallback\nmodel: model-one\n")
|
||||
second := digestFor("id: fallback\nmodel: model-two\n")
|
||||
if first == second {
|
||||
t.Fatalf("fallback profile digests = %q and %q, want content change", first, second)
|
||||
}
|
||||
if !strings.HasPrefix(first, "sha256:") || strings.Contains(first, "model-one") || strings.Contains(first, "profile.yaml") {
|
||||
t.Fatalf("fallback profile digest leaked source details: %q", first)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryCombinesNamespacedPromptSources(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||
@@ -196,3 +280,9 @@ output:
|
||||
repair_attempts: 0
|
||||
`
|
||||
}
|
||||
|
||||
type unreadableAssetFS struct{}
|
||||
|
||||
func (unreadableAssetFS) Open(name string) (fs.File, error) {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrPermission}
|
||||
}
|
||||
|
||||
@@ -37,12 +37,13 @@ type PromptKitClientConfig struct {
|
||||
}
|
||||
|
||||
type PromptKitClient struct {
|
||||
engine *promptkit.Engine
|
||||
recorder *LLMProfileRecorder
|
||||
profileDir string
|
||||
profileFile string
|
||||
localEndpoint string
|
||||
reasoningEffort *string
|
||||
engine *promptkit.Engine
|
||||
recorder *LLMProfileRecorder
|
||||
profileDir string
|
||||
profileFile string
|
||||
localEndpoint string
|
||||
fallbackProfileDigest string
|
||||
reasoningEffort *string
|
||||
}
|
||||
|
||||
type LLMProfileRecorder struct {
|
||||
@@ -69,7 +70,7 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options, err := cfg.Assets.PromptKitOptions()
|
||||
options, fallbackProfileDigest, err := cfg.Assets.promptKitOptions()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -94,12 +95,13 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
||||
reasoningEffort = &value
|
||||
}
|
||||
return &PromptKitClient{
|
||||
engine: engine,
|
||||
recorder: recorder,
|
||||
profileDir: profileSource.ProfileDir,
|
||||
profileFile: profileSource.ProfileFile,
|
||||
localEndpoint: profileSource.localEndpoint(),
|
||||
reasoningEffort: reasoningEffort,
|
||||
engine: engine,
|
||||
recorder: recorder,
|
||||
profileDir: profileSource.ProfileDir,
|
||||
profileFile: profileSource.ProfileFile,
|
||||
localEndpoint: profileSource.localEndpoint(),
|
||||
fallbackProfileDigest: fallbackProfileDigest,
|
||||
reasoningEffort: reasoningEffort,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -325,7 +327,7 @@ func (c *PromptKitClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint,
|
||||
if c == nil {
|
||||
return nil, nil
|
||||
}
|
||||
fingerprint, err := promptKitProfileFingerprint(c.profileDir, c.profileFile)
|
||||
fingerprint, err := promptKitProfileFingerprint(c.profileDir, c.profileFile, c.fallbackProfileDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -431,6 +431,84 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing.T) {
|
||||
assets := newTestPromptKitAssets(t)
|
||||
const profileID = "fallback-profile"
|
||||
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"profiles/fallback.yaml": {Data: []byte("id: " + profileID + "\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
|
||||
}, "profiles"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: assets,
|
||||
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
||||
}
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
ProfileID: profileID,
|
||||
SessionID: "fallback-profile-test",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
||||
}
|
||||
if response.ProfileID != profileID || response.Model != "fallback-model" {
|
||||
t.Fatalf("completion response = %#v, want fallback profile", response)
|
||||
}
|
||||
|
||||
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Assets: assets})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitProfileInspector() error = %v, want nil", err)
|
||||
}
|
||||
inspection, err := inspector.InspectProfile(context.Background(), profileID)
|
||||
if err != nil {
|
||||
t.Fatalf("InspectProfile() error = %v, want nil", err)
|
||||
}
|
||||
if inspection.ProfileID != profileID || inspection.Model != "fallback-model" {
|
||||
t.Fatalf("profile inspection = %#v, want fallback profile", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *testing.T) {
|
||||
fingerprintFor := func(content string) CheckpointFingerprint {
|
||||
t.Helper()
|
||||
assets := newTestPromptKitAssets(t)
|
||||
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"profiles/fallback.yaml": {Data: []byte(content)},
|
||||
}, "profiles"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: assets})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fingerprints, err := client.LLMCheckpointFingerprints()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitProfileFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want profile source identity", fingerprints)
|
||||
}
|
||||
return fingerprints[0]
|
||||
}
|
||||
|
||||
first := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-one\n")
|
||||
second := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-two\n")
|
||||
if first == second {
|
||||
t.Fatalf("checkpoint fingerprints = %#v and %#v, want fallback asset change", first, second)
|
||||
}
|
||||
if strings.Contains(first.Value, "model-one") || strings.Contains(first.Value, "fallback.yaml") {
|
||||
t.Fatalf("checkpoint fingerprint leaked fallback source details: %#v", first)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing.T) {
|
||||
const (
|
||||
firstEndpoint = "http://localhost:8000/v1"
|
||||
|
||||
@@ -20,9 +20,10 @@ const (
|
||||
promptKitBuiltinProfileCatalogID = "promptkit:v0.5.0:builtin-profiles"
|
||||
)
|
||||
|
||||
func promptKitProfileFingerprint(profileDir, profileFile string) (CheckpointFingerprint, error) {
|
||||
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
|
||||
hasher := sha256.New()
|
||||
writeFingerprintPart(hasher, []byte(promptKitBuiltinProfileCatalogID))
|
||||
writeFingerprintPart(hasher, []byte(strings.TrimSpace(fallbackProfileDigest)))
|
||||
|
||||
switch {
|
||||
case strings.TrimSpace(profileFile) != "":
|
||||
|
||||
@@ -15,6 +15,11 @@ type PromptKitProfileSourceConfig struct {
|
||||
LocalBackend *PromptKitLocalBackendConfig
|
||||
}
|
||||
|
||||
type PromptKitProfileInspectorConfig struct {
|
||||
Source PromptKitProfileSourceConfig
|
||||
Assets *AssetRegistry
|
||||
}
|
||||
|
||||
func (c PromptKitProfileSourceConfig) localEndpoint() string {
|
||||
if c.LocalBackend == nil {
|
||||
return ""
|
||||
@@ -50,11 +55,20 @@ func (e *PromptKitProfileInspectionError) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
func NewPromptKitProfileInspector(cfg PromptKitProfileSourceConfig) (*PromptKitProfileInspector, error) {
|
||||
source, options, err := promptKitProfileSourceEngineOptions(cfg)
|
||||
func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) {
|
||||
source, options, err := promptKitProfileSourceEngineOptions(cfg.Source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.Assets != nil {
|
||||
fallbackOption, hasFallback, err := cfg.Assets.promptKitFallbackProfileOption()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if hasFallback {
|
||||
options = append(options, fallbackOption)
|
||||
}
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: ".",
|
||||
ProfileDir: source.ProfileDir,
|
||||
|
||||
Reference in New Issue
Block a user